Merge pull request #15376 from overleaf/td-ide-page-working-editor

React IDE page: working editor

GitOrigin-RevId: 3ba8cb787a6f7f8435686d8962adb7444d09acb5

services/web/config/settings.defaults.js

@@ -836,6 +836,7 @@ module.exports = {
     sourceEditorExtensions: [],
     sourceEditorComponents: [],
     sourceEditorCompletionSources: [],
+    sourceEditorSymbolPalette: [],
     integrationLinkingWidgets: [],
     referenceLinkingWidgets: [],
     importProjectFromGithubModalWrapper: [],

services/web/frontend/js/features/editor-left-menu/context/project-settings-context.tsx

@@ -1,5 +1,4 @@
-import { createContext, useContext, useMemo } from 'react'
+import { createContext, FC, useContext, useMemo } from 'react'
-import type { PropsWithChildren } from 'react'
 import useProjectWideSettings from '../hooks/use-project-wide-settings'
 import useUserWideSettings from '../hooks/use-user-wide-settings'
 import useProjectWideSettingsSocketListener from '../hooks/use-project-wide-settings-socket-listener'
@@ -36,9 +35,7 @@ export const ProjectSettingsContext = createContext<
   ProjectSettingsContextValue | undefined
 >(undefined)
 
-export function ProjectSettingsProvider({
+export const ProjectSettingsProvider: FC = ({ children }) => {
-  children,
-}: PropsWithChildren<Record<string, never>>) {
   const {
     compiler,
     setCompiler,

services/web/frontend/js/features/ide-react/components/alerts/alerts.tsx

@@ -3,6 +3,7 @@ import { LostConnectionAlert } from './lost-connection-alert'
 import { useConnectionContext } from '@/features/ide-react/context/connection-context'
 import { debugging } from '@/utils/debugging'
 import { Alert } from 'react-bootstrap'
+import useScopeValue from '@/shared/hooks/use-scope-value'
 
 // TODO SavingNotificationController, SystemMessagesController, out-of-sync modal
 export function Alerts() {
@@ -15,8 +16,7 @@ export function Alerts() {
     secondsUntilReconnect,
   } = useConnectionContext()
 
-  // TODO: Get this from a context
+  const [synctexError] = useScopeValue('sync_tex_error')
-  const synctexError = false
 
   return (
     <div className="global-alerts">

services/web/frontend/js/features/ide-react/components/editor-and-sidebar.tsx

@@ -0,0 +1,69 @@
+import React, { useCallback, useState } from 'react'
+import TwoColumnMainContent from '@/features/ide-react/components/layout/two-column-main-content'
+import Editor from '@/features/ide-react/components/editor/editor'
+import EditorSidebar from '@/features/ide-react/components/editor-sidebar'
+import customLocalStorage from '@/infrastructure/local-storage'
+import { useProjectContext } from '@/shared/context/project-context'
+import { FileTreeFindResult } from '@/features/ide-react/types/file-tree'
+
+type EditorAndSidebarProps = {
+  shouldPersistLayout: boolean
+  leftColumnDefaultSize: number
+  setLeftColumnDefaultSize: React.Dispatch<React.SetStateAction<number>>
+}
+
+export function EditorAndSidebar({
+  shouldPersistLayout,
+  leftColumnDefaultSize,
+  setLeftColumnDefaultSize,
+}: EditorAndSidebarProps) {
+  const [leftColumnIsOpen, setLeftColumnIsOpen] = useState(true)
+  const { rootDocId, _id: projectId } = useProjectContext()
+
+  const [openDocId, setOpenDocId] = useState(
+    () => customLocalStorage.getItem(`doc.open_id.${projectId}`) || rootDocId
+  )
+  const [fileTreeReady, setFileTreeReady] = useState(false)
+
+  const handleFileTreeInit = useCallback(() => {
+    setFileTreeReady(true)
+  }, [])
+
+  const handleFileTreeSelect = useCallback(
+    (selectedEntities: FileTreeFindResult[]) => {
+      const firstDocId =
+        selectedEntities.find(result => result.type === 'doc')?.entity._id ||
+        null
+      setOpenDocId(firstDocId)
+    },
+    []
+  )
+
+  const leftColumnContent = (
+    <EditorSidebar
+      shouldPersistLayout={shouldPersistLayout}
+      onFileTreeInit={handleFileTreeInit}
+      onFileTreeSelect={handleFileTreeSelect}
+    />
+  )
+  const rightColumnContent = (
+    <Editor
+      shouldPersistLayout={shouldPersistLayout}
+      openDocId={openDocId}
+      fileTreeReady={fileTreeReady}
+    />
+  )
+
+  return (
+    <TwoColumnMainContent
+      leftColumnId="editor-left-column"
+      leftColumnContent={leftColumnContent}
+      leftColumnDefaultSize={leftColumnDefaultSize}
+      setLeftColumnDefaultSize={setLeftColumnDefaultSize}
+      rightColumnContent={rightColumnContent}
+      leftColumnIsOpen={leftColumnIsOpen}
+      setLeftColumnIsOpen={setLeftColumnIsOpen}
+      shouldPersistLayout={shouldPersistLayout}
+    />
+  )
+}

services/web/frontend/js/features/ide-react/components/editor-sidebar.tsx

@@ -0,0 +1,34 @@
+import React from 'react'
+import { Panel, PanelGroup } from 'react-resizable-panels'
+import { VerticalResizeHandle } from '@/features/ide-react/components/resize/vertical-resize-handle'
+import { FileTree } from '@/features/ide-react/components/file-tree'
+import { FileTreeSelectHandler } from '@/features/ide-react/types/file-tree'
+
+type EditorSidebarProps = {
+  shouldPersistLayout: boolean
+  onFileTreeInit: () => void
+  onFileTreeSelect: FileTreeSelectHandler
+}
+
+export default function EditorSidebar({
+  shouldPersistLayout,
+  onFileTreeInit,
+  onFileTreeSelect,
+}: EditorSidebarProps) {
+  return (
+    <aside className="ide-react-placeholder-editor-sidebar">
+      <PanelGroup
+        autoSaveId={
+          shouldPersistLayout ? 'ide-react-editor-sidebar-layout' : undefined
+        }
+        direction="vertical"
+      >
+        <Panel defaultSize={75} className="ide-react-file-tree-panel">
+          <FileTree onInit={onFileTreeInit} onSelect={onFileTreeSelect} />
+        </Panel>
+        <VerticalResizeHandle />
+        <Panel defaultSize={25}>File outline placeholder</Panel>
+      </PanelGroup>
+    </aside>
+  )
+}

services/web/frontend/js/features/ide-react/components/editor/editor-pane.tsx

@@ -0,0 +1,80 @@
+import { Panel, PanelGroup } from 'react-resizable-panels'
+import { VerticalResizeHandle } from '@/features/ide-react/components/resize/vertical-resize-handle'
+import React, { ElementType, useEffect } from 'react'
+import useScopeValue from '@/shared/hooks/use-scope-value'
+import SourceEditor from '@/features/source-editor/components/source-editor'
+import {
+  EditorScopeValue,
+  useEditorManagerContext,
+} from '@/features/ide-react/context/editor-manager-context'
+import importOverleafModules from '../../../../../macros/import-overleaf-module.macro'
+import { EditorProps } from '@/features/ide-react/components/editor/editor'
+
+const symbolPaletteComponents = importOverleafModules(
+  'sourceEditorSymbolPalette'
+) as { import: { default: ElementType }; path: string }[]
+
+export function EditorPane({
+  shouldPersistLayout,
+  openDocId,
+  fileTreeReady,
+}: EditorProps) {
+  const { openDocId: openDocWithId } = useEditorManagerContext()
+
+  const [editor] = useScopeValue<EditorScopeValue>('editor')
+
+  useEffect(() => {
+    if (!fileTreeReady || !openDocId) {
+      return
+    }
+    openDocWithId(openDocId)
+  }, [fileTreeReady, openDocId, openDocWithId])
+
+  return (
+    <PanelGroup
+      autoSaveId={
+        shouldPersistLayout
+          ? 'ide-react-editor-and-symbol-palette-layout'
+          : undefined
+      }
+      direction="vertical"
+      units="pixels"
+    >
+      <Panel
+        id="editor"
+        order={1}
+        style={{
+          display: 'flex',
+          flexDirection: 'column',
+        }}
+      >
+        {!!editor.sharejs_doc &&
+        !editor.opening &&
+        editor.multiSelectedCount === 0 &&
+        !editor.error_state ? (
+          <SourceEditor />
+        ) : null}
+      </Panel>
+      {editor.showSymbolPalette ? (
+        <>
+          <VerticalResizeHandle id="editor-symbol-palette" />
+          <Panel
+            id="symbol-palette"
+            order={2}
+            defaultSize={250}
+            minSize={250}
+            maxSize={336}
+          >
+            <div className="ide-react-placeholder-symbol-palette">
+              {symbolPaletteComponents.map(
+                ({ import: { default: Component }, path }) => (
+                  <Component key={path} />
+                )
+              )}
+            </div>
+          </Panel>
+        </>
+      ) : null}
+    </PanelGroup>
+  )
+}

services/web/frontend/js/features/ide-react/components/editor/editor.tsx

@@ -0,0 +1,96 @@
+import { useRef } from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+  ImperativePanelHandle,
+  Panel,
+  PanelGroup,
+} from 'react-resizable-panels'
+import { HorizontalResizeHandle } from '@/features/ide-react/components/resize/horizontal-resize-handle'
+import { HorizontalToggler } from '@/features/ide-react/components/resize/horizontal-toggler'
+import useCollapsiblePanel from '@/features/ide-react/hooks/use-collapsible-panel'
+import { useLayoutContext } from '@/shared/context/layout-context'
+import { EditorPane } from '@/features/ide-react/components/editor/editor-pane'
+import PlaceholderFile from '@/features/ide-react/components/layout/placeholder/placeholder-file'
+import PlaceholderPdf from '@/features/ide-react/components/layout/placeholder/placeholder-pdf'
+
+export type EditorProps = {
+  shouldPersistLayout?: boolean
+  openDocId: string | null
+  fileTreeReady: boolean
+}
+
+export default function Editor({
+  shouldPersistLayout = false,
+  openDocId,
+  fileTreeReady,
+}: EditorProps) {
+  const { t } = useTranslation()
+  const { view, pdfLayout, changeLayout } = useLayoutContext()
+
+  const pdfPanelRef = useRef<ImperativePanelHandle>(null)
+  const isDualPane = pdfLayout === 'sideBySide'
+  const editorIsVisible = isDualPane || view === 'editor'
+  const pdfIsOpen = isDualPane || view === 'pdf'
+
+  useCollapsiblePanel(pdfIsOpen, pdfPanelRef)
+
+  if (view === 'file') {
+    return <PlaceholderFile />
+  }
+
+  if (view === 'history') {
+    return null
+  }
+
+  function setPdfIsOpen(isOpen: boolean) {
+    if (isOpen) {
+      changeLayout('sideBySide')
+    } else {
+      changeLayout('flat', 'editor')
+    }
+  }
+
+  return (
+    <PanelGroup
+      autoSaveId={
+        shouldPersistLayout ? 'ide-react-editor-and-pdf-layout' : undefined
+      }
+      direction="horizontal"
+    >
+      {editorIsVisible ? (
+        <Panel id="editor" order={1} defaultSize={50}>
+          <EditorPane
+            shouldPersistLayout={shouldPersistLayout}
+            openDocId={openDocId}
+            fileTreeReady={fileTreeReady}
+          />
+        </Panel>
+      ) : null}
+      {isDualPane ? (
+        <HorizontalResizeHandle>
+          <HorizontalToggler
+            id="editor-pdf"
+            togglerType="east"
+            isOpen={pdfIsOpen}
+            setIsOpen={isOpen => setPdfIsOpen(isOpen)}
+            tooltipWhenOpen={t('tooltip_hide_pdf')}
+            tooltipWhenClosed={t('tooltip_show_pdf')}
+          />
+        </HorizontalResizeHandle>
+      ) : null}
+      {pdfIsOpen ? (
+        <Panel
+          ref={pdfPanelRef}
+          id="pdf"
+          order={2}
+          defaultSize={50}
+          minSize={5}
+          collapsible
+          onCollapse={collapsed => setPdfIsOpen(!collapsed)}
+        >
+          <PlaceholderPdf />
+        </Panel>
+      ) : null}
+    </PanelGroup>
+  )
+}

services/web/frontend/js/features/ide-react/components/file-tree.tsx

@@ -0,0 +1,49 @@
+import FileTreeRoot from '@/features/file-tree/components/file-tree-root'
+import React, { useCallback, useState } from 'react'
+import { useUserContext } from '@/shared/context/user-context'
+import { useReferencesContext } from '@/features/ide-react/context/references-context'
+import { useIdeReactContext } from '@/features/ide-react/context/ide-react-context'
+import { useConnectionContext } from '@/features/ide-react/context/connection-context'
+import { FileTreeSelectHandler } from '@/features/ide-react/types/file-tree'
+import { RefProviders } from '../../../../../types/user'
+
+type FileTreeProps = {
+  onInit: () => void
+  onSelect: FileTreeSelectHandler
+}
+
+export function FileTree({ onInit, onSelect }: FileTreeProps) {
+  const user = useUserContext()
+  const { indexAllReferences } = useReferencesContext()
+  const { setStartedFreeTrial } = useIdeReactContext()
+  const { isConnected } = useConnectionContext()
+
+  const [refProviders, setRefProviders] = useState<RefProviders>(
+    () => user.refProviders || {}
+  )
+
+  function reindexReferences() {
+    indexAllReferences(true)
+  }
+
+  const setRefProviderEnabled = useCallback(
+    (provider: keyof RefProviders, value = true) => {
+      setRefProviders(refProviders => ({ ...refProviders, [provider]: value }))
+    },
+    []
+  )
+
+  return (
+    <div className="file-tree">
+      <FileTreeRoot
+        refProviders={refProviders}
+        reindexReferences={reindexReferences}
+        setRefProviderEnabled={setRefProviderEnabled}
+        setStartedFreeTrial={setStartedFreeTrial}
+        isConnected={isConnected}
+        onInit={onInit}
+        onSelect={onSelect}
+      />
+    </div>
+  )
+}

services/web/frontend/js/features/ide-react/components/header.tsx

@@ -0,0 +1,63 @@
+import React, { useCallback } from 'react'
+import ChatToggleButton from '@/features/editor-navigation-toolbar/components/chat-toggle-button'
+import HistoryToggleButton from '@/features/editor-navigation-toolbar/components/history-toggle-button'
+import LayoutDropdownButton from '@/features/editor-navigation-toolbar/components/layout-dropdown-button'
+import MenuButton from '@/features/editor-navigation-toolbar/components/menu-button'
+import { useLayoutContext } from '@/shared/context/layout-context'
+import { sendMB } from '@/infrastructure/event-tracking'
+import OnlineUsersWidget from '@/features/editor-navigation-toolbar/components/online-users-widget'
+import { useOnlineUsersContext } from '@/features/ide-react/context/online-users-context'
+
+type HeaderProps = {
+  chatIsOpen: boolean
+  setChatIsOpen: (chatIsOpen: boolean) => void
+  historyIsOpen: boolean
+  setHistoryIsOpen: (historyIsOpen: boolean) => void
+}
+
+export default function Header({
+  chatIsOpen,
+  setChatIsOpen,
+  historyIsOpen,
+  setHistoryIsOpen,
+}: HeaderProps) {
+  const { setLeftMenuShown } = useLayoutContext()
+  const { onlineUsersArray } = useOnlineUsersContext()
+
+  function toggleChatOpen() {
+    setChatIsOpen(!chatIsOpen)
+  }
+
+  function toggleHistoryOpen() {
+    setHistoryIsOpen(!historyIsOpen)
+  }
+
+  const handleShowLeftMenuClick = useCallback(() => {
+    sendMB('navigation-clicked-menu')
+    setLeftMenuShown(value => !value)
+  }, [setLeftMenuShown])
+
+  return (
+    <header className="toolbar toolbar-header">
+      <div className="toolbar-left">
+        <MenuButton onClick={handleShowLeftMenuClick} />
+      </div>
+      <div className="toolbar-right">
+        <OnlineUsersWidget
+          onlineUsers={onlineUsersArray}
+          goToUser={() => alert('Not implemented')}
+        />
+        <LayoutDropdownButton />
+        <HistoryToggleButton
+          historyIsOpen={historyIsOpen}
+          onClick={toggleHistoryOpen}
+        />
+        <ChatToggleButton
+          chatIsOpen={chatIsOpen}
+          onClick={toggleChatOpen}
+          unreadMessageCount={0}
+        />
+      </div>
+    </header>
+  )
+}

services/web/frontend/js/features/ide-react/components/layout/ide-page.tsx

@@ -1,12 +1,22 @@
-import LayoutWithPlaceholders from '@/features/ide-react/components/layout/layout-with-placeholders'
 import { useConnectionContext } from '@/features/ide-react/context/connection-context'
 import useEventListener from '@/shared/hooks/use-event-listener'
-import { useCallback, useEffect } from 'react'
+import { useCallback, useEffect, useState } from 'react'
 import { Alerts } from '@/features/ide-react/components/alerts/alerts'
+import { useLayoutContext } from '@/shared/context/layout-context'
+import PlaceholderChat from '@/features/ide-react/components/layout/placeholder/placeholder-chat'
+import PlaceholderHistory from '@/features/ide-react/components/layout/placeholder/placeholder-history'
+import MainLayout from '@/features/ide-react/components/layout/main-layout'
+import { EditorAndSidebar } from '@/features/ide-react/components/editor-and-sidebar'
+import Header from '@/features/ide-react/components/header'
+import EditorLeftMenu from '@/features/editor-left-menu/components/editor-left-menu'
+import { useLayoutEventTracking } from '@/features/ide-react/hooks/use-layout-event-tracking'
 
 // This is filled with placeholder content while the real content is migrated
 // away from Angular
 export default function IdePage() {
+  useLayoutEventTracking()
+
+  const [leftColumnDefaultSize, setLeftColumnDefaultSize] = useState(20)
   const { registerUserActivity } = useConnectionContext()
 
   // Inform the connection manager when the user is active
@@ -22,11 +32,50 @@ export default function IdePage() {
     return () => document.body.removeEventListener('click', listener)
   }, [listener])
 
+  const { chatIsOpen, setChatIsOpen, view, setView } = useLayoutContext()
+  const historyIsOpen = view === 'history'
+  const setHistoryIsOpen = useCallback(
+    (historyIsOpen: boolean) => {
+      setView(historyIsOpen ? 'history' : 'editor')
+    },
+    [setView]
+  )
+
+  const headerContent = (
+    <Header
+      chatIsOpen={chatIsOpen}
+      setChatIsOpen={setChatIsOpen}
+      historyIsOpen={historyIsOpen}
+      setHistoryIsOpen={setHistoryIsOpen}
+    />
+  )
+  const chatContent = <PlaceholderChat />
+
+  const mainContent = historyIsOpen ? (
+    <PlaceholderHistory
+      shouldPersistLayout
+      leftColumnDefaultSize={leftColumnDefaultSize}
+      setLeftColumnDefaultSize={setLeftColumnDefaultSize}
+    />
+  ) : (
+    <EditorAndSidebar
+      leftColumnDefaultSize={leftColumnDefaultSize}
+      setLeftColumnDefaultSize={setLeftColumnDefaultSize}
+      shouldPersistLayout
+    />
+  )
+
   return (
     <>
       <Alerts />
-      {/* TODO: Left menu will go here */}
+      <EditorLeftMenu />
-      <LayoutWithPlaceholders shouldPersistLayout />
+      <MainLayout
+        headerContent={headerContent}
+        chatContent={chatContent}
+        mainContent={mainContent}
+        chatIsOpen={chatIsOpen}
+        shouldPersistLayout
+      />
     </>
   )
 }

services/web/frontend/js/features/ide-react/components/layout/placeholder/placeholder-editor-sidebar.tsx

@@ -2,13 +2,13 @@ import React from 'react'
 import { Panel, PanelGroup } from 'react-resizable-panels'
 import { VerticalResizeHandle } from '@/features/ide-react/components/resize/vertical-resize-handle'
 
-type PlaceholderHeaderProps = {
+type PlaceholderEditorSidebarProps = {
   shouldPersistLayout: boolean
 }
 
 export default function PlaceholderEditorSidebar({
   shouldPersistLayout,
-}: PlaceholderHeaderProps) {
+}: PlaceholderEditorSidebarProps) {
   return (
     <aside className="ide-react-placeholder-editor-sidebar">
       <PanelGroup

services/web/frontend/js/features/ide-react/components/layout/placeholder/placeholder-file.tsx

@@ -0,0 +1,5 @@
+import React from 'react'
+
+export default function PlaceholderFile() {
+  return <div className="file-view full-size">File placeholder</div>
+}

services/web/frontend/js/features/ide-react/components/layout/placeholder/placeholder-header.tsx

@@ -1,12 +1,13 @@
 import React from 'react'
 import ChatToggleButton from '@/features/editor-navigation-toolbar/components/chat-toggle-button'
 import HistoryToggleButton from '@/features/editor-navigation-toolbar/components/history-toggle-button'
+import LayoutDropdownButton from '@/features/editor-navigation-toolbar/components/layout-dropdown-button'
 
 type PlaceholderHeaderProps = {
   chatIsOpen: boolean
   setChatIsOpen: (chatIsOpen: boolean) => void
   historyIsOpen: boolean
-  setHistoryIsOpen: (chatIsOpen: boolean) => void
+  setHistoryIsOpen: (historyIsOpen: boolean) => void
 }
 
 export default function PlaceholderHeader({
@@ -27,6 +28,7 @@ export default function PlaceholderHeader({
     <header className="toolbar toolbar-header">
       <div className="toolbar-left">Header placeholder</div>
       <div className="toolbar-right">
+        <LayoutDropdownButton />
         <HistoryToggleButton
           historyIsOpen={historyIsOpen}
           onClick={toggleHistoryOpen}

services/web/frontend/js/features/ide-react/components/layout/placeholder/placeholder-pdf.tsx

@@ -0,0 +1,5 @@
+import React from 'react'
+
+export default function PlaceholderPdf() {
+  return <div>PDF</div>
+}

services/web/frontend/js/features/ide-react/connection/connection-manager.ts

@@ -253,7 +253,7 @@ export class ConnectionManager extends Emitter<Events> {
     return true
   }
 
-  private disconnect() {
+  disconnect() {
     this.changeState({
       ...this.state,
       readyState: WebSocket.CLOSED,

services/web/frontend/js/features/ide-react/connection/editor-watchdog-manager.ts

@@ -0,0 +1,234 @@
+/*
+  Migrated from services/web/frontend/js/ide/connection/EditorWatchdogManager.js
+
+  EditorWatchdogManager is used for end-to-end checks of edits.
+
+
+  The editor UI is backed by Ace and CodeMirrors, which in turn are connected
+   to ShareJs documents in the frontend.
+  Edits propagate from the editor to ShareJs and are send through socket.io
+   and real-time to document-updater.
+  In document-updater edits are integrated into the document history and
+   a confirmation/rejection is sent back to the frontend.
+
+  Along the way things can get lost.
+  We have certain safe-guards in place, but are still getting occasional
+   reports of lost edits.
+
+  EditorWatchdogManager is implementing the basis for end-to-end checks on
+   two levels:
+
+  - local/ShareJsDoc: edits that pass-by a ShareJs document shall get
+     acknowledged eventually.
+  - global: any edits made in the editor shall get acknowledged eventually,
+     independent for which ShareJs document (potentially none) sees it.
+
+  How does this work?
+  ===================
+
+  The global check is using a global EditorWatchdogManager that is available
+   via the angular factory 'ide'.
+  Local/ShareJsDoc level checks will connect to the global instance.
+
+  Each EditorWatchdogManager keeps track of the oldest un-acknowledged edit.
+  When ever a ShareJs document receives an acknowledgement event, a local
+   EditorWatchdogManager will see it and also notify the global instance about
+   it.
+  The next edit cycle will clear the oldest un-acknowledged timestamp in case
+   a new ack has arrived, otherwise it will bark loud! via the timeout handler.
+
+  Scenarios
+  =========
+
+  - User opens the CodeMirror editor
+    - attach global check to new CM instance
+    - detach Ace from the local EditorWatchdogManager
+    - when the frontend attaches the CM instance to ShareJs, we also
+       attach it to the local EditorWatchdogManager
+      - the internal attach process writes the document content to the editor,
+         which in turn emits 'change' events. These event need to be excluded
+         from the watchdog. EditorWatchdogManager.ignoreEditsFor takes care
+         of that.
+  - User opens the Ace editor (again)
+    - (attach global check to the Ace editor, only one copy of Ace is around)
+    - detach local EditorWatchdogManager from CM
+    - likewise with CM, attach Ace to the local EditorWatchdogManager
+  - User makes an edit
+    - the editor will emit a 'change' event
+    - the global EditorWatchdogManager will process it first
+    - the local EditorWatchdogManager will process it next
+  - Document-updater confirms an edit
+    - the local EditorWatchdogManager will process it first, it passes it on to
+    - the global EditorWatchdogManager will process it next
+
+  Time
+  ====
+
+  The delay between edits and acks is measured using a monotonic clock:
+   `performance.now()`.
+  It is agnostic to system clock changes in either direction and timezone
+   changes do not affect it as well.
+  Roughly speaking, it is initialized with `0` when the `window` context is
+   created, before our JS app boots.
+  As per canIUse.com and MDN `performance.now()` is available to all supported
+   Browsers, including IE11.
+  See also: https://caniuse.com/?search=performance.now
+  See also: https://developer.mozilla.org/en-US/docs/Web/API/Performance/now
+ */
+
+import {
+  ChangeDescription,
+  EditorFacade,
+} from '../../source-editor/extensions/realtime'
+import { debugConsole } from '@/utils/debugging'
+
+// TIMEOUT specifies the timeout for edits into a single ShareJsDoc.
+const TIMEOUT = 60 * 1000
+// GLOBAL_TIMEOUT specifies the timeout for edits into any ShareJSDoc.
+const GLOBAL_TIMEOUT = TIMEOUT
+// REPORT_EVERY specifies how often we send events/report errors.
+const REPORT_EVERY = 60 * 1000
+
+const SCOPE_LOCAL = 'ShareJsDoc'
+const SCOPE_GLOBAL = 'global'
+
+type Scope = 'ShareJsDoc' | 'global'
+type Meta = {
+  scope: Scope
+  delay: number
+  lastAck: number
+  lastUnackedEdit: number
+}
+type TimeoutHandler = (meta: Meta) => void
+
+class Reporter {
+  private lastReport: number | null = null
+  private queue: Meta[] = []
+
+  // eslint-disable-next-line no-useless-constructor
+  constructor(private readonly onTimeoutHandler: TimeoutHandler) {}
+
+  private getMetaPreferLocal() {
+    for (const meta of this.queue) {
+      if (meta.scope === SCOPE_LOCAL) {
+        return meta
+      }
+    }
+    return this.queue.pop()
+  }
+
+  onTimeout(meta: Meta) {
+    // Collect all 'meta's for this update.
+    // global arrive before local ones, but we are eager to report local ones.
+    this.queue.push(meta)
+
+    setTimeout(() => {
+      // Another handler processed the 'meta' entry already
+      if (!this.queue.length) return
+
+      const maybeLocalMeta = this.getMetaPreferLocal()
+
+      // Discard other, newly arrived 'meta's
+      this.queue.length = 0
+
+      const now = Date.now()
+      // Do not flood the server with losing-edits events
+      const reportedRecently =
+        this.lastReport !== null && now - this.lastReport < REPORT_EVERY
+      if (!reportedRecently && maybeLocalMeta) {
+        this.lastReport = now
+        this.onTimeoutHandler(maybeLocalMeta)
+      }
+    })
+  }
+}
+
+export default class EditorWatchdogManager {
+  lastAck: number | null = null
+  reporter: Reporter
+  // eslint-disable-next-line no-use-before-define
+  parent?: EditorWatchdogManager
+  scope: Scope
+  timeout: number
+  lastUnackedEdit: number | null
+
+  constructor({
+    parent,
+    onTimeoutHandler,
+  }: {
+    parent?: EditorWatchdogManager
+    onTimeoutHandler?: TimeoutHandler
+  }) {
+    this.scope = parent ? SCOPE_LOCAL : SCOPE_GLOBAL
+    this.timeout = parent ? TIMEOUT : GLOBAL_TIMEOUT
+    this.parent = parent
+    if (parent) {
+      this.reporter = parent.reporter
+    } else if (onTimeoutHandler) {
+      this.reporter = new Reporter(onTimeoutHandler)
+    } else {
+      throw new Error('No parent or onTimeoutHandler')
+    }
+
+    this.lastAck = null
+    this.lastUnackedEdit = null
+  }
+
+  onAck() {
+    this.lastAck = performance.now()
+
+    // bubble up to globalEditorWatchdogManager
+    if (this.parent) this.parent.onAck()
+  }
+
+  onEdit() {
+    // Use timestamps to track the high-water mark of unacked edits
+    const now = performance.now()
+
+    // Discard the last unacked edit if there are now newer acks
+    // TODO Handle cases where lastAck and/or lastUnackedEdit are null more transparently
+    // @ts-ignore
+    if (this.lastAck > this.lastUnackedEdit) {
+      this.lastUnackedEdit = null
+    }
+    // Start tracking for this keypress if we aren't already tracking an
+    //  unacked edit
+    if (!this.lastUnackedEdit) {
+      this.lastUnackedEdit = now
+    }
+
+    // Report an error if the last tracked edit hasn't been cleared by an
+    //  ack from the server after a long time
+    const delay = now - this.lastUnackedEdit
+    if (delay > this.timeout) {
+      const timeOrigin = Date.now() - now
+      const scope = this.scope
+      const lastAck = this.lastAck ? timeOrigin + this.lastAck : 0
+      const lastUnackedEdit = timeOrigin + this.lastUnackedEdit
+      const meta: Meta = { scope, delay, lastAck, lastUnackedEdit }
+      this.log('timedOut', meta)
+      this.reporter.onTimeout(meta)
+    }
+  }
+
+  attachToEditor(editor: EditorFacade) {
+    this.log('attach to editor')
+    const onChange = (
+      _editor: EditorFacade,
+      changeDescription: ChangeDescription
+    ) => {
+      if (changeDescription.origin === 'remote') return
+      if (!(changeDescription.removed || changeDescription.inserted)) return
+      this.onEdit()
+    }
+    editor.on('change', onChange)
+    return () => {
+      this.log('detach from editor')
+      editor.off('change', onChange)
+    }
+  }
+
+  private log(...args: any[]) {
+    debugConsole.log(`[EditorWatchdogManager] ${this.scope}:`, ...args)
+  }
+}

services/web/frontend/js/features/ide-react/connection/types/socket.ts

@@ -21,6 +21,7 @@ export type Socket = {
     callback?: (error: Error, ...data: any[]) => void
   ): void
   socket: {
+    connected: boolean
     connect(): void
   }
   disconnect(): void

services/web/frontend/js/features/ide-react/context/connection-context.tsx

@@ -20,6 +20,7 @@ type ConnectionContextValue = {
   secondsUntilReconnect: () => number
   tryReconnectNow: () => void
   registerUserActivity: () => void
+  disconnect: () => void
 }
 
 const ConnectionContext = createContext<ConnectionContextValue | undefined>(
@@ -64,6 +65,10 @@ export const ConnectionProvider: FC = ({ children }) => {
     [connectionManager]
   )
 
+  const disconnect = useCallback(() => {
+    connectionManager.disconnect()
+  }, [connectionManager])
+
   const value = useMemo<ConnectionContextValue>(
     () => ({
       socket: connectionManager.socket,
@@ -73,6 +78,7 @@ export const ConnectionProvider: FC = ({ children }) => {
       secondsUntilReconnect,
       tryReconnectNow,
       registerUserActivity,
+      disconnect,
     }),
     [
       connectionManager.socket,
@@ -82,6 +88,7 @@ export const ConnectionProvider: FC = ({ children }) => {
       registerUserActivity,
       secondsUntilReconnect,
       tryReconnectNow,
+      disconnect,
     ]
   )
 

services/web/frontend/js/features/ide-react/context/editor-manager-context.tsx

@@ -0,0 +1,597 @@
+/* eslint-disable no-use-before-define */
+import { ReactScopeValueStore } from '@/features/ide-react/scope-value-store/react-scope-value-store'
+import {
+  createContext,
+  FC,
+  useCallback,
+  useContext,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from 'react'
+import customLocalStorage from '@/infrastructure/local-storage'
+import { sendMB } from '@/infrastructure/event-tracking'
+import useScopeValue from '@/shared/hooks/use-scope-value'
+import { useIdeContext } from '@/shared/context/ide-context'
+import { OpenDocuments } from '@/features/ide-react/editor/open-documents'
+import EditorWatchdogManager from '@/features/ide-react/connection/editor-watchdog-manager'
+import { useIdeReactContext } from '@/features/ide-react/context/ide-react-context'
+import { useConnectionContext } from '@/features/ide-react/context/connection-context'
+import { debugConsole } from '@/utils/debugging'
+import { Document } from '@/features/ide-react/editor/document'
+import { useLayoutContext } from '@/shared/context/layout-context'
+import { GotoLineOptions } from '@/features/ide-react/types/goto-line-options'
+import _ from 'lodash'
+import { Doc } from '../../../../../types/doc'
+import { useFileTreeData } from '@/shared/context/file-tree-data-context'
+import { findDocEntityById } from '@/features/ide-react/util/find-doc-entity-by-id'
+import useScopeEventEmitter from '@/shared/hooks/use-scope-event-emitter'
+
+interface GotoOffsetOptions {
+  gotoOffset: number
+}
+
+interface OpenDocOptions
+  extends Partial<GotoLineOptions>,
+    Partial<GotoOffsetOptions> {
+  gotoOffset?: number
+  forceReopen?: boolean
+}
+
+type EditorManager = {
+  getEditorType: () => 'cm6' | 'cm6-rich-text' | null
+  showSymbolPalette: boolean
+  currentDocument: Document
+  getCurrentDocValue: () => string | null
+  getCurrentDocId: () => string | null
+  startIgnoringExternalUpdates: () => void
+  stopIgnoringExternalUpdates: () => void
+  openDocId: (docId: string, options?: OpenDocOptions) => void
+  openDoc: (document: Document, options?: OpenDocOptions) => void
+  jumpToLine: (options: GotoLineOptions) => void
+}
+
+type DocumentOpenCallback = (error: Error | null, document?: Document) => void
+
+function hasGotoLine(options: OpenDocOptions): options is GotoLineOptions {
+  return typeof options.gotoLine === 'number'
+}
+
+function hasGotoOffset(options: OpenDocOptions): options is GotoOffsetOptions {
+  return typeof options.gotoOffset === 'number'
+}
+
+export type EditorScopeValue = {
+  showSymbolPalette: false
+  toggleSymbolPalette: () => void
+  sharejs_doc: Document | null
+  open_doc_id: string | null
+  open_doc_name: string | null
+  opening: boolean
+  trackChanges: boolean
+  wantTrackChanges: boolean
+  showVisual: boolean
+  newSourceEditor: boolean
+  multiSelectedCount: number
+  error_state: boolean
+}
+
+export function populateEditorScope(store: ReactScopeValueStore) {
+  const projectId = window.project_id
+
+  // This value is not used in the React code. It's just here to prevent errors
+  // from EditorProvider
+  store.set('state.loading', false)
+
+  store.set('project.name', null)
+
+  store.set('editor', {
+    showSymbolPalette: false,
+    toggleSymbolPalette: () => {},
+    sharejs_doc: null,
+    open_doc_id: null,
+    open_doc_name: null,
+    opening: true,
+    trackChanges: false,
+    wantTrackChanges: false,
+    // No Ace here
+    newSourceEditor: true,
+    // TODO: Ignore multiSelectedCount and just default it to zero for now until
+    // we get to the file tree. It seems to be stored in two places in the
+    // Angular scope and needs further investigation.
+    multiSelectedCount: 0,
+    error_state: false,
+  })
+  store.persisted('editor.showVisual', false, `editor.mode.${projectId}`, {
+    toPersisted: showVisual => (showVisual ? 'rich-text' : 'source'),
+    fromPersisted: mode => mode === 'rich-text',
+  })
+}
+
+const EditorManagerContext = createContext<EditorManager | undefined>(undefined)
+
+export const EditorManagerProvider: FC = ({ children }) => {
+  const ide = useIdeContext()
+  const { reportError, eventEmitter, eventLog, projectId } =
+    useIdeReactContext()
+  const { socket, disconnect } = useConnectionContext()
+  const { view, setView } = useLayoutContext()
+
+  const [showSymbolPalette, setShowSymbolPalette] = useScopeValue<boolean>(
+    'editor.showSymbolPalette'
+  )
+  const [showVisual] = useScopeValue<boolean>('editor.showVisual')
+  // eslint-disable-next-line no-unused-vars
+  const [currentDocument, setCurrentDocument] =
+    useScopeValue<Document>('editor.sharejs_doc')
+  const [openDocId, setOpenDocId] = useScopeValue<string | null>(
+    'editor.open_doc_id'
+  )
+  const [, setOpenDocName] = useScopeValue<string | null>(
+    'editor.open_doc_name'
+  )
+  const [, setOpening] = useScopeValue<boolean>('editor.opening')
+  const [, setIsInErrorState] = useScopeValue<boolean>('editor.error_state')
+  const [, setTrackChanges] = useScopeValue<boolean>('editor.trackChanges')
+  const [wantTrackChanges] = useScopeValue<boolean>('editor.wantTrackChanges')
+
+  const goToLineEmitter = useScopeEventEmitter('editor:gotoLine')
+
+  const { fileTreeData } = useFileTreeData()
+
+  // eslint-disable-next-line no-unused-vars
+  const [ignoringExternalUpdates, setIgnoringExternalUpdates] = useState(false)
+
+  const [globalEditorWatchdogManager] = useState(
+    () =>
+      new EditorWatchdogManager({
+        onTimeoutHandler: (meta: Record<string, any>) => {
+          sendMB('losing-edits', meta)
+          reportError('losing-edits', meta)
+        },
+      })
+  )
+
+  const [docTooLongErrorShown, setDocTooLongErrorShown] = useState(false)
+
+  const [openDocs] = useState(
+    () =>
+      new OpenDocuments(
+        socket,
+        globalEditorWatchdogManager,
+        eventEmitter,
+        eventLog
+      )
+  )
+
+  const editorOpenDocEpochRef = useRef(0)
+
+  // TODO: This looks dodgy because it wraps a state setter and is itself
+  // stored in React state in the scope store. The problem is that it needs to
+  // be exposed via the scope store because some components access it that way;
+  // it would be better to simply access it from a context, but the current
+  // implementation in EditorManager interacts with Angular scope to update
+  // the layout. Once Angular is gone, this can become a context method.
+  useEffect(() => {
+    ide.scopeStore.set('editor.toggleSymbolPalette', () => {
+      setShowSymbolPalette(show => {
+        const newValue = !show
+        sendMB(newValue ? 'symbol-palette-show' : 'symbol-palette-hide')
+        return newValue
+      })
+    })
+  }, [ide.scopeStore, setShowSymbolPalette])
+
+  const getEditorType = useCallback(() => {
+    if (!currentDocument) {
+      return null
+    }
+
+    return showVisual ? 'cm6-rich-text' : 'cm6'
+  }, [currentDocument, showVisual])
+
+  const getCurrentDocValue = useCallback(() => {
+    return currentDocument?.getSnapshot() ?? null
+  }, [currentDocument])
+
+  const getCurrentDocId = useCallback(() => openDocId, [openDocId])
+
+  const startIgnoringExternalUpdates = useCallback(
+    () => setIgnoringExternalUpdates(true),
+    []
+  )
+  const stopIgnoringExternalUpdates = useCallback(
+    () => setIgnoringExternalUpdates(false),
+    []
+  )
+
+  // Ignore insertSymbol from Angular EditorManager because it's only required
+  // for Ace.
+
+  const jumpToLine = useCallback(
+    (options: GotoLineOptions) => {
+      goToLineEmitter(
+        options.gotoLine,
+        options.gotoColumn ?? 0,
+        options.syncToPdf ?? false
+      )
+    },
+    [goToLineEmitter]
+  )
+
+  const unbindFromDocumentEvents = (document: Document) => {
+    document.off()
+  }
+
+  const openDocWithId = useCallback(
+    (docId: string, options: OpenDocOptions = {}) => {
+      const doc = findDocEntityById(fileTreeData, docId)
+      if (!doc) {
+        return
+      }
+      openDoc(doc, options)
+    },
+    // @ts-ignore
+    [fileTreeData, openDoc]
+  )
+
+  // @ts-ignore
+  const attachErrorHandlerToDocument = useCallback(
+    (doc: Doc, document: Document) => {
+      document.on(
+        'error',
+        (
+          error: Error | string,
+          meta?: Record<string, any>,
+          editorContent?: string
+        ) => {
+          const message =
+            typeof error === 'string' ? error : error?.message ?? ''
+          if (/maxDocLength/.test(message)) {
+            setDocTooLongErrorShown(true)
+            openDoc(doc, { forceReopen: true })
+
+            // TODO: MIGRATION: Show generic modal here
+            // const genericMessageModal = this.ide.showGenericMessageModal(
+            //   'Document Too Long',
+            //   'Sorry, this file is too long to be edited manually. Please upload it directly.'
+            // )
+            // genericMessageModal.result.finally(() => {
+            //   this.$scope.docTooLongErrorShown = false
+            // })
+          } else if (/too many comments or tracked changes/.test(message)) {
+            // TODO: MIGRATION: Show generic modal here
+            // this.ide.showGenericMessageModal(
+            //   'Too many comments or tracked changes',
+            //   'Sorry, this file has too many comments or tracked changes. Please try accepting or rejecting some existing changes, or resolving and deleting some comments.'
+            // )
+          } else if (!docTooLongErrorShown) {
+            // Do not allow this doc to open another error modal.
+            document.off('error')
+
+            // Preserve the sharejs contents before the teardown.
+            editorContent =
+              typeof editorContent === 'string'
+                ? editorContent
+                : document.doc?._doc.snapshot
+
+            // Tear down the ShareJsDoc.
+            if (document.doc) document.doc.clearInflightAndPendingOps()
+
+            // Do not re-join after re-connecting.
+            document.leaveAndCleanUp()
+
+            disconnect()
+            reportError(error, meta)
+
+            // Tell the user about the error state.
+            setIsInErrorState(true)
+
+            // TODO: MIGRATION: Show out-of-sync modal
+            // this.ide.showOutOfSyncModal(
+            //   'Out of sync',
+            //   "Sorry, this file has gone out of sync and we need to do a full refresh. <br> <a target='_blank' rel='noopener noreferrer' href='/learn/Kb/Editor_out_of_sync_problems'>Please see this help guide for more information</a>",
+            //   editorContent
+            // )
+
+            // Do not forceReopen the document.
+            return
+          }
+
+          eventEmitter.once('project:joined', () => {
+            openDoc(doc, { forceReopen: true })
+          })
+        }
+      )
+    },
+    [
+      disconnect,
+      docTooLongErrorShown,
+      eventEmitter,
+      // @ts-ignore
+      openDoc,
+      reportError,
+      setIsInErrorState,
+    ]
+  )
+
+  const bindToDocumentEvents = useCallback(
+    (doc: Doc, document: Document) => {
+      attachErrorHandlerToDocument(doc, document)
+
+      // TODO: MIGRATION: Add a type for `update`
+      document.on('externalUpdate', (update: any) => {
+        if (ignoringExternalUpdates) {
+          return
+        }
+        if (
+          _.property(['meta', 'type'])(update) === 'external' &&
+          _.property(['meta', 'source'])(update) === 'git-bridge'
+        ) {
+          // eslint-disable-next-line no-useless-return
+          return
+        }
+        // TODO: MIGRATION: Show generic modal here
+        // this.ide.showGenericMessageModal(
+        //   'Document Updated Externally',
+        //   'This document was just updated externally. Any recent changes you have made may have been overwritten. To see previous versions please look in the history.'
+        // )
+      })
+    },
+    [attachErrorHandlerToDocument, ignoringExternalUpdates]
+  )
+
+  const syncTimeoutRef = useRef<number | null>(null)
+
+  const syncTrackChangesState = useCallback(
+    (doc: Document) => {
+      if (!doc) {
+        return
+      }
+
+      if (syncTimeoutRef.current) {
+        window.clearTimeout(syncTimeoutRef.current)
+        syncTimeoutRef.current = null
+      }
+
+      const want = wantTrackChanges
+      const have = doc.getTrackingChanges()
+      if (wantTrackChanges === have) {
+        setTrackChanges(want)
+        return
+      }
+
+      const tryToggle = () => {
+        const saved = doc.getInflightOp() == null && doc.getPendingOp() == null
+        if (saved) {
+          doc.setTrackingChanges(wantTrackChanges)
+          setTrackChanges(want)
+        } else {
+          syncTimeoutRef.current = window.setTimeout(tryToggle, 100)
+        }
+      }
+
+      tryToggle()
+    },
+    [setTrackChanges, wantTrackChanges]
+  )
+
+  const doOpenNewDocument = useCallback(
+    (doc: Doc, callback: DocumentOpenCallback) => {
+      debugConsole.log('[doOpenNewDocument] Opening...')
+      const newDocument = openDocs.getDocument(doc._id)
+      if (!newDocument) {
+        debugConsole.error(`No open document with ID '${doc._id}' found`)
+        return
+      }
+      const preJoinEpoch = ++editorOpenDocEpochRef.current
+      newDocument.join(error => {
+        if (error) {
+          debugConsole.log(
+            `[doOpenNewDocument] error joining doc ${doc._id}`,
+            error
+          )
+          callback(error)
+        }
+        if (editorOpenDocEpochRef.current !== preJoinEpoch) {
+          debugConsole.log(
+            `[openNewDocument] editorOpenDocEpoch mismatch ${editorOpenDocEpochRef.current} vs ${preJoinEpoch}`
+          )
+          newDocument.leaveAndCleanUp()
+          return callback(new Error('another document was loaded'))
+        }
+        bindToDocumentEvents(doc, newDocument)
+        return callback(null, newDocument)
+      })
+    },
+    [bindToDocumentEvents, openDocs]
+  )
+
+  // @ts-ignore
+  const openNewDocument = useCallback(
+    (doc: Doc, callback: DocumentOpenCallback) => {
+      // Leave the current document
+      //  - when we are opening a different new one, to avoid race conditions
+      //     between leaving and joining the same document
+      //  - when the current one has pending ops that need flushing, to avoid
+      //     race conditions from cleanup
+      const currentDocId = currentDocument?.doc_id
+      const hasBufferedOps = currentDocument?.hasBufferedOps()
+      const changingDoc = currentDocument && currentDocId !== doc._id
+      if (changingDoc || hasBufferedOps) {
+        debugConsole.log('[openNewDocument] Leaving existing open doc...')
+
+        // Do not trigger any UI changes from remote operations
+        unbindFromDocumentEvents(currentDocument)
+
+        // Keep listening for out-of-sync and similar errors.
+        attachErrorHandlerToDocument(doc, currentDocument)
+
+        // Teardown the Document -> ShareJsDoc -> sharejs doc
+        // By the time this completes, the Document instance is no longer
+        //  registered in OpenDocuments and doOpenNewDocument can start
+        //  from scratch -- read: no corrupted internal state.
+        const preLeaveEpoch = ++editorOpenDocEpochRef.current
+
+        currentDocument.leaveAndCleanUp(error => {
+          if (error) {
+            debugConsole.log(
+              `[_openNewDocument] error leaving doc ${currentDocId}`,
+              error
+            )
+            return callback(error)
+          }
+          if (editorOpenDocEpochRef.current !== preLeaveEpoch) {
+            debugConsole.log(
+              `[openNewDocument] editorOpenDocEpoch mismatch ${editorOpenDocEpochRef.current} vs ${preLeaveEpoch}`
+            )
+            callback(new Error('another document was loaded'))
+          }
+          doOpenNewDocument(doc, callback)
+        })
+      } else {
+        doOpenNewDocument(doc, callback)
+      }
+    },
+    [attachErrorHandlerToDocument, doOpenNewDocument, currentDocument]
+  )
+
+  // @ts-ignore
+  const openDoc = useCallback(
+    (doc: Doc, options: OpenDocOptions = {}) => {
+      debugConsole.log(`[openDoc] Opening ${doc._id}`)
+      if (view === 'editor') {
+        // store position of previous doc before switching docs
+        eventEmitter.emit('store-doc-position')
+      }
+      setView('editor')
+
+      const done = (isNewDoc: boolean) => {
+        window.dispatchEvent(
+          new CustomEvent('doc:after-opened', { detail: isNewDoc })
+        )
+        if (hasGotoLine(options)) {
+          // In CM6, jump to the line again after a stored scroll position has been restored
+          if (isNewDoc) {
+            window.addEventListener(
+              'editor:scroll-position-restored',
+              () => jumpToLine(options),
+              { once: true }
+            )
+          }
+        } else if (hasGotoOffset(options)) {
+          window.setTimeout(() => {
+            eventEmitter.emit('editor:gotoOffset', options.gotoOffset)
+          })
+        }
+      }
+
+      // If we already have the document open we can return at this point.
+      // Note: only use forceReopen:true to override this when the document is
+      // out of sync and needs to be reloaded from the server.
+      if (doc._id === openDocId && !options.forceReopen) {
+        window.dispatchEvent(
+          new CustomEvent('editor.openDoc', { detail: doc._id })
+        )
+        done(false)
+        return
+      }
+
+      // We're now either opening a new document or reloading a broken one.
+      setOpenDocId(doc._id)
+      setOpenDocName(doc.name)
+      customLocalStorage.setItem(`doc.open_id.${projectId}`, doc._id)
+
+      setOpening(true)
+
+      openNewDocument(doc, (error: Error | null, document: Document) => {
+        if (error?.message === 'another document was loaded') {
+          debugConsole.log(
+            `[openDoc] another document was loaded while ${doc._id} was loading`
+          )
+          return
+        }
+        if (error != null) {
+          // TODO: MIGRATION: Show generic modal here
+          // this.ide.showGenericMessageModal(
+          //   'Error opening document',
+          //   'Sorry, something went wrong opening this document. Please try again.'
+          // )
+          return
+        }
+
+        syncTrackChangesState(document)
+
+        eventEmitter.emit('doc:opened')
+
+        setOpening(false)
+        setCurrentDocument(document)
+        done(true)
+      })
+    },
+    [
+      eventEmitter,
+      jumpToLine,
+      openDocId,
+      openNewDocument,
+      projectId,
+      setCurrentDocument,
+      setOpenDocId,
+      setOpenDocName,
+      setOpening,
+      setView,
+      syncTrackChangesState,
+      view,
+    ]
+  )
+
+  const editorManager = useMemo(
+    () => ({
+      getEditorType,
+      showSymbolPalette,
+      currentDocument,
+      getCurrentDocValue,
+      getCurrentDocId,
+      startIgnoringExternalUpdates,
+      stopIgnoringExternalUpdates,
+      openDocId: openDocWithId,
+      openDoc,
+      jumpToLine,
+    }),
+    [
+      getEditorType,
+      showSymbolPalette,
+      currentDocument,
+      getCurrentDocValue,
+      getCurrentDocId,
+      startIgnoringExternalUpdates,
+      stopIgnoringExternalUpdates,
+      openDocWithId,
+      openDoc,
+      jumpToLine,
+    ]
+  )
+
+  // Expose editorManager via ide object because some React code relies on it,
+  // for now
+  ide.editorManager = editorManager
+
+  return (
+    <EditorManagerContext.Provider value={editorManager}>
+      {children}
+    </EditorManagerContext.Provider>
+  )
+}
+
+export function useEditorManagerContext(): EditorManager {
+  const context = useContext(EditorManagerContext)
+
+  if (!context) {
+    throw new Error(
+      'useEditorManagerContext is only available inside EditorManagerProvider'
+    )
+  }
+
+  return context
+}

services/web/frontend/js/features/ide-react/context/ide-react-context.tsx

@@ -1,12 +1,14 @@
-import {
+import React, {
   createContext,
   useContext,
   useState,
   FC,
   useMemo,
   useEffect,
+  useCallback,
 } from 'react'
 import { ReactScopeValueStore } from '@/features/ide-react/scope-value-store/react-scope-value-store'
+import populateLayoutScope from '@/features/ide-react/scope-adapters/layout-context-adapter'
 import { IdeProvider } from '@/shared/context/ide-context'
 import {
   createIdeEventEmitter,
@@ -15,17 +17,37 @@ import {
 import { JoinProjectPayload } from '@/features/ide-react/connection/join-project-payload'
 import { useConnectionContext } from '@/features/ide-react/context/connection-context'
 import { getMockIde } from '@/shared/context/mock/mock-ide'
+import { populateEditorScope } from '@/features/ide-react/context/editor-manager-context'
+import { debugConsole } from '@/utils/debugging'
+import { postJSON } from '@/infrastructure/fetch-json'
+import { EventLog } from '@/features/ide-react/editor/event-log'
+import { populateSettingsScope } from '@/features/ide-react/scope-adapters/settings-adapter'
+import { populateOnlineUsersScope } from '@/features/ide-react/context/online-users-context'
 import { ReactScopeEventEmitter } from '@/features/ide-react/scope-event-emitter/react-scope-event-emitter'
+import { populateReferenceScope } from '@/features/ide-react/context/references-context'
 
 type IdeReactContextValue = {
   projectId: string
   eventEmitter: IdeEventEmitter
+  eventLog: EventLog
+  startedFreeTrial: boolean
+  setStartedFreeTrial: React.Dispatch<
+    React.SetStateAction<IdeReactContextValue['startedFreeTrial']>
+  >
+  reportError: (error: any, meta?: Record<string, any>) => void
 }
 
-const IdeReactContext = createContext<IdeReactContextValue | null>(null)
+const IdeReactContext = createContext<IdeReactContextValue | undefined>(
+  undefined
+)
+
+function showGenericMessageModal(title: string, message: string) {
+  debugConsole.log('*** showGenericMessageModal ***', title, message)
+}
 
 function populateIdeReactScope(store: ReactScopeValueStore) {
   store.set('sync_tex_error', false)
+  store.set('settings', window.userSettings)
 }
 
 function populateProjectScope(store: ReactScopeValueStore) {
@@ -33,6 +55,14 @@ function populateProjectScope(store: ReactScopeValueStore) {
   store.set('permissionsLevel', 'readOnly')
 }
 
+function populatePdfScope(store: ReactScopeValueStore) {
+  store.allowNonExistentPath('pdf', true)
+}
+
+function populateFileTreeScope(store: ReactScopeValueStore) {
+  store.set('docs', [])
+}
+
 function createReactScopeValueStore() {
   const scopeStore = new ReactScopeValueStore()
 
@@ -42,7 +72,17 @@ function createReactScopeValueStore() {
   // initialization code together with the context and would only populate
   // necessary values in the store, but this is simpler for now
   populateIdeReactScope(scopeStore)
+  populateEditorScope(scopeStore)
+  populateLayoutScope(scopeStore)
   populateProjectScope(scopeStore)
+  populatePdfScope(scopeStore)
+  populateSettingsScope(scopeStore)
+  populateOnlineUsersScope(scopeStore)
+  populateReferenceScope(scopeStore)
+  populateFileTreeScope(scopeStore)
+
+  scopeStore.allowNonExistentPath('hasLintingError')
+  scopeStore.allowNonExistentPath('loadingThreads')
 
   return scopeStore
 }
@@ -55,24 +95,42 @@ export const IdeReactProvider: FC = ({ children }) => {
   const [scopeEventEmitter] = useState(
     () => new ReactScopeEventEmitter(eventEmitter)
   )
+  const [eventLog] = useState(() => new EventLog())
+  const [startedFreeTrial, setStartedFreeTrial] = useState(false)
 
   const { socket } = useConnectionContext()
 
-  // Fire project:joined event
+  const reportError = useCallback(
-  useEffect(() => {
+    (error: any, meta?: Record<string, any>) => {
-    function handleJoinProjectResponse({
+      const metadata = {
-      project,
+        ...meta,
-      permissionsLevel,
+        user_id: window.user_id,
-    }: JoinProjectPayload) {
+        project_id: projectId,
-      eventEmitter.emit('project:joined', { project, permissionsLevel })
+        // @ts-ignore
-    }
+        client_id: socket.socket.sessionid,
+        // @ts-ignore
+        transport: socket.socket.transport,
+        client_now: new Date(),
+      }
 
-    socket.on('joinProjectResponse', handleJoinProjectResponse)
+      const errorObj: Record<string, any> = {}
-
+      if (typeof error === 'object') {
-    return () => {
+        for (const key of Object.getOwnPropertyNames(error)) {
-      socket.removeListener('joinProjectResponse', handleJoinProjectResponse)
+          errorObj[key] = error[key]
-    }
+        }
-  }, [socket, eventEmitter])
+      } else if (typeof error === 'string') {
+        errorObj.message = error
+      }
+      return postJSON('/error/client', {
+        body: {
+          error: errorObj,
+          meta: metadata,
+          _csrf: window.csrfToken,
+        },
+      })
+    },
+    [socket.socket]
+  )
 
   // Populate scope values when joining project, then fire project:joined event
   useEffect(() => {
@@ -98,15 +156,28 @@ export const IdeReactProvider: FC = ({ children }) => {
     return {
       ...getMockIde(),
       socket,
+      showGenericMessageModal,
+      reportError,
+      // TODO: MIGRATION: Remove this once it's no longer used
+      fileTreeManager: {
+        findEntityByPath: () => null,
+        selectEntity: () => {},
+        getPreviewByPath: () => null,
+        getRootDocDirname: () => '',
+      },
     }
-  }, [socket])
+  }, [socket, reportError])
 
   const value = useMemo(
     () => ({
       eventEmitter,
+      eventLog,
+      startedFreeTrial,
+      setStartedFreeTrial,
       projectId,
+      reportError,
     }),
-    [eventEmitter]
+    [eventEmitter, eventLog, reportError, startedFreeTrial]
   )
 
   return (

services/web/frontend/js/features/ide-react/context/metadata-context.tsx

@@ -0,0 +1,228 @@
+import {
+  createContext,
+  useContext,
+  useEffect,
+  FC,
+  useCallback,
+  useMemo,
+  useState,
+  useRef,
+} from 'react'
+import { useIdeReactContext } from '@/features/ide-react/context/ide-react-context'
+import { useConnectionContext } from '@/features/ide-react/context/connection-context'
+import { useEditorManagerContext } from '@/features/ide-react/context/editor-manager-context'
+import _ from 'lodash'
+import { getJSON, postJSON } from '@/infrastructure/fetch-json'
+import { useOnlineUsersContext } from '@/features/ide-react/context/online-users-context'
+import { useEditorContext } from '@/shared/context/editor-context'
+import { useIdeContext } from '@/shared/context/ide-context'
+import useSocketListener from '@/features/ide-react/hooks/use-socket-listener'
+import useEventListener from '@/shared/hooks/use-event-listener'
+
+type DocumentMetadata = {
+  labels: string[]
+  packages: Record<string, any>
+}
+
+type DocumentsMetadata = Record<string, DocumentMetadata>
+
+type MetadataContextValue = {
+  metadata: {
+    state: {
+      documents: DocumentsMetadata
+    }
+    getAllLabels: () => DocumentMetadata['labels']
+    getAllPackages: () => DocumentMetadata['packages']
+  }
+}
+
+type DocMetadataResponse = { docId: string; meta: DocumentMetadata }
+
+const MetadataContext = createContext<MetadataContextValue | undefined>(
+  undefined
+)
+
+export const MetadataProvider: FC = ({ children }) => {
+  const ide = useIdeContext()
+  const { eventEmitter, projectId } = useIdeReactContext()
+  const { socket } = useConnectionContext()
+  const { onlineUsersCount } = useOnlineUsersContext()
+  const { permissionsLevel } = useEditorContext()
+  const { currentDocument } = useEditorManagerContext()
+
+  const [documents, setDocuments] = useState<DocumentsMetadata>({})
+
+  const debouncerRef = useRef<Map<string, number>>(new Map()) // DocId => Timeout
+
+  useEffect(() => {
+    eventEmitter.on('entity:deleted', entity => {
+      if (entity.type === 'doc') {
+        setDocuments(documents => _.omit(documents, entity.id))
+      }
+    })
+  }, [eventEmitter])
+
+  useEffect(() => {
+    window.dispatchEvent(
+      new CustomEvent('project:metadata', { detail: documents })
+    )
+  }, [documents])
+
+  const onBroadcastDocMeta = useCallback((data: DocMetadataResponse) => {
+    const { docId, meta } = data
+    if (docId != null && meta != null) {
+      setDocuments(documents => ({ ...documents, [docId]: meta }))
+    }
+  }, [])
+
+  const getAllLabels = useCallback(
+    () =>
+      _.flattenDeep(
+        Array.from(Object.values(documents)).map(meta => meta.labels)
+      ),
+    [documents]
+  )
+
+  const getAllPackages = useCallback(() => {
+    const packageCommandMapping: Record<string, any> = {}
+    for (const meta of Object.values(documents)) {
+      for (const [packageName, commandSnippets] of Object.entries(
+        meta.packages
+      )) {
+        packageCommandMapping[packageName] = commandSnippets
+      }
+    }
+    return packageCommandMapping
+  }, [documents])
+
+  const loadProjectMetaFromServer = useCallback(() => {
+    getJSON(`/project/${projectId}/metadata`).then(
+      (response: { projectMeta: DocumentsMetadata }) => {
+        const { projectMeta } = response
+        if (projectMeta) {
+          setDocuments(projectMeta)
+        }
+      }
+    )
+  }, [projectId])
+
+  const loadDocMetaFromServer = useCallback(
+    (docId: string) => {
+      // Don't broadcast metadata when there are no other users in the
+      // project.
+      const broadcast = onlineUsersCount > 0
+      postJSON(`/project/${projectId}/doc/${docId}/metadata`, {
+        body: {
+          broadcast,
+        },
+      }).then((response: DocMetadataResponse) => {
+        if (!broadcast && response) {
+          // handle the POST response like a broadcast event when there are no
+          // other users in the project.
+          onBroadcastDocMeta(response)
+        }
+      })
+    },
+    [onBroadcastDocMeta, onlineUsersCount, projectId]
+  )
+
+  const scheduleLoadDocMetaFromServer = useCallback(
+    (docId: string) => {
+      if (permissionsLevel === 'readOnly') {
+        // The POST request is blocked for users without write permission.
+        // The user will not be able to consume the metadata for edits anyway.
+        return
+      }
+      // Debounce loading labels with a timeout
+      const existingTimeout = debouncerRef.current.get(docId)
+
+      if (existingTimeout != null) {
+        window.clearTimeout(existingTimeout)
+        debouncerRef.current.delete(docId)
+      }
+
+      debouncerRef.current.set(
+        docId,
+        window.setTimeout(() => {
+          // TODO: wait for the document to be saved?
+          loadDocMetaFromServer(docId)
+          debouncerRef.current.delete(docId)
+        }, 2000)
+      )
+    },
+    [loadDocMetaFromServer, permissionsLevel]
+  )
+
+  const handleBroadcastDocMeta = useCallback(
+    (data: DocMetadataResponse) => {
+      onBroadcastDocMeta(data)
+    },
+    [onBroadcastDocMeta]
+  )
+
+  useSocketListener(socket, 'broadcastDocMeta', handleBroadcastDocMeta)
+
+  const handleMetadataOutdated = useCallback(() => {
+    if (currentDocument) {
+      scheduleLoadDocMetaFromServer(currentDocument.doc_id)
+    }
+  }, [currentDocument, scheduleLoadDocMetaFromServer])
+
+  useEventListener('editor:metadata-outdated', handleMetadataOutdated)
+
+  useEffect(() => {
+    eventEmitter.once('project:joined', ({ project }) => {
+      if (project.deletedByExternalDataSource) {
+        // TODO: MIGRATION: Show generic message modal here
+        /*
+        ide.showGenericMessageModal(
+          'Project Renamed or Deleted',
+          `\
+This project has either been renamed or deleted by an external data source such as Dropbox.
+We don't want to delete your data on Overleaf, so this project still contains your history and collaborators.
+If the project has been renamed please look in your project list for a new project under the new name.\
+`
+        )
+*/
+      }
+      window.setTimeout(() => {
+        if (permissionsLevel !== 'readOnly') {
+          loadProjectMetaFromServer()
+        }
+      }, 200)
+    })
+  }, [eventEmitter, loadProjectMetaFromServer, permissionsLevel])
+
+  const value = useMemo<MetadataContextValue>(
+    () => ({
+      metadata: {
+        state: { documents },
+        getAllLabels,
+        getAllPackages,
+      },
+    }),
+    [documents, getAllLabels, getAllPackages]
+  )
+
+  // Expose metadataManager via ide object because useCodeMirrorScope relies on
+  // it, for now
+  ide.metadataManager = value
+
+  return (
+    <MetadataContext.Provider value={value}>
+      {children}
+    </MetadataContext.Provider>
+  )
+}
+
+export function useMetadataContext(): MetadataContextValue {
+  const context = useContext(MetadataContext)
+
+  if (!context) {
+    throw new Error(
+      'useMetadataContext is only available inside MetadataProvider'
+    )
+  }
+
+  return context
+}

services/web/frontend/js/features/ide-react/context/online-users-context.tsx

@@ -0,0 +1,297 @@
+import {
+  createContext,
+  useContext,
+  useEffect,
+  FC,
+  useCallback,
+  useMemo,
+  useState,
+} from 'react'
+import { ReactScopeValueStore } from '@/features/ide-react/scope-value-store/react-scope-value-store'
+import { useIdeReactContext } from '@/features/ide-react/context/ide-react-context'
+import { useConnectionContext } from '@/features/ide-react/context/connection-context'
+import useScopeValue from '@/shared/hooks/use-scope-value'
+import ColorManager from '@/ide/colors/ColorManager'
+import { CursorPosition } from '@/features/ide-react/types/cursor-position'
+import { omit } from 'lodash'
+import { Doc } from '../../../../../types/doc'
+import { useFileTreeData } from '@/shared/context/file-tree-data-context'
+import { findDocEntityById } from '@/features/ide-react/util/find-doc-entity-by-id'
+import useSocketListener from '@/features/ide-react/hooks/use-socket-listener'
+
+type OnlineUser = {
+  id: string
+  user_id: string
+  email: string
+  name: string
+  initial?: string
+  doc_id?: string
+  doc?: Doc | null
+  row?: number
+  column?: number
+}
+
+type ConnectedUser = {
+  client_id: string
+  user_id: string
+  email: string
+  first_name: string
+  last_name: string
+  cursorData?: {
+    doc_id: string
+    row: number
+    column: number
+  }
+}
+
+type CursorHighlight = {
+  label: string
+  cursor: {
+    row: number
+    column: number
+  }
+  hue: number
+}
+
+type OnlineUsersContextValue = {
+  onlineUsers: Record<string, OnlineUser>
+  onlineUserCursorHighlights: Record<string, CursorHighlight[]>
+  onlineUsersArray: OnlineUser[]
+  onlineUsersCount: number
+}
+
+export function populateOnlineUsersScope(store: ReactScopeValueStore) {
+  store.set('onlineUsers', {})
+  store.set('onlineUserCursorHighlights', {})
+  store.set('onlineUsersArray', [])
+  store.set('onlineUsersCount', 0)
+}
+
+const OnlineUsersContext = createContext<OnlineUsersContextValue | undefined>(
+  undefined
+)
+
+export const OnlineUsersProvider: FC = ({ children }) => {
+  const { eventEmitter } = useIdeReactContext()
+  const { socket } = useConnectionContext()
+  const [openDocId] = useScopeValue<string | null>('editor.open_doc_id')
+  const { fileTreeData } = useFileTreeData()
+
+  const [onlineUsers, setOnlineUsers] =
+    useScopeValue<OnlineUsersContextValue['onlineUsers']>('onlineUsers')
+  const [onlineUserCursorHighlights, setOnlineUserCursorHighlights] =
+    useScopeValue<OnlineUsersContextValue['onlineUserCursorHighlights']>(
+      'onlineUserCursorHighlights'
+    )
+  const [onlineUsersArray, setOnlineUsersArray] =
+    useScopeValue<OnlineUsersContextValue['onlineUsersArray']>(
+      'onlineUsersArray'
+    )
+  const [onlineUsersCount, setOnlineUsersCount] =
+    useScopeValue<OnlineUsersContextValue['onlineUsersCount']>(
+      'onlineUsersCount'
+    )
+
+  const [currentPosition, setCurrentPosition] = useState<CursorPosition | null>(
+    null
+  )
+  const [cursorUpdateInterval, setCursorUpdateInterval] = useState(500)
+
+  const calculateValues = useCallback(
+    (onlineUsers: OnlineUsersContextValue['onlineUsers']) => {
+      const decoratedOnlineUsers: OnlineUsersContextValue['onlineUsers'] = {}
+      const onlineUsersArray: OnlineUser[] = []
+      const onlineUserCursorHighlights: OnlineUsersContextValue['onlineUserCursorHighlights'] =
+        {}
+
+      for (const [clientId, user] of Object.entries(onlineUsers)) {
+        const decoratedUser = { ...user }
+        const docId = user.doc_id
+        if (docId) {
+          decoratedUser.doc = findDocEntityById(fileTreeData, docId)
+        }
+
+        // If the user's name is empty use their email as display name
+        // Otherwise they're probably an anonymous user
+        if (user.name === null || user.name.trim().length === 0) {
+          decoratedUser.name = user.email ? user.email.trim() : 'Anonymous'
+        }
+
+        decoratedUser.initial = user.name?.[0]
+        if (!decoratedUser.initial || decoratedUser.initial === ' ') {
+          decoratedUser.initial = '?'
+        }
+
+        onlineUsersArray.push(decoratedUser)
+        decoratedOnlineUsers[clientId] = decoratedUser
+
+        if (docId == null || user.row == null || user.column == null) {
+          continue
+        }
+        if (!onlineUserCursorHighlights[docId]) {
+          onlineUserCursorHighlights[docId] = []
+        }
+        onlineUserCursorHighlights[docId].push({
+          label: user.name,
+          cursor: {
+            row: user.row,
+            column: user.column,
+          },
+          hue: ColorManager.getHueForUserId(user.user_id),
+        })
+      }
+
+      const cursorUpdateInterval =
+        onlineUsersArray.length > 0 ? 500 : 60 * 1000 * 5
+
+      return {
+        onlineUsers: decoratedOnlineUsers,
+        onlineUsersArray,
+        onlineUserCursorHighlights,
+        cursorUpdateInterval,
+      }
+    },
+    [fileTreeData]
+  )
+
+  const setAllValues = useCallback(
+    (newOnlineUsers: OnlineUsersContextValue['onlineUsers']) => {
+      const values = calculateValues(newOnlineUsers)
+      setOnlineUsers(values.onlineUsers)
+      setOnlineUsersArray(values.onlineUsersArray)
+      setOnlineUsersCount(values.onlineUsersArray.length)
+      setOnlineUserCursorHighlights(values.onlineUserCursorHighlights)
+      setCursorUpdateInterval(values.cursorUpdateInterval)
+    },
+    [
+      calculateValues,
+      setOnlineUserCursorHighlights,
+      setOnlineUsers,
+      setOnlineUsersArray,
+      setOnlineUsersCount,
+    ]
+  )
+
+  useEffect(() => {
+    const handleProjectJoined = () => {
+      socket.emit(
+        'clientTracking.getConnectedUsers',
+        // eslint-disable-next-line n/handle-callback-err
+        (error: Error, connectedUsers: ConnectedUser[]) => {
+          // TODO: MIGRATION: Handle error (although the original code doesn't)
+          const newOnlineUsers: OnlineUsersContextValue['onlineUsers'] = {}
+          for (const user of connectedUsers) {
+            if (user.client_id === socket.publicId) {
+              // Don't store myself
+              continue
+            }
+            // Store data in the same format returned by clientTracking.clientUpdated
+            newOnlineUsers[user.client_id] = {
+              id: user.client_id,
+              user_id: user.user_id,
+              email: user.email,
+              name: `${user.first_name} ${user.last_name}`,
+              doc_id: user.cursorData?.doc_id,
+              row: user.cursorData?.row,
+              column: user.cursorData?.column,
+            }
+          }
+          setAllValues(newOnlineUsers)
+        }
+      )
+    }
+    eventEmitter.on('project:joined', handleProjectJoined)
+
+    return () => {
+      eventEmitter.off('project:joined', handleProjectJoined)
+    }
+  }, [eventEmitter, setAllValues, setOnlineUsers, socket])
+
+  // Track the position of the main cursor
+  useEffect(() => {
+    const handleCursorUpdate = (position: CursorPosition | null) => {
+      if (position) {
+        setCurrentPosition(position)
+      }
+    }
+
+    eventEmitter.on('cursor:editor:update', handleCursorUpdate)
+
+    return () => {
+      eventEmitter.off('cursor:editor:update', handleCursorUpdate)
+    }
+  }, [cursorUpdateInterval, eventEmitter])
+
+  // Send the latest position to other clients when currentPosition changes
+  useEffect(() => {
+    const timer = window.setTimeout(() => {
+      socket.emit('clientTracking.updatePosition', {
+        row: currentPosition?.row,
+        column: currentPosition?.column,
+        doc_id: openDocId,
+      })
+    }, cursorUpdateInterval)
+
+    return () => {
+      window.clearTimeout(timer)
+    }
+  }, [currentPosition, cursorUpdateInterval, openDocId, socket])
+
+  const handleClientUpdated = useCallback(
+    (client: OnlineUser) => {
+      // Check it's not me!
+      if (client.id !== socket.publicId) {
+        setAllValues({ ...onlineUsers, [client.id]: client })
+      }
+    },
+    [onlineUsers, setAllValues, socket.publicId]
+  )
+
+  useSocketListener(socket, 'clientTracking.clientUpdated', handleClientUpdated)
+
+  const handleClientDisconnected = useCallback(
+    (clientId: string) => {
+      setAllValues(omit(onlineUsers, clientId))
+    },
+    [onlineUsers, setAllValues]
+  )
+
+  useSocketListener(
+    socket,
+    'clientTracking.clientDisconnected',
+    handleClientDisconnected
+  )
+
+  const value = useMemo<OnlineUsersContextValue>(
+    () => ({
+      onlineUsers,
+      onlineUsersArray,
+      onlineUserCursorHighlights,
+      onlineUsersCount,
+    }),
+    [
+      onlineUsers,
+      onlineUsersArray,
+      onlineUserCursorHighlights,
+      onlineUsersCount,
+    ]
+  )
+
+  return (
+    <OnlineUsersContext.Provider value={value}>
+      {children}
+    </OnlineUsersContext.Provider>
+  )
+}
+
+export function useOnlineUsersContext(): OnlineUsersContextValue {
+  const context = useContext(OnlineUsersContext)
+
+  if (!context) {
+    throw new Error(
+      'useOnlineUsersContext is only available inside OnlineUsersProvider'
+    )
+  }
+
+  return context
+}

services/web/frontend/js/features/ide-react/context/react-context-root.tsx

@@ -1,17 +1,58 @@
-import { ConnectionProvider } from './connection-context'
 import { FC } from 'react'
+import { ConnectionProvider } from './connection-context'
 import { IdeReactProvider } from '@/features/ide-react/context/ide-react-context'
+import { LayoutProvider } from '@/shared/context/layout-context'
+import { DetachProvider } from '@/shared/context/detach-context'
 import { ProjectProvider } from '@/shared/context/project-context'
+import { ProjectSettingsProvider } from '@/features/editor-left-menu/context/project-settings-context'
+import { EditorProvider } from '@/shared/context/editor-context'
 import { UserProvider } from '@/shared/context/user-context'
+import { EditorManagerProvider } from '@/features/ide-react/context/editor-manager-context'
+import { FileTreeDataProvider } from '@/shared/context/file-tree-data-context'
+import { DetachCompileProvider } from '@/shared/context/detach-compile-context'
+import { ChatProvider } from '@/features/chat/context/chat-context'
+import { LocalCompileProvider } from '@/shared/context/local-compile-context'
+import { OnlineUsersProvider } from '@/features/ide-react/context/online-users-context'
+import { MetadataProvider } from '@/features/ide-react/context/metadata-context'
+import { ReferencesProvider } from '@/features/ide-react/context/references-context'
+import { SplitTestProvider } from '@/shared/context/split-test-context'
 
 export const ReactContextRoot: FC = ({ children }) => {
   return (
-    <ConnectionProvider>
+    <SplitTestProvider>
-      <IdeReactProvider>
+      <ConnectionProvider>
-        <UserProvider>
+        <IdeReactProvider>
-          <ProjectProvider>{children}</ProjectProvider>
+          <UserProvider>
-        </UserProvider>
+            <ProjectProvider>
-      </IdeReactProvider>
+              <FileTreeDataProvider>
-    </ConnectionProvider>
+                <ReferencesProvider>
+                  <DetachProvider>
+                    <EditorProvider>
+                      <ProjectSettingsProvider>
+                        <LayoutProvider>
+                          <LocalCompileProvider>
+                            <DetachCompileProvider>
+                              <ChatProvider>
+                                <EditorManagerProvider>
+                                  <OnlineUsersProvider>
+                                    <MetadataProvider>
+                                      {children}
+                                    </MetadataProvider>
+                                  </OnlineUsersProvider>
+                                </EditorManagerProvider>
+                              </ChatProvider>
+                            </DetachCompileProvider>
+                          </LocalCompileProvider>
+                        </LayoutProvider>
+                      </ProjectSettingsProvider>
+                    </EditorProvider>
+                  </DetachProvider>
+                </ReferencesProvider>
+              </FileTreeDataProvider>
+            </ProjectProvider>
+          </UserProvider>
+        </IdeReactProvider>
+      </ConnectionProvider>
+    </SplitTestProvider>
   )
 }

services/web/frontend/js/features/ide-react/context/references-context.tsx

@@ -0,0 +1,197 @@
+// @ts-ignore
+import CryptoJSSHA1 from 'crypto-js/sha1'
+import {
+  createContext,
+  useContext,
+  useEffect,
+  FC,
+  useCallback,
+  useMemo,
+  useState,
+} from 'react'
+import { useIdeReactContext } from '@/features/ide-react/context/ide-react-context'
+import { useConnectionContext } from '@/features/ide-react/context/connection-context'
+import _ from 'lodash'
+import { postJSON } from '@/infrastructure/fetch-json'
+import { ShareJsDoc } from '@/features/ide-react/editor/share-js-doc'
+import useScopeValue from '@/shared/hooks/use-scope-value'
+import { ReactScopeValueStore } from '@/features/ide-react/scope-value-store/react-scope-value-store'
+import { useFileTreeData } from '@/shared/context/file-tree-data-context'
+import { findDocEntityById } from '@/features/ide-react/util/find-doc-entity-by-id'
+
+type References = {
+  keys: string[]
+}
+
+type ReferencesContextValue = {
+  indexReferencesIfDocModified: (
+    doc: ShareJsDoc,
+    shouldBroadcast: boolean
+  ) => void
+  indexReferences: (docIds: string[], shouldBroadcast: boolean) => void
+  indexAllReferences: (shouldBroadcast: boolean) => void
+}
+
+type IndexReferencesResponse = References
+
+const ReferencesContext = createContext<ReferencesContextValue | undefined>(
+  undefined
+)
+
+export function populateReferenceScope(store: ReactScopeValueStore) {
+  store.set('$root._references', { keys: [] })
+}
+
+export const ReferencesProvider: FC = ({ children }) => {
+  const { fileTreeData } = useFileTreeData()
+  const { eventEmitter, projectId } = useIdeReactContext()
+  const { socket } = useConnectionContext()
+
+  const [references, setReferences] =
+    useScopeValue<References>('$root._references')
+
+  const [existingIndexHash, setExistingIndexHash] = useState<
+    Record<string, { hash: string; timestamp: number }>
+  >({})
+
+  const storeReferencesKeys = useCallback(
+    (newKeys: string[], replaceExistingKeys: boolean) => {
+      const oldKeys = references.keys
+      const keys = replaceExistingKeys ? newKeys : _.union(oldKeys, newKeys)
+      window.dispatchEvent(
+        new CustomEvent('project:references', {
+          detail: keys,
+        })
+      )
+      setReferences({ keys })
+    },
+    [references.keys, setReferences]
+  )
+
+  const indexReferences = useCallback(
+    (docIds: string[], shouldBroadcast: boolean) => {
+      postJSON(`/project/${projectId}/references/index`, {
+        body: {
+          docIds,
+          shouldBroadcast,
+        },
+      }).then((response: IndexReferencesResponse) => {
+        storeReferencesKeys(response.keys, false)
+      })
+    },
+    [projectId, storeReferencesKeys]
+  )
+
+  const indexAllReferences = useCallback(
+    (shouldBroadcast: boolean) => {
+      postJSON(`/project/${projectId}/references/indexAll`, {
+        body: {
+          shouldBroadcast,
+        },
+      }).then((response: IndexReferencesResponse) => {
+        storeReferencesKeys(response.keys, true)
+      })
+    },
+    [projectId, storeReferencesKeys]
+  )
+
+  const indexReferencesIfDocModified = useCallback(
+    (doc: ShareJsDoc, shouldBroadcast: boolean) => {
+      // avoid reindexing references if the bib file has not changed since the
+      // last time they were indexed
+      const docId = doc.doc_id
+      const snapshot = doc._doc.snapshot
+      const now = Date.now()
+      const sha1 = CryptoJSSHA1(
+        'blob ' + snapshot.length + '\x00' + snapshot
+      ).toString()
+      const CACHE_LIFETIME = 6 * 3600 * 1000 // allow reindexing every 6 hours
+      const cacheEntry = existingIndexHash[docId]
+      const isCached =
+        cacheEntry &&
+        cacheEntry.timestamp > now - CACHE_LIFETIME &&
+        cacheEntry.hash === sha1
+      if (!isCached) {
+        indexReferences([docId], shouldBroadcast)
+        setExistingIndexHash(existingIndexHash => ({
+          ...existingIndexHash,
+          [docId]: { hash: sha1, timestamp: now },
+        }))
+      }
+    },
+    [existingIndexHash, indexReferences]
+  )
+
+  useEffect(() => {
+    const handleDocClosed = (doc: ShareJsDoc) => {
+      if (
+        doc.doc_id &&
+        findDocEntityById(fileTreeData, doc.doc_id)?.name?.endsWith('.bib')
+      ) {
+        indexReferencesIfDocModified(doc, true)
+      }
+    }
+
+    eventEmitter.on('document:closed', handleDocClosed)
+
+    return () => {
+      eventEmitter.off('document:closed', handleDocClosed)
+    }
+  }, [eventEmitter, fileTreeData, indexReferencesIfDocModified])
+
+  useEffect(() => {
+    const handleShouldReindex = () => {
+      indexAllReferences(true)
+    }
+
+    eventEmitter.on('references:should-reindex', handleShouldReindex)
+
+    return () => {
+      eventEmitter.off('references:should-reindex', handleShouldReindex)
+    }
+  }, [eventEmitter, indexAllReferences])
+
+  useEffect(() => {
+    const handleProjectJoined = () => {
+      // We only need to grab the references when the editor first loads,
+      // not on every reconnect
+      socket.on('references:keys:updated', (keys, allDocs) =>
+        storeReferencesKeys(keys, allDocs)
+      )
+      indexAllReferences(false)
+    }
+
+    eventEmitter.once('project:joined', handleProjectJoined)
+
+    return () => {
+      eventEmitter.off('project:joined', handleProjectJoined)
+    }
+  }, [eventEmitter, indexAllReferences, socket, storeReferencesKeys])
+
+  const value = useMemo<ReferencesContextValue>(
+    () => ({
+      indexReferencesIfDocModified,
+      indexReferences,
+      indexAllReferences,
+    }),
+    [indexReferencesIfDocModified, indexReferences, indexAllReferences]
+  )
+
+  return (
+    <ReferencesContext.Provider value={value}>
+      {children}
+    </ReferencesContext.Provider>
+  )
+}
+
+export function useReferencesContext(): ReferencesContextValue {
+  const context = useContext(ReferencesContext)
+
+  if (!context) {
+    throw new Error(
+      'useReferencesContext is only available inside ReferencesProvider'
+    )
+  }
+
+  return context
+}

services/web/frontend/js/features/ide-react/create-ide-event-emitter.ts

@@ -1,19 +1,33 @@
 import { Emitter } from 'strict-event-emitter'
 import { Project } from '../../../../types/project'
 import { PermissionsLevel } from '@/features/ide-react/types/permissions-level'
+import { ShareJsDoc } from '@/features/ide-react/editor/share-js-doc'
 import { GotoLineOptions } from '@/features/ide-react/types/goto-line-options'
 import { CursorPosition } from '@/features/ide-react/types/cursor-position'
 
 export type IdeEvents = {
   'project:joined': [{ project: Project; permissionsLevel: PermissionsLevel }]
 
+  // TODO: MIGRATION: This doesn't seem to be used. Investigate whether it can be removed
+  'document:opened': [doc: ShareJsDoc]
+
+  'document:closed': [doc: ShareJsDoc]
+  'doc:changed': [{ doc_id: string }]
+  'doc:saved': [{ doc_id: string }]
+  'doc:opened': []
+  'ide:opAcknowledged': [{ doc_id: string; op: any }]
+  'store-doc-position': []
   'editor:gotoOffset': [gotoOffset: number]
   'editor:gotoLine': [options: GotoLineOptions]
-  'outline-toggled': [isOpen: boolean]
   'cursor:editor:update': [position: CursorPosition]
+  'outline-toggled': [isOpen: boolean]
   'cursor:editor:syncToPdf': []
   'scroll:editor:update': []
   'comment:start_adding': []
+  'references:should-reindex': []
+
+  // TODO: MIGRATION: Create a proper type for entity when migrating the file tree
+  'entity:deleted': [entity: Record<string, any>]
 }
 
 export type IdeEventEmitter = Emitter<IdeEvents>

services/web/frontend/js/features/ide-react/editor/document.ts

@@ -0,0 +1,706 @@
+/* eslint-disable
+    camelcase,
+    n/handle-callback-err,
+    max-len,
+*/
+// Migrated from services/web/frontend/js/ide/editor/Document.js
+
+// @ts-ignore
+import RangesTracker from '@overleaf/ranges-tracker'
+import { ShareJsDoc } from './share-js-doc'
+import { debugConsole } from '@/utils/debugging'
+import { Socket } from '@/features/ide-react/connection/types/socket'
+import { IdeEventEmitter } from '@/features/ide-react/create-ide-event-emitter'
+import { EditorFacade } from '@/features/source-editor/extensions/realtime'
+import { EventLog } from '@/features/ide-react/editor/event-log'
+import EditorWatchdogManager from '@/features/ide-react/connection/editor-watchdog-manager'
+import EventEmitter from '@/utils/EventEmitter'
+import {
+  AnyOperation,
+  Change,
+  CommentOperation,
+} from '../../../../../types/change'
+import {
+  isCommentOperation,
+  isDeleteOperation,
+  isInsertOperation,
+} from '@/utils/operations'
+import { decodeUtf8 } from '@/utils/decode-utf8'
+import {
+  ShareJsOperation,
+  TrackChangesIdSeeds,
+  Version,
+} from '@/features/ide-react/editor/types/document'
+
+const MAX_PENDING_OP_SIZE = 64
+
+type JoinCallback = (error?: Error) => void
+type LeaveCallback = JoinCallback
+
+type Update = Record<string, any>
+
+type Message = {
+  meta: {
+    tc: string
+    user_id: string
+  }
+}
+
+type ErrorMetadata = Record<string, any>
+
+function getOpSize(op: AnyOperation) {
+  if (isInsertOperation(op)) {
+    return op.i.length
+  }
+  if (isDeleteOperation(op)) {
+    return op.d.length
+  }
+  return 0
+}
+
+function getShareJsOpSize(shareJsOp: ShareJsOperation) {
+  return shareJsOp.reduce((total, op) => total + getOpSize(op), 0)
+}
+
+export class Document extends EventEmitter {
+  private connected: boolean
+  private wantToBeJoined = false
+  private chaosMonkeyTimer: number | null = null
+  private track_changes_as: string | null = null
+
+  private joinCallbacks: JoinCallback[] = []
+  private leaveCallbacks: LeaveCallback[] = []
+
+  doc?: ShareJsDoc
+  cm6?: EditorFacade
+  oldInflightOp?: ShareJsOperation
+  ranges: RangesTracker
+  joined = false
+
+  // This is set and read in useCodeMirrorScope
+  docName = ''
+
+  constructor(
+    readonly doc_id: string,
+    readonly socket: Socket,
+    private readonly globalEditorWatchdogManager: EditorWatchdogManager,
+    private readonly ideEventEmitter: IdeEventEmitter,
+    private readonly eventLog: EventLog
+  ) {
+    super()
+    this.connected = this.socket.socket.connected
+    this.bindToEditorEvents()
+    this.bindToSocketEvents()
+  }
+
+  attachToCM6(cm6: EditorFacade) {
+    this.cm6 = cm6
+    if (this.doc) {
+      this.doc.attachToCM6(this.cm6)
+    }
+    if (this.cm6) {
+      this.cm6.on('change', this.checkConsistency)
+    }
+    if (this.doc) {
+      this.ideEventEmitter.emit('document:opened', this.doc)
+    }
+  }
+
+  detachFromCM6() {
+    if (this.doc) {
+      this.doc.detachFromCM6()
+    }
+    if (this.cm6) {
+      this.cm6.off('change', this.checkConsistency)
+    }
+    delete this.cm6
+    this.clearChaosMonkey()
+    if (this.doc) {
+      this.ideEventEmitter.emit('document:closed', this.doc)
+    }
+  }
+
+  submitOp(...ops: AnyOperation[]) {
+    this.doc?.submitOp(ops)
+  }
+
+  private checkConsistency = (editor: EditorFacade) => {
+    // We've been seeing a lot of errors when I think there shouldn't be
+    // any, which may be related to this check happening before the change is
+    // applied. If we use a timeout, hopefully we can reduce this.
+    window.setTimeout(() => {
+      const editorValue = editor?.getValue()
+      const sharejsValue = this.doc?.getSnapshot()
+      if (editorValue !== sharejsValue) {
+        return this.onError(
+          new Error('Editor text does not match server text'),
+          {},
+          editorValue
+        )
+      }
+    }, 0)
+  }
+
+  getSnapshot() {
+    return this.doc?.getSnapshot()
+  }
+
+  getType() {
+    return this.doc?.getType()
+  }
+
+  getInflightOp(): ShareJsOperation | undefined {
+    return this.doc?.getInflightOp()
+  }
+
+  getPendingOp(): ShareJsOperation | undefined {
+    return this.doc?.getPendingOp()
+  }
+
+  getRecentAck() {
+    return this.doc?.getRecentAck()
+  }
+
+  hasBufferedOps() {
+    return this.doc?.hasBufferedOps()
+  }
+
+  setTrackingChanges(track_changes: boolean) {
+    if (this.doc) {
+      this.doc.track_changes = track_changes
+    }
+  }
+
+  getTrackingChanges() {
+    return !!this.doc?.track_changes
+  }
+
+  setTrackChangesIdSeeds(id_seeds: TrackChangesIdSeeds) {
+    if (this.doc) {
+      this.doc.track_changes_id_seeds = id_seeds
+    }
+  }
+
+  private onUpdateAppliedHandler = (update: any) => this.onUpdateApplied(update)
+
+  // TODO: MIGRATION: Create proper types for error and message
+  private onErrorHandler = (error: Error, message: { doc_id: string }) => {
+    // 'otUpdateError' are emitted per doc socket.io room, hence we can be
+    //  sure that message.doc_id exists.
+    if (message.doc_id !== this.doc_id) {
+      // This error is for another doc. Do not action it. We could open
+      //  a modal that has the wrong context on it.
+      return
+    }
+    this.onError(error, message)
+  }
+
+  private onDisconnectHandler = () => this.onDisconnect()
+
+  private bindToSocketEvents() {
+    this.socket.on('otUpdateApplied', this.onUpdateAppliedHandler)
+    this.socket.on('otUpdateError', this.onErrorHandler)
+    return this.socket.on('disconnect', this.onDisconnectHandler)
+  }
+
+  private unBindFromSocketEvents() {
+    this.socket.removeListener('otUpdateApplied', this.onUpdateAppliedHandler)
+    this.socket.removeListener('otUpdateError', this.onErrorHandler)
+    return this.socket.removeListener('disconnect', this.onDisconnectHandler)
+  }
+
+  private bindToEditorEvents() {
+    this.ideEventEmitter.on('project:joined', this.onReconnect)
+  }
+
+  private unBindFromEditorEvents() {
+    this.ideEventEmitter.off('project:joined', this.onReconnect)
+  }
+
+  leaveAndCleanUp(cb?: (error?: Error) => void) {
+    return this.leave((error?: Error) => {
+      this.cleanUp()
+      if (cb) cb(error)
+    })
+  }
+
+  join(callback?: JoinCallback) {
+    this.wantToBeJoined = true
+    this.cancelLeave()
+    if (this.connected) {
+      this.joinDoc(callback)
+    } else if (callback) {
+      this.joinCallbacks.push(callback)
+    }
+  }
+
+  leave(callback?: LeaveCallback) {
+    this.flush() // force an immediate flush when leaving document
+    this.wantToBeJoined = false
+    this.cancelJoin()
+    if (this.doc?.hasBufferedOps()) {
+      debugConsole.log(
+        '[leave] Doc has buffered ops, pushing callback for later'
+      )
+      if (callback) {
+        this.leaveCallbacks.push(callback)
+      }
+    } else if (!this.connected) {
+      debugConsole.log('[leave] Not connected, returning now')
+      callback?.()
+    } else {
+      debugConsole.log('[leave] Leaving now')
+      this.leaveDoc(callback)
+    }
+  }
+
+  flush() {
+    return this.doc?.flushPendingOps()
+  }
+
+  chaosMonkey(line = 0, char = 'a') {
+    const orig = char
+    let copy: string | null = null
+    let pos = 0
+    const timer = () => {
+      if (copy == null || !copy.length) {
+        copy = orig.slice() + ' ' + new Date() + '\n'
+        line += Math.random() > 0.1 ? 1 : -2
+        if (line < 0) {
+          line = 0
+        }
+        pos = 0
+      }
+      char = copy[0]
+      copy = copy.slice(1)
+      if (this.cm6) {
+        this.cm6.view.dispatch({
+          changes: {
+            from: Math.min(pos, this.cm6.view.state.doc.length),
+            insert: char,
+          },
+        })
+      }
+      pos += 1
+      this.chaosMonkeyTimer = window.setTimeout(
+        timer,
+        100 + (Math.random() < 0.1 ? 1000 : 0)
+      )
+    }
+    timer()
+  }
+
+  clearChaosMonkey() {
+    const timer = this.chaosMonkeyTimer
+    if (timer) {
+      this.chaosMonkeyTimer = null
+      window.clearTimeout(timer)
+    }
+  }
+
+  pollSavedStatus() {
+    // returns false if doc has ops waiting to be acknowledged or
+    // sent that haven't changed since the last time we checked.
+    // Otherwise returns true.
+    let saved
+    const inflightOp = this.getInflightOp()
+    const pendingOp = this.getPendingOp()
+    const recentAck = this.getRecentAck()
+    const pendingOpSize = pendingOp ? getShareJsOpSize(pendingOp) : 0
+    if (inflightOp == null && pendingOp == null) {
+      // There's nothing going on, this is OK.
+      saved = true
+      debugConsole.log('[pollSavedStatus] no inflight or pending ops')
+    } else if (inflightOp && inflightOp === this.oldInflightOp) {
+      // The same inflight op has been sitting unacked since we
+      // last checked, this is bad.
+      saved = false
+      debugConsole.log('[pollSavedStatus] inflight op is same as before')
+    } else if (
+      pendingOp != null &&
+      recentAck &&
+      pendingOpSize < MAX_PENDING_OP_SIZE
+    ) {
+      // There is an op waiting to go to server but it is small and
+      // within the flushDelay, this is OK for now.
+      saved = true
+      debugConsole.log(
+        '[pollSavedStatus] pending op (small with recent ack) assume ok',
+        pendingOp,
+        pendingOpSize
+      )
+    } else {
+      // In any other situation, assume the document is unsaved.
+      saved = false
+      debugConsole.log(
+        `[pollSavedStatus] assuming not saved (inflightOp?: ${
+          inflightOp != null
+        }, pendingOp?: ${pendingOp != null})`
+      )
+    }
+
+    this.oldInflightOp = inflightOp
+    return saved
+  }
+
+  private cancelLeave() {
+    this.leaveCallbacks = []
+  }
+
+  private cancelJoin() {
+    this.joinCallbacks = []
+  }
+
+  private onUpdateApplied(update: Update) {
+    this.eventLog.pushEvent('received-update', {
+      doc_id: this.doc_id,
+      remote_doc_id: update?.doc,
+      wantToBeJoined: this.wantToBeJoined,
+      update,
+      hasDoc: !!this.doc,
+    })
+
+    if (update?.doc === this.doc_id && this.doc != null) {
+      this.eventLog.pushEvent('received-update:processing', {
+        update,
+      })
+      // FIXME: change this back to processUpdateFromServer when redis fixed
+      this.doc.processUpdateFromServerInOrder(update)
+
+      if (!this.wantToBeJoined) {
+        return this.leave()
+      }
+    }
+  }
+
+  private onDisconnect() {
+    debugConsole.log('[onDisconnect] disconnecting')
+    this.connected = false
+    this.joined = false
+    return this.doc != null
+      ? this.doc.updateConnectionState('disconnected')
+      : undefined
+  }
+
+  private onReconnect = () => {
+    debugConsole.log('[onReconnect] reconnected (joined project)')
+    this.eventLog.pushEvent('reconnected:afterJoinProject')
+
+    this.connected = true
+    if (this.wantToBeJoined || this.doc?.hasBufferedOps()) {
+      debugConsole.log(
+        `[onReconnect] Rejoining (wantToBeJoined: ${
+          this.wantToBeJoined
+        } OR hasBufferedOps: ${this.doc?.hasBufferedOps()})`
+      )
+      this.joinDoc((error?: Error) => {
+        if (error) {
+          this.onError(error)
+          return
+        }
+        this.doc?.updateConnectionState('ok')
+        this.doc?.flushPendingOps()
+        this.callJoinCallbacks()
+      })
+    }
+  }
+
+  private callJoinCallbacks() {
+    for (const callback of this.joinCallbacks) {
+      callback()
+    }
+    this.joinCallbacks = []
+  }
+
+  private joinDoc(callback?: JoinCallback) {
+    if (this.doc) {
+      this.eventLog.pushEvent('joinDoc:existing', {
+        doc_id: this.doc_id,
+        version: this.doc.getVersion(),
+      })
+      return this.socket.emit(
+        'joinDoc',
+        this.doc_id,
+        this.doc.getVersion(),
+        { encodeRanges: true },
+        (error, docLines, version, updates, ranges) => {
+          if (error) {
+            callback?.(error)
+            return
+          }
+          this.joined = true
+          this.doc?.catchUp(updates)
+          this.decodeRanges(ranges)
+          this.catchUpRanges(ranges?.changes, ranges?.comments)
+          callback?.()
+        }
+      )
+    } else {
+      this.eventLog.pushEvent('joinDoc:new', {
+        doc_id: this.doc_id,
+      })
+      this.socket.emit(
+        'joinDoc',
+        this.doc_id,
+        { encodeRanges: true },
+        (error, docLines, version, updates, ranges) => {
+          if (error) {
+            callback?.(error)
+            return
+          }
+          this.joined = true
+          this.eventLog.pushEvent('joinDoc:inited', {
+            doc_id: this.doc_id,
+            version,
+          })
+          this.doc = new ShareJsDoc(
+            this.doc_id,
+            docLines,
+            version,
+            this.socket,
+            this.globalEditorWatchdogManager,
+            this.ideEventEmitter,
+            this.eventLog
+          )
+          this.decodeRanges(ranges)
+          this.ranges = new RangesTracker(ranges?.changes, ranges?.comments)
+          this.bindToShareJsDocEvents()
+          callback?.()
+        }
+      )
+    }
+  }
+
+  private decodeRanges(ranges: RangesTracker) {
+    try {
+      if (ranges.changes) {
+        for (const change of ranges.changes) {
+          if (isInsertOperation(change.op)) {
+            change.op.i = decodeUtf8(change.op.i)
+          }
+          if (isDeleteOperation(change.op)) {
+            change.op.d = decodeUtf8(change.op.d)
+          }
+        }
+      }
+      return (() => {
+        if (!ranges.comments) {
+          return []
+        }
+        return ranges.comments.map((comment: Change<CommentOperation>) =>
+          comment.op.c != null
+            ? (comment.op.c = decodeUtf8(comment.op.c))
+            : undefined
+        )
+      })()
+    } catch (err) {
+      debugConsole.error(err)
+    }
+  }
+
+  private leaveDoc(callback?: LeaveCallback) {
+    this.eventLog.pushEvent('leaveDoc', {
+      doc_id: this.doc_id,
+    })
+    debugConsole.log('[leaveDoc] Sending leaveDoc request')
+    this.socket.emit('leaveDoc', this.doc_id, error => {
+      if (error) {
+        callback?.(error)
+        return
+      }
+      this.joined = false
+      for (const leaveCallback of this.leaveCallbacks) {
+        debugConsole.log('[_leaveDoc] Calling buffered callback', leaveCallback)
+        leaveCallback(error)
+      }
+      this.leaveCallbacks = []
+      callback?.()
+    })
+  }
+
+  cleanUp() {
+    // if we arrive here from _onError the pending and inflight ops will have been cleared
+    if (this.hasBufferedOps()) {
+      debugConsole.log(
+        `[cleanUp] Document (${this.doc_id}) has buffered ops, refusing to remove from openDocs`
+      )
+      return // return immediately, do not unbind from events
+    } else {
+      this.emit('detach', this.doc_id)
+    }
+    this.unBindFromEditorEvents()
+    this.unBindFromSocketEvents()
+  }
+
+  private bindToShareJsDocEvents() {
+    if (!this.doc) {
+      return
+    }
+
+    this.doc.on('error', (error: Error, meta: ErrorMetadata) =>
+      this.onError(error, meta)
+    )
+    this.doc.on('externalUpdate', (update: Update) => {
+      this.eventLog.pushEvent('externalUpdate', { doc_id: this.doc_id })
+      return this.trigger('externalUpdate', update)
+    })
+    this.doc.on('remoteop', (...ops: AnyOperation[]) => {
+      this.eventLog.pushEvent('remoteop', { doc_id: this.doc_id })
+      return this.trigger('remoteop', ...ops)
+    })
+    this.doc.on('op:sent', (op: AnyOperation) => {
+      this.eventLog.pushEvent('op:sent', {
+        doc_id: this.doc_id,
+        op,
+      })
+      return this.trigger('op:sent')
+    })
+    this.doc.on('op:acknowledged', (op: AnyOperation) => {
+      this.eventLog.pushEvent('op:acknowledged', {
+        doc_id: this.doc_id,
+        op,
+      })
+      this.ideEventEmitter.emit('ide:opAcknowledged', {
+        doc_id: this.doc_id,
+        op,
+      })
+      return this.trigger('op:acknowledged')
+    })
+    this.doc.on('op:timeout', (op: AnyOperation) => {
+      this.eventLog.pushEvent('op:timeout', {
+        doc_id: this.doc_id,
+        op,
+      })
+      this.trigger('op:timeout')
+      return this.onError(new Error('op timed out'))
+    })
+    this.doc.on(
+      'flush',
+      (inflightOp: AnyOperation, pendingOp: AnyOperation, version: Version) => {
+        return this.eventLog.pushEvent('flush', {
+          doc_id: this.doc_id,
+          inflightOp,
+          pendingOp,
+          v: version,
+        })
+      }
+    )
+
+    let docChangedTimeout: number | null = null
+    this.doc.on(
+      'change',
+      (ops: AnyOperation[], oldSnapshot: any, msg: Message) => {
+        this.applyOpsToRanges(ops, msg)
+        if (docChangedTimeout) {
+          window.clearTimeout(docChangedTimeout)
+        }
+        docChangedTimeout = window.setTimeout(() => {
+          window.dispatchEvent(
+            new CustomEvent('doc:changed', { detail: { id: this.doc_id } })
+          )
+          this.ideEventEmitter.emit('doc:changed', { doc_id: this.doc_id })
+        }, 50)
+      }
+    )
+
+    this.doc.on('flipped_pending_to_inflight', () => {
+      return this.trigger('flipped_pending_to_inflight')
+    })
+
+    let docSavedTimeout: number | null
+    this.doc.on('saved', () => {
+      if (docSavedTimeout) {
+        window.clearTimeout(docSavedTimeout)
+      }
+      docSavedTimeout = window.setTimeout(() => {
+        window.dispatchEvent(
+          new CustomEvent('doc:saved', { detail: { id: this.doc_id } })
+        )
+        this.ideEventEmitter.emit('doc:saved', { doc_id: this.doc_id })
+      }, 50)
+    })
+  }
+
+  private onError(
+    error: Error,
+    meta: ErrorMetadata = {},
+    editorContent?: string
+  ) {
+    meta.doc_id = this.doc_id
+    debugConsole.log('ShareJS error', error, meta)
+    if (error.message === 'no project_id found on client') {
+      debugConsole.log('ignoring error, will wait to join project')
+      return
+    }
+    if (this.doc) {
+      this.doc.clearInflightAndPendingOps()
+    }
+    this.trigger('error', error, meta, editorContent)
+    // The clean-up should run after the error is triggered because the error triggers a
+    // disconnect. If we run the clean-up first, we remove our event handlers and miss
+    // the disconnect event, which means we try to leaveDoc when the connection comes back.
+    // This could interfere with the new connection of a new instance of this document.
+    this.cleanUp()
+  }
+
+  private applyOpsToRanges(ops: AnyOperation[], msg?: Message) {
+    let old_id_seed
+    let track_changes_as = null
+    const remote_op = msg != null
+    if (remote_op && msg?.meta.tc) {
+      old_id_seed = this.ranges.getIdSeed()
+      this.ranges.setIdSeed(msg.meta.tc)
+      track_changes_as = msg.meta.user_id
+    } else if (!remote_op && this.track_changes_as != null) {
+      track_changes_as = this.track_changes_as
+    }
+    this.ranges.track_changes = track_changes_as != null
+    for (const op of this.filterOps(ops)) {
+      this.ranges.applyOp(op, { user_id: track_changes_as })
+    }
+    if (old_id_seed != null) {
+      this.ranges.setIdSeed(old_id_seed)
+    }
+    if (remote_op) {
+      // With remote ops, the editor hasn't been updated when we receive this
+      // op, so defer updating track changes until it has
+      return window.setTimeout(() => this.emit('ranges:dirty'))
+    } else {
+      return this.emit('ranges:dirty')
+    }
+  }
+
+  private catchUpRanges(changes: Change[], comments: CommentOperation[]) {
+    // We've just been given the current server's ranges, but need to apply any local ops we have.
+    // Reset to the server state then apply our local ops again.
+    if (changes == null) {
+      changes = []
+    }
+    if (comments == null) {
+      comments = []
+    }
+    this.emit('ranges:clear')
+    this.ranges.changes = changes
+    this.ranges.comments = comments
+    this.ranges.track_changes = this.doc?.track_changes
+    for (const op of this.filterOps(this.doc?.getInflightOp() || [])) {
+      this.ranges.setIdSeed(this.doc?.track_changes_id_seeds?.inflight)
+      this.ranges.applyOp(op, { user_id: this.track_changes_as })
+    }
+    for (const op of this.filterOps(this.doc?.getPendingOp() || [])) {
+      this.ranges.setIdSeed(this.doc?.track_changes_id_seeds?.pending)
+      this.ranges.applyOp(op, { user_id: this.track_changes_as })
+    }
+    return this.emit('ranges:redraw')
+  }
+
+  private filterOps(ops: AnyOperation[]) {
+    // Read-only token users can't see/edit comment, so we filter out comment
+    // ops to avoid highlighting comment ranges.
+    if (window.isRestrictedTokenMember) {
+      return ops.filter(op => !isCommentOperation(op))
+    } else {
+      return ops
+    }
+  }
+}

services/web/frontend/js/features/ide-react/editor/event-log.ts

@@ -0,0 +1,16 @@
+import { debugConsole } from '@/utils/debugging'
+
+type EditorEvent = { type: string; meta: unknown; date: Date }
+
+// Record events and then do nothing with them.
+export class EventLog {
+  private recentEvents: EditorEvent[] = []
+
+  pushEvent = (type: string, meta: unknown = {}) => {
+    debugConsole.log('event', type, meta)
+    this.recentEvents.push({ type, meta, date: new Date() })
+    if (this.recentEvents.length > 100) {
+      return this.recentEvents.shift()
+    }
+  }
+}

services/web/frontend/js/features/ide-react/editor/open-documents.ts

@@ -0,0 +1,75 @@
+// Migrated from static methods of Document in Document.js
+
+import { Document } from '@/features/ide-react/editor/document'
+import { debugConsole } from '@/utils/debugging'
+import { Socket } from '@/features/ide-react/connection/types/socket'
+import { IdeEventEmitter } from '@/features/ide-react/create-ide-event-emitter'
+import { EventLog } from '@/features/ide-react/editor/event-log'
+import EditorWatchdogManager from '@/features/ide-react/connection/editor-watchdog-manager'
+
+export class OpenDocuments {
+  private openDocs = new Map<string, Document>()
+
+  // eslint-disable-next-line no-useless-constructor
+  constructor(
+    private readonly socket: Socket,
+    private readonly globalEditorWatchdogManager: EditorWatchdogManager,
+    private readonly events: IdeEventEmitter,
+    private readonly eventLog: EventLog
+  ) {}
+
+  getDocument(docId: string) {
+    // Try to clean up existing docs before reopening them. If the doc has no
+    // buffered ops then it will be deleted by _cleanup() and a new instance
+    // of the document created below. This prevents us trying to follow the
+    // joinDoc:existing code path on an existing doc that doesn't have any
+    // local changes and getting an error if its version is too old.
+    if (this.openDocs.has(docId)) {
+      debugConsole.log(
+        `[getDocument] Cleaning up existing document instance for ${docId}`
+      )
+      this.openDocs.get(docId)?.cleanUp()
+    }
+    if (!this.openDocs.has(docId)) {
+      debugConsole.log(
+        `[getDocument] Creating new document instance for ${docId}`
+      )
+      this.createDoc(docId)
+    } else {
+      debugConsole.log(
+        `[getDocument] Returning existing document instance for ${docId}`
+      )
+    }
+    return this.openDocs.get(docId)
+  }
+
+  private createDoc(docId: string) {
+    const doc = new Document(
+      docId,
+      this.socket,
+      this.globalEditorWatchdogManager,
+      this.events,
+      this.eventLog
+    )
+    this.openDocs.set(docId, doc)
+    doc.on('detach', () => {
+      debugConsole.log(
+        `[detach] Removing document with ID (${docId}) from openDocs`
+      )
+      doc.off('detach')
+      this.openDocs.delete(docId)
+    })
+  }
+
+  private docsArray() {
+    return Array.from(this.openDocs.values())
+  }
+
+  hasUnsavedChanges() {
+    return this.docsArray().some(doc => doc.hasBufferedOps())
+  }
+
+  flushAll() {
+    return this.docsArray().map(doc => doc.flush())
+  }
+}

services/web/frontend/js/features/ide-react/editor/share-js-doc.ts

@@ -0,0 +1,437 @@
+/* eslint-disable camelcase */
+// Migrated from services/web/frontend/js/ide/editor/ShareJsDoc.js
+
+import EventEmitter from '../../../utils/EventEmitter'
+import { Doc } from '@/vendor/libs/sharejs'
+import { Socket } from '@/features/ide-react/connection/types/socket'
+import { debugConsole } from '@/utils/debugging'
+import { decodeUtf8 } from '@/utils/decode-utf8'
+import { IdeEventEmitter } from '@/features/ide-react/create-ide-event-emitter'
+import { EventLog } from '@/features/ide-react/editor/event-log'
+import EditorWatchdogManager from '@/features/ide-react/connection/editor-watchdog-manager'
+import {
+  Message,
+  ShareJsConnectionState,
+  ShareJsOperation,
+  TrackChangesIdSeeds,
+} from '@/features/ide-react/editor/types/document'
+import { EditorFacade } from '@/features/source-editor/extensions/realtime'
+
+// All times below are in milliseconds
+const SINGLE_USER_FLUSH_DELAY = 2000
+const MULTI_USER_FLUSH_DELAY = 500
+const INFLIGHT_OP_TIMEOUT = 5000 // Retry sending ops after 5 seconds without an ack
+const WAIT_FOR_CONNECTION_TIMEOUT = 500
+const FATAL_OP_TIMEOUT = 30000
+
+type Update = Record<string, any>
+
+type Connection = {
+  send: (update: Update) => void
+  state: ShareJsConnectionState
+  id: string
+}
+
+export class ShareJsDoc extends EventEmitter {
+  type: string
+  track_changes = false
+  track_changes_id_seeds: TrackChangesIdSeeds | null = null
+  connection: Connection
+
+  // @ts-ignore
+  _doc: Doc
+  private editorWatchdogManager: EditorWatchdogManager
+  private lastAcked: Date | null = null
+  private queuedMessageTimer: number | null = null
+  private queuedMessages: Message[] = []
+  private detachEditorWatchdogManager: (() => void) | null = null
+  private _timeoutTimer: number | null = null
+
+  constructor(
+    readonly doc_id: string,
+    docLines: string[],
+    version: number,
+    readonly socket: Socket,
+    private readonly globalEditorWatchdogManager: EditorWatchdogManager,
+    private readonly eventEmitter: IdeEventEmitter,
+    private readonly eventLog: EventLog
+  ) {
+    super()
+    this.type = 'text'
+    // Decode any binary bits of data
+    const snapshot = docLines.map(line => decodeUtf8(line)).join('\n')
+
+    this.connection = {
+      send: (update: Update) => {
+        this.startInflightOpTimeout(update)
+        // TODO: MIGRATION: Work out whether we can get rid of this. It looks as
+        // though it's here for debugging and isn't used
+
+        // if (
+        //   window.disconnectOnUpdate != null &&
+        //   Math.random() < window.disconnectOnUpdate
+        // ) {
+        //   debugConsole.log('Disconnecting on update', update)
+        //   this.socket.disconnect()
+        // }
+        // if (window.dropUpdates != null && Math.random() < window.dropUpdates) {
+        //   debugConsole.log('Simulating a lost update', update)
+        //   return
+        // }
+        if (this.track_changes && this.track_changes_id_seeds) {
+          if (update.meta == null) {
+            update.meta = {}
+          }
+          update.meta.tc = this.track_changes_id_seeds.inflight
+        }
+        return this.socket.emit(
+          'applyOtUpdate',
+          this.doc_id,
+          update,
+          (error: Error) => {
+            if (error != null) {
+              this.handleError(error)
+            }
+          }
+        )
+      },
+      state: 'ok',
+      id: this.socket.publicId,
+    }
+
+    this._doc = new Doc(this.connection, this.doc_id, {
+      type: this.type,
+    })
+    this._doc.setFlushDelay(SINGLE_USER_FLUSH_DELAY)
+    this._doc.on('change', (...args: any[]) => {
+      return this.trigger('change', ...args)
+    })
+    this.editorWatchdogManager = new EditorWatchdogManager({
+      parent: globalEditorWatchdogManager,
+    })
+    this._doc.on('acknowledge', () => {
+      this.lastAcked = new Date() // note time of last ack from server for an op we sent
+      this.editorWatchdogManager.onAck() // keep track of last ack globally
+      return this.trigger('acknowledge')
+    })
+    this._doc.on('remoteop', (...args: any[]) => {
+      // As soon as we're working with a collaborator, start sending
+      // ops more frequently for low latency.
+      this._doc.setFlushDelay(MULTI_USER_FLUSH_DELAY)
+      return this.trigger('remoteop', ...args)
+    })
+    this._doc.on('flipped_pending_to_inflight', () => {
+      return this.trigger('flipped_pending_to_inflight')
+    })
+    this._doc.on('saved', () => {
+      return this.trigger('saved')
+    })
+    this._doc.on('error', (e: Error) => {
+      return this.handleError(e)
+    })
+
+    this.bindToDocChanges(this._doc)
+
+    this.processUpdateFromServer({
+      open: true,
+      v: version,
+      snapshot,
+    })
+    this.removeCarriageReturnCharFromShareJsDoc()
+  }
+
+  private removeCarriageReturnCharFromShareJsDoc() {
+    const doc = this._doc
+    if (doc.snapshot.indexOf('\r') === -1) {
+      return
+    }
+    this.eventLog.pushEvent('remove-carriage-return-char', {
+      doc_id: this.doc_id,
+    })
+    let nextPos
+    while ((nextPos = doc.snapshot.indexOf('\r')) !== -1) {
+      debugConsole.log('[ShareJsDoc] remove-carriage-return-char', nextPos)
+      doc.del(nextPos, 1)
+    }
+  }
+
+  submitOp(op: ShareJsOperation) {
+    this._doc.submitOp(op)
+  }
+
+  // The following code puts out of order messages into a queue
+  // so that they can be processed in order.  This is a workaround
+  // for messages being delayed by redis cluster.
+  // FIXME: REMOVE THIS WHEN REDIS PUBSUB IS SENDING MESSAGES IN ORDER
+  private isAheadOfExpectedVersion(message: Message) {
+    return this._doc.version > 0 && message.v > this._doc.version
+  }
+
+  private pushOntoQueue(message: Message) {
+    debugConsole.log(`[processUpdate] push onto queue ${message.v}`)
+    // set a timer so that we never leave messages in the queue indefinitely
+    if (!this.queuedMessageTimer) {
+      this.queuedMessageTimer = window.setTimeout(() => {
+        debugConsole.log(`[processUpdate] queue timeout fired for ${message.v}`)
+        // force the message to be processed after the timeout,
+        // it will cause an error if the missing update has not arrived
+        this.processUpdateFromServer(message)
+      }, INFLIGHT_OP_TIMEOUT)
+    }
+    this.queuedMessages.push(message)
+    // keep the queue in order, lowest version first
+    this.queuedMessages.sort(function (a, b) {
+      return a.v - b.v
+    })
+  }
+
+  private clearQueue() {
+    this.queuedMessages = []
+  }
+
+  private processQueue() {
+    if (this.queuedMessages.length > 0) {
+      const nextAvailableVersion = this.queuedMessages[0].v
+      if (nextAvailableVersion > this._doc.version) {
+        // there are updates we still can't apply yet
+      } else {
+        // there's a version we can accept on the queue, apply it
+        debugConsole.log(
+          `[processUpdate] taken from queue ${nextAvailableVersion}`
+        )
+        const message = this.queuedMessages.shift()
+        if (message) {
+          this.processUpdateFromServerInOrder(message)
+        }
+        // clear the pending timer if the queue has now been cleared
+        if (this.queuedMessages.length === 0 && this.queuedMessageTimer) {
+          debugConsole.log('[processUpdate] queue is empty, cleared timeout')
+          window.clearTimeout(this.queuedMessageTimer)
+          this.queuedMessageTimer = null
+        }
+      }
+    }
+  }
+
+  // FIXME: This is the new method which reorders incoming updates if needed
+  // called from document.ts
+  processUpdateFromServerInOrder(message: Message) {
+    // Is this update ahead of the next expected update?
+    // If so, put it on a queue to be handled later.
+    if (this.isAheadOfExpectedVersion(message)) {
+      this.pushOntoQueue(message)
+      return // defer processing this update for now
+    }
+    const error = this.processUpdateFromServer(message)
+    if (
+      error instanceof Error &&
+      error.message === 'Invalid version from server'
+    ) {
+      // if there was an error, abandon the queued updates ahead of this one
+      this.clearQueue()
+      return
+    }
+    // Do we have any messages queued up?
+    // find the next message if available
+    this.processQueue()
+  }
+
+  // FIXME: This is the original method. Switch back to this when redis
+  // issues are resolved.
+  processUpdateFromServer(message: Message) {
+    try {
+      this._doc._onMessage(message)
+    } catch (error) {
+      // Version mismatches are thrown as errors
+      debugConsole.log(error)
+      this.handleError(error)
+      return error // return the error for queue handling
+    }
+
+    if (message.meta?.type === 'external') {
+      return this.trigger('externalUpdate', message)
+    }
+  }
+
+  catchUp(updates: Message[]) {
+    return updates.map(update => {
+      update.v = this._doc.version
+      update.doc = this.doc_id
+      return this.processUpdateFromServer(update)
+    })
+  }
+
+  getSnapshot() {
+    return this._doc.snapshot as string | undefined
+  }
+
+  getVersion() {
+    return this._doc.version
+  }
+
+  getType() {
+    return this.type
+  }
+
+  clearInflightAndPendingOps() {
+    this.clearFatalTimeoutTimer()
+    this._doc.inflightOp = null
+    this._doc.inflightCallbacks = []
+    this._doc.pendingOp = null
+    return (this._doc.pendingCallbacks = [])
+  }
+
+  flushPendingOps() {
+    // This will flush any ops that are pending.
+    // If there is an inflight op it will do nothing.
+    return this._doc.flush()
+  }
+
+  updateConnectionState(state: ShareJsConnectionState) {
+    debugConsole.log(`[updateConnectionState] Setting state to ${state}`)
+    this.connection.state = state
+    this.connection.id = this.socket.publicId
+    this._doc.autoOpen = false
+    this._doc._connectionStateChanged(state)
+    return (this.lastAcked = null) // reset the last ack time when connection changes
+  }
+
+  hasBufferedOps() {
+    return this._doc.inflightOp != null || this._doc.pendingOp != null
+  }
+
+  getInflightOp() {
+    return this._doc.inflightOp
+  }
+
+  getPendingOp() {
+    return this._doc.pendingOp
+  }
+
+  getRecentAck() {
+    // check if we have received an ack recently (within a factor of two of the single user flush delay)
+    return (
+      this.lastAcked !== null &&
+      Date.now() - this.lastAcked.getTime() < 2 * SINGLE_USER_FLUSH_DELAY
+    )
+  }
+
+  private attachEditorWatchdogManager(editor: EditorFacade) {
+    // end-to-end check for edits -> acks, for this very ShareJsdoc
+    // This will catch a broken connection and missing UX-blocker for the
+    //  user, allowing them to keep editing.
+    this.detachEditorWatchdogManager =
+      this.editorWatchdogManager.attachToEditor(editor)
+  }
+
+  private attachToEditor(editor: EditorFacade, attachToShareJs: () => void) {
+    this.attachEditorWatchdogManager(editor)
+
+    attachToShareJs()
+  }
+
+  private maybeDetachEditorWatchdogManager() {
+    // a failed attach attempt may lead to a missing cleanup handler
+    if (this.detachEditorWatchdogManager) {
+      this.detachEditorWatchdogManager()
+      this.detachEditorWatchdogManager = null
+    }
+  }
+
+  attachToCM6(cm6: EditorFacade) {
+    this.attachToEditor(cm6, () => {
+      // @ts-ignore
+      cm6.attachShareJs(this._doc, window.maxDocLength)
+    })
+  }
+
+  detachFromCM6() {
+    this.maybeDetachEditorWatchdogManager()
+    if (this._doc.detach_cm6) {
+      this._doc.detach_cm6()
+    }
+  }
+
+  private startInflightOpTimeout(update: Update) {
+    this.startFatalTimeoutTimer(update)
+    const retryOp = () => {
+      // Only send the update again if inflightOp is still populated
+      // This can be cleared when hard reloading the document in which
+      // case we don't want to keep trying to send it.
+      debugConsole.log('[inflightOpTimeout] Trying op again')
+      if (this._doc.inflightOp != null) {
+        // When there is a socket.io disconnect, @_doc.inflightSubmittedIds
+        // is updated with the socket.io client id of the current op in flight
+        // (meta.source of the op).
+        // @connection.id is the client id of the current socket.io session.
+        // So we need both depending on whether the op was submitted before
+        // one or more disconnects, or if it was submitted during the current session.
+        update.dupIfSource = [
+          this.connection.id,
+          ...Array.from(this._doc.inflightSubmittedIds),
+        ]
+
+        // We must be joined to a project for applyOtUpdate to work on the real-time
+        // service, so don't send an op if we're not. Connection state is set to 'ok'
+        // when we've joined the project
+        if (this.connection.state !== 'ok') {
+          debugConsole.log(
+            '[inflightOpTimeout] Not connected, retrying in 0.5s'
+          )
+          window.setTimeout(retryOp, WAIT_FOR_CONNECTION_TIMEOUT)
+        } else {
+          debugConsole.log('[inflightOpTimeout] Sending')
+          return this.connection.send(update)
+        }
+      }
+    }
+
+    const timer = window.setTimeout(retryOp, INFLIGHT_OP_TIMEOUT)
+    return this._doc.inflightCallbacks.push(() => {
+      this.clearFatalTimeoutTimer()
+      window.clearTimeout(timer)
+    }) // 30 seconds
+  }
+
+  private startFatalTimeoutTimer(update: Update) {
+    // If an op doesn't get acked within FATAL_OP_TIMEOUT, something has
+    // gone unrecoverably wrong (the op will have been retried multiple times)
+    if (this._timeoutTimer != null) {
+      return
+    }
+    return (this._timeoutTimer = window.setTimeout(() => {
+      this.clearFatalTimeoutTimer()
+      return this.trigger('op:timeout', update)
+    }, FATAL_OP_TIMEOUT))
+  }
+
+  private clearFatalTimeoutTimer() {
+    if (this._timeoutTimer == null) {
+      return
+    }
+    clearTimeout(this._timeoutTimer)
+    return (this._timeoutTimer = null)
+  }
+
+  private handleError(error: unknown, meta = {}) {
+    return this.trigger('error', error, meta)
+  }
+
+  // @ts-ignore
+  private bindToDocChanges(doc: Doc) {
+    const { submitOp } = doc
+    doc.submitOp = (op: ShareJsOperation, callback?: () => void) => {
+      this.trigger('op:sent', op)
+      doc.pendingCallbacks.push(() => {
+        return this.trigger('op:acknowledged', op)
+      })
+      return submitOp.call(doc, op, callback)
+    }
+
+    const { flush } = doc
+    doc.flush = () => {
+      this.trigger('flush', doc.inflightOp, doc.pendingOp, doc.version)
+      return flush.call(doc)
+    }
+  }
+}

services/web/frontend/js/features/ide-react/editor/types/document.ts

@@ -0,0 +1,21 @@
+import { AnyOperation } from '../../../../../../types/change'
+
+export type Version = number
+
+export type ShareJsConnectionState = 'ok' | 'disconnected' | 'stopped'
+
+export type ShareJsOperation = AnyOperation[]
+
+export type TrackChangesIdSeeds = { inflight: string; pending: string }
+
+export type Message = Record<string, any>
+// TODO: MIGRATION: Make an accurate and more specific type
+// {
+//   v: Version
+//   open?: boolean
+//   meta?: {
+//     type: string
+//   }
+//   doc?: string
+//   snapshot?: string
+// }

services/web/frontend/js/features/ide-react/hooks/use-layout-event-tracking.ts

@@ -0,0 +1,25 @@
+import { useLayoutContext } from '@/shared/context/layout-context'
+import { useEffect } from 'react'
+import { sendMBOnce } from '@/infrastructure/event-tracking'
+
+export function useLayoutEventTracking() {
+  const { view, leftMenuShown, chatIsOpen } = useLayoutContext()
+
+  useEffect(() => {
+    if (view !== 'editor' && view !== 'pdf') {
+      sendMBOnce(`ide-open-view-${view}-once`)
+    }
+  }, [view])
+
+  useEffect(() => {
+    if (leftMenuShown) {
+      sendMBOnce(`ide-open-left-menu-once`)
+    }
+  }, [leftMenuShown])
+
+  useEffect(() => {
+    if (chatIsOpen) {
+      sendMBOnce(`ide-open-chat-once`)
+    }
+  }, [chatIsOpen])
+}

services/web/frontend/js/features/ide-react/hooks/use-socket-listener.ts

@@ -0,0 +1,18 @@
+import { useEffect } from 'react'
+import { Socket } from '@/features/ide-react/connection/types/socket'
+
+type SocketOnParams = Parameters<Socket['on']>
+
+export default function useSocketListener(
+  socket: Socket,
+  event: SocketOnParams[0],
+  listener: SocketOnParams[1]
+) {
+  useEffect(() => {
+    socket.on(event, listener)
+
+    return () => {
+      socket.removeListener(event, listener)
+    }
+  }, [event, listener, socket])
+}

services/web/frontend/js/features/ide-react/scope-adapters/layout-context-adapter.ts

@@ -0,0 +1,18 @@
+import { ReactScopeValueStore } from '../scope-value-store/react-scope-value-store'
+import getMeta from '@/utils/meta'
+
+const reviewPanelStorageKey = `ui.reviewPanelOpen.${getMeta('ol-project_id')}`
+
+export default function populateLayoutScope(store: ReactScopeValueStore) {
+  store.set('ui.view', 'editor')
+
+  // TODO: Find out what this needs to do and make it do it
+  store.set('toggleHistory', () => {})
+
+  store.set('openFile', null)
+  store.set('ui.chatOpen', false)
+  store.persisted('ui.reviewPanelOpen', false, reviewPanelStorageKey)
+  store.set('ui.leftMenuShown', false)
+  store.set('ui.pdfLayout', 'sideBySide')
+  store.set('ui.loadingStyleSheet', false)
+}

services/web/frontend/js/features/ide-react/scope-adapters/settings-adapter.ts

@@ -0,0 +1,5 @@
+import { ReactScopeValueStore } from '@/features/ide-react/scope-value-store/react-scope-value-store'
+
+export function populateSettingsScope(store: ReactScopeValueStore) {
+  store.set('settings', window.userSettings)
+}

services/web/frontend/js/features/ide-react/types/file-tree.ts

@@ -0,0 +1,38 @@
+import { FileRef } from '../../../../../types/file-ref'
+import { Folder } from '../../../../../types/folder'
+import { Doc } from '../../../../../types/doc'
+
+export type FileTreeFolderFindResultType = 'folder' | 'doc' | 'fileRef'
+
+interface BaseFileTreeFindResult<T> {
+  type: FileTreeFolderFindResultType
+  entity: T
+  parent: T[]
+  parentFolderId: string
+  path: string
+  index: number
+}
+
+export interface FileTreeFolderFindResult
+  extends BaseFileTreeFindResult<Folder> {
+  type: 'folder'
+}
+
+export interface FileTreeDocumentFindResult
+  extends BaseFileTreeFindResult<Doc> {
+  type: 'doc'
+}
+
+export interface FileTreeFileRefFindResult
+  extends BaseFileTreeFindResult<FileRef> {
+  type: 'fileRef'
+}
+
+export type FileTreeFindResult =
+  | FileTreeFolderFindResult
+  | FileTreeDocumentFindResult
+  | FileTreeFileRefFindResult
+
+export type FileTreeSelectHandler = (
+  selectedEntities: FileTreeFindResult[]
+) => void

services/web/frontend/js/features/ide-react/util/find-doc-entity-by-id.ts

@@ -0,0 +1,11 @@
+import { findInTree } from '@/features/file-tree/util/find-in-tree'
+import { Folder } from '../../../../../types/folder'
+import { Doc } from '../../../../../types/doc'
+
+export function findDocEntityById(fileTreeData: Folder, docId: string) {
+  const item = findInTree(fileTreeData, docId)
+  if (!item || item.type !== 'doc') {
+    return null
+  }
+  return item.entity as Doc
+}

services/web/frontend/js/features/source-editor/components/codemirror-editor.tsx

@@ -20,6 +20,7 @@ import importOverleafModules from '../../../../macros/import-overleaf-module.mac
 import { FigureModal } from './figure-modal/figure-modal'
 import ReviewPanel from './review-panel/review-panel'
 import getMeta from '../../../utils/meta'
+import { useIdeContext } from '@/shared/context/ide-context'
 
 const sourceEditorComponents = importOverleafModules(
   'sourceEditorComponents'
@@ -33,6 +34,7 @@ function CodeMirrorEditor() {
 
   const isMounted = useIsMounted()
   const isReviewPanelReact = getMeta('ol-isReviewPanelReact')
+  const { isReactIde } = useIdeContext()
 
   // create the view using the initial state and intercept transactions
   const viewRef = useRef<EditorView | null>(null)
@@ -62,7 +64,7 @@ function CodeMirrorEditor() {
         <CodeMirrorSearch />
         <CodeMirrorToolbar />
         <CodeMirrorCommandTooltip />
-        {isReviewPanelReact && <ReviewPanel />}
+        {isReviewPanelReact && !isReactIde && <ReviewPanel />}
         {sourceEditorComponents.map(
           ({ import: { default: Component }, path }) => (
             <Component key={path} />

services/web/frontend/js/features/source-editor/extensions/realtime.ts

@@ -19,8 +19,17 @@ import { debugConsole } from '@/utils/debugging'
  *   - frontend/js/ide/editor/Document.js
  *   - frontend/js/ide/editor/ShareJsDoc.js
  *   - frontend/js/ide/connection/EditorWatchdogManager.js
+ *   - frontend/js/features/ide-react/editor/document.ts
+ *   - frontend/js/features/ide-react/editor/share-js-doc.ts
+ *   - frontend/js/features/ide-react/connection/editor-watchdog-manager.js
  */
 
+export type ChangeDescription = {
+  origin: 'remote' | 'undo' | 'reject' | undefined
+  inserted: boolean
+  removed: boolean
+}
+
 /**
  * A custom extension that connects the CodeMirror 6 editor to the currently open ShareJS document.
  */
@@ -203,7 +212,13 @@ export class EditorFacade extends EventEmitter {
             // TODO: mapPos instead?
             positionShift = positionShift - removedLength + insertedLength
 
-            this.emit('change', this, { origin, inserted, removed })
+            const changeDescription: ChangeDescription = {
+              origin,
+              inserted,
+              removed,
+            }
+
+            this.emit('change', this, changeDescription)
           }
         )
       }

services/web/frontend/js/features/source-editor/extensions/track-changes.ts

@@ -22,15 +22,14 @@ import {
 } from './changes/comments'
 import { invertedEffects } from '@codemirror/commands'
 import { CurrentDoc } from '../../../../../types/current-doc'
-import {
+import { Change, DeleteOperation } from '../../../../../types/change'
-  Change,
-  ChangeOperation,
-  CommentOperation,
-  DeleteOperation,
-  Operation,
-} from '../../../../../types/change'
 import { ChangeManager } from './changes/change-manager'
 import { debugConsole } from '@/utils/debugging'
+import {
+  isChangeOperation,
+  isCommentOperation,
+  isDeleteOperation,
+} from '@/utils/operations'
 
 const clearChangesEffect = StateEffect.define()
 const buildChangesEffect = StateEffect.define()
@@ -246,13 +245,6 @@ class ChangeCalloutWidget extends WidgetType {
   }
 }
 
-// const isInsertOperation = (op: Operation): op is InsertOperation => 'i' in op
-const isChangeOperation = (op: Operation): op is ChangeOperation =>
-  'c' in op && 't' in op
-const isCommentOperation = (op: Operation): op is CommentOperation =>
-  'c' in op && !('t' in op)
-const isDeleteOperation = (op: Operation): op is DeleteOperation => 'd' in op
-
 const createChangeRange = (change: Change, currentDoc: CurrentDoc) => {
   const { id, metadata, op } = change
 

services/web/frontend/js/features/source-editor/hooks/use-codemirror-scope.ts

@@ -89,6 +89,8 @@ function useCodeMirrorScope(view: EditorView) {
   const [visual] = useScopeValue<boolean>('editor.showVisual')
   const reactReviewPanel: boolean = getMeta('ol-isReviewPanelReact')
 
+  const [references] = useScopeValue<{ keys: string[] }>('$root._references')
+
   // build the translation phrases
   const phrases = usePhrases()
 
@@ -206,7 +208,7 @@ function useCodeMirrorScope(view: EditorView) {
   // TODO: read this data from the scope?
   const metadataRef = useRef({
     documents: ide.metadataManager.metadata.state.documents,
-    references: ide.$scope.$root._references.keys,
+    references: references.keys,
     fileTreeData,
   })
 

services/web/frontend/js/utils/decode-utf8.ts

@@ -0,0 +1,5 @@
+// TODO: MIGRATION: Can we use TextDecoder now? https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder
+// See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html
+export function decodeUtf8(text: string) {
+  return decodeURIComponent(escape(text))
+}

services/web/frontend/js/utils/operations.ts

@@ -0,0 +1,16 @@
+import {
+  ChangeOperation,
+  CommentOperation,
+  DeleteOperation,
+  InsertOperation,
+  Operation,
+} from '../../../types/change'
+
+export const isInsertOperation = (op: Operation): op is InsertOperation =>
+  'i' in op
+export const isChangeOperation = (op: Operation): op is ChangeOperation =>
+  'c' in op && 't' in op
+export const isCommentOperation = (op: Operation): op is CommentOperation =>
+  'c' in op && !('t' in op)
+export const isDeleteOperation = (op: Operation): op is DeleteOperation =>
+  'd' in op

services/web/frontend/stylesheets/app/editor/ide-react.less

@@ -95,3 +95,7 @@
   background-color: @symbol-palette-bg;
   color: var(--neutral-20);
 }
+
+.ide-react-file-tree-panel {
+  display: flex;
+}

services/web/types/change.ts

@@ -20,14 +20,18 @@ export interface CommentOperation extends Operation {
   c: string
 }
 
-export type AnyOperation =
+export type NonCommentOperation =
   | InsertOperation
   | ChangeOperation
   | DeleteOperation
-  | CommentOperation
+
+export type AnyOperation = NonCommentOperation | CommentOperation
 
 export type Change<T extends AnyOperation = AnyOperation> = {
   id: string
-  metadata?: string
+  metadata?: {
+    user_id: string
+    ts: Date
+  }
   op: T
 }

services/web/types/current-doc.ts

@@ -12,6 +12,10 @@ import {
 
 // type for the Document class in ide/editor/Document.js
 // note: this is a custom EventEmitter class
+
+// TODO: MIGRATION: This doesn't match the type for
+// ide-react/editor/document.ts, which has a nullable `ranges` property and some
+// other quirks. They should match.
 export interface CurrentDoc extends EventEmitter {
   doc_id: string
   docName: string

services/web/types/project.ts

@@ -27,4 +27,5 @@ export type Project = {
   invites: ProjectInvite[]
   rootDoc_id?: string
   rootFolder?: Folder[]
+  deletedByExternalDataSource?: boolean
 }

services/web/types/user.ts

@@ -1,3 +1,8 @@
+export type RefProviders = {
+  mendeley?: boolean
+  zotero?: boolean
+}
+
 export type User = {
   id: string
   email: string
@@ -19,6 +24,7 @@ export type User = {
     versioning?: boolean
     zotero?: boolean
   }
+  refProviders?: RefProviders
 }
 
 export type MongoUser = Pick<User, Exclude<keyof User, 'id'>> & { _id: string }

services/web/types/window.ts

@@ -3,6 +3,7 @@ import { OAuthProviders } from './oauth-providers'
 import { OverallThemeMeta } from './project-settings'
 import { User } from './user'
 import 'recurly__recurly-js'
+import { UserSettings } from '@/features/editor-left-menu/utils/api'
 
 declare global {
   // eslint-disable-next-line no-unused-vars
@@ -10,6 +11,7 @@ declare global {
     csrfToken: string
     user: User
     user_id?: string
+    userSettings: UserSettings
     oauthProviders: OAuthProviders
     thirdPartyIds: Record<string, string>
     metaAttributesCache: Map<string, any>