From 01439641ca0a969e49aeeabea03d34658d580219 Mon Sep 17 00:00:00 2001 From: Tim Down <158919+timdown@users.noreply.github.com> Date: Thu, 26 Oct 2023 09:57:00 +0100 Subject: [PATCH] Merge pull request #15376 from overleaf/td-ide-page-working-editor React IDE page: working editor GitOrigin-RevId: 3ba8cb787a6f7f8435686d8962adb7444d09acb5 --- services/web/config/settings.defaults.js | 1 + .../context/project-settings-context.tsx | 7 +- .../ide-react/components/alerts/alerts.tsx | 4 +- .../components/editor-and-sidebar.tsx | 69 ++ .../ide-react/components/editor-sidebar.tsx | 34 + .../components/editor/editor-pane.tsx | 80 ++ .../ide-react/components/editor/editor.tsx | 96 +++ .../ide-react/components/file-tree.tsx | 49 ++ .../features/ide-react/components/header.tsx | 63 ++ .../ide-react/components/layout/ide-page.tsx | 57 +- .../placeholder-editor-sidebar.tsx | 4 +- .../layout/placeholder/placeholder-file.tsx | 5 + .../layout/placeholder/placeholder-header.tsx | 4 +- .../layout/placeholder/placeholder-pdf.tsx | 5 + .../connection/connection-manager.ts | 2 +- .../connection/editor-watchdog-manager.ts | 234 ++++++ .../ide-react/connection/types/socket.ts | 1 + .../ide-react/context/connection-context.tsx | 7 + .../context/editor-manager-context.tsx | 597 +++++++++++++++ .../ide-react/context/ide-react-context.tsx | 107 ++- .../ide-react/context/metadata-context.tsx | 228 ++++++ .../context/online-users-context.tsx | 297 ++++++++ .../ide-react/context/react-context-root.tsx | 57 +- .../ide-react/context/references-context.tsx | 197 +++++ .../ide-react/create-ide-event-emitter.ts | 16 +- .../js/features/ide-react/editor/document.ts | 706 ++++++++++++++++++ .../js/features/ide-react/editor/event-log.ts | 16 + .../ide-react/editor/open-documents.ts | 75 ++ .../features/ide-react/editor/share-js-doc.ts | 437 +++++++++++ .../ide-react/editor/types/document.ts | 21 + .../hooks/use-layout-event-tracking.ts | 25 + .../ide-react/hooks/use-socket-listener.ts | 18 + .../scope-adapters/layout-context-adapter.ts | 18 + .../scope-adapters/settings-adapter.ts | 5 + .../js/features/ide-react/types/file-tree.ts | 38 + .../ide-react/util/find-doc-entity-by-id.ts | 11 + .../components/codemirror-editor.tsx | 4 +- .../source-editor/extensions/realtime.ts | 17 +- .../source-editor/extensions/track-changes.ts | 20 +- .../hooks/use-codemirror-scope.ts | 4 +- services/web/frontend/js/utils/decode-utf8.ts | 5 + services/web/frontend/js/utils/operations.ts | 16 + .../stylesheets/app/editor/ide-react.less | 4 + services/web/types/change.ts | 10 +- services/web/types/current-doc.ts | 4 + services/web/types/project.ts | 1 + services/web/types/user.ts | 6 + services/web/types/window.ts | 2 + 48 files changed, 3622 insertions(+), 62 deletions(-) create mode 100644 services/web/frontend/js/features/ide-react/components/editor-and-sidebar.tsx create mode 100644 services/web/frontend/js/features/ide-react/components/editor-sidebar.tsx create mode 100644 services/web/frontend/js/features/ide-react/components/editor/editor-pane.tsx create mode 100644 services/web/frontend/js/features/ide-react/components/editor/editor.tsx create mode 100644 services/web/frontend/js/features/ide-react/components/file-tree.tsx create mode 100644 services/web/frontend/js/features/ide-react/components/header.tsx create mode 100644 services/web/frontend/js/features/ide-react/components/layout/placeholder/placeholder-file.tsx create mode 100644 services/web/frontend/js/features/ide-react/components/layout/placeholder/placeholder-pdf.tsx create mode 100644 services/web/frontend/js/features/ide-react/connection/editor-watchdog-manager.ts create mode 100644 services/web/frontend/js/features/ide-react/context/editor-manager-context.tsx create mode 100644 services/web/frontend/js/features/ide-react/context/metadata-context.tsx create mode 100644 services/web/frontend/js/features/ide-react/context/online-users-context.tsx create mode 100644 services/web/frontend/js/features/ide-react/context/references-context.tsx create mode 100644 services/web/frontend/js/features/ide-react/editor/document.ts create mode 100644 services/web/frontend/js/features/ide-react/editor/event-log.ts create mode 100644 services/web/frontend/js/features/ide-react/editor/open-documents.ts create mode 100644 services/web/frontend/js/features/ide-react/editor/share-js-doc.ts create mode 100644 services/web/frontend/js/features/ide-react/editor/types/document.ts create mode 100644 services/web/frontend/js/features/ide-react/hooks/use-layout-event-tracking.ts create mode 100644 services/web/frontend/js/features/ide-react/hooks/use-socket-listener.ts create mode 100644 services/web/frontend/js/features/ide-react/scope-adapters/layout-context-adapter.ts create mode 100644 services/web/frontend/js/features/ide-react/scope-adapters/settings-adapter.ts create mode 100644 services/web/frontend/js/features/ide-react/types/file-tree.ts create mode 100644 services/web/frontend/js/features/ide-react/util/find-doc-entity-by-id.ts create mode 100644 services/web/frontend/js/utils/decode-utf8.ts create mode 100644 services/web/frontend/js/utils/operations.ts diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index f16b204bbb..585a0cfb70 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -836,6 +836,7 @@ module.exports = { sourceEditorExtensions: [], sourceEditorComponents: [], sourceEditorCompletionSources: [], + sourceEditorSymbolPalette: [], integrationLinkingWidgets: [], referenceLinkingWidgets: [], importProjectFromGithubModalWrapper: [], diff --git a/services/web/frontend/js/features/editor-left-menu/context/project-settings-context.tsx b/services/web/frontend/js/features/editor-left-menu/context/project-settings-context.tsx index 520d7901b4..a44e76d50e 100644 --- a/services/web/frontend/js/features/editor-left-menu/context/project-settings-context.tsx +++ b/services/web/frontend/js/features/editor-left-menu/context/project-settings-context.tsx @@ -1,5 +1,4 @@ -import { createContext, useContext, useMemo } from 'react' -import type { PropsWithChildren } from 'react' +import { createContext, FC, useContext, useMemo } from 'react' import useProjectWideSettings from '../hooks/use-project-wide-settings' import useUserWideSettings from '../hooks/use-user-wide-settings' import useProjectWideSettingsSocketListener from '../hooks/use-project-wide-settings-socket-listener' @@ -36,9 +35,7 @@ export const ProjectSettingsContext = createContext< ProjectSettingsContextValue | undefined >(undefined) -export function ProjectSettingsProvider({ - children, -}: PropsWithChildren>) { +export const ProjectSettingsProvider: FC = ({ children }) => { const { compiler, setCompiler, diff --git a/services/web/frontend/js/features/ide-react/components/alerts/alerts.tsx b/services/web/frontend/js/features/ide-react/components/alerts/alerts.tsx index f76544df50..c0fb7394ae 100644 --- a/services/web/frontend/js/features/ide-react/components/alerts/alerts.tsx +++ b/services/web/frontend/js/features/ide-react/components/alerts/alerts.tsx @@ -3,6 +3,7 @@ import { LostConnectionAlert } from './lost-connection-alert' import { useConnectionContext } from '@/features/ide-react/context/connection-context' import { debugging } from '@/utils/debugging' import { Alert } from 'react-bootstrap' +import useScopeValue from '@/shared/hooks/use-scope-value' // TODO SavingNotificationController, SystemMessagesController, out-of-sync modal export function Alerts() { @@ -15,8 +16,7 @@ export function Alerts() { secondsUntilReconnect, } = useConnectionContext() - // TODO: Get this from a context - const synctexError = false + const [synctexError] = useScopeValue('sync_tex_error') return (
diff --git a/services/web/frontend/js/features/ide-react/components/editor-and-sidebar.tsx b/services/web/frontend/js/features/ide-react/components/editor-and-sidebar.tsx new file mode 100644 index 0000000000..103aae86b5 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/components/editor-and-sidebar.tsx @@ -0,0 +1,69 @@ +import React, { useCallback, useState } from 'react' +import TwoColumnMainContent from '@/features/ide-react/components/layout/two-column-main-content' +import Editor from '@/features/ide-react/components/editor/editor' +import EditorSidebar from '@/features/ide-react/components/editor-sidebar' +import customLocalStorage from '@/infrastructure/local-storage' +import { useProjectContext } from '@/shared/context/project-context' +import { FileTreeFindResult } from '@/features/ide-react/types/file-tree' + +type EditorAndSidebarProps = { + shouldPersistLayout: boolean + leftColumnDefaultSize: number + setLeftColumnDefaultSize: React.Dispatch> +} + +export function EditorAndSidebar({ + shouldPersistLayout, + leftColumnDefaultSize, + setLeftColumnDefaultSize, +}: EditorAndSidebarProps) { + const [leftColumnIsOpen, setLeftColumnIsOpen] = useState(true) + const { rootDocId, _id: projectId } = useProjectContext() + + const [openDocId, setOpenDocId] = useState( + () => customLocalStorage.getItem(`doc.open_id.${projectId}`) || rootDocId + ) + const [fileTreeReady, setFileTreeReady] = useState(false) + + const handleFileTreeInit = useCallback(() => { + setFileTreeReady(true) + }, []) + + const handleFileTreeSelect = useCallback( + (selectedEntities: FileTreeFindResult[]) => { + const firstDocId = + selectedEntities.find(result => result.type === 'doc')?.entity._id || + null + setOpenDocId(firstDocId) + }, + [] + ) + + const leftColumnContent = ( + + ) + const rightColumnContent = ( + + ) + + return ( + + ) +} diff --git a/services/web/frontend/js/features/ide-react/components/editor-sidebar.tsx b/services/web/frontend/js/features/ide-react/components/editor-sidebar.tsx new file mode 100644 index 0000000000..63ffed6568 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/components/editor-sidebar.tsx @@ -0,0 +1,34 @@ +import React from 'react' +import { Panel, PanelGroup } from 'react-resizable-panels' +import { VerticalResizeHandle } from '@/features/ide-react/components/resize/vertical-resize-handle' +import { FileTree } from '@/features/ide-react/components/file-tree' +import { FileTreeSelectHandler } from '@/features/ide-react/types/file-tree' + +type EditorSidebarProps = { + shouldPersistLayout: boolean + onFileTreeInit: () => void + onFileTreeSelect: FileTreeSelectHandler +} + +export default function EditorSidebar({ + shouldPersistLayout, + onFileTreeInit, + onFileTreeSelect, +}: EditorSidebarProps) { + return ( + + ) +} diff --git a/services/web/frontend/js/features/ide-react/components/editor/editor-pane.tsx b/services/web/frontend/js/features/ide-react/components/editor/editor-pane.tsx new file mode 100644 index 0000000000..c42cc78983 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/components/editor/editor-pane.tsx @@ -0,0 +1,80 @@ +import { Panel, PanelGroup } from 'react-resizable-panels' +import { VerticalResizeHandle } from '@/features/ide-react/components/resize/vertical-resize-handle' +import React, { ElementType, useEffect } from 'react' +import useScopeValue from '@/shared/hooks/use-scope-value' +import SourceEditor from '@/features/source-editor/components/source-editor' +import { + EditorScopeValue, + useEditorManagerContext, +} from '@/features/ide-react/context/editor-manager-context' +import importOverleafModules from '../../../../../macros/import-overleaf-module.macro' +import { EditorProps } from '@/features/ide-react/components/editor/editor' + +const symbolPaletteComponents = importOverleafModules( + 'sourceEditorSymbolPalette' +) as { import: { default: ElementType }; path: string }[] + +export function EditorPane({ + shouldPersistLayout, + openDocId, + fileTreeReady, +}: EditorProps) { + const { openDocId: openDocWithId } = useEditorManagerContext() + + const [editor] = useScopeValue('editor') + + useEffect(() => { + if (!fileTreeReady || !openDocId) { + return + } + openDocWithId(openDocId) + }, [fileTreeReady, openDocId, openDocWithId]) + + return ( + + + {!!editor.sharejs_doc && + !editor.opening && + editor.multiSelectedCount === 0 && + !editor.error_state ? ( + + ) : null} + + {editor.showSymbolPalette ? ( + <> + + +
+ {symbolPaletteComponents.map( + ({ import: { default: Component }, path }) => ( + + ) + )} +
+ + + ) : null} + + ) +} diff --git a/services/web/frontend/js/features/ide-react/components/editor/editor.tsx b/services/web/frontend/js/features/ide-react/components/editor/editor.tsx new file mode 100644 index 0000000000..91a869ca1e --- /dev/null +++ b/services/web/frontend/js/features/ide-react/components/editor/editor.tsx @@ -0,0 +1,96 @@ +import { useRef } from 'react' +import { useTranslation } from 'react-i18next' +import { + ImperativePanelHandle, + Panel, + PanelGroup, +} from 'react-resizable-panels' +import { HorizontalResizeHandle } from '@/features/ide-react/components/resize/horizontal-resize-handle' +import { HorizontalToggler } from '@/features/ide-react/components/resize/horizontal-toggler' +import useCollapsiblePanel from '@/features/ide-react/hooks/use-collapsible-panel' +import { useLayoutContext } from '@/shared/context/layout-context' +import { EditorPane } from '@/features/ide-react/components/editor/editor-pane' +import PlaceholderFile from '@/features/ide-react/components/layout/placeholder/placeholder-file' +import PlaceholderPdf from '@/features/ide-react/components/layout/placeholder/placeholder-pdf' + +export type EditorProps = { + shouldPersistLayout?: boolean + openDocId: string | null + fileTreeReady: boolean +} + +export default function Editor({ + shouldPersistLayout = false, + openDocId, + fileTreeReady, +}: EditorProps) { + const { t } = useTranslation() + const { view, pdfLayout, changeLayout } = useLayoutContext() + + const pdfPanelRef = useRef(null) + const isDualPane = pdfLayout === 'sideBySide' + const editorIsVisible = isDualPane || view === 'editor' + const pdfIsOpen = isDualPane || view === 'pdf' + + useCollapsiblePanel(pdfIsOpen, pdfPanelRef) + + if (view === 'file') { + return + } + + if (view === 'history') { + return null + } + + function setPdfIsOpen(isOpen: boolean) { + if (isOpen) { + changeLayout('sideBySide') + } else { + changeLayout('flat', 'editor') + } + } + + return ( + + {editorIsVisible ? ( + + + + ) : null} + {isDualPane ? ( + + setPdfIsOpen(isOpen)} + tooltipWhenOpen={t('tooltip_hide_pdf')} + tooltipWhenClosed={t('tooltip_show_pdf')} + /> + + ) : null} + {pdfIsOpen ? ( + setPdfIsOpen(!collapsed)} + > + + + ) : null} + + ) +} diff --git a/services/web/frontend/js/features/ide-react/components/file-tree.tsx b/services/web/frontend/js/features/ide-react/components/file-tree.tsx new file mode 100644 index 0000000000..6a1e497743 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/components/file-tree.tsx @@ -0,0 +1,49 @@ +import FileTreeRoot from '@/features/file-tree/components/file-tree-root' +import React, { useCallback, useState } from 'react' +import { useUserContext } from '@/shared/context/user-context' +import { useReferencesContext } from '@/features/ide-react/context/references-context' +import { useIdeReactContext } from '@/features/ide-react/context/ide-react-context' +import { useConnectionContext } from '@/features/ide-react/context/connection-context' +import { FileTreeSelectHandler } from '@/features/ide-react/types/file-tree' +import { RefProviders } from '../../../../../types/user' + +type FileTreeProps = { + onInit: () => void + onSelect: FileTreeSelectHandler +} + +export function FileTree({ onInit, onSelect }: FileTreeProps) { + const user = useUserContext() + const { indexAllReferences } = useReferencesContext() + const { setStartedFreeTrial } = useIdeReactContext() + const { isConnected } = useConnectionContext() + + const [refProviders, setRefProviders] = useState( + () => user.refProviders || {} + ) + + function reindexReferences() { + indexAllReferences(true) + } + + const setRefProviderEnabled = useCallback( + (provider: keyof RefProviders, value = true) => { + setRefProviders(refProviders => ({ ...refProviders, [provider]: value })) + }, + [] + ) + + return ( +
+ +
+ ) +} diff --git a/services/web/frontend/js/features/ide-react/components/header.tsx b/services/web/frontend/js/features/ide-react/components/header.tsx new file mode 100644 index 0000000000..11fd158705 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/components/header.tsx @@ -0,0 +1,63 @@ +import React, { useCallback } from 'react' +import ChatToggleButton from '@/features/editor-navigation-toolbar/components/chat-toggle-button' +import HistoryToggleButton from '@/features/editor-navigation-toolbar/components/history-toggle-button' +import LayoutDropdownButton from '@/features/editor-navigation-toolbar/components/layout-dropdown-button' +import MenuButton from '@/features/editor-navigation-toolbar/components/menu-button' +import { useLayoutContext } from '@/shared/context/layout-context' +import { sendMB } from '@/infrastructure/event-tracking' +import OnlineUsersWidget from '@/features/editor-navigation-toolbar/components/online-users-widget' +import { useOnlineUsersContext } from '@/features/ide-react/context/online-users-context' + +type HeaderProps = { + chatIsOpen: boolean + setChatIsOpen: (chatIsOpen: boolean) => void + historyIsOpen: boolean + setHistoryIsOpen: (historyIsOpen: boolean) => void +} + +export default function Header({ + chatIsOpen, + setChatIsOpen, + historyIsOpen, + setHistoryIsOpen, +}: HeaderProps) { + const { setLeftMenuShown } = useLayoutContext() + const { onlineUsersArray } = useOnlineUsersContext() + + function toggleChatOpen() { + setChatIsOpen(!chatIsOpen) + } + + function toggleHistoryOpen() { + setHistoryIsOpen(!historyIsOpen) + } + + const handleShowLeftMenuClick = useCallback(() => { + sendMB('navigation-clicked-menu') + setLeftMenuShown(value => !value) + }, [setLeftMenuShown]) + + return ( +
+
+ +
+
+ alert('Not implemented')} + /> + + + +
+
+ ) +} diff --git a/services/web/frontend/js/features/ide-react/components/layout/ide-page.tsx b/services/web/frontend/js/features/ide-react/components/layout/ide-page.tsx index 90c5185acb..3f2643b863 100644 --- a/services/web/frontend/js/features/ide-react/components/layout/ide-page.tsx +++ b/services/web/frontend/js/features/ide-react/components/layout/ide-page.tsx @@ -1,12 +1,22 @@ -import LayoutWithPlaceholders from '@/features/ide-react/components/layout/layout-with-placeholders' import { useConnectionContext } from '@/features/ide-react/context/connection-context' import useEventListener from '@/shared/hooks/use-event-listener' -import { useCallback, useEffect } from 'react' +import { useCallback, useEffect, useState } from 'react' import { Alerts } from '@/features/ide-react/components/alerts/alerts' +import { useLayoutContext } from '@/shared/context/layout-context' +import PlaceholderChat from '@/features/ide-react/components/layout/placeholder/placeholder-chat' +import PlaceholderHistory from '@/features/ide-react/components/layout/placeholder/placeholder-history' +import MainLayout from '@/features/ide-react/components/layout/main-layout' +import { EditorAndSidebar } from '@/features/ide-react/components/editor-and-sidebar' +import Header from '@/features/ide-react/components/header' +import EditorLeftMenu from '@/features/editor-left-menu/components/editor-left-menu' +import { useLayoutEventTracking } from '@/features/ide-react/hooks/use-layout-event-tracking' // This is filled with placeholder content while the real content is migrated // away from Angular export default function IdePage() { + useLayoutEventTracking() + + const [leftColumnDefaultSize, setLeftColumnDefaultSize] = useState(20) const { registerUserActivity } = useConnectionContext() // Inform the connection manager when the user is active @@ -22,11 +32,50 @@ export default function IdePage() { return () => document.body.removeEventListener('click', listener) }, [listener]) + const { chatIsOpen, setChatIsOpen, view, setView } = useLayoutContext() + const historyIsOpen = view === 'history' + const setHistoryIsOpen = useCallback( + (historyIsOpen: boolean) => { + setView(historyIsOpen ? 'history' : 'editor') + }, + [setView] + ) + + const headerContent = ( +
+ ) + const chatContent = + + const mainContent = historyIsOpen ? ( + + ) : ( + + ) + return ( <> - {/* TODO: Left menu will go here */} - + + ) } diff --git a/services/web/frontend/js/features/ide-react/components/layout/placeholder/placeholder-editor-sidebar.tsx b/services/web/frontend/js/features/ide-react/components/layout/placeholder/placeholder-editor-sidebar.tsx index c6f91c6691..b6ca7a73b3 100644 --- a/services/web/frontend/js/features/ide-react/components/layout/placeholder/placeholder-editor-sidebar.tsx +++ b/services/web/frontend/js/features/ide-react/components/layout/placeholder/placeholder-editor-sidebar.tsx @@ -2,13 +2,13 @@ import React from 'react' import { Panel, PanelGroup } from 'react-resizable-panels' import { VerticalResizeHandle } from '@/features/ide-react/components/resize/vertical-resize-handle' -type PlaceholderHeaderProps = { +type PlaceholderEditorSidebarProps = { shouldPersistLayout: boolean } export default function PlaceholderEditorSidebar({ shouldPersistLayout, -}: PlaceholderHeaderProps) { +}: PlaceholderEditorSidebarProps) { return (
+} diff --git a/services/web/frontend/js/features/ide-react/components/layout/placeholder/placeholder-header.tsx b/services/web/frontend/js/features/ide-react/components/layout/placeholder/placeholder-header.tsx index 88ca322af0..2e1190bf95 100644 --- a/services/web/frontend/js/features/ide-react/components/layout/placeholder/placeholder-header.tsx +++ b/services/web/frontend/js/features/ide-react/components/layout/placeholder/placeholder-header.tsx @@ -1,12 +1,13 @@ import React from 'react' import ChatToggleButton from '@/features/editor-navigation-toolbar/components/chat-toggle-button' import HistoryToggleButton from '@/features/editor-navigation-toolbar/components/history-toggle-button' +import LayoutDropdownButton from '@/features/editor-navigation-toolbar/components/layout-dropdown-button' type PlaceholderHeaderProps = { chatIsOpen: boolean setChatIsOpen: (chatIsOpen: boolean) => void historyIsOpen: boolean - setHistoryIsOpen: (chatIsOpen: boolean) => void + setHistoryIsOpen: (historyIsOpen: boolean) => void } export default function PlaceholderHeader({ @@ -27,6 +28,7 @@ export default function PlaceholderHeader({
Header placeholder
+ PDF
+} diff --git a/services/web/frontend/js/features/ide-react/connection/connection-manager.ts b/services/web/frontend/js/features/ide-react/connection/connection-manager.ts index 61edcab6c4..80bdd3ef5b 100644 --- a/services/web/frontend/js/features/ide-react/connection/connection-manager.ts +++ b/services/web/frontend/js/features/ide-react/connection/connection-manager.ts @@ -253,7 +253,7 @@ export class ConnectionManager extends Emitter { return true } - private disconnect() { + disconnect() { this.changeState({ ...this.state, readyState: WebSocket.CLOSED, diff --git a/services/web/frontend/js/features/ide-react/connection/editor-watchdog-manager.ts b/services/web/frontend/js/features/ide-react/connection/editor-watchdog-manager.ts new file mode 100644 index 0000000000..d6e5a2bd50 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/connection/editor-watchdog-manager.ts @@ -0,0 +1,234 @@ +/* + Migrated from services/web/frontend/js/ide/connection/EditorWatchdogManager.js + + EditorWatchdogManager is used for end-to-end checks of edits. + + + The editor UI is backed by Ace and CodeMirrors, which in turn are connected + to ShareJs documents in the frontend. + Edits propagate from the editor to ShareJs and are send through socket.io + and real-time to document-updater. + In document-updater edits are integrated into the document history and + a confirmation/rejection is sent back to the frontend. + + Along the way things can get lost. + We have certain safe-guards in place, but are still getting occasional + reports of lost edits. + + EditorWatchdogManager is implementing the basis for end-to-end checks on + two levels: + + - local/ShareJsDoc: edits that pass-by a ShareJs document shall get + acknowledged eventually. + - global: any edits made in the editor shall get acknowledged eventually, + independent for which ShareJs document (potentially none) sees it. + + How does this work? + =================== + + The global check is using a global EditorWatchdogManager that is available + via the angular factory 'ide'. + Local/ShareJsDoc level checks will connect to the global instance. + + Each EditorWatchdogManager keeps track of the oldest un-acknowledged edit. + When ever a ShareJs document receives an acknowledgement event, a local + EditorWatchdogManager will see it and also notify the global instance about + it. + The next edit cycle will clear the oldest un-acknowledged timestamp in case + a new ack has arrived, otherwise it will bark loud! via the timeout handler. + + Scenarios + ========= + + - User opens the CodeMirror editor + - attach global check to new CM instance + - detach Ace from the local EditorWatchdogManager + - when the frontend attaches the CM instance to ShareJs, we also + attach it to the local EditorWatchdogManager + - the internal attach process writes the document content to the editor, + which in turn emits 'change' events. These event need to be excluded + from the watchdog. EditorWatchdogManager.ignoreEditsFor takes care + of that. + - User opens the Ace editor (again) + - (attach global check to the Ace editor, only one copy of Ace is around) + - detach local EditorWatchdogManager from CM + - likewise with CM, attach Ace to the local EditorWatchdogManager + - User makes an edit + - the editor will emit a 'change' event + - the global EditorWatchdogManager will process it first + - the local EditorWatchdogManager will process it next + - Document-updater confirms an edit + - the local EditorWatchdogManager will process it first, it passes it on to + - the global EditorWatchdogManager will process it next + + Time + ==== + + The delay between edits and acks is measured using a monotonic clock: + `performance.now()`. + It is agnostic to system clock changes in either direction and timezone + changes do not affect it as well. + Roughly speaking, it is initialized with `0` when the `window` context is + created, before our JS app boots. + As per canIUse.com and MDN `performance.now()` is available to all supported + Browsers, including IE11. + See also: https://caniuse.com/?search=performance.now + See also: https://developer.mozilla.org/en-US/docs/Web/API/Performance/now + */ + +import { + ChangeDescription, + EditorFacade, +} from '../../source-editor/extensions/realtime' +import { debugConsole } from '@/utils/debugging' + +// TIMEOUT specifies the timeout for edits into a single ShareJsDoc. +const TIMEOUT = 60 * 1000 +// GLOBAL_TIMEOUT specifies the timeout for edits into any ShareJSDoc. +const GLOBAL_TIMEOUT = TIMEOUT +// REPORT_EVERY specifies how often we send events/report errors. +const REPORT_EVERY = 60 * 1000 + +const SCOPE_LOCAL = 'ShareJsDoc' +const SCOPE_GLOBAL = 'global' + +type Scope = 'ShareJsDoc' | 'global' +type Meta = { + scope: Scope + delay: number + lastAck: number + lastUnackedEdit: number +} +type TimeoutHandler = (meta: Meta) => void + +class Reporter { + private lastReport: number | null = null + private queue: Meta[] = [] + + // eslint-disable-next-line no-useless-constructor + constructor(private readonly onTimeoutHandler: TimeoutHandler) {} + + private getMetaPreferLocal() { + for (const meta of this.queue) { + if (meta.scope === SCOPE_LOCAL) { + return meta + } + } + return this.queue.pop() + } + + onTimeout(meta: Meta) { + // Collect all 'meta's for this update. + // global arrive before local ones, but we are eager to report local ones. + this.queue.push(meta) + + setTimeout(() => { + // Another handler processed the 'meta' entry already + if (!this.queue.length) return + + const maybeLocalMeta = this.getMetaPreferLocal() + + // Discard other, newly arrived 'meta's + this.queue.length = 0 + + const now = Date.now() + // Do not flood the server with losing-edits events + const reportedRecently = + this.lastReport !== null && now - this.lastReport < REPORT_EVERY + if (!reportedRecently && maybeLocalMeta) { + this.lastReport = now + this.onTimeoutHandler(maybeLocalMeta) + } + }) + } +} + +export default class EditorWatchdogManager { + lastAck: number | null = null + reporter: Reporter + // eslint-disable-next-line no-use-before-define + parent?: EditorWatchdogManager + scope: Scope + timeout: number + lastUnackedEdit: number | null + + constructor({ + parent, + onTimeoutHandler, + }: { + parent?: EditorWatchdogManager + onTimeoutHandler?: TimeoutHandler + }) { + this.scope = parent ? SCOPE_LOCAL : SCOPE_GLOBAL + this.timeout = parent ? TIMEOUT : GLOBAL_TIMEOUT + this.parent = parent + if (parent) { + this.reporter = parent.reporter + } else if (onTimeoutHandler) { + this.reporter = new Reporter(onTimeoutHandler) + } else { + throw new Error('No parent or onTimeoutHandler') + } + + this.lastAck = null + this.lastUnackedEdit = null + } + + onAck() { + this.lastAck = performance.now() + + // bubble up to globalEditorWatchdogManager + if (this.parent) this.parent.onAck() + } + + onEdit() { + // Use timestamps to track the high-water mark of unacked edits + const now = performance.now() + + // Discard the last unacked edit if there are now newer acks + // TODO Handle cases where lastAck and/or lastUnackedEdit are null more transparently + // @ts-ignore + if (this.lastAck > this.lastUnackedEdit) { + this.lastUnackedEdit = null + } + // Start tracking for this keypress if we aren't already tracking an + // unacked edit + if (!this.lastUnackedEdit) { + this.lastUnackedEdit = now + } + + // Report an error if the last tracked edit hasn't been cleared by an + // ack from the server after a long time + const delay = now - this.lastUnackedEdit + if (delay > this.timeout) { + const timeOrigin = Date.now() - now + const scope = this.scope + const lastAck = this.lastAck ? timeOrigin + this.lastAck : 0 + const lastUnackedEdit = timeOrigin + this.lastUnackedEdit + const meta: Meta = { scope, delay, lastAck, lastUnackedEdit } + this.log('timedOut', meta) + this.reporter.onTimeout(meta) + } + } + + attachToEditor(editor: EditorFacade) { + this.log('attach to editor') + const onChange = ( + _editor: EditorFacade, + changeDescription: ChangeDescription + ) => { + if (changeDescription.origin === 'remote') return + if (!(changeDescription.removed || changeDescription.inserted)) return + this.onEdit() + } + editor.on('change', onChange) + return () => { + this.log('detach from editor') + editor.off('change', onChange) + } + } + + private log(...args: any[]) { + debugConsole.log(`[EditorWatchdogManager] ${this.scope}:`, ...args) + } +} diff --git a/services/web/frontend/js/features/ide-react/connection/types/socket.ts b/services/web/frontend/js/features/ide-react/connection/types/socket.ts index 3f0cc97840..4f9ec128a5 100644 --- a/services/web/frontend/js/features/ide-react/connection/types/socket.ts +++ b/services/web/frontend/js/features/ide-react/connection/types/socket.ts @@ -21,6 +21,7 @@ export type Socket = { callback?: (error: Error, ...data: any[]) => void ): void socket: { + connected: boolean connect(): void } disconnect(): void diff --git a/services/web/frontend/js/features/ide-react/context/connection-context.tsx b/services/web/frontend/js/features/ide-react/context/connection-context.tsx index c39d982ed9..8ad7c25396 100644 --- a/services/web/frontend/js/features/ide-react/context/connection-context.tsx +++ b/services/web/frontend/js/features/ide-react/context/connection-context.tsx @@ -20,6 +20,7 @@ type ConnectionContextValue = { secondsUntilReconnect: () => number tryReconnectNow: () => void registerUserActivity: () => void + disconnect: () => void } const ConnectionContext = createContext( @@ -64,6 +65,10 @@ export const ConnectionProvider: FC = ({ children }) => { [connectionManager] ) + const disconnect = useCallback(() => { + connectionManager.disconnect() + }, [connectionManager]) + const value = useMemo( () => ({ socket: connectionManager.socket, @@ -73,6 +78,7 @@ export const ConnectionProvider: FC = ({ children }) => { secondsUntilReconnect, tryReconnectNow, registerUserActivity, + disconnect, }), [ connectionManager.socket, @@ -82,6 +88,7 @@ export const ConnectionProvider: FC = ({ children }) => { registerUserActivity, secondsUntilReconnect, tryReconnectNow, + disconnect, ] ) diff --git a/services/web/frontend/js/features/ide-react/context/editor-manager-context.tsx b/services/web/frontend/js/features/ide-react/context/editor-manager-context.tsx new file mode 100644 index 0000000000..5531844a96 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/context/editor-manager-context.tsx @@ -0,0 +1,597 @@ +/* eslint-disable no-use-before-define */ +import { ReactScopeValueStore } from '@/features/ide-react/scope-value-store/react-scope-value-store' +import { + createContext, + FC, + useCallback, + useContext, + useEffect, + useMemo, + useRef, + useState, +} from 'react' +import customLocalStorage from '@/infrastructure/local-storage' +import { sendMB } from '@/infrastructure/event-tracking' +import useScopeValue from '@/shared/hooks/use-scope-value' +import { useIdeContext } from '@/shared/context/ide-context' +import { OpenDocuments } from '@/features/ide-react/editor/open-documents' +import EditorWatchdogManager from '@/features/ide-react/connection/editor-watchdog-manager' +import { useIdeReactContext } from '@/features/ide-react/context/ide-react-context' +import { useConnectionContext } from '@/features/ide-react/context/connection-context' +import { debugConsole } from '@/utils/debugging' +import { Document } from '@/features/ide-react/editor/document' +import { useLayoutContext } from '@/shared/context/layout-context' +import { GotoLineOptions } from '@/features/ide-react/types/goto-line-options' +import _ from 'lodash' +import { Doc } from '../../../../../types/doc' +import { useFileTreeData } from '@/shared/context/file-tree-data-context' +import { findDocEntityById } from '@/features/ide-react/util/find-doc-entity-by-id' +import useScopeEventEmitter from '@/shared/hooks/use-scope-event-emitter' + +interface GotoOffsetOptions { + gotoOffset: number +} + +interface OpenDocOptions + extends Partial, + Partial { + gotoOffset?: number + forceReopen?: boolean +} + +type EditorManager = { + getEditorType: () => 'cm6' | 'cm6-rich-text' | null + showSymbolPalette: boolean + currentDocument: Document + getCurrentDocValue: () => string | null + getCurrentDocId: () => string | null + startIgnoringExternalUpdates: () => void + stopIgnoringExternalUpdates: () => void + openDocId: (docId: string, options?: OpenDocOptions) => void + openDoc: (document: Document, options?: OpenDocOptions) => void + jumpToLine: (options: GotoLineOptions) => void +} + +type DocumentOpenCallback = (error: Error | null, document?: Document) => void + +function hasGotoLine(options: OpenDocOptions): options is GotoLineOptions { + return typeof options.gotoLine === 'number' +} + +function hasGotoOffset(options: OpenDocOptions): options is GotoOffsetOptions { + return typeof options.gotoOffset === 'number' +} + +export type EditorScopeValue = { + showSymbolPalette: false + toggleSymbolPalette: () => void + sharejs_doc: Document | null + open_doc_id: string | null + open_doc_name: string | null + opening: boolean + trackChanges: boolean + wantTrackChanges: boolean + showVisual: boolean + newSourceEditor: boolean + multiSelectedCount: number + error_state: boolean +} + +export function populateEditorScope(store: ReactScopeValueStore) { + const projectId = window.project_id + + // This value is not used in the React code. It's just here to prevent errors + // from EditorProvider + store.set('state.loading', false) + + store.set('project.name', null) + + store.set('editor', { + showSymbolPalette: false, + toggleSymbolPalette: () => {}, + sharejs_doc: null, + open_doc_id: null, + open_doc_name: null, + opening: true, + trackChanges: false, + wantTrackChanges: false, + // No Ace here + newSourceEditor: true, + // TODO: Ignore multiSelectedCount and just default it to zero for now until + // we get to the file tree. It seems to be stored in two places in the + // Angular scope and needs further investigation. + multiSelectedCount: 0, + error_state: false, + }) + store.persisted('editor.showVisual', false, `editor.mode.${projectId}`, { + toPersisted: showVisual => (showVisual ? 'rich-text' : 'source'), + fromPersisted: mode => mode === 'rich-text', + }) +} + +const EditorManagerContext = createContext(undefined) + +export const EditorManagerProvider: FC = ({ children }) => { + const ide = useIdeContext() + const { reportError, eventEmitter, eventLog, projectId } = + useIdeReactContext() + const { socket, disconnect } = useConnectionContext() + const { view, setView } = useLayoutContext() + + const [showSymbolPalette, setShowSymbolPalette] = useScopeValue( + 'editor.showSymbolPalette' + ) + const [showVisual] = useScopeValue('editor.showVisual') + // eslint-disable-next-line no-unused-vars + const [currentDocument, setCurrentDocument] = + useScopeValue('editor.sharejs_doc') + const [openDocId, setOpenDocId] = useScopeValue( + 'editor.open_doc_id' + ) + const [, setOpenDocName] = useScopeValue( + 'editor.open_doc_name' + ) + const [, setOpening] = useScopeValue('editor.opening') + const [, setIsInErrorState] = useScopeValue('editor.error_state') + const [, setTrackChanges] = useScopeValue('editor.trackChanges') + const [wantTrackChanges] = useScopeValue('editor.wantTrackChanges') + + const goToLineEmitter = useScopeEventEmitter('editor:gotoLine') + + const { fileTreeData } = useFileTreeData() + + // eslint-disable-next-line no-unused-vars + const [ignoringExternalUpdates, setIgnoringExternalUpdates] = useState(false) + + const [globalEditorWatchdogManager] = useState( + () => + new EditorWatchdogManager({ + onTimeoutHandler: (meta: Record) => { + sendMB('losing-edits', meta) + reportError('losing-edits', meta) + }, + }) + ) + + const [docTooLongErrorShown, setDocTooLongErrorShown] = useState(false) + + const [openDocs] = useState( + () => + new OpenDocuments( + socket, + globalEditorWatchdogManager, + eventEmitter, + eventLog + ) + ) + + const editorOpenDocEpochRef = useRef(0) + + // TODO: This looks dodgy because it wraps a state setter and is itself + // stored in React state in the scope store. The problem is that it needs to + // be exposed via the scope store because some components access it that way; + // it would be better to simply access it from a context, but the current + // implementation in EditorManager interacts with Angular scope to update + // the layout. Once Angular is gone, this can become a context method. + useEffect(() => { + ide.scopeStore.set('editor.toggleSymbolPalette', () => { + setShowSymbolPalette(show => { + const newValue = !show + sendMB(newValue ? 'symbol-palette-show' : 'symbol-palette-hide') + return newValue + }) + }) + }, [ide.scopeStore, setShowSymbolPalette]) + + const getEditorType = useCallback(() => { + if (!currentDocument) { + return null + } + + return showVisual ? 'cm6-rich-text' : 'cm6' + }, [currentDocument, showVisual]) + + const getCurrentDocValue = useCallback(() => { + return currentDocument?.getSnapshot() ?? null + }, [currentDocument]) + + const getCurrentDocId = useCallback(() => openDocId, [openDocId]) + + const startIgnoringExternalUpdates = useCallback( + () => setIgnoringExternalUpdates(true), + [] + ) + const stopIgnoringExternalUpdates = useCallback( + () => setIgnoringExternalUpdates(false), + [] + ) + + // Ignore insertSymbol from Angular EditorManager because it's only required + // for Ace. + + const jumpToLine = useCallback( + (options: GotoLineOptions) => { + goToLineEmitter( + options.gotoLine, + options.gotoColumn ?? 0, + options.syncToPdf ?? false + ) + }, + [goToLineEmitter] + ) + + const unbindFromDocumentEvents = (document: Document) => { + document.off() + } + + const openDocWithId = useCallback( + (docId: string, options: OpenDocOptions = {}) => { + const doc = findDocEntityById(fileTreeData, docId) + if (!doc) { + return + } + openDoc(doc, options) + }, + // @ts-ignore + [fileTreeData, openDoc] + ) + + // @ts-ignore + const attachErrorHandlerToDocument = useCallback( + (doc: Doc, document: Document) => { + document.on( + 'error', + ( + error: Error | string, + meta?: Record, + editorContent?: string + ) => { + const message = + typeof error === 'string' ? error : error?.message ?? '' + if (/maxDocLength/.test(message)) { + setDocTooLongErrorShown(true) + openDoc(doc, { forceReopen: true }) + + // TODO: MIGRATION: Show generic modal here + // const genericMessageModal = this.ide.showGenericMessageModal( + // 'Document Too Long', + // 'Sorry, this file is too long to be edited manually. Please upload it directly.' + // ) + // genericMessageModal.result.finally(() => { + // this.$scope.docTooLongErrorShown = false + // }) + } else if (/too many comments or tracked changes/.test(message)) { + // TODO: MIGRATION: Show generic modal here + // this.ide.showGenericMessageModal( + // 'Too many comments or tracked changes', + // 'Sorry, this file has too many comments or tracked changes. Please try accepting or rejecting some existing changes, or resolving and deleting some comments.' + // ) + } else if (!docTooLongErrorShown) { + // Do not allow this doc to open another error modal. + document.off('error') + + // Preserve the sharejs contents before the teardown. + editorContent = + typeof editorContent === 'string' + ? editorContent + : document.doc?._doc.snapshot + + // Tear down the ShareJsDoc. + if (document.doc) document.doc.clearInflightAndPendingOps() + + // Do not re-join after re-connecting. + document.leaveAndCleanUp() + + disconnect() + reportError(error, meta) + + // Tell the user about the error state. + setIsInErrorState(true) + + // TODO: MIGRATION: Show out-of-sync modal + // this.ide.showOutOfSyncModal( + // 'Out of sync', + // "Sorry, this file has gone out of sync and we need to do a full refresh.
Please see this help guide for more information", + // editorContent + // ) + + // Do not forceReopen the document. + return + } + + eventEmitter.once('project:joined', () => { + openDoc(doc, { forceReopen: true }) + }) + } + ) + }, + [ + disconnect, + docTooLongErrorShown, + eventEmitter, + // @ts-ignore + openDoc, + reportError, + setIsInErrorState, + ] + ) + + const bindToDocumentEvents = useCallback( + (doc: Doc, document: Document) => { + attachErrorHandlerToDocument(doc, document) + + // TODO: MIGRATION: Add a type for `update` + document.on('externalUpdate', (update: any) => { + if (ignoringExternalUpdates) { + return + } + if ( + _.property(['meta', 'type'])(update) === 'external' && + _.property(['meta', 'source'])(update) === 'git-bridge' + ) { + // eslint-disable-next-line no-useless-return + return + } + // TODO: MIGRATION: Show generic modal here + // this.ide.showGenericMessageModal( + // 'Document Updated Externally', + // 'This document was just updated externally. Any recent changes you have made may have been overwritten. To see previous versions please look in the history.' + // ) + }) + }, + [attachErrorHandlerToDocument, ignoringExternalUpdates] + ) + + const syncTimeoutRef = useRef(null) + + const syncTrackChangesState = useCallback( + (doc: Document) => { + if (!doc) { + return + } + + if (syncTimeoutRef.current) { + window.clearTimeout(syncTimeoutRef.current) + syncTimeoutRef.current = null + } + + const want = wantTrackChanges + const have = doc.getTrackingChanges() + if (wantTrackChanges === have) { + setTrackChanges(want) + return + } + + const tryToggle = () => { + const saved = doc.getInflightOp() == null && doc.getPendingOp() == null + if (saved) { + doc.setTrackingChanges(wantTrackChanges) + setTrackChanges(want) + } else { + syncTimeoutRef.current = window.setTimeout(tryToggle, 100) + } + } + + tryToggle() + }, + [setTrackChanges, wantTrackChanges] + ) + + const doOpenNewDocument = useCallback( + (doc: Doc, callback: DocumentOpenCallback) => { + debugConsole.log('[doOpenNewDocument] Opening...') + const newDocument = openDocs.getDocument(doc._id) + if (!newDocument) { + debugConsole.error(`No open document with ID '${doc._id}' found`) + return + } + const preJoinEpoch = ++editorOpenDocEpochRef.current + newDocument.join(error => { + if (error) { + debugConsole.log( + `[doOpenNewDocument] error joining doc ${doc._id}`, + error + ) + callback(error) + } + if (editorOpenDocEpochRef.current !== preJoinEpoch) { + debugConsole.log( + `[openNewDocument] editorOpenDocEpoch mismatch ${editorOpenDocEpochRef.current} vs ${preJoinEpoch}` + ) + newDocument.leaveAndCleanUp() + return callback(new Error('another document was loaded')) + } + bindToDocumentEvents(doc, newDocument) + return callback(null, newDocument) + }) + }, + [bindToDocumentEvents, openDocs] + ) + + // @ts-ignore + const openNewDocument = useCallback( + (doc: Doc, callback: DocumentOpenCallback) => { + // Leave the current document + // - when we are opening a different new one, to avoid race conditions + // between leaving and joining the same document + // - when the current one has pending ops that need flushing, to avoid + // race conditions from cleanup + const currentDocId = currentDocument?.doc_id + const hasBufferedOps = currentDocument?.hasBufferedOps() + const changingDoc = currentDocument && currentDocId !== doc._id + if (changingDoc || hasBufferedOps) { + debugConsole.log('[openNewDocument] Leaving existing open doc...') + + // Do not trigger any UI changes from remote operations + unbindFromDocumentEvents(currentDocument) + + // Keep listening for out-of-sync and similar errors. + attachErrorHandlerToDocument(doc, currentDocument) + + // Teardown the Document -> ShareJsDoc -> sharejs doc + // By the time this completes, the Document instance is no longer + // registered in OpenDocuments and doOpenNewDocument can start + // from scratch -- read: no corrupted internal state. + const preLeaveEpoch = ++editorOpenDocEpochRef.current + + currentDocument.leaveAndCleanUp(error => { + if (error) { + debugConsole.log( + `[_openNewDocument] error leaving doc ${currentDocId}`, + error + ) + return callback(error) + } + if (editorOpenDocEpochRef.current !== preLeaveEpoch) { + debugConsole.log( + `[openNewDocument] editorOpenDocEpoch mismatch ${editorOpenDocEpochRef.current} vs ${preLeaveEpoch}` + ) + callback(new Error('another document was loaded')) + } + doOpenNewDocument(doc, callback) + }) + } else { + doOpenNewDocument(doc, callback) + } + }, + [attachErrorHandlerToDocument, doOpenNewDocument, currentDocument] + ) + + // @ts-ignore + const openDoc = useCallback( + (doc: Doc, options: OpenDocOptions = {}) => { + debugConsole.log(`[openDoc] Opening ${doc._id}`) + if (view === 'editor') { + // store position of previous doc before switching docs + eventEmitter.emit('store-doc-position') + } + setView('editor') + + const done = (isNewDoc: boolean) => { + window.dispatchEvent( + new CustomEvent('doc:after-opened', { detail: isNewDoc }) + ) + if (hasGotoLine(options)) { + // In CM6, jump to the line again after a stored scroll position has been restored + if (isNewDoc) { + window.addEventListener( + 'editor:scroll-position-restored', + () => jumpToLine(options), + { once: true } + ) + } + } else if (hasGotoOffset(options)) { + window.setTimeout(() => { + eventEmitter.emit('editor:gotoOffset', options.gotoOffset) + }) + } + } + + // If we already have the document open we can return at this point. + // Note: only use forceReopen:true to override this when the document is + // out of sync and needs to be reloaded from the server. + if (doc._id === openDocId && !options.forceReopen) { + window.dispatchEvent( + new CustomEvent('editor.openDoc', { detail: doc._id }) + ) + done(false) + return + } + + // We're now either opening a new document or reloading a broken one. + setOpenDocId(doc._id) + setOpenDocName(doc.name) + customLocalStorage.setItem(`doc.open_id.${projectId}`, doc._id) + + setOpening(true) + + openNewDocument(doc, (error: Error | null, document: Document) => { + if (error?.message === 'another document was loaded') { + debugConsole.log( + `[openDoc] another document was loaded while ${doc._id} was loading` + ) + return + } + if (error != null) { + // TODO: MIGRATION: Show generic modal here + // this.ide.showGenericMessageModal( + // 'Error opening document', + // 'Sorry, something went wrong opening this document. Please try again.' + // ) + return + } + + syncTrackChangesState(document) + + eventEmitter.emit('doc:opened') + + setOpening(false) + setCurrentDocument(document) + done(true) + }) + }, + [ + eventEmitter, + jumpToLine, + openDocId, + openNewDocument, + projectId, + setCurrentDocument, + setOpenDocId, + setOpenDocName, + setOpening, + setView, + syncTrackChangesState, + view, + ] + ) + + const editorManager = useMemo( + () => ({ + getEditorType, + showSymbolPalette, + currentDocument, + getCurrentDocValue, + getCurrentDocId, + startIgnoringExternalUpdates, + stopIgnoringExternalUpdates, + openDocId: openDocWithId, + openDoc, + jumpToLine, + }), + [ + getEditorType, + showSymbolPalette, + currentDocument, + getCurrentDocValue, + getCurrentDocId, + startIgnoringExternalUpdates, + stopIgnoringExternalUpdates, + openDocWithId, + openDoc, + jumpToLine, + ] + ) + + // Expose editorManager via ide object because some React code relies on it, + // for now + ide.editorManager = editorManager + + return ( + + {children} + + ) +} + +export function useEditorManagerContext(): EditorManager { + const context = useContext(EditorManagerContext) + + if (!context) { + throw new Error( + 'useEditorManagerContext is only available inside EditorManagerProvider' + ) + } + + return context +} diff --git a/services/web/frontend/js/features/ide-react/context/ide-react-context.tsx b/services/web/frontend/js/features/ide-react/context/ide-react-context.tsx index 247019c302..d8c263b95b 100644 --- a/services/web/frontend/js/features/ide-react/context/ide-react-context.tsx +++ b/services/web/frontend/js/features/ide-react/context/ide-react-context.tsx @@ -1,12 +1,14 @@ -import { +import React, { createContext, useContext, useState, FC, useMemo, useEffect, + useCallback, } from 'react' import { ReactScopeValueStore } from '@/features/ide-react/scope-value-store/react-scope-value-store' +import populateLayoutScope from '@/features/ide-react/scope-adapters/layout-context-adapter' import { IdeProvider } from '@/shared/context/ide-context' import { createIdeEventEmitter, @@ -15,17 +17,37 @@ import { import { JoinProjectPayload } from '@/features/ide-react/connection/join-project-payload' import { useConnectionContext } from '@/features/ide-react/context/connection-context' import { getMockIde } from '@/shared/context/mock/mock-ide' +import { populateEditorScope } from '@/features/ide-react/context/editor-manager-context' +import { debugConsole } from '@/utils/debugging' +import { postJSON } from '@/infrastructure/fetch-json' +import { EventLog } from '@/features/ide-react/editor/event-log' +import { populateSettingsScope } from '@/features/ide-react/scope-adapters/settings-adapter' +import { populateOnlineUsersScope } from '@/features/ide-react/context/online-users-context' import { ReactScopeEventEmitter } from '@/features/ide-react/scope-event-emitter/react-scope-event-emitter' +import { populateReferenceScope } from '@/features/ide-react/context/references-context' type IdeReactContextValue = { projectId: string eventEmitter: IdeEventEmitter + eventLog: EventLog + startedFreeTrial: boolean + setStartedFreeTrial: React.Dispatch< + React.SetStateAction + > + reportError: (error: any, meta?: Record) => void } -const IdeReactContext = createContext(null) +const IdeReactContext = createContext( + undefined +) + +function showGenericMessageModal(title: string, message: string) { + debugConsole.log('*** showGenericMessageModal ***', title, message) +} function populateIdeReactScope(store: ReactScopeValueStore) { store.set('sync_tex_error', false) + store.set('settings', window.userSettings) } function populateProjectScope(store: ReactScopeValueStore) { @@ -33,6 +55,14 @@ function populateProjectScope(store: ReactScopeValueStore) { store.set('permissionsLevel', 'readOnly') } +function populatePdfScope(store: ReactScopeValueStore) { + store.allowNonExistentPath('pdf', true) +} + +function populateFileTreeScope(store: ReactScopeValueStore) { + store.set('docs', []) +} + function createReactScopeValueStore() { const scopeStore = new ReactScopeValueStore() @@ -42,7 +72,17 @@ function createReactScopeValueStore() { // initialization code together with the context and would only populate // necessary values in the store, but this is simpler for now populateIdeReactScope(scopeStore) + populateEditorScope(scopeStore) + populateLayoutScope(scopeStore) populateProjectScope(scopeStore) + populatePdfScope(scopeStore) + populateSettingsScope(scopeStore) + populateOnlineUsersScope(scopeStore) + populateReferenceScope(scopeStore) + populateFileTreeScope(scopeStore) + + scopeStore.allowNonExistentPath('hasLintingError') + scopeStore.allowNonExistentPath('loadingThreads') return scopeStore } @@ -55,24 +95,42 @@ export const IdeReactProvider: FC = ({ children }) => { const [scopeEventEmitter] = useState( () => new ReactScopeEventEmitter(eventEmitter) ) + const [eventLog] = useState(() => new EventLog()) + const [startedFreeTrial, setStartedFreeTrial] = useState(false) const { socket } = useConnectionContext() - // Fire project:joined event - useEffect(() => { - function handleJoinProjectResponse({ - project, - permissionsLevel, - }: JoinProjectPayload) { - eventEmitter.emit('project:joined', { project, permissionsLevel }) - } + const reportError = useCallback( + (error: any, meta?: Record) => { + const metadata = { + ...meta, + user_id: window.user_id, + project_id: projectId, + // @ts-ignore + client_id: socket.socket.sessionid, + // @ts-ignore + transport: socket.socket.transport, + client_now: new Date(), + } - socket.on('joinProjectResponse', handleJoinProjectResponse) - - return () => { - socket.removeListener('joinProjectResponse', handleJoinProjectResponse) - } - }, [socket, eventEmitter]) + const errorObj: Record = {} + if (typeof error === 'object') { + for (const key of Object.getOwnPropertyNames(error)) { + errorObj[key] = error[key] + } + } else if (typeof error === 'string') { + errorObj.message = error + } + return postJSON('/error/client', { + body: { + error: errorObj, + meta: metadata, + _csrf: window.csrfToken, + }, + }) + }, + [socket.socket] + ) // Populate scope values when joining project, then fire project:joined event useEffect(() => { @@ -98,15 +156,28 @@ export const IdeReactProvider: FC = ({ children }) => { return { ...getMockIde(), socket, + showGenericMessageModal, + reportError, + // TODO: MIGRATION: Remove this once it's no longer used + fileTreeManager: { + findEntityByPath: () => null, + selectEntity: () => {}, + getPreviewByPath: () => null, + getRootDocDirname: () => '', + }, } - }, [socket]) + }, [socket, reportError]) const value = useMemo( () => ({ eventEmitter, + eventLog, + startedFreeTrial, + setStartedFreeTrial, projectId, + reportError, }), - [eventEmitter] + [eventEmitter, eventLog, reportError, startedFreeTrial] ) return ( diff --git a/services/web/frontend/js/features/ide-react/context/metadata-context.tsx b/services/web/frontend/js/features/ide-react/context/metadata-context.tsx new file mode 100644 index 0000000000..c3c34072af --- /dev/null +++ b/services/web/frontend/js/features/ide-react/context/metadata-context.tsx @@ -0,0 +1,228 @@ +import { + createContext, + useContext, + useEffect, + FC, + useCallback, + useMemo, + useState, + useRef, +} from 'react' +import { useIdeReactContext } from '@/features/ide-react/context/ide-react-context' +import { useConnectionContext } from '@/features/ide-react/context/connection-context' +import { useEditorManagerContext } from '@/features/ide-react/context/editor-manager-context' +import _ from 'lodash' +import { getJSON, postJSON } from '@/infrastructure/fetch-json' +import { useOnlineUsersContext } from '@/features/ide-react/context/online-users-context' +import { useEditorContext } from '@/shared/context/editor-context' +import { useIdeContext } from '@/shared/context/ide-context' +import useSocketListener from '@/features/ide-react/hooks/use-socket-listener' +import useEventListener from '@/shared/hooks/use-event-listener' + +type DocumentMetadata = { + labels: string[] + packages: Record +} + +type DocumentsMetadata = Record + +type MetadataContextValue = { + metadata: { + state: { + documents: DocumentsMetadata + } + getAllLabels: () => DocumentMetadata['labels'] + getAllPackages: () => DocumentMetadata['packages'] + } +} + +type DocMetadataResponse = { docId: string; meta: DocumentMetadata } + +const MetadataContext = createContext( + undefined +) + +export const MetadataProvider: FC = ({ children }) => { + const ide = useIdeContext() + const { eventEmitter, projectId } = useIdeReactContext() + const { socket } = useConnectionContext() + const { onlineUsersCount } = useOnlineUsersContext() + const { permissionsLevel } = useEditorContext() + const { currentDocument } = useEditorManagerContext() + + const [documents, setDocuments] = useState({}) + + const debouncerRef = useRef>(new Map()) // DocId => Timeout + + useEffect(() => { + eventEmitter.on('entity:deleted', entity => { + if (entity.type === 'doc') { + setDocuments(documents => _.omit(documents, entity.id)) + } + }) + }, [eventEmitter]) + + useEffect(() => { + window.dispatchEvent( + new CustomEvent('project:metadata', { detail: documents }) + ) + }, [documents]) + + const onBroadcastDocMeta = useCallback((data: DocMetadataResponse) => { + const { docId, meta } = data + if (docId != null && meta != null) { + setDocuments(documents => ({ ...documents, [docId]: meta })) + } + }, []) + + const getAllLabels = useCallback( + () => + _.flattenDeep( + Array.from(Object.values(documents)).map(meta => meta.labels) + ), + [documents] + ) + + const getAllPackages = useCallback(() => { + const packageCommandMapping: Record = {} + for (const meta of Object.values(documents)) { + for (const [packageName, commandSnippets] of Object.entries( + meta.packages + )) { + packageCommandMapping[packageName] = commandSnippets + } + } + return packageCommandMapping + }, [documents]) + + const loadProjectMetaFromServer = useCallback(() => { + getJSON(`/project/${projectId}/metadata`).then( + (response: { projectMeta: DocumentsMetadata }) => { + const { projectMeta } = response + if (projectMeta) { + setDocuments(projectMeta) + } + } + ) + }, [projectId]) + + const loadDocMetaFromServer = useCallback( + (docId: string) => { + // Don't broadcast metadata when there are no other users in the + // project. + const broadcast = onlineUsersCount > 0 + postJSON(`/project/${projectId}/doc/${docId}/metadata`, { + body: { + broadcast, + }, + }).then((response: DocMetadataResponse) => { + if (!broadcast && response) { + // handle the POST response like a broadcast event when there are no + // other users in the project. + onBroadcastDocMeta(response) + } + }) + }, + [onBroadcastDocMeta, onlineUsersCount, projectId] + ) + + const scheduleLoadDocMetaFromServer = useCallback( + (docId: string) => { + if (permissionsLevel === 'readOnly') { + // The POST request is blocked for users without write permission. + // The user will not be able to consume the metadata for edits anyway. + return + } + // Debounce loading labels with a timeout + const existingTimeout = debouncerRef.current.get(docId) + + if (existingTimeout != null) { + window.clearTimeout(existingTimeout) + debouncerRef.current.delete(docId) + } + + debouncerRef.current.set( + docId, + window.setTimeout(() => { + // TODO: wait for the document to be saved? + loadDocMetaFromServer(docId) + debouncerRef.current.delete(docId) + }, 2000) + ) + }, + [loadDocMetaFromServer, permissionsLevel] + ) + + const handleBroadcastDocMeta = useCallback( + (data: DocMetadataResponse) => { + onBroadcastDocMeta(data) + }, + [onBroadcastDocMeta] + ) + + useSocketListener(socket, 'broadcastDocMeta', handleBroadcastDocMeta) + + const handleMetadataOutdated = useCallback(() => { + if (currentDocument) { + scheduleLoadDocMetaFromServer(currentDocument.doc_id) + } + }, [currentDocument, scheduleLoadDocMetaFromServer]) + + useEventListener('editor:metadata-outdated', handleMetadataOutdated) + + useEffect(() => { + eventEmitter.once('project:joined', ({ project }) => { + if (project.deletedByExternalDataSource) { + // TODO: MIGRATION: Show generic message modal here + /* + ide.showGenericMessageModal( + 'Project Renamed or Deleted', + `\ +This project has either been renamed or deleted by an external data source such as Dropbox. +We don't want to delete your data on Overleaf, so this project still contains your history and collaborators. +If the project has been renamed please look in your project list for a new project under the new name.\ +` + ) +*/ + } + window.setTimeout(() => { + if (permissionsLevel !== 'readOnly') { + loadProjectMetaFromServer() + } + }, 200) + }) + }, [eventEmitter, loadProjectMetaFromServer, permissionsLevel]) + + const value = useMemo( + () => ({ + metadata: { + state: { documents }, + getAllLabels, + getAllPackages, + }, + }), + [documents, getAllLabels, getAllPackages] + ) + + // Expose metadataManager via ide object because useCodeMirrorScope relies on + // it, for now + ide.metadataManager = value + + return ( + + {children} + + ) +} + +export function useMetadataContext(): MetadataContextValue { + const context = useContext(MetadataContext) + + if (!context) { + throw new Error( + 'useMetadataContext is only available inside MetadataProvider' + ) + } + + return context +} diff --git a/services/web/frontend/js/features/ide-react/context/online-users-context.tsx b/services/web/frontend/js/features/ide-react/context/online-users-context.tsx new file mode 100644 index 0000000000..afa0107737 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/context/online-users-context.tsx @@ -0,0 +1,297 @@ +import { + createContext, + useContext, + useEffect, + FC, + useCallback, + useMemo, + useState, +} from 'react' +import { ReactScopeValueStore } from '@/features/ide-react/scope-value-store/react-scope-value-store' +import { useIdeReactContext } from '@/features/ide-react/context/ide-react-context' +import { useConnectionContext } from '@/features/ide-react/context/connection-context' +import useScopeValue from '@/shared/hooks/use-scope-value' +import ColorManager from '@/ide/colors/ColorManager' +import { CursorPosition } from '@/features/ide-react/types/cursor-position' +import { omit } from 'lodash' +import { Doc } from '../../../../../types/doc' +import { useFileTreeData } from '@/shared/context/file-tree-data-context' +import { findDocEntityById } from '@/features/ide-react/util/find-doc-entity-by-id' +import useSocketListener from '@/features/ide-react/hooks/use-socket-listener' + +type OnlineUser = { + id: string + user_id: string + email: string + name: string + initial?: string + doc_id?: string + doc?: Doc | null + row?: number + column?: number +} + +type ConnectedUser = { + client_id: string + user_id: string + email: string + first_name: string + last_name: string + cursorData?: { + doc_id: string + row: number + column: number + } +} + +type CursorHighlight = { + label: string + cursor: { + row: number + column: number + } + hue: number +} + +type OnlineUsersContextValue = { + onlineUsers: Record + onlineUserCursorHighlights: Record + onlineUsersArray: OnlineUser[] + onlineUsersCount: number +} + +export function populateOnlineUsersScope(store: ReactScopeValueStore) { + store.set('onlineUsers', {}) + store.set('onlineUserCursorHighlights', {}) + store.set('onlineUsersArray', []) + store.set('onlineUsersCount', 0) +} + +const OnlineUsersContext = createContext( + undefined +) + +export const OnlineUsersProvider: FC = ({ children }) => { + const { eventEmitter } = useIdeReactContext() + const { socket } = useConnectionContext() + const [openDocId] = useScopeValue('editor.open_doc_id') + const { fileTreeData } = useFileTreeData() + + const [onlineUsers, setOnlineUsers] = + useScopeValue('onlineUsers') + const [onlineUserCursorHighlights, setOnlineUserCursorHighlights] = + useScopeValue( + 'onlineUserCursorHighlights' + ) + const [onlineUsersArray, setOnlineUsersArray] = + useScopeValue( + 'onlineUsersArray' + ) + const [onlineUsersCount, setOnlineUsersCount] = + useScopeValue( + 'onlineUsersCount' + ) + + const [currentPosition, setCurrentPosition] = useState( + null + ) + const [cursorUpdateInterval, setCursorUpdateInterval] = useState(500) + + const calculateValues = useCallback( + (onlineUsers: OnlineUsersContextValue['onlineUsers']) => { + const decoratedOnlineUsers: OnlineUsersContextValue['onlineUsers'] = {} + const onlineUsersArray: OnlineUser[] = [] + const onlineUserCursorHighlights: OnlineUsersContextValue['onlineUserCursorHighlights'] = + {} + + for (const [clientId, user] of Object.entries(onlineUsers)) { + const decoratedUser = { ...user } + const docId = user.doc_id + if (docId) { + decoratedUser.doc = findDocEntityById(fileTreeData, docId) + } + + // If the user's name is empty use their email as display name + // Otherwise they're probably an anonymous user + if (user.name === null || user.name.trim().length === 0) { + decoratedUser.name = user.email ? user.email.trim() : 'Anonymous' + } + + decoratedUser.initial = user.name?.[0] + if (!decoratedUser.initial || decoratedUser.initial === ' ') { + decoratedUser.initial = '?' + } + + onlineUsersArray.push(decoratedUser) + decoratedOnlineUsers[clientId] = decoratedUser + + if (docId == null || user.row == null || user.column == null) { + continue + } + if (!onlineUserCursorHighlights[docId]) { + onlineUserCursorHighlights[docId] = [] + } + onlineUserCursorHighlights[docId].push({ + label: user.name, + cursor: { + row: user.row, + column: user.column, + }, + hue: ColorManager.getHueForUserId(user.user_id), + }) + } + + const cursorUpdateInterval = + onlineUsersArray.length > 0 ? 500 : 60 * 1000 * 5 + + return { + onlineUsers: decoratedOnlineUsers, + onlineUsersArray, + onlineUserCursorHighlights, + cursorUpdateInterval, + } + }, + [fileTreeData] + ) + + const setAllValues = useCallback( + (newOnlineUsers: OnlineUsersContextValue['onlineUsers']) => { + const values = calculateValues(newOnlineUsers) + setOnlineUsers(values.onlineUsers) + setOnlineUsersArray(values.onlineUsersArray) + setOnlineUsersCount(values.onlineUsersArray.length) + setOnlineUserCursorHighlights(values.onlineUserCursorHighlights) + setCursorUpdateInterval(values.cursorUpdateInterval) + }, + [ + calculateValues, + setOnlineUserCursorHighlights, + setOnlineUsers, + setOnlineUsersArray, + setOnlineUsersCount, + ] + ) + + useEffect(() => { + const handleProjectJoined = () => { + socket.emit( + 'clientTracking.getConnectedUsers', + // eslint-disable-next-line n/handle-callback-err + (error: Error, connectedUsers: ConnectedUser[]) => { + // TODO: MIGRATION: Handle error (although the original code doesn't) + const newOnlineUsers: OnlineUsersContextValue['onlineUsers'] = {} + for (const user of connectedUsers) { + if (user.client_id === socket.publicId) { + // Don't store myself + continue + } + // Store data in the same format returned by clientTracking.clientUpdated + newOnlineUsers[user.client_id] = { + id: user.client_id, + user_id: user.user_id, + email: user.email, + name: `${user.first_name} ${user.last_name}`, + doc_id: user.cursorData?.doc_id, + row: user.cursorData?.row, + column: user.cursorData?.column, + } + } + setAllValues(newOnlineUsers) + } + ) + } + eventEmitter.on('project:joined', handleProjectJoined) + + return () => { + eventEmitter.off('project:joined', handleProjectJoined) + } + }, [eventEmitter, setAllValues, setOnlineUsers, socket]) + + // Track the position of the main cursor + useEffect(() => { + const handleCursorUpdate = (position: CursorPosition | null) => { + if (position) { + setCurrentPosition(position) + } + } + + eventEmitter.on('cursor:editor:update', handleCursorUpdate) + + return () => { + eventEmitter.off('cursor:editor:update', handleCursorUpdate) + } + }, [cursorUpdateInterval, eventEmitter]) + + // Send the latest position to other clients when currentPosition changes + useEffect(() => { + const timer = window.setTimeout(() => { + socket.emit('clientTracking.updatePosition', { + row: currentPosition?.row, + column: currentPosition?.column, + doc_id: openDocId, + }) + }, cursorUpdateInterval) + + return () => { + window.clearTimeout(timer) + } + }, [currentPosition, cursorUpdateInterval, openDocId, socket]) + + const handleClientUpdated = useCallback( + (client: OnlineUser) => { + // Check it's not me! + if (client.id !== socket.publicId) { + setAllValues({ ...onlineUsers, [client.id]: client }) + } + }, + [onlineUsers, setAllValues, socket.publicId] + ) + + useSocketListener(socket, 'clientTracking.clientUpdated', handleClientUpdated) + + const handleClientDisconnected = useCallback( + (clientId: string) => { + setAllValues(omit(onlineUsers, clientId)) + }, + [onlineUsers, setAllValues] + ) + + useSocketListener( + socket, + 'clientTracking.clientDisconnected', + handleClientDisconnected + ) + + const value = useMemo( + () => ({ + onlineUsers, + onlineUsersArray, + onlineUserCursorHighlights, + onlineUsersCount, + }), + [ + onlineUsers, + onlineUsersArray, + onlineUserCursorHighlights, + onlineUsersCount, + ] + ) + + return ( + + {children} + + ) +} + +export function useOnlineUsersContext(): OnlineUsersContextValue { + const context = useContext(OnlineUsersContext) + + if (!context) { + throw new Error( + 'useOnlineUsersContext is only available inside OnlineUsersProvider' + ) + } + + return context +} diff --git a/services/web/frontend/js/features/ide-react/context/react-context-root.tsx b/services/web/frontend/js/features/ide-react/context/react-context-root.tsx index acd64d9cd3..0d3e80dfdc 100644 --- a/services/web/frontend/js/features/ide-react/context/react-context-root.tsx +++ b/services/web/frontend/js/features/ide-react/context/react-context-root.tsx @@ -1,17 +1,58 @@ -import { ConnectionProvider } from './connection-context' import { FC } from 'react' +import { ConnectionProvider } from './connection-context' import { IdeReactProvider } from '@/features/ide-react/context/ide-react-context' +import { LayoutProvider } from '@/shared/context/layout-context' +import { DetachProvider } from '@/shared/context/detach-context' import { ProjectProvider } from '@/shared/context/project-context' +import { ProjectSettingsProvider } from '@/features/editor-left-menu/context/project-settings-context' +import { EditorProvider } from '@/shared/context/editor-context' import { UserProvider } from '@/shared/context/user-context' +import { EditorManagerProvider } from '@/features/ide-react/context/editor-manager-context' +import { FileTreeDataProvider } from '@/shared/context/file-tree-data-context' +import { DetachCompileProvider } from '@/shared/context/detach-compile-context' +import { ChatProvider } from '@/features/chat/context/chat-context' +import { LocalCompileProvider } from '@/shared/context/local-compile-context' +import { OnlineUsersProvider } from '@/features/ide-react/context/online-users-context' +import { MetadataProvider } from '@/features/ide-react/context/metadata-context' +import { ReferencesProvider } from '@/features/ide-react/context/references-context' +import { SplitTestProvider } from '@/shared/context/split-test-context' export const ReactContextRoot: FC = ({ children }) => { return ( - - - - {children} - - - + + + + + + + + + + + + + + + + + + {children} + + + + + + + + + + + + + + + + + ) } diff --git a/services/web/frontend/js/features/ide-react/context/references-context.tsx b/services/web/frontend/js/features/ide-react/context/references-context.tsx new file mode 100644 index 0000000000..0364060b19 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/context/references-context.tsx @@ -0,0 +1,197 @@ +// @ts-ignore +import CryptoJSSHA1 from 'crypto-js/sha1' +import { + createContext, + useContext, + useEffect, + FC, + useCallback, + useMemo, + useState, +} from 'react' +import { useIdeReactContext } from '@/features/ide-react/context/ide-react-context' +import { useConnectionContext } from '@/features/ide-react/context/connection-context' +import _ from 'lodash' +import { postJSON } from '@/infrastructure/fetch-json' +import { ShareJsDoc } from '@/features/ide-react/editor/share-js-doc' +import useScopeValue from '@/shared/hooks/use-scope-value' +import { ReactScopeValueStore } from '@/features/ide-react/scope-value-store/react-scope-value-store' +import { useFileTreeData } from '@/shared/context/file-tree-data-context' +import { findDocEntityById } from '@/features/ide-react/util/find-doc-entity-by-id' + +type References = { + keys: string[] +} + +type ReferencesContextValue = { + indexReferencesIfDocModified: ( + doc: ShareJsDoc, + shouldBroadcast: boolean + ) => void + indexReferences: (docIds: string[], shouldBroadcast: boolean) => void + indexAllReferences: (shouldBroadcast: boolean) => void +} + +type IndexReferencesResponse = References + +const ReferencesContext = createContext( + undefined +) + +export function populateReferenceScope(store: ReactScopeValueStore) { + store.set('$root._references', { keys: [] }) +} + +export const ReferencesProvider: FC = ({ children }) => { + const { fileTreeData } = useFileTreeData() + const { eventEmitter, projectId } = useIdeReactContext() + const { socket } = useConnectionContext() + + const [references, setReferences] = + useScopeValue('$root._references') + + const [existingIndexHash, setExistingIndexHash] = useState< + Record + >({}) + + const storeReferencesKeys = useCallback( + (newKeys: string[], replaceExistingKeys: boolean) => { + const oldKeys = references.keys + const keys = replaceExistingKeys ? newKeys : _.union(oldKeys, newKeys) + window.dispatchEvent( + new CustomEvent('project:references', { + detail: keys, + }) + ) + setReferences({ keys }) + }, + [references.keys, setReferences] + ) + + const indexReferences = useCallback( + (docIds: string[], shouldBroadcast: boolean) => { + postJSON(`/project/${projectId}/references/index`, { + body: { + docIds, + shouldBroadcast, + }, + }).then((response: IndexReferencesResponse) => { + storeReferencesKeys(response.keys, false) + }) + }, + [projectId, storeReferencesKeys] + ) + + const indexAllReferences = useCallback( + (shouldBroadcast: boolean) => { + postJSON(`/project/${projectId}/references/indexAll`, { + body: { + shouldBroadcast, + }, + }).then((response: IndexReferencesResponse) => { + storeReferencesKeys(response.keys, true) + }) + }, + [projectId, storeReferencesKeys] + ) + + const indexReferencesIfDocModified = useCallback( + (doc: ShareJsDoc, shouldBroadcast: boolean) => { + // avoid reindexing references if the bib file has not changed since the + // last time they were indexed + const docId = doc.doc_id + const snapshot = doc._doc.snapshot + const now = Date.now() + const sha1 = CryptoJSSHA1( + 'blob ' + snapshot.length + '\x00' + snapshot + ).toString() + const CACHE_LIFETIME = 6 * 3600 * 1000 // allow reindexing every 6 hours + const cacheEntry = existingIndexHash[docId] + const isCached = + cacheEntry && + cacheEntry.timestamp > now - CACHE_LIFETIME && + cacheEntry.hash === sha1 + if (!isCached) { + indexReferences([docId], shouldBroadcast) + setExistingIndexHash(existingIndexHash => ({ + ...existingIndexHash, + [docId]: { hash: sha1, timestamp: now }, + })) + } + }, + [existingIndexHash, indexReferences] + ) + + useEffect(() => { + const handleDocClosed = (doc: ShareJsDoc) => { + if ( + doc.doc_id && + findDocEntityById(fileTreeData, doc.doc_id)?.name?.endsWith('.bib') + ) { + indexReferencesIfDocModified(doc, true) + } + } + + eventEmitter.on('document:closed', handleDocClosed) + + return () => { + eventEmitter.off('document:closed', handleDocClosed) + } + }, [eventEmitter, fileTreeData, indexReferencesIfDocModified]) + + useEffect(() => { + const handleShouldReindex = () => { + indexAllReferences(true) + } + + eventEmitter.on('references:should-reindex', handleShouldReindex) + + return () => { + eventEmitter.off('references:should-reindex', handleShouldReindex) + } + }, [eventEmitter, indexAllReferences]) + + useEffect(() => { + const handleProjectJoined = () => { + // We only need to grab the references when the editor first loads, + // not on every reconnect + socket.on('references:keys:updated', (keys, allDocs) => + storeReferencesKeys(keys, allDocs) + ) + indexAllReferences(false) + } + + eventEmitter.once('project:joined', handleProjectJoined) + + return () => { + eventEmitter.off('project:joined', handleProjectJoined) + } + }, [eventEmitter, indexAllReferences, socket, storeReferencesKeys]) + + const value = useMemo( + () => ({ + indexReferencesIfDocModified, + indexReferences, + indexAllReferences, + }), + [indexReferencesIfDocModified, indexReferences, indexAllReferences] + ) + + return ( + + {children} + + ) +} + +export function useReferencesContext(): ReferencesContextValue { + const context = useContext(ReferencesContext) + + if (!context) { + throw new Error( + 'useReferencesContext is only available inside ReferencesProvider' + ) + } + + return context +} diff --git a/services/web/frontend/js/features/ide-react/create-ide-event-emitter.ts b/services/web/frontend/js/features/ide-react/create-ide-event-emitter.ts index a8c8ef7131..7d5dfb88e3 100644 --- a/services/web/frontend/js/features/ide-react/create-ide-event-emitter.ts +++ b/services/web/frontend/js/features/ide-react/create-ide-event-emitter.ts @@ -1,19 +1,33 @@ import { Emitter } from 'strict-event-emitter' import { Project } from '../../../../types/project' import { PermissionsLevel } from '@/features/ide-react/types/permissions-level' +import { ShareJsDoc } from '@/features/ide-react/editor/share-js-doc' import { GotoLineOptions } from '@/features/ide-react/types/goto-line-options' import { CursorPosition } from '@/features/ide-react/types/cursor-position' export type IdeEvents = { 'project:joined': [{ project: Project; permissionsLevel: PermissionsLevel }] + // TODO: MIGRATION: This doesn't seem to be used. Investigate whether it can be removed + 'document:opened': [doc: ShareJsDoc] + + 'document:closed': [doc: ShareJsDoc] + 'doc:changed': [{ doc_id: string }] + 'doc:saved': [{ doc_id: string }] + 'doc:opened': [] + 'ide:opAcknowledged': [{ doc_id: string; op: any }] + 'store-doc-position': [] 'editor:gotoOffset': [gotoOffset: number] 'editor:gotoLine': [options: GotoLineOptions] - 'outline-toggled': [isOpen: boolean] 'cursor:editor:update': [position: CursorPosition] + 'outline-toggled': [isOpen: boolean] 'cursor:editor:syncToPdf': [] 'scroll:editor:update': [] 'comment:start_adding': [] + 'references:should-reindex': [] + + // TODO: MIGRATION: Create a proper type for entity when migrating the file tree + 'entity:deleted': [entity: Record] } export type IdeEventEmitter = Emitter diff --git a/services/web/frontend/js/features/ide-react/editor/document.ts b/services/web/frontend/js/features/ide-react/editor/document.ts new file mode 100644 index 0000000000..5d84909f54 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/editor/document.ts @@ -0,0 +1,706 @@ +/* eslint-disable + camelcase, + n/handle-callback-err, + max-len, +*/ +// Migrated from services/web/frontend/js/ide/editor/Document.js + +// @ts-ignore +import RangesTracker from '@overleaf/ranges-tracker' +import { ShareJsDoc } from './share-js-doc' +import { debugConsole } from '@/utils/debugging' +import { Socket } from '@/features/ide-react/connection/types/socket' +import { IdeEventEmitter } from '@/features/ide-react/create-ide-event-emitter' +import { EditorFacade } from '@/features/source-editor/extensions/realtime' +import { EventLog } from '@/features/ide-react/editor/event-log' +import EditorWatchdogManager from '@/features/ide-react/connection/editor-watchdog-manager' +import EventEmitter from '@/utils/EventEmitter' +import { + AnyOperation, + Change, + CommentOperation, +} from '../../../../../types/change' +import { + isCommentOperation, + isDeleteOperation, + isInsertOperation, +} from '@/utils/operations' +import { decodeUtf8 } from '@/utils/decode-utf8' +import { + ShareJsOperation, + TrackChangesIdSeeds, + Version, +} from '@/features/ide-react/editor/types/document' + +const MAX_PENDING_OP_SIZE = 64 + +type JoinCallback = (error?: Error) => void +type LeaveCallback = JoinCallback + +type Update = Record + +type Message = { + meta: { + tc: string + user_id: string + } +} + +type ErrorMetadata = Record + +function getOpSize(op: AnyOperation) { + if (isInsertOperation(op)) { + return op.i.length + } + if (isDeleteOperation(op)) { + return op.d.length + } + return 0 +} + +function getShareJsOpSize(shareJsOp: ShareJsOperation) { + return shareJsOp.reduce((total, op) => total + getOpSize(op), 0) +} + +export class Document extends EventEmitter { + private connected: boolean + private wantToBeJoined = false + private chaosMonkeyTimer: number | null = null + private track_changes_as: string | null = null + + private joinCallbacks: JoinCallback[] = [] + private leaveCallbacks: LeaveCallback[] = [] + + doc?: ShareJsDoc + cm6?: EditorFacade + oldInflightOp?: ShareJsOperation + ranges: RangesTracker + joined = false + + // This is set and read in useCodeMirrorScope + docName = '' + + constructor( + readonly doc_id: string, + readonly socket: Socket, + private readonly globalEditorWatchdogManager: EditorWatchdogManager, + private readonly ideEventEmitter: IdeEventEmitter, + private readonly eventLog: EventLog + ) { + super() + this.connected = this.socket.socket.connected + this.bindToEditorEvents() + this.bindToSocketEvents() + } + + attachToCM6(cm6: EditorFacade) { + this.cm6 = cm6 + if (this.doc) { + this.doc.attachToCM6(this.cm6) + } + if (this.cm6) { + this.cm6.on('change', this.checkConsistency) + } + if (this.doc) { + this.ideEventEmitter.emit('document:opened', this.doc) + } + } + + detachFromCM6() { + if (this.doc) { + this.doc.detachFromCM6() + } + if (this.cm6) { + this.cm6.off('change', this.checkConsistency) + } + delete this.cm6 + this.clearChaosMonkey() + if (this.doc) { + this.ideEventEmitter.emit('document:closed', this.doc) + } + } + + submitOp(...ops: AnyOperation[]) { + this.doc?.submitOp(ops) + } + + private checkConsistency = (editor: EditorFacade) => { + // We've been seeing a lot of errors when I think there shouldn't be + // any, which may be related to this check happening before the change is + // applied. If we use a timeout, hopefully we can reduce this. + window.setTimeout(() => { + const editorValue = editor?.getValue() + const sharejsValue = this.doc?.getSnapshot() + if (editorValue !== sharejsValue) { + return this.onError( + new Error('Editor text does not match server text'), + {}, + editorValue + ) + } + }, 0) + } + + getSnapshot() { + return this.doc?.getSnapshot() + } + + getType() { + return this.doc?.getType() + } + + getInflightOp(): ShareJsOperation | undefined { + return this.doc?.getInflightOp() + } + + getPendingOp(): ShareJsOperation | undefined { + return this.doc?.getPendingOp() + } + + getRecentAck() { + return this.doc?.getRecentAck() + } + + hasBufferedOps() { + return this.doc?.hasBufferedOps() + } + + setTrackingChanges(track_changes: boolean) { + if (this.doc) { + this.doc.track_changes = track_changes + } + } + + getTrackingChanges() { + return !!this.doc?.track_changes + } + + setTrackChangesIdSeeds(id_seeds: TrackChangesIdSeeds) { + if (this.doc) { + this.doc.track_changes_id_seeds = id_seeds + } + } + + private onUpdateAppliedHandler = (update: any) => this.onUpdateApplied(update) + + // TODO: MIGRATION: Create proper types for error and message + private onErrorHandler = (error: Error, message: { doc_id: string }) => { + // 'otUpdateError' are emitted per doc socket.io room, hence we can be + // sure that message.doc_id exists. + if (message.doc_id !== this.doc_id) { + // This error is for another doc. Do not action it. We could open + // a modal that has the wrong context on it. + return + } + this.onError(error, message) + } + + private onDisconnectHandler = () => this.onDisconnect() + + private bindToSocketEvents() { + this.socket.on('otUpdateApplied', this.onUpdateAppliedHandler) + this.socket.on('otUpdateError', this.onErrorHandler) + return this.socket.on('disconnect', this.onDisconnectHandler) + } + + private unBindFromSocketEvents() { + this.socket.removeListener('otUpdateApplied', this.onUpdateAppliedHandler) + this.socket.removeListener('otUpdateError', this.onErrorHandler) + return this.socket.removeListener('disconnect', this.onDisconnectHandler) + } + + private bindToEditorEvents() { + this.ideEventEmitter.on('project:joined', this.onReconnect) + } + + private unBindFromEditorEvents() { + this.ideEventEmitter.off('project:joined', this.onReconnect) + } + + leaveAndCleanUp(cb?: (error?: Error) => void) { + return this.leave((error?: Error) => { + this.cleanUp() + if (cb) cb(error) + }) + } + + join(callback?: JoinCallback) { + this.wantToBeJoined = true + this.cancelLeave() + if (this.connected) { + this.joinDoc(callback) + } else if (callback) { + this.joinCallbacks.push(callback) + } + } + + leave(callback?: LeaveCallback) { + this.flush() // force an immediate flush when leaving document + this.wantToBeJoined = false + this.cancelJoin() + if (this.doc?.hasBufferedOps()) { + debugConsole.log( + '[leave] Doc has buffered ops, pushing callback for later' + ) + if (callback) { + this.leaveCallbacks.push(callback) + } + } else if (!this.connected) { + debugConsole.log('[leave] Not connected, returning now') + callback?.() + } else { + debugConsole.log('[leave] Leaving now') + this.leaveDoc(callback) + } + } + + flush() { + return this.doc?.flushPendingOps() + } + + chaosMonkey(line = 0, char = 'a') { + const orig = char + let copy: string | null = null + let pos = 0 + const timer = () => { + if (copy == null || !copy.length) { + copy = orig.slice() + ' ' + new Date() + '\n' + line += Math.random() > 0.1 ? 1 : -2 + if (line < 0) { + line = 0 + } + pos = 0 + } + char = copy[0] + copy = copy.slice(1) + if (this.cm6) { + this.cm6.view.dispatch({ + changes: { + from: Math.min(pos, this.cm6.view.state.doc.length), + insert: char, + }, + }) + } + pos += 1 + this.chaosMonkeyTimer = window.setTimeout( + timer, + 100 + (Math.random() < 0.1 ? 1000 : 0) + ) + } + timer() + } + + clearChaosMonkey() { + const timer = this.chaosMonkeyTimer + if (timer) { + this.chaosMonkeyTimer = null + window.clearTimeout(timer) + } + } + + pollSavedStatus() { + // returns false if doc has ops waiting to be acknowledged or + // sent that haven't changed since the last time we checked. + // Otherwise returns true. + let saved + const inflightOp = this.getInflightOp() + const pendingOp = this.getPendingOp() + const recentAck = this.getRecentAck() + const pendingOpSize = pendingOp ? getShareJsOpSize(pendingOp) : 0 + if (inflightOp == null && pendingOp == null) { + // There's nothing going on, this is OK. + saved = true + debugConsole.log('[pollSavedStatus] no inflight or pending ops') + } else if (inflightOp && inflightOp === this.oldInflightOp) { + // The same inflight op has been sitting unacked since we + // last checked, this is bad. + saved = false + debugConsole.log('[pollSavedStatus] inflight op is same as before') + } else if ( + pendingOp != null && + recentAck && + pendingOpSize < MAX_PENDING_OP_SIZE + ) { + // There is an op waiting to go to server but it is small and + // within the flushDelay, this is OK for now. + saved = true + debugConsole.log( + '[pollSavedStatus] pending op (small with recent ack) assume ok', + pendingOp, + pendingOpSize + ) + } else { + // In any other situation, assume the document is unsaved. + saved = false + debugConsole.log( + `[pollSavedStatus] assuming not saved (inflightOp?: ${ + inflightOp != null + }, pendingOp?: ${pendingOp != null})` + ) + } + + this.oldInflightOp = inflightOp + return saved + } + + private cancelLeave() { + this.leaveCallbacks = [] + } + + private cancelJoin() { + this.joinCallbacks = [] + } + + private onUpdateApplied(update: Update) { + this.eventLog.pushEvent('received-update', { + doc_id: this.doc_id, + remote_doc_id: update?.doc, + wantToBeJoined: this.wantToBeJoined, + update, + hasDoc: !!this.doc, + }) + + if (update?.doc === this.doc_id && this.doc != null) { + this.eventLog.pushEvent('received-update:processing', { + update, + }) + // FIXME: change this back to processUpdateFromServer when redis fixed + this.doc.processUpdateFromServerInOrder(update) + + if (!this.wantToBeJoined) { + return this.leave() + } + } + } + + private onDisconnect() { + debugConsole.log('[onDisconnect] disconnecting') + this.connected = false + this.joined = false + return this.doc != null + ? this.doc.updateConnectionState('disconnected') + : undefined + } + + private onReconnect = () => { + debugConsole.log('[onReconnect] reconnected (joined project)') + this.eventLog.pushEvent('reconnected:afterJoinProject') + + this.connected = true + if (this.wantToBeJoined || this.doc?.hasBufferedOps()) { + debugConsole.log( + `[onReconnect] Rejoining (wantToBeJoined: ${ + this.wantToBeJoined + } OR hasBufferedOps: ${this.doc?.hasBufferedOps()})` + ) + this.joinDoc((error?: Error) => { + if (error) { + this.onError(error) + return + } + this.doc?.updateConnectionState('ok') + this.doc?.flushPendingOps() + this.callJoinCallbacks() + }) + } + } + + private callJoinCallbacks() { + for (const callback of this.joinCallbacks) { + callback() + } + this.joinCallbacks = [] + } + + private joinDoc(callback?: JoinCallback) { + if (this.doc) { + this.eventLog.pushEvent('joinDoc:existing', { + doc_id: this.doc_id, + version: this.doc.getVersion(), + }) + return this.socket.emit( + 'joinDoc', + this.doc_id, + this.doc.getVersion(), + { encodeRanges: true }, + (error, docLines, version, updates, ranges) => { + if (error) { + callback?.(error) + return + } + this.joined = true + this.doc?.catchUp(updates) + this.decodeRanges(ranges) + this.catchUpRanges(ranges?.changes, ranges?.comments) + callback?.() + } + ) + } else { + this.eventLog.pushEvent('joinDoc:new', { + doc_id: this.doc_id, + }) + this.socket.emit( + 'joinDoc', + this.doc_id, + { encodeRanges: true }, + (error, docLines, version, updates, ranges) => { + if (error) { + callback?.(error) + return + } + this.joined = true + this.eventLog.pushEvent('joinDoc:inited', { + doc_id: this.doc_id, + version, + }) + this.doc = new ShareJsDoc( + this.doc_id, + docLines, + version, + this.socket, + this.globalEditorWatchdogManager, + this.ideEventEmitter, + this.eventLog + ) + this.decodeRanges(ranges) + this.ranges = new RangesTracker(ranges?.changes, ranges?.comments) + this.bindToShareJsDocEvents() + callback?.() + } + ) + } + } + + private decodeRanges(ranges: RangesTracker) { + try { + if (ranges.changes) { + for (const change of ranges.changes) { + if (isInsertOperation(change.op)) { + change.op.i = decodeUtf8(change.op.i) + } + if (isDeleteOperation(change.op)) { + change.op.d = decodeUtf8(change.op.d) + } + } + } + return (() => { + if (!ranges.comments) { + return [] + } + return ranges.comments.map((comment: Change) => + comment.op.c != null + ? (comment.op.c = decodeUtf8(comment.op.c)) + : undefined + ) + })() + } catch (err) { + debugConsole.error(err) + } + } + + private leaveDoc(callback?: LeaveCallback) { + this.eventLog.pushEvent('leaveDoc', { + doc_id: this.doc_id, + }) + debugConsole.log('[leaveDoc] Sending leaveDoc request') + this.socket.emit('leaveDoc', this.doc_id, error => { + if (error) { + callback?.(error) + return + } + this.joined = false + for (const leaveCallback of this.leaveCallbacks) { + debugConsole.log('[_leaveDoc] Calling buffered callback', leaveCallback) + leaveCallback(error) + } + this.leaveCallbacks = [] + callback?.() + }) + } + + cleanUp() { + // if we arrive here from _onError the pending and inflight ops will have been cleared + if (this.hasBufferedOps()) { + debugConsole.log( + `[cleanUp] Document (${this.doc_id}) has buffered ops, refusing to remove from openDocs` + ) + return // return immediately, do not unbind from events + } else { + this.emit('detach', this.doc_id) + } + this.unBindFromEditorEvents() + this.unBindFromSocketEvents() + } + + private bindToShareJsDocEvents() { + if (!this.doc) { + return + } + + this.doc.on('error', (error: Error, meta: ErrorMetadata) => + this.onError(error, meta) + ) + this.doc.on('externalUpdate', (update: Update) => { + this.eventLog.pushEvent('externalUpdate', { doc_id: this.doc_id }) + return this.trigger('externalUpdate', update) + }) + this.doc.on('remoteop', (...ops: AnyOperation[]) => { + this.eventLog.pushEvent('remoteop', { doc_id: this.doc_id }) + return this.trigger('remoteop', ...ops) + }) + this.doc.on('op:sent', (op: AnyOperation) => { + this.eventLog.pushEvent('op:sent', { + doc_id: this.doc_id, + op, + }) + return this.trigger('op:sent') + }) + this.doc.on('op:acknowledged', (op: AnyOperation) => { + this.eventLog.pushEvent('op:acknowledged', { + doc_id: this.doc_id, + op, + }) + this.ideEventEmitter.emit('ide:opAcknowledged', { + doc_id: this.doc_id, + op, + }) + return this.trigger('op:acknowledged') + }) + this.doc.on('op:timeout', (op: AnyOperation) => { + this.eventLog.pushEvent('op:timeout', { + doc_id: this.doc_id, + op, + }) + this.trigger('op:timeout') + return this.onError(new Error('op timed out')) + }) + this.doc.on( + 'flush', + (inflightOp: AnyOperation, pendingOp: AnyOperation, version: Version) => { + return this.eventLog.pushEvent('flush', { + doc_id: this.doc_id, + inflightOp, + pendingOp, + v: version, + }) + } + ) + + let docChangedTimeout: number | null = null + this.doc.on( + 'change', + (ops: AnyOperation[], oldSnapshot: any, msg: Message) => { + this.applyOpsToRanges(ops, msg) + if (docChangedTimeout) { + window.clearTimeout(docChangedTimeout) + } + docChangedTimeout = window.setTimeout(() => { + window.dispatchEvent( + new CustomEvent('doc:changed', { detail: { id: this.doc_id } }) + ) + this.ideEventEmitter.emit('doc:changed', { doc_id: this.doc_id }) + }, 50) + } + ) + + this.doc.on('flipped_pending_to_inflight', () => { + return this.trigger('flipped_pending_to_inflight') + }) + + let docSavedTimeout: number | null + this.doc.on('saved', () => { + if (docSavedTimeout) { + window.clearTimeout(docSavedTimeout) + } + docSavedTimeout = window.setTimeout(() => { + window.dispatchEvent( + new CustomEvent('doc:saved', { detail: { id: this.doc_id } }) + ) + this.ideEventEmitter.emit('doc:saved', { doc_id: this.doc_id }) + }, 50) + }) + } + + private onError( + error: Error, + meta: ErrorMetadata = {}, + editorContent?: string + ) { + meta.doc_id = this.doc_id + debugConsole.log('ShareJS error', error, meta) + if (error.message === 'no project_id found on client') { + debugConsole.log('ignoring error, will wait to join project') + return + } + if (this.doc) { + this.doc.clearInflightAndPendingOps() + } + this.trigger('error', error, meta, editorContent) + // The clean-up should run after the error is triggered because the error triggers a + // disconnect. If we run the clean-up first, we remove our event handlers and miss + // the disconnect event, which means we try to leaveDoc when the connection comes back. + // This could interfere with the new connection of a new instance of this document. + this.cleanUp() + } + + private applyOpsToRanges(ops: AnyOperation[], msg?: Message) { + let old_id_seed + let track_changes_as = null + const remote_op = msg != null + if (remote_op && msg?.meta.tc) { + old_id_seed = this.ranges.getIdSeed() + this.ranges.setIdSeed(msg.meta.tc) + track_changes_as = msg.meta.user_id + } else if (!remote_op && this.track_changes_as != null) { + track_changes_as = this.track_changes_as + } + this.ranges.track_changes = track_changes_as != null + for (const op of this.filterOps(ops)) { + this.ranges.applyOp(op, { user_id: track_changes_as }) + } + if (old_id_seed != null) { + this.ranges.setIdSeed(old_id_seed) + } + if (remote_op) { + // With remote ops, the editor hasn't been updated when we receive this + // op, so defer updating track changes until it has + return window.setTimeout(() => this.emit('ranges:dirty')) + } else { + return this.emit('ranges:dirty') + } + } + + private catchUpRanges(changes: Change[], comments: CommentOperation[]) { + // We've just been given the current server's ranges, but need to apply any local ops we have. + // Reset to the server state then apply our local ops again. + if (changes == null) { + changes = [] + } + if (comments == null) { + comments = [] + } + this.emit('ranges:clear') + this.ranges.changes = changes + this.ranges.comments = comments + this.ranges.track_changes = this.doc?.track_changes + for (const op of this.filterOps(this.doc?.getInflightOp() || [])) { + this.ranges.setIdSeed(this.doc?.track_changes_id_seeds?.inflight) + this.ranges.applyOp(op, { user_id: this.track_changes_as }) + } + for (const op of this.filterOps(this.doc?.getPendingOp() || [])) { + this.ranges.setIdSeed(this.doc?.track_changes_id_seeds?.pending) + this.ranges.applyOp(op, { user_id: this.track_changes_as }) + } + return this.emit('ranges:redraw') + } + + private filterOps(ops: AnyOperation[]) { + // Read-only token users can't see/edit comment, so we filter out comment + // ops to avoid highlighting comment ranges. + if (window.isRestrictedTokenMember) { + return ops.filter(op => !isCommentOperation(op)) + } else { + return ops + } + } +} diff --git a/services/web/frontend/js/features/ide-react/editor/event-log.ts b/services/web/frontend/js/features/ide-react/editor/event-log.ts new file mode 100644 index 0000000000..bd65de621c --- /dev/null +++ b/services/web/frontend/js/features/ide-react/editor/event-log.ts @@ -0,0 +1,16 @@ +import { debugConsole } from '@/utils/debugging' + +type EditorEvent = { type: string; meta: unknown; date: Date } + +// Record events and then do nothing with them. +export class EventLog { + private recentEvents: EditorEvent[] = [] + + pushEvent = (type: string, meta: unknown = {}) => { + debugConsole.log('event', type, meta) + this.recentEvents.push({ type, meta, date: new Date() }) + if (this.recentEvents.length > 100) { + return this.recentEvents.shift() + } + } +} diff --git a/services/web/frontend/js/features/ide-react/editor/open-documents.ts b/services/web/frontend/js/features/ide-react/editor/open-documents.ts new file mode 100644 index 0000000000..eef90b5d82 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/editor/open-documents.ts @@ -0,0 +1,75 @@ +// Migrated from static methods of Document in Document.js + +import { Document } from '@/features/ide-react/editor/document' +import { debugConsole } from '@/utils/debugging' +import { Socket } from '@/features/ide-react/connection/types/socket' +import { IdeEventEmitter } from '@/features/ide-react/create-ide-event-emitter' +import { EventLog } from '@/features/ide-react/editor/event-log' +import EditorWatchdogManager from '@/features/ide-react/connection/editor-watchdog-manager' + +export class OpenDocuments { + private openDocs = new Map() + + // eslint-disable-next-line no-useless-constructor + constructor( + private readonly socket: Socket, + private readonly globalEditorWatchdogManager: EditorWatchdogManager, + private readonly events: IdeEventEmitter, + private readonly eventLog: EventLog + ) {} + + getDocument(docId: string) { + // Try to clean up existing docs before reopening them. If the doc has no + // buffered ops then it will be deleted by _cleanup() and a new instance + // of the document created below. This prevents us trying to follow the + // joinDoc:existing code path on an existing doc that doesn't have any + // local changes and getting an error if its version is too old. + if (this.openDocs.has(docId)) { + debugConsole.log( + `[getDocument] Cleaning up existing document instance for ${docId}` + ) + this.openDocs.get(docId)?.cleanUp() + } + if (!this.openDocs.has(docId)) { + debugConsole.log( + `[getDocument] Creating new document instance for ${docId}` + ) + this.createDoc(docId) + } else { + debugConsole.log( + `[getDocument] Returning existing document instance for ${docId}` + ) + } + return this.openDocs.get(docId) + } + + private createDoc(docId: string) { + const doc = new Document( + docId, + this.socket, + this.globalEditorWatchdogManager, + this.events, + this.eventLog + ) + this.openDocs.set(docId, doc) + doc.on('detach', () => { + debugConsole.log( + `[detach] Removing document with ID (${docId}) from openDocs` + ) + doc.off('detach') + this.openDocs.delete(docId) + }) + } + + private docsArray() { + return Array.from(this.openDocs.values()) + } + + hasUnsavedChanges() { + return this.docsArray().some(doc => doc.hasBufferedOps()) + } + + flushAll() { + return this.docsArray().map(doc => doc.flush()) + } +} diff --git a/services/web/frontend/js/features/ide-react/editor/share-js-doc.ts b/services/web/frontend/js/features/ide-react/editor/share-js-doc.ts new file mode 100644 index 0000000000..506dcd7205 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/editor/share-js-doc.ts @@ -0,0 +1,437 @@ +/* eslint-disable camelcase */ +// Migrated from services/web/frontend/js/ide/editor/ShareJsDoc.js + +import EventEmitter from '../../../utils/EventEmitter' +import { Doc } from '@/vendor/libs/sharejs' +import { Socket } from '@/features/ide-react/connection/types/socket' +import { debugConsole } from '@/utils/debugging' +import { decodeUtf8 } from '@/utils/decode-utf8' +import { IdeEventEmitter } from '@/features/ide-react/create-ide-event-emitter' +import { EventLog } from '@/features/ide-react/editor/event-log' +import EditorWatchdogManager from '@/features/ide-react/connection/editor-watchdog-manager' +import { + Message, + ShareJsConnectionState, + ShareJsOperation, + TrackChangesIdSeeds, +} from '@/features/ide-react/editor/types/document' +import { EditorFacade } from '@/features/source-editor/extensions/realtime' + +// All times below are in milliseconds +const SINGLE_USER_FLUSH_DELAY = 2000 +const MULTI_USER_FLUSH_DELAY = 500 +const INFLIGHT_OP_TIMEOUT = 5000 // Retry sending ops after 5 seconds without an ack +const WAIT_FOR_CONNECTION_TIMEOUT = 500 +const FATAL_OP_TIMEOUT = 30000 + +type Update = Record + +type Connection = { + send: (update: Update) => void + state: ShareJsConnectionState + id: string +} + +export class ShareJsDoc extends EventEmitter { + type: string + track_changes = false + track_changes_id_seeds: TrackChangesIdSeeds | null = null + connection: Connection + + // @ts-ignore + _doc: Doc + private editorWatchdogManager: EditorWatchdogManager + private lastAcked: Date | null = null + private queuedMessageTimer: number | null = null + private queuedMessages: Message[] = [] + private detachEditorWatchdogManager: (() => void) | null = null + private _timeoutTimer: number | null = null + + constructor( + readonly doc_id: string, + docLines: string[], + version: number, + readonly socket: Socket, + private readonly globalEditorWatchdogManager: EditorWatchdogManager, + private readonly eventEmitter: IdeEventEmitter, + private readonly eventLog: EventLog + ) { + super() + this.type = 'text' + // Decode any binary bits of data + const snapshot = docLines.map(line => decodeUtf8(line)).join('\n') + + this.connection = { + send: (update: Update) => { + this.startInflightOpTimeout(update) + // TODO: MIGRATION: Work out whether we can get rid of this. It looks as + // though it's here for debugging and isn't used + + // if ( + // window.disconnectOnUpdate != null && + // Math.random() < window.disconnectOnUpdate + // ) { + // debugConsole.log('Disconnecting on update', update) + // this.socket.disconnect() + // } + // if (window.dropUpdates != null && Math.random() < window.dropUpdates) { + // debugConsole.log('Simulating a lost update', update) + // return + // } + if (this.track_changes && this.track_changes_id_seeds) { + if (update.meta == null) { + update.meta = {} + } + update.meta.tc = this.track_changes_id_seeds.inflight + } + return this.socket.emit( + 'applyOtUpdate', + this.doc_id, + update, + (error: Error) => { + if (error != null) { + this.handleError(error) + } + } + ) + }, + state: 'ok', + id: this.socket.publicId, + } + + this._doc = new Doc(this.connection, this.doc_id, { + type: this.type, + }) + this._doc.setFlushDelay(SINGLE_USER_FLUSH_DELAY) + this._doc.on('change', (...args: any[]) => { + return this.trigger('change', ...args) + }) + this.editorWatchdogManager = new EditorWatchdogManager({ + parent: globalEditorWatchdogManager, + }) + this._doc.on('acknowledge', () => { + this.lastAcked = new Date() // note time of last ack from server for an op we sent + this.editorWatchdogManager.onAck() // keep track of last ack globally + return this.trigger('acknowledge') + }) + this._doc.on('remoteop', (...args: any[]) => { + // As soon as we're working with a collaborator, start sending + // ops more frequently for low latency. + this._doc.setFlushDelay(MULTI_USER_FLUSH_DELAY) + return this.trigger('remoteop', ...args) + }) + this._doc.on('flipped_pending_to_inflight', () => { + return this.trigger('flipped_pending_to_inflight') + }) + this._doc.on('saved', () => { + return this.trigger('saved') + }) + this._doc.on('error', (e: Error) => { + return this.handleError(e) + }) + + this.bindToDocChanges(this._doc) + + this.processUpdateFromServer({ + open: true, + v: version, + snapshot, + }) + this.removeCarriageReturnCharFromShareJsDoc() + } + + private removeCarriageReturnCharFromShareJsDoc() { + const doc = this._doc + if (doc.snapshot.indexOf('\r') === -1) { + return + } + this.eventLog.pushEvent('remove-carriage-return-char', { + doc_id: this.doc_id, + }) + let nextPos + while ((nextPos = doc.snapshot.indexOf('\r')) !== -1) { + debugConsole.log('[ShareJsDoc] remove-carriage-return-char', nextPos) + doc.del(nextPos, 1) + } + } + + submitOp(op: ShareJsOperation) { + this._doc.submitOp(op) + } + + // The following code puts out of order messages into a queue + // so that they can be processed in order. This is a workaround + // for messages being delayed by redis cluster. + // FIXME: REMOVE THIS WHEN REDIS PUBSUB IS SENDING MESSAGES IN ORDER + private isAheadOfExpectedVersion(message: Message) { + return this._doc.version > 0 && message.v > this._doc.version + } + + private pushOntoQueue(message: Message) { + debugConsole.log(`[processUpdate] push onto queue ${message.v}`) + // set a timer so that we never leave messages in the queue indefinitely + if (!this.queuedMessageTimer) { + this.queuedMessageTimer = window.setTimeout(() => { + debugConsole.log(`[processUpdate] queue timeout fired for ${message.v}`) + // force the message to be processed after the timeout, + // it will cause an error if the missing update has not arrived + this.processUpdateFromServer(message) + }, INFLIGHT_OP_TIMEOUT) + } + this.queuedMessages.push(message) + // keep the queue in order, lowest version first + this.queuedMessages.sort(function (a, b) { + return a.v - b.v + }) + } + + private clearQueue() { + this.queuedMessages = [] + } + + private processQueue() { + if (this.queuedMessages.length > 0) { + const nextAvailableVersion = this.queuedMessages[0].v + if (nextAvailableVersion > this._doc.version) { + // there are updates we still can't apply yet + } else { + // there's a version we can accept on the queue, apply it + debugConsole.log( + `[processUpdate] taken from queue ${nextAvailableVersion}` + ) + const message = this.queuedMessages.shift() + if (message) { + this.processUpdateFromServerInOrder(message) + } + // clear the pending timer if the queue has now been cleared + if (this.queuedMessages.length === 0 && this.queuedMessageTimer) { + debugConsole.log('[processUpdate] queue is empty, cleared timeout') + window.clearTimeout(this.queuedMessageTimer) + this.queuedMessageTimer = null + } + } + } + } + + // FIXME: This is the new method which reorders incoming updates if needed + // called from document.ts + processUpdateFromServerInOrder(message: Message) { + // Is this update ahead of the next expected update? + // If so, put it on a queue to be handled later. + if (this.isAheadOfExpectedVersion(message)) { + this.pushOntoQueue(message) + return // defer processing this update for now + } + const error = this.processUpdateFromServer(message) + if ( + error instanceof Error && + error.message === 'Invalid version from server' + ) { + // if there was an error, abandon the queued updates ahead of this one + this.clearQueue() + return + } + // Do we have any messages queued up? + // find the next message if available + this.processQueue() + } + + // FIXME: This is the original method. Switch back to this when redis + // issues are resolved. + processUpdateFromServer(message: Message) { + try { + this._doc._onMessage(message) + } catch (error) { + // Version mismatches are thrown as errors + debugConsole.log(error) + this.handleError(error) + return error // return the error for queue handling + } + + if (message.meta?.type === 'external') { + return this.trigger('externalUpdate', message) + } + } + + catchUp(updates: Message[]) { + return updates.map(update => { + update.v = this._doc.version + update.doc = this.doc_id + return this.processUpdateFromServer(update) + }) + } + + getSnapshot() { + return this._doc.snapshot as string | undefined + } + + getVersion() { + return this._doc.version + } + + getType() { + return this.type + } + + clearInflightAndPendingOps() { + this.clearFatalTimeoutTimer() + this._doc.inflightOp = null + this._doc.inflightCallbacks = [] + this._doc.pendingOp = null + return (this._doc.pendingCallbacks = []) + } + + flushPendingOps() { + // This will flush any ops that are pending. + // If there is an inflight op it will do nothing. + return this._doc.flush() + } + + updateConnectionState(state: ShareJsConnectionState) { + debugConsole.log(`[updateConnectionState] Setting state to ${state}`) + this.connection.state = state + this.connection.id = this.socket.publicId + this._doc.autoOpen = false + this._doc._connectionStateChanged(state) + return (this.lastAcked = null) // reset the last ack time when connection changes + } + + hasBufferedOps() { + return this._doc.inflightOp != null || this._doc.pendingOp != null + } + + getInflightOp() { + return this._doc.inflightOp + } + + getPendingOp() { + return this._doc.pendingOp + } + + getRecentAck() { + // check if we have received an ack recently (within a factor of two of the single user flush delay) + return ( + this.lastAcked !== null && + Date.now() - this.lastAcked.getTime() < 2 * SINGLE_USER_FLUSH_DELAY + ) + } + + private attachEditorWatchdogManager(editor: EditorFacade) { + // end-to-end check for edits -> acks, for this very ShareJsdoc + // This will catch a broken connection and missing UX-blocker for the + // user, allowing them to keep editing. + this.detachEditorWatchdogManager = + this.editorWatchdogManager.attachToEditor(editor) + } + + private attachToEditor(editor: EditorFacade, attachToShareJs: () => void) { + this.attachEditorWatchdogManager(editor) + + attachToShareJs() + } + + private maybeDetachEditorWatchdogManager() { + // a failed attach attempt may lead to a missing cleanup handler + if (this.detachEditorWatchdogManager) { + this.detachEditorWatchdogManager() + this.detachEditorWatchdogManager = null + } + } + + attachToCM6(cm6: EditorFacade) { + this.attachToEditor(cm6, () => { + // @ts-ignore + cm6.attachShareJs(this._doc, window.maxDocLength) + }) + } + + detachFromCM6() { + this.maybeDetachEditorWatchdogManager() + if (this._doc.detach_cm6) { + this._doc.detach_cm6() + } + } + + private startInflightOpTimeout(update: Update) { + this.startFatalTimeoutTimer(update) + const retryOp = () => { + // Only send the update again if inflightOp is still populated + // This can be cleared when hard reloading the document in which + // case we don't want to keep trying to send it. + debugConsole.log('[inflightOpTimeout] Trying op again') + if (this._doc.inflightOp != null) { + // When there is a socket.io disconnect, @_doc.inflightSubmittedIds + // is updated with the socket.io client id of the current op in flight + // (meta.source of the op). + // @connection.id is the client id of the current socket.io session. + // So we need both depending on whether the op was submitted before + // one or more disconnects, or if it was submitted during the current session. + update.dupIfSource = [ + this.connection.id, + ...Array.from(this._doc.inflightSubmittedIds), + ] + + // We must be joined to a project for applyOtUpdate to work on the real-time + // service, so don't send an op if we're not. Connection state is set to 'ok' + // when we've joined the project + if (this.connection.state !== 'ok') { + debugConsole.log( + '[inflightOpTimeout] Not connected, retrying in 0.5s' + ) + window.setTimeout(retryOp, WAIT_FOR_CONNECTION_TIMEOUT) + } else { + debugConsole.log('[inflightOpTimeout] Sending') + return this.connection.send(update) + } + } + } + + const timer = window.setTimeout(retryOp, INFLIGHT_OP_TIMEOUT) + return this._doc.inflightCallbacks.push(() => { + this.clearFatalTimeoutTimer() + window.clearTimeout(timer) + }) // 30 seconds + } + + private startFatalTimeoutTimer(update: Update) { + // If an op doesn't get acked within FATAL_OP_TIMEOUT, something has + // gone unrecoverably wrong (the op will have been retried multiple times) + if (this._timeoutTimer != null) { + return + } + return (this._timeoutTimer = window.setTimeout(() => { + this.clearFatalTimeoutTimer() + return this.trigger('op:timeout', update) + }, FATAL_OP_TIMEOUT)) + } + + private clearFatalTimeoutTimer() { + if (this._timeoutTimer == null) { + return + } + clearTimeout(this._timeoutTimer) + return (this._timeoutTimer = null) + } + + private handleError(error: unknown, meta = {}) { + return this.trigger('error', error, meta) + } + + // @ts-ignore + private bindToDocChanges(doc: Doc) { + const { submitOp } = doc + doc.submitOp = (op: ShareJsOperation, callback?: () => void) => { + this.trigger('op:sent', op) + doc.pendingCallbacks.push(() => { + return this.trigger('op:acknowledged', op) + }) + return submitOp.call(doc, op, callback) + } + + const { flush } = doc + doc.flush = () => { + this.trigger('flush', doc.inflightOp, doc.pendingOp, doc.version) + return flush.call(doc) + } + } +} diff --git a/services/web/frontend/js/features/ide-react/editor/types/document.ts b/services/web/frontend/js/features/ide-react/editor/types/document.ts new file mode 100644 index 0000000000..744d8f1b5b --- /dev/null +++ b/services/web/frontend/js/features/ide-react/editor/types/document.ts @@ -0,0 +1,21 @@ +import { AnyOperation } from '../../../../../../types/change' + +export type Version = number + +export type ShareJsConnectionState = 'ok' | 'disconnected' | 'stopped' + +export type ShareJsOperation = AnyOperation[] + +export type TrackChangesIdSeeds = { inflight: string; pending: string } + +export type Message = Record +// TODO: MIGRATION: Make an accurate and more specific type +// { +// v: Version +// open?: boolean +// meta?: { +// type: string +// } +// doc?: string +// snapshot?: string +// } diff --git a/services/web/frontend/js/features/ide-react/hooks/use-layout-event-tracking.ts b/services/web/frontend/js/features/ide-react/hooks/use-layout-event-tracking.ts new file mode 100644 index 0000000000..a1031950e9 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/hooks/use-layout-event-tracking.ts @@ -0,0 +1,25 @@ +import { useLayoutContext } from '@/shared/context/layout-context' +import { useEffect } from 'react' +import { sendMBOnce } from '@/infrastructure/event-tracking' + +export function useLayoutEventTracking() { + const { view, leftMenuShown, chatIsOpen } = useLayoutContext() + + useEffect(() => { + if (view !== 'editor' && view !== 'pdf') { + sendMBOnce(`ide-open-view-${view}-once`) + } + }, [view]) + + useEffect(() => { + if (leftMenuShown) { + sendMBOnce(`ide-open-left-menu-once`) + } + }, [leftMenuShown]) + + useEffect(() => { + if (chatIsOpen) { + sendMBOnce(`ide-open-chat-once`) + } + }, [chatIsOpen]) +} diff --git a/services/web/frontend/js/features/ide-react/hooks/use-socket-listener.ts b/services/web/frontend/js/features/ide-react/hooks/use-socket-listener.ts new file mode 100644 index 0000000000..26eed61005 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/hooks/use-socket-listener.ts @@ -0,0 +1,18 @@ +import { useEffect } from 'react' +import { Socket } from '@/features/ide-react/connection/types/socket' + +type SocketOnParams = Parameters + +export default function useSocketListener( + socket: Socket, + event: SocketOnParams[0], + listener: SocketOnParams[1] +) { + useEffect(() => { + socket.on(event, listener) + + return () => { + socket.removeListener(event, listener) + } + }, [event, listener, socket]) +} diff --git a/services/web/frontend/js/features/ide-react/scope-adapters/layout-context-adapter.ts b/services/web/frontend/js/features/ide-react/scope-adapters/layout-context-adapter.ts new file mode 100644 index 0000000000..9214988d77 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/scope-adapters/layout-context-adapter.ts @@ -0,0 +1,18 @@ +import { ReactScopeValueStore } from '../scope-value-store/react-scope-value-store' +import getMeta from '@/utils/meta' + +const reviewPanelStorageKey = `ui.reviewPanelOpen.${getMeta('ol-project_id')}` + +export default function populateLayoutScope(store: ReactScopeValueStore) { + store.set('ui.view', 'editor') + + // TODO: Find out what this needs to do and make it do it + store.set('toggleHistory', () => {}) + + store.set('openFile', null) + store.set('ui.chatOpen', false) + store.persisted('ui.reviewPanelOpen', false, reviewPanelStorageKey) + store.set('ui.leftMenuShown', false) + store.set('ui.pdfLayout', 'sideBySide') + store.set('ui.loadingStyleSheet', false) +} diff --git a/services/web/frontend/js/features/ide-react/scope-adapters/settings-adapter.ts b/services/web/frontend/js/features/ide-react/scope-adapters/settings-adapter.ts new file mode 100644 index 0000000000..fae072502b --- /dev/null +++ b/services/web/frontend/js/features/ide-react/scope-adapters/settings-adapter.ts @@ -0,0 +1,5 @@ +import { ReactScopeValueStore } from '@/features/ide-react/scope-value-store/react-scope-value-store' + +export function populateSettingsScope(store: ReactScopeValueStore) { + store.set('settings', window.userSettings) +} diff --git a/services/web/frontend/js/features/ide-react/types/file-tree.ts b/services/web/frontend/js/features/ide-react/types/file-tree.ts new file mode 100644 index 0000000000..e636a39157 --- /dev/null +++ b/services/web/frontend/js/features/ide-react/types/file-tree.ts @@ -0,0 +1,38 @@ +import { FileRef } from '../../../../../types/file-ref' +import { Folder } from '../../../../../types/folder' +import { Doc } from '../../../../../types/doc' + +export type FileTreeFolderFindResultType = 'folder' | 'doc' | 'fileRef' + +interface BaseFileTreeFindResult { + type: FileTreeFolderFindResultType + entity: T + parent: T[] + parentFolderId: string + path: string + index: number +} + +export interface FileTreeFolderFindResult + extends BaseFileTreeFindResult { + type: 'folder' +} + +export interface FileTreeDocumentFindResult + extends BaseFileTreeFindResult { + type: 'doc' +} + +export interface FileTreeFileRefFindResult + extends BaseFileTreeFindResult { + type: 'fileRef' +} + +export type FileTreeFindResult = + | FileTreeFolderFindResult + | FileTreeDocumentFindResult + | FileTreeFileRefFindResult + +export type FileTreeSelectHandler = ( + selectedEntities: FileTreeFindResult[] +) => void diff --git a/services/web/frontend/js/features/ide-react/util/find-doc-entity-by-id.ts b/services/web/frontend/js/features/ide-react/util/find-doc-entity-by-id.ts new file mode 100644 index 0000000000..202d29a60f --- /dev/null +++ b/services/web/frontend/js/features/ide-react/util/find-doc-entity-by-id.ts @@ -0,0 +1,11 @@ +import { findInTree } from '@/features/file-tree/util/find-in-tree' +import { Folder } from '../../../../../types/folder' +import { Doc } from '../../../../../types/doc' + +export function findDocEntityById(fileTreeData: Folder, docId: string) { + const item = findInTree(fileTreeData, docId) + if (!item || item.type !== 'doc') { + return null + } + return item.entity as Doc +} diff --git a/services/web/frontend/js/features/source-editor/components/codemirror-editor.tsx b/services/web/frontend/js/features/source-editor/components/codemirror-editor.tsx index e80f54cd48..84d058c349 100644 --- a/services/web/frontend/js/features/source-editor/components/codemirror-editor.tsx +++ b/services/web/frontend/js/features/source-editor/components/codemirror-editor.tsx @@ -20,6 +20,7 @@ import importOverleafModules from '../../../../macros/import-overleaf-module.mac import { FigureModal } from './figure-modal/figure-modal' import ReviewPanel from './review-panel/review-panel' import getMeta from '../../../utils/meta' +import { useIdeContext } from '@/shared/context/ide-context' const sourceEditorComponents = importOverleafModules( 'sourceEditorComponents' @@ -33,6 +34,7 @@ function CodeMirrorEditor() { const isMounted = useIsMounted() const isReviewPanelReact = getMeta('ol-isReviewPanelReact') + const { isReactIde } = useIdeContext() // create the view using the initial state and intercept transactions const viewRef = useRef(null) @@ -62,7 +64,7 @@ function CodeMirrorEditor() { - {isReviewPanelReact && } + {isReviewPanelReact && !isReactIde && } {sourceEditorComponents.map( ({ import: { default: Component }, path }) => ( diff --git a/services/web/frontend/js/features/source-editor/extensions/realtime.ts b/services/web/frontend/js/features/source-editor/extensions/realtime.ts index a2946b57bf..e26ad30f7b 100644 --- a/services/web/frontend/js/features/source-editor/extensions/realtime.ts +++ b/services/web/frontend/js/features/source-editor/extensions/realtime.ts @@ -19,8 +19,17 @@ import { debugConsole } from '@/utils/debugging' * - frontend/js/ide/editor/Document.js * - frontend/js/ide/editor/ShareJsDoc.js * - frontend/js/ide/connection/EditorWatchdogManager.js + * - frontend/js/features/ide-react/editor/document.ts + * - frontend/js/features/ide-react/editor/share-js-doc.ts + * - frontend/js/features/ide-react/connection/editor-watchdog-manager.js */ +export type ChangeDescription = { + origin: 'remote' | 'undo' | 'reject' | undefined + inserted: boolean + removed: boolean +} + /** * A custom extension that connects the CodeMirror 6 editor to the currently open ShareJS document. */ @@ -203,7 +212,13 @@ export class EditorFacade extends EventEmitter { // TODO: mapPos instead? positionShift = positionShift - removedLength + insertedLength - this.emit('change', this, { origin, inserted, removed }) + const changeDescription: ChangeDescription = { + origin, + inserted, + removed, + } + + this.emit('change', this, changeDescription) } ) } diff --git a/services/web/frontend/js/features/source-editor/extensions/track-changes.ts b/services/web/frontend/js/features/source-editor/extensions/track-changes.ts index 2a2c64c46d..c9f1fe4d6a 100644 --- a/services/web/frontend/js/features/source-editor/extensions/track-changes.ts +++ b/services/web/frontend/js/features/source-editor/extensions/track-changes.ts @@ -22,15 +22,14 @@ import { } from './changes/comments' import { invertedEffects } from '@codemirror/commands' import { CurrentDoc } from '../../../../../types/current-doc' -import { - Change, - ChangeOperation, - CommentOperation, - DeleteOperation, - Operation, -} from '../../../../../types/change' +import { Change, DeleteOperation } from '../../../../../types/change' import { ChangeManager } from './changes/change-manager' import { debugConsole } from '@/utils/debugging' +import { + isChangeOperation, + isCommentOperation, + isDeleteOperation, +} from '@/utils/operations' const clearChangesEffect = StateEffect.define() const buildChangesEffect = StateEffect.define() @@ -246,13 +245,6 @@ class ChangeCalloutWidget extends WidgetType { } } -// const isInsertOperation = (op: Operation): op is InsertOperation => 'i' in op -const isChangeOperation = (op: Operation): op is ChangeOperation => - 'c' in op && 't' in op -const isCommentOperation = (op: Operation): op is CommentOperation => - 'c' in op && !('t' in op) -const isDeleteOperation = (op: Operation): op is DeleteOperation => 'd' in op - const createChangeRange = (change: Change, currentDoc: CurrentDoc) => { const { id, metadata, op } = change diff --git a/services/web/frontend/js/features/source-editor/hooks/use-codemirror-scope.ts b/services/web/frontend/js/features/source-editor/hooks/use-codemirror-scope.ts index 404e9f1588..6b585b4a1b 100644 --- a/services/web/frontend/js/features/source-editor/hooks/use-codemirror-scope.ts +++ b/services/web/frontend/js/features/source-editor/hooks/use-codemirror-scope.ts @@ -89,6 +89,8 @@ function useCodeMirrorScope(view: EditorView) { const [visual] = useScopeValue('editor.showVisual') const reactReviewPanel: boolean = getMeta('ol-isReviewPanelReact') + const [references] = useScopeValue<{ keys: string[] }>('$root._references') + // build the translation phrases const phrases = usePhrases() @@ -206,7 +208,7 @@ function useCodeMirrorScope(view: EditorView) { // TODO: read this data from the scope? const metadataRef = useRef({ documents: ide.metadataManager.metadata.state.documents, - references: ide.$scope.$root._references.keys, + references: references.keys, fileTreeData, }) diff --git a/services/web/frontend/js/utils/decode-utf8.ts b/services/web/frontend/js/utils/decode-utf8.ts new file mode 100644 index 0000000000..1cd38d6eaf --- /dev/null +++ b/services/web/frontend/js/utils/decode-utf8.ts @@ -0,0 +1,5 @@ +// TODO: MIGRATION: Can we use TextDecoder now? https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder +// See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html +export function decodeUtf8(text: string) { + return decodeURIComponent(escape(text)) +} diff --git a/services/web/frontend/js/utils/operations.ts b/services/web/frontend/js/utils/operations.ts new file mode 100644 index 0000000000..d5c26a6513 --- /dev/null +++ b/services/web/frontend/js/utils/operations.ts @@ -0,0 +1,16 @@ +import { + ChangeOperation, + CommentOperation, + DeleteOperation, + InsertOperation, + Operation, +} from '../../../types/change' + +export const isInsertOperation = (op: Operation): op is InsertOperation => + 'i' in op +export const isChangeOperation = (op: Operation): op is ChangeOperation => + 'c' in op && 't' in op +export const isCommentOperation = (op: Operation): op is CommentOperation => + 'c' in op && !('t' in op) +export const isDeleteOperation = (op: Operation): op is DeleteOperation => + 'd' in op diff --git a/services/web/frontend/stylesheets/app/editor/ide-react.less b/services/web/frontend/stylesheets/app/editor/ide-react.less index 9ad548f92b..ec513769cc 100644 --- a/services/web/frontend/stylesheets/app/editor/ide-react.less +++ b/services/web/frontend/stylesheets/app/editor/ide-react.less @@ -95,3 +95,7 @@ background-color: @symbol-palette-bg; color: var(--neutral-20); } + +.ide-react-file-tree-panel { + display: flex; +} diff --git a/services/web/types/change.ts b/services/web/types/change.ts index 65b3c1ccf8..4a50aa9bf5 100644 --- a/services/web/types/change.ts +++ b/services/web/types/change.ts @@ -20,14 +20,18 @@ export interface CommentOperation extends Operation { c: string } -export type AnyOperation = +export type NonCommentOperation = | InsertOperation | ChangeOperation | DeleteOperation - | CommentOperation + +export type AnyOperation = NonCommentOperation | CommentOperation export type Change = { id: string - metadata?: string + metadata?: { + user_id: string + ts: Date + } op: T } diff --git a/services/web/types/current-doc.ts b/services/web/types/current-doc.ts index 3482a58d68..d6a77ac1c6 100644 --- a/services/web/types/current-doc.ts +++ b/services/web/types/current-doc.ts @@ -12,6 +12,10 @@ import { // type for the Document class in ide/editor/Document.js // note: this is a custom EventEmitter class + +// TODO: MIGRATION: This doesn't match the type for +// ide-react/editor/document.ts, which has a nullable `ranges` property and some +// other quirks. They should match. export interface CurrentDoc extends EventEmitter { doc_id: string docName: string diff --git a/services/web/types/project.ts b/services/web/types/project.ts index e80f466d05..c85e2ba005 100644 --- a/services/web/types/project.ts +++ b/services/web/types/project.ts @@ -27,4 +27,5 @@ export type Project = { invites: ProjectInvite[] rootDoc_id?: string rootFolder?: Folder[] + deletedByExternalDataSource?: boolean } diff --git a/services/web/types/user.ts b/services/web/types/user.ts index 9478fba88e..2e5b6da088 100644 --- a/services/web/types/user.ts +++ b/services/web/types/user.ts @@ -1,3 +1,8 @@ +export type RefProviders = { + mendeley?: boolean + zotero?: boolean +} + export type User = { id: string email: string @@ -19,6 +24,7 @@ export type User = { versioning?: boolean zotero?: boolean } + refProviders?: RefProviders } export type MongoUser = Pick> & { _id: string } diff --git a/services/web/types/window.ts b/services/web/types/window.ts index ee97878621..7dff27c5f0 100644 --- a/services/web/types/window.ts +++ b/services/web/types/window.ts @@ -3,6 +3,7 @@ import { OAuthProviders } from './oauth-providers' import { OverallThemeMeta } from './project-settings' import { User } from './user' import 'recurly__recurly-js' +import { UserSettings } from '@/features/editor-left-menu/utils/api' declare global { // eslint-disable-next-line no-unused-vars @@ -10,6 +11,7 @@ declare global { csrfToken: string user: User user_id?: string + userSettings: UserSettings oauthProviders: OAuthProviders thirdPartyIds: Record metaAttributesCache: Map