From a826677225a49ba53cc100a6201cee31b158b9e0 Mon Sep 17 00:00:00 2001 From: Tilman Vatteroth Date: Fri, 24 Mar 2023 10:26:49 +0100 Subject: [PATCH] refactor: save ydoc state in the database, so it can be restored easier By storing the ydoc state in the database we can reconnect lost clients easier and enable offline editing because we continue using the crdt data that has been used by the client before the connection loss. Signed-off-by: Tilman Vatteroth --- .../realtime-note/realtime-connection.spec.ts | 10 +-- .../realtime-note/realtime-connection.ts | 6 +- .../realtime-note/realtime-note-store.spec.ts | 17 +++++ .../realtime-note/realtime-note-store.ts | 15 +++- .../realtime-note.service.spec.ts | 40 ++++++++++- .../realtime-note/realtime-note.service.ts | 10 ++- .../realtime-note/realtime-note.spec.ts | 14 +--- .../realtime/realtime-note/realtime-note.ts | 11 ++- backend/src/revisions/revision.entity.ts | 5 ++ backend/src/revisions/revisions.service.ts | 3 +- .../mocked-backend-message-transporter.ts | 11 ++- commons/src/y-doc-sync/realtime-doc.test.ts | 28 ++++++-- commons/src/y-doc-sync/realtime-doc.ts | 72 ++++++++++++++++--- .../src/y-doc-sync/y-doc-sync-adapter.test.ts | 29 ++++---- commons/src/y-doc-sync/y-doc-sync-adapter.ts | 68 +++++++----------- .../y-doc-sync/y-doc-sync-client-adapter.ts | 18 +++-- .../y-doc-sync/y-doc-sync-server-adapter.ts | 8 ++- .../document-sync/y-text-sync-view-plugin.ts | 12 +--- .../editor-page/editor-pane/editor-pane.tsx | 12 ++-- .../hooks/yjs/use-bind-y-text-to-redux.ts | 14 ++-- .../yjs/use-code-mirror-yjs-extension.ts | 19 ++--- .../hooks/yjs/use-markdown-content-y-text.ts | 18 ----- .../hooks/yjs/use-realtime-connection.ts | 2 +- .../editor-pane/hooks/yjs/use-realtime-doc.ts | 20 ++++++ .../yjs/use-y-doc-sync-client-adapter.ts | 13 ++-- .../editor-pane/hooks/yjs/use-y-doc.ts | 30 -------- 26 files changed, 301 insertions(+), 204 deletions(-) delete mode 100644 frontend/src/components/editor-page/editor-pane/hooks/yjs/use-markdown-content-y-text.ts create mode 100644 frontend/src/components/editor-page/editor-pane/hooks/yjs/use-realtime-doc.ts delete mode 100644 frontend/src/components/editor-page/editor-pane/hooks/yjs/use-y-doc.ts diff --git a/backend/src/realtime/realtime-note/realtime-connection.spec.ts b/backend/src/realtime/realtime-note/realtime-connection.spec.ts index 8cbd7954d..d2754f5bf 100644 --- a/backend/src/realtime/realtime-note/realtime-connection.spec.ts +++ b/backend/src/realtime/realtime-note/realtime-connection.spec.ts @@ -27,11 +27,7 @@ jest.mock( ({ ...jest.requireActual('@hedgedoc/commons'), // eslint-disable-next-line @typescript-eslint/naming-convention - YDocSyncServerAdapter: jest.fn(() => - Mock.of({ - setYDoc: jest.fn(), - }), - ), + YDocSyncServerAdapter: jest.fn(() => Mock.of({})), } as Record), ); @@ -86,9 +82,7 @@ describe('websocket connection', () => { }); it('returns the correct sync adapter', () => { - const yDocSyncServerAdapter = Mock.of({ - setYDoc: jest.fn(), - }); + const yDocSyncServerAdapter = Mock.of({}); jest .spyOn(HedgeDocCommonsModule, 'YDocSyncServerAdapter') .mockImplementation(() => yDocSyncServerAdapter); diff --git a/backend/src/realtime/realtime-note/realtime-connection.ts b/backend/src/realtime/realtime-note/realtime-connection.ts index f43765f51..3aae04137 100644 --- a/backend/src/realtime/realtime-note/realtime-connection.ts +++ b/backend/src/realtime/realtime-note/realtime-connection.ts @@ -41,8 +41,10 @@ export class RealtimeConnection { this.transporter.on('disconnected', () => { realtimeNote.removeClient(this); }); - this.yDocSyncAdapter = new YDocSyncServerAdapter(this.transporter); - this.yDocSyncAdapter.setYDoc(realtimeNote.getRealtimeDoc()); + this.yDocSyncAdapter = new YDocSyncServerAdapter( + this.transporter, + realtimeNote.getRealtimeDoc(), + ); this.realtimeUserStateAdapter = new RealtimeUserStatusAdapter( this.user?.username ?? null, this.getDisplayName(), diff --git a/backend/src/realtime/realtime-note/realtime-note-store.spec.ts b/backend/src/realtime/realtime-note/realtime-note-store.spec.ts index 89071bf0f..4ebf7343d 100644 --- a/backend/src/realtime/realtime-note/realtime-note-store.spec.ts +++ b/backend/src/realtime/realtime-note/realtime-note-store.spec.ts @@ -41,6 +41,23 @@ describe('RealtimeNoteStore', () => { expect(realtimeNoteConstructorSpy).toHaveBeenCalledWith( mockedNote, mockedContent, + undefined, + ); + expect(realtimeNoteStore.find(mockedNoteId)).toBe(mockedRealtimeNote); + expect(realtimeNoteStore.getAllRealtimeNotes()).toStrictEqual([ + mockedRealtimeNote, + ]); + }); + + it("can create a new realtime note with a yjs state if it doesn't exist yet", () => { + const initialYjsState = [1, 2, 3]; + expect( + realtimeNoteStore.create(mockedNote, mockedContent, initialYjsState), + ).toBe(mockedRealtimeNote); + expect(realtimeNoteConstructorSpy).toHaveBeenCalledWith( + mockedNote, + mockedContent, + initialYjsState, ); expect(realtimeNoteStore.find(mockedNoteId)).toBe(mockedRealtimeNote); expect(realtimeNoteStore.getAllRealtimeNotes()).toStrictEqual([ diff --git a/backend/src/realtime/realtime-note/realtime-note-store.ts b/backend/src/realtime/realtime-note/realtime-note-store.ts index 1758c65e2..ed26f7eb7 100644 --- a/backend/src/realtime/realtime-note/realtime-note-store.ts +++ b/backend/src/realtime/realtime-note/realtime-note-store.ts @@ -16,15 +16,24 @@ export class RealtimeNoteStore { * Creates a new {@link RealtimeNote} for the given {@link Note} and memorizes it. * * @param note The note for which the realtime note should be created - * @param initialContent The initial content for the realtime note + * @param initialTextContent the initial text content of realtime doc + * @param initialYjsState the initial yjs state. If provided this will be used instead of the text content * @throws Error if there is already an realtime note for the given note. * @return The created realtime note */ - public create(note: Note, initialContent: string): RealtimeNote { + public create( + note: Note, + initialTextContent: string, + initialYjsState?: number[], + ): RealtimeNote { if (this.noteIdToRealtimeNote.has(note.id)) { throw new Error(`Realtime note for note ${note.id} already exists.`); } - const realtimeNote = new RealtimeNote(note, initialContent); + const realtimeNote = new RealtimeNote( + note, + initialTextContent, + initialYjsState, + ); realtimeNote.on('destroy', () => { this.noteIdToRealtimeNote.delete(note.id); }); diff --git a/backend/src/realtime/realtime-note/realtime-note.service.spec.ts b/backend/src/realtime/realtime-note/realtime-note.service.spec.ts index 9a399af0d..696db4493 100644 --- a/backend/src/realtime/realtime-note/realtime-note.service.spec.ts +++ b/backend/src/realtime/realtime-note/realtime-note.service.spec.ts @@ -17,6 +17,7 @@ import { RealtimeNoteService } from './realtime-note.service'; describe('RealtimeNoteService', () => { const mockedContent = 'mockedContent'; + const mockedYjsState = [1, 2, 3]; const mockedNoteId = 4711; let note: Note; let realtimeNote: RealtimeNote; @@ -37,7 +38,10 @@ describe('RealtimeNoteService', () => { jest.useFakeTimers(); }); - function mockGetLatestRevision(latestRevisionExists: boolean) { + function mockGetLatestRevision( + latestRevisionExists: boolean, + hasYjsState = false, + ) { jest .spyOn(revisionsService, 'getLatestRevision') .mockImplementation((note: Note) => @@ -45,6 +49,7 @@ describe('RealtimeNoteService', () => { ? Promise.resolve( Mock.of({ content: mockedContent, + ...(hasYjsState ? { yjsStateVector: mockedYjsState } : {}), }), ) : Promise.reject('Revision for note mockedNoteId not found.'), @@ -106,7 +111,32 @@ describe('RealtimeNoteService', () => { ).resolves.toBe(realtimeNote); expect(realtimeNoteStore.find).toHaveBeenCalledWith(mockedNoteId); - expect(realtimeNoteStore.create).toHaveBeenCalledWith(note, mockedContent); + expect(realtimeNoteStore.create).toHaveBeenCalledWith( + note, + mockedContent, + undefined, + ); + expect(setIntervalSpy).not.toHaveBeenCalled(); + }); + + it("creates a new realtime note with a yjs state if it doesn't exist yet", async () => { + mockGetLatestRevision(true, true); + jest.spyOn(realtimeNoteStore, 'find').mockImplementation(() => undefined); + jest + .spyOn(realtimeNoteStore, 'create') + .mockImplementation(() => realtimeNote); + mockedAppConfig.persistInterval = 0; + + await expect( + realtimeNoteService.getOrCreateRealtimeNote(note), + ).resolves.toBe(realtimeNote); + + expect(realtimeNoteStore.find).toHaveBeenCalledWith(mockedNoteId); + expect(realtimeNoteStore.create).toHaveBeenCalledWith( + note, + mockedContent, + mockedYjsState, + ); expect(setIntervalSpy).not.toHaveBeenCalled(); }); @@ -192,7 +222,11 @@ describe('RealtimeNoteService', () => { .mockImplementation(() => Promise.resolve(Mock.of())); realtimeNote.emit('beforeDestroy'); - expect(createRevisionSpy).toHaveBeenCalledWith(note, mockedContent); + expect(createRevisionSpy).toHaveBeenCalledWith( + note, + mockedContent, + expect.any(Array), + ); }); it('destroys every realtime note on application shutdown', () => { diff --git a/backend/src/realtime/realtime-note/realtime-note.service.ts b/backend/src/realtime/realtime-note/realtime-note.service.ts index a4d0f982a..6ed01892e 100644 --- a/backend/src/realtime/realtime-note/realtime-note.service.ts +++ b/backend/src/realtime/realtime-note/realtime-note.service.ts @@ -43,6 +43,7 @@ export class RealtimeNoteService implements BeforeApplicationShutdown { .createRevision( realtimeNote.getNote(), realtimeNote.getRealtimeDoc().getCurrentContent(), + realtimeNote.getRealtimeDoc().encodeStateAsUpdate(), ) .catch((reason) => this.logger.error(reason)); } @@ -68,9 +69,12 @@ export class RealtimeNoteService implements BeforeApplicationShutdown { * @return The created realtime note */ private async createNewRealtimeNote(note: Note): Promise { - const initialContent = (await this.revisionsService.getLatestRevision(note)) - .content; - const realtimeNote = this.realtimeNoteStore.create(note, initialContent); + const lastRevision = await this.revisionsService.getLatestRevision(note); + const realtimeNote = this.realtimeNoteStore.create( + note, + lastRevision.content, + lastRevision.yjsStateVector ?? undefined, + ); realtimeNote.on('beforeDestroy', () => { this.saveRealtimeNote(realtimeNote); }); diff --git a/backend/src/realtime/realtime-note/realtime-note.spec.ts b/backend/src/realtime/realtime-note/realtime-note.spec.ts index 2ad20e5b7..f1807473a 100644 --- a/backend/src/realtime/realtime-note/realtime-note.spec.ts +++ b/backend/src/realtime/realtime-note/realtime-note.spec.ts @@ -4,15 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only */ import { MessageType, RealtimeDoc } from '@hedgedoc/commons'; -import * as hedgedocCommonsModule from '@hedgedoc/commons'; import { Mock } from 'ts-mockery'; import { Note } from '../../notes/note.entity'; import { RealtimeNote } from './realtime-note'; import { MockConnectionBuilder } from './test-utils/mock-connection'; -jest.mock('@hedgedoc/commons'); - describe('realtime note', () => { let mockedNote: Note; @@ -40,18 +37,13 @@ describe('realtime note', () => { expect(sut.hasConnections()).toBeFalsy(); }); - it('creates a y-doc', () => { + it('creates a realtime doc', () => { const initialContent = 'nothing'; - const mockedDoc = new RealtimeDoc(initialContent); - const docSpy = jest - .spyOn(hedgedocCommonsModule, 'RealtimeDoc') - .mockReturnValue(mockedDoc); const sut = new RealtimeNote(mockedNote, initialContent); - expect(docSpy).toHaveBeenCalledWith(initialContent); - expect(sut.getRealtimeDoc()).toBe(mockedDoc); + expect(sut.getRealtimeDoc() instanceof RealtimeDoc).toBeTruthy(); }); - it('destroys y-doc on self-destruction', () => { + it('destroys realtime doc on self-destruction', () => { const sut = new RealtimeNote(mockedNote, 'nothing'); const docDestroy = jest.spyOn(sut.getRealtimeDoc(), 'destroy'); sut.destroy(); diff --git a/backend/src/realtime/realtime-note/realtime-note.ts b/backend/src/realtime/realtime-note/realtime-note.ts index f6a1d8912..9c80b9238 100644 --- a/backend/src/realtime/realtime-note/realtime-note.ts +++ b/backend/src/realtime/realtime-note/realtime-note.ts @@ -28,12 +28,17 @@ export class RealtimeNote extends EventEmitter2 { private readonly clients = new Set(); private isClosing = false; - constructor(private readonly note: Note, initialContent: string) { + constructor( + private readonly note: Note, + initialTextContent: string, + initialYjsState?: number[], + ) { super(); this.logger = new Logger(`${RealtimeNote.name} ${note.id}`); - this.doc = new RealtimeDoc(initialContent); + this.doc = new RealtimeDoc(initialTextContent, initialYjsState); + const length = this.doc.getCurrentContent().length; this.logger.debug( - `New realtime session for note ${note.id} created. Length of initial content: ${initialContent.length} characters`, + `New realtime session for note ${note.id} created. Length of initial content: ${length} characters`, ); } diff --git a/backend/src/revisions/revision.entity.ts b/backend/src/revisions/revision.entity.ts index 9858889ef..8607c34bc 100644 --- a/backend/src/revisions/revision.entity.ts +++ b/backend/src/revisions/revision.entity.ts @@ -48,6 +48,9 @@ export class Revision { @Column() length: number; + @Column('simple-array', { nullable: true }) + yjsStateVector: null | number[]; + /** * Date at which the revision was created. */ @@ -74,6 +77,7 @@ export class Revision { content: string, patch: string, note: Note, + yjsStateVector?: number[], ): Omit { const newRevision = new Revision(); newRevision.patch = patch; @@ -81,6 +85,7 @@ export class Revision { newRevision.length = content.length; newRevision.note = Promise.resolve(note); newRevision.edits = Promise.resolve([]); + newRevision.yjsStateVector = yjsStateVector ?? null; return newRevision; } } diff --git a/backend/src/revisions/revisions.service.ts b/backend/src/revisions/revisions.service.ts index e39ec2ad6..d36c8c474 100644 --- a/backend/src/revisions/revisions.service.ts +++ b/backend/src/revisions/revisions.service.ts @@ -145,6 +145,7 @@ export class RevisionsService { async createRevision( note: Note, newContent: string, + yjsStateVector?: number[], ): Promise { // TODO: Save metadata const latestRevision = await this.getLatestRevision(note); @@ -157,7 +158,7 @@ export class RevisionsService { latestRevision.content, newContent, ); - const revision = Revision.create(newContent, patch, note); + const revision = Revision.create(newContent, patch, note, yjsStateVector); return await this.revisionRepository.save(revision); } } diff --git a/commons/src/message-transporters/mocked-backend-message-transporter.ts b/commons/src/message-transporters/mocked-backend-message-transporter.ts index 031da80c0..ce79f8e7e 100644 --- a/commons/src/message-transporters/mocked-backend-message-transporter.ts +++ b/commons/src/message-transporters/mocked-backend-message-transporter.ts @@ -6,14 +6,13 @@ import { RealtimeDoc } from '../y-doc-sync/realtime-doc.js' import { ConnectionState, MessageTransporter } from './message-transporter.js' import { Message, MessageType } from './message.js' -import { Doc, encodeStateAsUpdate } from 'yjs' /** * A mocked connection that doesn't send or receive any data and is instantly ready. * The only exception is the note content state request that is answered with the given initial content. */ export class MockedBackendMessageTransporter extends MessageTransporter { - private readonly doc: Doc + private readonly doc: RealtimeDoc private connected = true @@ -41,10 +40,10 @@ export class MockedBackendMessageTransporter extends MessageTransporter { sendMessage(content: Message) { if (content.type === MessageType.NOTE_CONTENT_STATE_REQUEST) { setTimeout(() => { - const payload = Array.from( - encodeStateAsUpdate(this.doc, new Uint8Array(content.payload)) - ) - this.receiveMessage({ type: MessageType.NOTE_CONTENT_UPDATE, payload }) + this.receiveMessage({ + type: MessageType.NOTE_CONTENT_UPDATE, + payload: this.doc.encodeStateAsUpdate(content.payload) + }) }, 10) } } diff --git a/commons/src/y-doc-sync/realtime-doc.test.ts b/commons/src/y-doc-sync/realtime-doc.test.ts index f0eee8180..78afe474d 100644 --- a/commons/src/y-doc-sync/realtime-doc.test.ts +++ b/commons/src/y-doc-sync/realtime-doc.test.ts @@ -6,11 +6,31 @@ import { RealtimeDoc } from './realtime-doc.js' import { describe, expect, it } from '@jest/globals' -describe('websocket-doc', () => { - it('saves the initial content', () => { +describe('realtime doc', () => { + it('saves an initial text content correctly', () => { const textContent = 'textContent' - const websocketDoc = new RealtimeDoc(textContent) + const realtimeDoc = new RealtimeDoc(textContent) + expect(realtimeDoc.getCurrentContent()).toBe(textContent) + }) - expect(websocketDoc.getCurrentContent()).toBe(textContent) + it('will initialize an empty text if no initial content is given', () => { + const realtimeDoc = new RealtimeDoc() + expect(realtimeDoc.getCurrentContent()).toBe('') + }) + + it('restores a yjs state vector update correctly', () => { + const realtimeDoc = new RealtimeDoc( + 'notTheVectorText', + [ + 1, 1, 221, 208, 165, 230, 3, 0, 4, 1, 15, 109, 97, 114, 107, 100, 111, + 119, 110, 67, 111, 110, 116, 101, 110, 116, 32, 116, 101, 120, 116, 67, + 111, 110, 116, 101, 110, 116, 70, 114, 111, 109, 83, 116, 97, 116, 101, + 86, 101, 99, 116, 111, 114, 85, 112, 100, 97, 116, 101, 0 + ] + ) + + expect(realtimeDoc.getCurrentContent()).toBe( + 'textContentFromStateVectorUpdate' + ) }) }) diff --git a/commons/src/y-doc-sync/realtime-doc.ts b/commons/src/y-doc-sync/realtime-doc.ts index 13bd7bdf8..53e5afc9c 100644 --- a/commons/src/y-doc-sync/realtime-doc.ts +++ b/commons/src/y-doc-sync/realtime-doc.ts @@ -3,27 +3,52 @@ * * SPDX-License-Identifier: AGPL-3.0-only */ -import { Doc } from 'yjs' -import { Text as YText } from 'yjs' +import { EventEmitter2 } from 'eventemitter2' +import type { EventMap } from 'eventemitter2' +import { + applyUpdate, + Doc, + encodeStateAsUpdate, + encodeStateVector, + Text as YText +} from 'yjs' const MARKDOWN_CONTENT_CHANNEL_NAME = 'markdownContent' +export interface RealtimeDocEvents extends EventMap { + update: (update: number[], origin: unknown) => void +} + /** * This is the implementation of {@link Doc YDoc} which includes additional handlers for message sending and receiving. */ -export class RealtimeDoc extends Doc { +export class RealtimeDoc extends EventEmitter2 { + private doc: Doc = new Doc() + private readonly docUpdateListener: ( + update: Uint8Array, + origin: unknown + ) => void + /** * Creates a new instance. * * The new instance is filled with the given initial content. * - * @param initialContent - the initial content of the {@link Doc YDoc} + * @param initialTextContent the initial text content of the {@link Doc YDoc} + * @param initialYjsState the initial yjs state. If provided this will be used instead of the text content */ - constructor(initialContent?: string) { + constructor(initialTextContent?: string, initialYjsState?: number[]) { super() - if (initialContent) { - this.getMarkdownContentChannel().insert(0, initialContent) + if (initialYjsState) { + this.applyUpdate(initialYjsState, this) + } else if (initialTextContent) { + this.getMarkdownContentChannel().insert(0, initialTextContent) } + + this.docUpdateListener = (update, origin) => { + this.emit('update', Array.from(update), origin) + } + this.doc.on('update', this.docUpdateListener) } /** @@ -32,7 +57,7 @@ export class RealtimeDoc extends Doc { * @return The markdown channel */ public getMarkdownContentChannel(): YText { - return this.getText(MARKDOWN_CONTENT_CHANNEL_NAME) + return this.doc.getText(MARKDOWN_CONTENT_CHANNEL_NAME) } /** @@ -45,4 +70,35 @@ export class RealtimeDoc extends Doc { public getCurrentContent(): string { return this.getMarkdownContentChannel().toString() } + + /** + * Encodes the current state of the doc as update so it can be applied to other y-docs. + * + * @param encodedTargetStateVector The current state vector of the other y-doc. If provided the update will contain only the differences. + */ + public encodeStateAsUpdate(encodedTargetStateVector?: number[]): number[] { + const update = encodedTargetStateVector + ? new Uint8Array(encodedTargetStateVector) + : undefined + return Array.from(encodeStateAsUpdate(this.doc, update)) + } + + public destroy(): void { + this.doc.off('update', this.docUpdateListener) + this.doc.destroy() + } + + /** + * Applies an update to the y-doc. + * + * @param payload The update to apply + * @param origin A reference that triggered the update + */ + public applyUpdate(payload: number[], origin: unknown): void { + applyUpdate(this.doc, new Uint8Array(payload), origin) + } + + public encodeStateVector(): number[] { + return Array.from(encodeStateVector(this.doc)) + } } diff --git a/commons/src/y-doc-sync/y-doc-sync-adapter.test.ts b/commons/src/y-doc-sync/y-doc-sync-adapter.test.ts index ce14ea802..dbb09aa54 100644 --- a/commons/src/y-doc-sync/y-doc-sync-adapter.test.ts +++ b/commons/src/y-doc-sync/y-doc-sync-adapter.test.ts @@ -68,10 +68,10 @@ describe('message transporter', () => { console.debug('s>2 is connected') ) - docServer.on('update', (update: Uint8Array, origin: unknown) => { + docServer.on('update', (update: number[], origin: unknown) => { const message: Message = { type: MessageType.NOTE_CONTENT_UPDATE, - payload: Array.from(update) + payload: update } if (origin !== transporterServerTo1) { console.debug('YDoc on Server updated. Sending to Client 1') @@ -82,32 +82,35 @@ describe('message transporter', () => { transporterServerTo2.sendMessage(message) } }) - docClient1.on('update', (update: Uint8Array, origin: unknown) => { + docClient1.on('update', (update: number[], origin: unknown) => { if (origin !== transporterClient1) { console.debug('YDoc on client 1 updated. Sending to Server') } }) - docClient2.on('update', (update: Uint8Array, origin: unknown) => { + docClient2.on('update', (update: number[], origin: unknown) => { if (origin !== transporterClient2) { console.debug('YDoc on client 2 updated. Sending to Server') } }) - const yDocSyncAdapter1 = new YDocSyncClientAdapter(transporterClient1) - yDocSyncAdapter1.setYDoc(docClient1) - - const yDocSyncAdapter2 = new YDocSyncClientAdapter(transporterClient2) - yDocSyncAdapter2.setYDoc(docClient2) + const yDocSyncAdapter1 = new YDocSyncClientAdapter( + transporterClient1, + docClient1 + ) + const yDocSyncAdapter2 = new YDocSyncClientAdapter( + transporterClient2, + docClient2 + ) const yDocSyncAdapterServerTo1 = new YDocSyncServerAdapter( - transporterServerTo1 + transporterServerTo1, + docServer ) - yDocSyncAdapterServerTo1.setYDoc(docServer) const yDocSyncAdapterServerTo2 = new YDocSyncServerAdapter( - transporterServerTo2 + transporterServerTo2, + docServer ) - yDocSyncAdapterServerTo2.setYDoc(docServer) const waitForClient1Sync = new Promise((resolve) => { yDocSyncAdapter1.doAsSoonAsSynced(() => { diff --git a/commons/src/y-doc-sync/y-doc-sync-adapter.ts b/commons/src/y-doc-sync/y-doc-sync-adapter.ts index 65b32a347..3d2778a2e 100644 --- a/commons/src/y-doc-sync/y-doc-sync-adapter.ts +++ b/commons/src/y-doc-sync/y-doc-sync-adapter.ts @@ -5,26 +5,37 @@ */ import { MessageTransporter } from '../message-transporters/message-transporter.js' import { Message, MessageType } from '../message-transporters/message.js' +import { RealtimeDoc } from './realtime-doc.js' import { Listener } from 'eventemitter2' import { EventEmitter2 } from 'eventemitter2' -import { applyUpdate, Doc, encodeStateAsUpdate, encodeStateVector } from 'yjs' type EventMap = Record<'synced' | 'desynced', () => void> /** - * Sends and processes messages that are used to first-synchronize and update a {@link Doc y-doc}. + * Sends and processes messages that are used to first-synchronize and update a {@link RealtimeDoc y-doc}. */ export abstract class YDocSyncAdapter { public readonly eventEmitter = new EventEmitter2() - protected doc: Doc | undefined - - private destroyYDocUpdateCallback: undefined | (() => void) - private destroyEventListenerCallback: undefined | (() => void) + private readonly yDocUpdateListener: Listener + private readonly destroyEventListenerCallback: undefined | (() => void) private synced = false - constructor(protected readonly messageTransporter: MessageTransporter) { - this.bindDocumentSyncMessageEvents() + constructor( + protected readonly messageTransporter: MessageTransporter, + protected readonly doc: RealtimeDoc + ) { + this.yDocUpdateListener = doc.on( + 'update', + (update, origin) => { + this.processDocUpdate(update, origin) + }, + { + objectify: true + } + ) as Listener + + this.destroyEventListenerCallback = this.bindDocumentSyncMessageEvents() } /** @@ -51,39 +62,19 @@ export abstract class YDocSyncAdapter { return this.synced } - /** - * Sets the {@link Doc y-doc} that should be synchronized. - * - * @param doc the doc to synchronize. - */ - public setYDoc(doc: Doc | undefined): void { - this.doc = doc - - this.destroyYDocUpdateCallback?.() - if (!doc) { - return - } - const yDocUpdateCallback = this.processDocUpdate.bind(this) - doc.on('update', yDocUpdateCallback) - this.destroyYDocUpdateCallback = () => doc.off('update', yDocUpdateCallback) - this.eventEmitter.emit('desynced') - } - public destroy(): void { - this.destroyYDocUpdateCallback?.() + this.yDocUpdateListener.off() this.destroyEventListenerCallback?.() } - protected bindDocumentSyncMessageEvents(): void { + protected bindDocumentSyncMessageEvents(): () => void { const stateRequestListener = this.messageTransporter.on( MessageType.NOTE_CONTENT_STATE_REQUEST, (payload) => { if (this.doc) { this.messageTransporter.sendMessage({ type: MessageType.NOTE_CONTENT_UPDATE, - payload: Array.from( - encodeStateAsUpdate(this.doc, new Uint8Array(payload.payload)) - ) + payload: this.doc.encodeStateAsUpdate(payload.payload) }) } }, @@ -95,35 +86,30 @@ export abstract class YDocSyncAdapter { () => { this.synced = false this.eventEmitter.emit('desynced') - this.destroy() }, { objectify: true } ) as Listener const noteContentUpdateListener = this.messageTransporter.on( MessageType.NOTE_CONTENT_UPDATE, - (payload) => { - if (this.doc) { - applyUpdate(this.doc, new Uint8Array(payload.payload), this) - } - }, + (payload) => this.doc.applyUpdate(payload.payload, this), { objectify: true } ) as Listener - this.destroyEventListenerCallback = () => { + return () => { stateRequestListener.off() disconnectedListener.off() noteContentUpdateListener.off() } } - private processDocUpdate(update: Uint8Array, origin: unknown): void { + private processDocUpdate(update: number[], origin: unknown): void { if (!this.isSynced() || origin === this) { return } const message: Message = { type: MessageType.NOTE_CONTENT_UPDATE, - payload: Array.from(update) + payload: update } this.messageTransporter.sendMessage(message) @@ -141,7 +127,7 @@ export abstract class YDocSyncAdapter { if (this.doc) { this.messageTransporter.sendMessage({ type: MessageType.NOTE_CONTENT_STATE_REQUEST, - payload: Array.from(encodeStateVector(this.doc)) + payload: this.doc.encodeStateVector() }) } } diff --git a/commons/src/y-doc-sync/y-doc-sync-client-adapter.ts b/commons/src/y-doc-sync/y-doc-sync-client-adapter.ts index 0332d9a3b..7bf140d0f 100644 --- a/commons/src/y-doc-sync/y-doc-sync-client-adapter.ts +++ b/commons/src/y-doc-sync/y-doc-sync-client-adapter.ts @@ -5,13 +5,23 @@ */ import { MessageType } from '../message-transporters/message.js' import { YDocSyncAdapter } from './y-doc-sync-adapter.js' +import { Listener } from 'eventemitter2' export class YDocSyncClientAdapter extends YDocSyncAdapter { protected bindDocumentSyncMessageEvents() { - super.bindDocumentSyncMessageEvents() + const destroyCallback = super.bindDocumentSyncMessageEvents() - this.messageTransporter.on(MessageType.NOTE_CONTENT_UPDATE, () => { - this.markAsSynced() - }) + const noteContentUpdateListener = this.messageTransporter.on( + MessageType.NOTE_CONTENT_UPDATE, + () => { + this.markAsSynced() + }, + { objectify: true } + ) as Listener + + return () => { + destroyCallback() + noteContentUpdateListener.off() + } } } diff --git a/commons/src/y-doc-sync/y-doc-sync-server-adapter.ts b/commons/src/y-doc-sync/y-doc-sync-server-adapter.ts index 54df1395e..a1c9c1d1c 100644 --- a/commons/src/y-doc-sync/y-doc-sync-server-adapter.ts +++ b/commons/src/y-doc-sync/y-doc-sync-server-adapter.ts @@ -4,11 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only */ import { MessageTransporter } from '../message-transporters/message-transporter.js' +import { RealtimeDoc } from './realtime-doc.js' import { YDocSyncAdapter } from './y-doc-sync-adapter.js' export class YDocSyncServerAdapter extends YDocSyncAdapter { - constructor(readonly messageTransporter: MessageTransporter) { - super(messageTransporter) + constructor( + readonly messageTransporter: MessageTransporter, + readonly doc: RealtimeDoc + ) { + super(messageTransporter, doc) this.markAsSynced() } } diff --git a/frontend/src/components/editor-page/editor-pane/codemirror-extensions/document-sync/y-text-sync-view-plugin.ts b/frontend/src/components/editor-page/editor-pane/codemirror-extensions/document-sync/y-text-sync-view-plugin.ts index d84f96e09..974331939 100644 --- a/frontend/src/components/editor-page/editor-pane/codemirror-extensions/document-sync/y-text-sync-view-plugin.ts +++ b/frontend/src/components/editor-page/editor-pane/codemirror-extensions/document-sync/y-text-sync-view-plugin.ts @@ -17,7 +17,6 @@ const syncAnnotation = Annotation.define() */ export class YTextSyncViewPlugin implements PluginValue { private readonly observer: YTextSyncViewPlugin['onYTextUpdate'] - private firstUpdate = true constructor(private view: EditorView, private readonly yText: YText, pluginLoaded: () => void) { this.observer = this.onYTextUpdate.bind(this) @@ -47,16 +46,7 @@ export class YTextSyncViewPlugin implements PluginValue { }, [[], 0] as [ChangeSpec[], number] ) - return this.addDeleteAllChanges(changes) - } - - private addDeleteAllChanges(changes: ChangeSpec[]): ChangeSpec[] { - if (this.firstUpdate) { - this.firstUpdate = false - return [{ from: 0, to: this.view.state.doc.length, insert: '' }, ...changes] - } else { - return changes - } + return changes } public update(update: ViewUpdate): void { diff --git a/frontend/src/components/editor-page/editor-pane/editor-pane.tsx b/frontend/src/components/editor-page/editor-pane/editor-pane.tsx index 9dfdd7172..72ad77c34 100644 --- a/frontend/src/components/editor-page/editor-pane/editor-pane.tsx +++ b/frontend/src/components/editor-page/editor-pane/editor-pane.tsx @@ -21,12 +21,11 @@ import { useCursorActivityCallback } from './hooks/use-cursor-activity-callback' import { useUpdateCodeMirrorReference } from './hooks/use-update-code-mirror-reference' import { useBindYTextToRedux } from './hooks/yjs/use-bind-y-text-to-redux' import { useCodeMirrorYjsExtension } from './hooks/yjs/use-code-mirror-yjs-extension' -import { useMarkdownContentYText } from './hooks/yjs/use-markdown-content-y-text' import { useOnMetadataUpdated } from './hooks/yjs/use-on-metadata-updated' import { useOnNoteDeleted } from './hooks/yjs/use-on-note-deleted' import { useRealtimeConnection } from './hooks/yjs/use-realtime-connection' +import { useRealtimeDoc } from './hooks/yjs/use-realtime-doc' import { useReceiveRealtimeUsers } from './hooks/yjs/use-receive-realtime-users' -import { useYDoc } from './hooks/yjs/use-y-doc' import { useYDocSyncClientAdapter } from './hooks/yjs/use-y-doc-sync-client-adapter' import { useLinter } from './linter/linter' import { MaxLengthWarning } from './max-length-warning/max-length-warning' @@ -57,8 +56,7 @@ export const EditorPane: React.FC = ({ scrollState, onScroll, o useApplyScrollState(scrollState) const messageTransporter = useRealtimeConnection() - const yDoc = useYDoc(messageTransporter) - const yText = useMarkdownContentYText(yDoc) + const realtimeDoc = useRealtimeDoc() const editorScrollExtension = useCodeMirrorScrollWatchExtension(onScroll) const tablePasteExtensions = useCodeMirrorTablePasteExtension() const fileInsertExtension = useCodeMirrorFileInsertExtension() @@ -70,13 +68,13 @@ export const EditorPane: React.FC = ({ scrollState, onScroll, o const linterExtension = useLinter() - const syncAdapter = useYDocSyncClientAdapter(messageTransporter, yDoc) - const yjsExtension = useCodeMirrorYjsExtension(yText, syncAdapter) + const syncAdapter = useYDocSyncClientAdapter(messageTransporter, realtimeDoc) + const yjsExtension = useCodeMirrorYjsExtension(realtimeDoc, syncAdapter) useOnMetadataUpdated(messageTransporter) useOnNoteDeleted(messageTransporter) - useBindYTextToRedux(yText) + useBindYTextToRedux(realtimeDoc) useReceiveRealtimeUsers(messageTransporter) const extensions = useMemo( diff --git a/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-bind-y-text-to-redux.ts b/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-bind-y-text-to-redux.ts index f7181cb6c..6b4518a00 100644 --- a/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-bind-y-text-to-redux.ts +++ b/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-bind-y-text-to-redux.ts @@ -4,21 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only */ import { setNoteContent } from '../../../../../redux/note-details/methods' +import type { RealtimeDoc } from '@hedgedoc/commons' import { useEffect } from 'react' -import type { YText } from 'yjs/dist/src/types/YText' /** - * One-Way-synchronizes the text of the given {@link YText y-text} into the global application state. + * One-Way-synchronizes the text of the markdown content channel from the given {@link RealtimeDoc realtime doc} into the global application state. * - * @param yText The source text + * @param realtimeDoc The {@link RealtimeDoc realtime doc} that contains the markdown content */ -export const useBindYTextToRedux = (yText: YText | undefined): void => { +export const useBindYTextToRedux = (realtimeDoc: RealtimeDoc): void => { useEffect(() => { - if (!yText) { - return - } + const yText = realtimeDoc.getMarkdownContentChannel() const yTextCallback = () => setNoteContent(yText.toString()) yText.observe(yTextCallback) return () => yText.unobserve(yTextCallback) - }, [yText]) + }, [realtimeDoc]) } diff --git a/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-code-mirror-yjs-extension.ts b/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-code-mirror-yjs-extension.ts index 5825354d3..fcc2e0a5f 100644 --- a/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-code-mirror-yjs-extension.ts +++ b/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-code-mirror-yjs-extension.ts @@ -8,30 +8,33 @@ import { YTextSyncViewPlugin } from '../../codemirror-extensions/document-sync/y import type { Extension } from '@codemirror/state' import { ViewPlugin } from '@codemirror/view' import type { YDocSyncClientAdapter } from '@hedgedoc/commons' +import type { RealtimeDoc } from '@hedgedoc/commons' import { useEffect, useMemo, useState } from 'react' -import type { Text as YText } from 'yjs' /** * Creates a {@link Extension code mirror extension} that synchronizes an editor with the given {@link YText ytext}. * - * @param yText The source and target for the editor content + * @param doc The {@link RealtimeDoc realtime doc} that contains the markdown content text channel * @param syncAdapter The sync adapter that processes the communication for content synchronisation. * @return the created extension */ -export const useCodeMirrorYjsExtension = (yText: YText | undefined, syncAdapter: YDocSyncClientAdapter): Extension => { +export const useCodeMirrorYjsExtension = (doc: RealtimeDoc, syncAdapter: YDocSyncClientAdapter): Extension => { const [editorReady, setEditorReady] = useState(false) const synchronized = useApplicationState((state) => state.realtimeStatus.isSynced) const connected = useApplicationState((state) => state.realtimeStatus.isConnected) useEffect(() => { - if (editorReady && connected && !synchronized && yText) { + if (editorReady && connected && !synchronized) { syncAdapter.requestDocumentState() } - }, [connected, editorReady, syncAdapter, synchronized, yText]) + }, [connected, editorReady, syncAdapter, synchronized]) return useMemo( - () => - yText ? [ViewPlugin.define((view) => new YTextSyncViewPlugin(view, yText, () => setEditorReady(true)))] : [], - [yText] + () => [ + ViewPlugin.define( + (view) => new YTextSyncViewPlugin(view, doc.getMarkdownContentChannel(), () => setEditorReady(true)) + ) + ], + [doc] ) } diff --git a/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-markdown-content-y-text.ts b/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-markdown-content-y-text.ts deleted file mode 100644 index 7124c3349..000000000 --- a/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-markdown-content-y-text.ts +++ /dev/null @@ -1,18 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2022 The HedgeDoc developers (see AUTHORS file) - * - * SPDX-License-Identifier: AGPL-3.0-only - */ -import type { RealtimeDoc } from '@hedgedoc/commons' -import { useMemo } from 'react' -import type { Text as YText } from 'yjs' - -/** - * Extracts the y-text channel that saves the markdown content from the given yDoc. - * - * @param yDoc The yjs document from which the yText should be extracted - * @return the extracted yText channel - */ -export const useMarkdownContentYText = (yDoc: RealtimeDoc | undefined): YText | undefined => { - return useMemo(() => yDoc?.getMarkdownContentChannel(), [yDoc]) -} diff --git a/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-realtime-connection.ts b/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-realtime-connection.ts index aab476948..dacdd29c6 100644 --- a/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-realtime-connection.ts +++ b/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-realtime-connection.ts @@ -95,7 +95,7 @@ export const useRealtimeConnection = (): MessageTransporter => { }, [messageTransporter]) useEffect(() => { - const connectedListener = messageTransporter.doAsSoonAsConnected(() => setRealtimeConnectionState(true)) + const connectedListener = messageTransporter.doAsSoonAsReady(() => setRealtimeConnectionState(true)) const disconnectedListener = messageTransporter.on('disconnected', () => setRealtimeConnectionState(false), { objectify: true }) as Listener diff --git a/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-realtime-doc.ts b/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-realtime-doc.ts new file mode 100644 index 000000000..05d3638b0 --- /dev/null +++ b/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-realtime-doc.ts @@ -0,0 +1,20 @@ +/* + * SPDX-FileCopyrightText: 2022 The HedgeDoc developers (see AUTHORS file) + * + * SPDX-License-Identifier: AGPL-3.0-only + */ +import { RealtimeDoc } from '@hedgedoc/commons' +import { useEffect, useMemo } from 'react' + +/** + * Creates a new {@link RealtimeDoc y-doc}. + * + * @return The created {@link RealtimeDoc y-doc} + */ +export const useRealtimeDoc = (): RealtimeDoc => { + const doc = useMemo(() => new RealtimeDoc(), []) + + useEffect(() => () => doc.destroy(), [doc]) + + return doc +} diff --git a/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-y-doc-sync-client-adapter.ts b/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-y-doc-sync-client-adapter.ts index 60d47477f..ba7647797 100644 --- a/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-y-doc-sync-client-adapter.ts +++ b/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-y-doc-sync-client-adapter.ts @@ -5,11 +5,10 @@ */ import { setRealtimeSyncedState } from '../../../../../redux/realtime/methods' import { Logger } from '../../../../../utils/logger' -import type { MessageTransporter } from '@hedgedoc/commons' +import type { MessageTransporter, RealtimeDoc } from '@hedgedoc/commons' import { YDocSyncClientAdapter } from '@hedgedoc/commons' import type { Listener } from 'eventemitter2' import { useEffect, useMemo } from 'react' -import type { Doc } from 'yjs' const logger = new Logger('useYDocSyncClient') @@ -17,18 +16,14 @@ const logger = new Logger('useYDocSyncClient') * Creates a {@link YDocSyncClientAdapter} and mirrors its sync state to the global application state. * * @param messageTransporter The {@link MessageTransporter message transporter} that sends and receives messages for the synchronisation - * @param yDoc The {@link Doc y-doc} that should be synchronized + * @param doc The {@link RealtimeDoc realtime doc} that should be synchronized * @return the created adapter */ export const useYDocSyncClientAdapter = ( messageTransporter: MessageTransporter, - yDoc: Doc | undefined + doc: RealtimeDoc ): YDocSyncClientAdapter => { - const syncAdapter = useMemo(() => new YDocSyncClientAdapter(messageTransporter), [messageTransporter]) - - useEffect(() => { - syncAdapter.setYDoc(yDoc) - }, [syncAdapter, yDoc]) + const syncAdapter = useMemo(() => new YDocSyncClientAdapter(messageTransporter, doc), [doc, messageTransporter]) useEffect(() => { const onceSyncedListener = syncAdapter.doAsSoonAsSynced(() => { diff --git a/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-y-doc.ts b/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-y-doc.ts deleted file mode 100644 index 5501d50ce..000000000 --- a/frontend/src/components/editor-page/editor-pane/hooks/yjs/use-y-doc.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2022 The HedgeDoc developers (see AUTHORS file) - * - * SPDX-License-Identifier: AGPL-3.0-only - */ -import type { MessageTransporter } from '@hedgedoc/commons' -import { RealtimeDoc } from '@hedgedoc/commons' -import { useEffect, useState } from 'react' - -/** - * Creates a new {@link RealtimeDoc y-doc}. - * - * @return The created {@link RealtimeDoc y-doc} - */ -export const useYDoc = (messageTransporter: MessageTransporter): RealtimeDoc | undefined => { - const [yDoc, setYDoc] = useState() - - useEffect(() => { - messageTransporter.doAsSoonAsConnected(() => { - setYDoc(new RealtimeDoc()) - }) - messageTransporter.on('disconnected', () => { - setYDoc(undefined) - }) - }, [messageTransporter]) - - useEffect(() => () => yDoc?.destroy(), [yDoc]) - - return yDoc -}