refactor: save ydoc state in the database, so it can be restored easier

By storing the ydoc state in the database we can reconnect lost clients easier
and enable offline editing because we continue using the crdt data that has been
used by the client before the connection loss.

Signed-off-by: Tilman Vatteroth <git@tilmanvatteroth.de>
This commit is contained in:
Tilman Vatteroth 2023-03-24 10:26:49 +01:00
parent 4707540237
commit a826677225
26 changed files with 301 additions and 204 deletions

View file

@ -27,11 +27,7 @@ jest.mock(
({ ({
...jest.requireActual('@hedgedoc/commons'), ...jest.requireActual('@hedgedoc/commons'),
// eslint-disable-next-line @typescript-eslint/naming-convention // eslint-disable-next-line @typescript-eslint/naming-convention
YDocSyncServerAdapter: jest.fn(() => YDocSyncServerAdapter: jest.fn(() => Mock.of<YDocSyncServerAdapter>({})),
Mock.of<YDocSyncServerAdapter>({
setYDoc: jest.fn(),
}),
),
} as Record<string, unknown>), } as Record<string, unknown>),
); );
@ -86,9 +82,7 @@ describe('websocket connection', () => {
}); });
it('returns the correct sync adapter', () => { it('returns the correct sync adapter', () => {
const yDocSyncServerAdapter = Mock.of<YDocSyncServerAdapter>({ const yDocSyncServerAdapter = Mock.of<YDocSyncServerAdapter>({});
setYDoc: jest.fn(),
});
jest jest
.spyOn(HedgeDocCommonsModule, 'YDocSyncServerAdapter') .spyOn(HedgeDocCommonsModule, 'YDocSyncServerAdapter')
.mockImplementation(() => yDocSyncServerAdapter); .mockImplementation(() => yDocSyncServerAdapter);

View file

@ -41,8 +41,10 @@ export class RealtimeConnection {
this.transporter.on('disconnected', () => { this.transporter.on('disconnected', () => {
realtimeNote.removeClient(this); realtimeNote.removeClient(this);
}); });
this.yDocSyncAdapter = new YDocSyncServerAdapter(this.transporter); this.yDocSyncAdapter = new YDocSyncServerAdapter(
this.yDocSyncAdapter.setYDoc(realtimeNote.getRealtimeDoc()); this.transporter,
realtimeNote.getRealtimeDoc(),
);
this.realtimeUserStateAdapter = new RealtimeUserStatusAdapter( this.realtimeUserStateAdapter = new RealtimeUserStatusAdapter(
this.user?.username ?? null, this.user?.username ?? null,
this.getDisplayName(), this.getDisplayName(),

View file

@ -41,6 +41,23 @@ describe('RealtimeNoteStore', () => {
expect(realtimeNoteConstructorSpy).toHaveBeenCalledWith( expect(realtimeNoteConstructorSpy).toHaveBeenCalledWith(
mockedNote, mockedNote,
mockedContent, mockedContent,
undefined,
);
expect(realtimeNoteStore.find(mockedNoteId)).toBe(mockedRealtimeNote);
expect(realtimeNoteStore.getAllRealtimeNotes()).toStrictEqual([
mockedRealtimeNote,
]);
});
it("can create a new realtime note with a yjs state if it doesn't exist yet", () => {
const initialYjsState = [1, 2, 3];
expect(
realtimeNoteStore.create(mockedNote, mockedContent, initialYjsState),
).toBe(mockedRealtimeNote);
expect(realtimeNoteConstructorSpy).toHaveBeenCalledWith(
mockedNote,
mockedContent,
initialYjsState,
); );
expect(realtimeNoteStore.find(mockedNoteId)).toBe(mockedRealtimeNote); expect(realtimeNoteStore.find(mockedNoteId)).toBe(mockedRealtimeNote);
expect(realtimeNoteStore.getAllRealtimeNotes()).toStrictEqual([ expect(realtimeNoteStore.getAllRealtimeNotes()).toStrictEqual([

View file

@ -16,15 +16,24 @@ export class RealtimeNoteStore {
* Creates a new {@link RealtimeNote} for the given {@link Note} and memorizes it. * Creates a new {@link RealtimeNote} for the given {@link Note} and memorizes it.
* *
* @param note The note for which the realtime note should be created * @param note The note for which the realtime note should be created
* @param initialContent The initial content for the realtime note * @param initialTextContent the initial text content of realtime doc
* @param initialYjsState the initial yjs state. If provided this will be used instead of the text content
* @throws Error if there is already an realtime note for the given note. * @throws Error if there is already an realtime note for the given note.
* @return The created realtime note * @return The created realtime note
*/ */
public create(note: Note, initialContent: string): RealtimeNote { public create(
note: Note,
initialTextContent: string,
initialYjsState?: number[],
): RealtimeNote {
if (this.noteIdToRealtimeNote.has(note.id)) { if (this.noteIdToRealtimeNote.has(note.id)) {
throw new Error(`Realtime note for note ${note.id} already exists.`); throw new Error(`Realtime note for note ${note.id} already exists.`);
} }
const realtimeNote = new RealtimeNote(note, initialContent); const realtimeNote = new RealtimeNote(
note,
initialTextContent,
initialYjsState,
);
realtimeNote.on('destroy', () => { realtimeNote.on('destroy', () => {
this.noteIdToRealtimeNote.delete(note.id); this.noteIdToRealtimeNote.delete(note.id);
}); });

View file

@ -17,6 +17,7 @@ import { RealtimeNoteService } from './realtime-note.service';
describe('RealtimeNoteService', () => { describe('RealtimeNoteService', () => {
const mockedContent = 'mockedContent'; const mockedContent = 'mockedContent';
const mockedYjsState = [1, 2, 3];
const mockedNoteId = 4711; const mockedNoteId = 4711;
let note: Note; let note: Note;
let realtimeNote: RealtimeNote; let realtimeNote: RealtimeNote;
@ -37,7 +38,10 @@ describe('RealtimeNoteService', () => {
jest.useFakeTimers(); jest.useFakeTimers();
}); });
function mockGetLatestRevision(latestRevisionExists: boolean) { function mockGetLatestRevision(
latestRevisionExists: boolean,
hasYjsState = false,
) {
jest jest
.spyOn(revisionsService, 'getLatestRevision') .spyOn(revisionsService, 'getLatestRevision')
.mockImplementation((note: Note) => .mockImplementation((note: Note) =>
@ -45,6 +49,7 @@ describe('RealtimeNoteService', () => {
? Promise.resolve( ? Promise.resolve(
Mock.of<Revision>({ Mock.of<Revision>({
content: mockedContent, content: mockedContent,
...(hasYjsState ? { yjsStateVector: mockedYjsState } : {}),
}), }),
) )
: Promise.reject('Revision for note mockedNoteId not found.'), : Promise.reject('Revision for note mockedNoteId not found.'),
@ -106,7 +111,32 @@ describe('RealtimeNoteService', () => {
).resolves.toBe(realtimeNote); ).resolves.toBe(realtimeNote);
expect(realtimeNoteStore.find).toHaveBeenCalledWith(mockedNoteId); expect(realtimeNoteStore.find).toHaveBeenCalledWith(mockedNoteId);
expect(realtimeNoteStore.create).toHaveBeenCalledWith(note, mockedContent); expect(realtimeNoteStore.create).toHaveBeenCalledWith(
note,
mockedContent,
undefined,
);
expect(setIntervalSpy).not.toHaveBeenCalled();
});
it("creates a new realtime note with a yjs state if it doesn't exist yet", async () => {
mockGetLatestRevision(true, true);
jest.spyOn(realtimeNoteStore, 'find').mockImplementation(() => undefined);
jest
.spyOn(realtimeNoteStore, 'create')
.mockImplementation(() => realtimeNote);
mockedAppConfig.persistInterval = 0;
await expect(
realtimeNoteService.getOrCreateRealtimeNote(note),
).resolves.toBe(realtimeNote);
expect(realtimeNoteStore.find).toHaveBeenCalledWith(mockedNoteId);
expect(realtimeNoteStore.create).toHaveBeenCalledWith(
note,
mockedContent,
mockedYjsState,
);
expect(setIntervalSpy).not.toHaveBeenCalled(); expect(setIntervalSpy).not.toHaveBeenCalled();
}); });
@ -192,7 +222,11 @@ describe('RealtimeNoteService', () => {
.mockImplementation(() => Promise.resolve(Mock.of<Revision>())); .mockImplementation(() => Promise.resolve(Mock.of<Revision>()));
realtimeNote.emit('beforeDestroy'); realtimeNote.emit('beforeDestroy');
expect(createRevisionSpy).toHaveBeenCalledWith(note, mockedContent); expect(createRevisionSpy).toHaveBeenCalledWith(
note,
mockedContent,
expect.any(Array),
);
}); });
it('destroys every realtime note on application shutdown', () => { it('destroys every realtime note on application shutdown', () => {

View file

@ -43,6 +43,7 @@ export class RealtimeNoteService implements BeforeApplicationShutdown {
.createRevision( .createRevision(
realtimeNote.getNote(), realtimeNote.getNote(),
realtimeNote.getRealtimeDoc().getCurrentContent(), realtimeNote.getRealtimeDoc().getCurrentContent(),
realtimeNote.getRealtimeDoc().encodeStateAsUpdate(),
) )
.catch((reason) => this.logger.error(reason)); .catch((reason) => this.logger.error(reason));
} }
@ -68,9 +69,12 @@ export class RealtimeNoteService implements BeforeApplicationShutdown {
* @return The created realtime note * @return The created realtime note
*/ */
private async createNewRealtimeNote(note: Note): Promise<RealtimeNote> { private async createNewRealtimeNote(note: Note): Promise<RealtimeNote> {
const initialContent = (await this.revisionsService.getLatestRevision(note)) const lastRevision = await this.revisionsService.getLatestRevision(note);
.content; const realtimeNote = this.realtimeNoteStore.create(
const realtimeNote = this.realtimeNoteStore.create(note, initialContent); note,
lastRevision.content,
lastRevision.yjsStateVector ?? undefined,
);
realtimeNote.on('beforeDestroy', () => { realtimeNote.on('beforeDestroy', () => {
this.saveRealtimeNote(realtimeNote); this.saveRealtimeNote(realtimeNote);
}); });

View file

@ -4,15 +4,12 @@
* SPDX-License-Identifier: AGPL-3.0-only * SPDX-License-Identifier: AGPL-3.0-only
*/ */
import { MessageType, RealtimeDoc } from '@hedgedoc/commons'; import { MessageType, RealtimeDoc } from '@hedgedoc/commons';
import * as hedgedocCommonsModule from '@hedgedoc/commons';
import { Mock } from 'ts-mockery'; import { Mock } from 'ts-mockery';
import { Note } from '../../notes/note.entity'; import { Note } from '../../notes/note.entity';
import { RealtimeNote } from './realtime-note'; import { RealtimeNote } from './realtime-note';
import { MockConnectionBuilder } from './test-utils/mock-connection'; import { MockConnectionBuilder } from './test-utils/mock-connection';
jest.mock('@hedgedoc/commons');
describe('realtime note', () => { describe('realtime note', () => {
let mockedNote: Note; let mockedNote: Note;
@ -40,18 +37,13 @@ describe('realtime note', () => {
expect(sut.hasConnections()).toBeFalsy(); expect(sut.hasConnections()).toBeFalsy();
}); });
it('creates a y-doc', () => { it('creates a realtime doc', () => {
const initialContent = 'nothing'; const initialContent = 'nothing';
const mockedDoc = new RealtimeDoc(initialContent);
const docSpy = jest
.spyOn(hedgedocCommonsModule, 'RealtimeDoc')
.mockReturnValue(mockedDoc);
const sut = new RealtimeNote(mockedNote, initialContent); const sut = new RealtimeNote(mockedNote, initialContent);
expect(docSpy).toHaveBeenCalledWith(initialContent); expect(sut.getRealtimeDoc() instanceof RealtimeDoc).toBeTruthy();
expect(sut.getRealtimeDoc()).toBe(mockedDoc);
}); });
it('destroys y-doc on self-destruction', () => { it('destroys realtime doc on self-destruction', () => {
const sut = new RealtimeNote(mockedNote, 'nothing'); const sut = new RealtimeNote(mockedNote, 'nothing');
const docDestroy = jest.spyOn(sut.getRealtimeDoc(), 'destroy'); const docDestroy = jest.spyOn(sut.getRealtimeDoc(), 'destroy');
sut.destroy(); sut.destroy();

View file

@ -28,12 +28,17 @@ export class RealtimeNote extends EventEmitter2<RealtimeNoteEventMap> {
private readonly clients = new Set<RealtimeConnection>(); private readonly clients = new Set<RealtimeConnection>();
private isClosing = false; private isClosing = false;
constructor(private readonly note: Note, initialContent: string) { constructor(
private readonly note: Note,
initialTextContent: string,
initialYjsState?: number[],
) {
super(); super();
this.logger = new Logger(`${RealtimeNote.name} ${note.id}`); this.logger = new Logger(`${RealtimeNote.name} ${note.id}`);
this.doc = new RealtimeDoc(initialContent); this.doc = new RealtimeDoc(initialTextContent, initialYjsState);
const length = this.doc.getCurrentContent().length;
this.logger.debug( this.logger.debug(
`New realtime session for note ${note.id} created. Length of initial content: ${initialContent.length} characters`, `New realtime session for note ${note.id} created. Length of initial content: ${length} characters`,
); );
} }

View file

@ -48,6 +48,9 @@ export class Revision {
@Column() @Column()
length: number; length: number;
@Column('simple-array', { nullable: true })
yjsStateVector: null | number[];
/** /**
* Date at which the revision was created. * Date at which the revision was created.
*/ */
@ -74,6 +77,7 @@ export class Revision {
content: string, content: string,
patch: string, patch: string,
note: Note, note: Note,
yjsStateVector?: number[],
): Omit<Revision, 'id' | 'createdAt'> { ): Omit<Revision, 'id' | 'createdAt'> {
const newRevision = new Revision(); const newRevision = new Revision();
newRevision.patch = patch; newRevision.patch = patch;
@ -81,6 +85,7 @@ export class Revision {
newRevision.length = content.length; newRevision.length = content.length;
newRevision.note = Promise.resolve(note); newRevision.note = Promise.resolve(note);
newRevision.edits = Promise.resolve([]); newRevision.edits = Promise.resolve([]);
newRevision.yjsStateVector = yjsStateVector ?? null;
return newRevision; return newRevision;
} }
} }

View file

@ -145,6 +145,7 @@ export class RevisionsService {
async createRevision( async createRevision(
note: Note, note: Note,
newContent: string, newContent: string,
yjsStateVector?: number[],
): Promise<Revision | undefined> { ): Promise<Revision | undefined> {
// TODO: Save metadata // TODO: Save metadata
const latestRevision = await this.getLatestRevision(note); const latestRevision = await this.getLatestRevision(note);
@ -157,7 +158,7 @@ export class RevisionsService {
latestRevision.content, latestRevision.content,
newContent, newContent,
); );
const revision = Revision.create(newContent, patch, note); const revision = Revision.create(newContent, patch, note, yjsStateVector);
return await this.revisionRepository.save(revision); return await this.revisionRepository.save(revision);
} }
} }

View file

@ -6,14 +6,13 @@
import { RealtimeDoc } from '../y-doc-sync/realtime-doc.js' import { RealtimeDoc } from '../y-doc-sync/realtime-doc.js'
import { ConnectionState, MessageTransporter } from './message-transporter.js' import { ConnectionState, MessageTransporter } from './message-transporter.js'
import { Message, MessageType } from './message.js' import { Message, MessageType } from './message.js'
import { Doc, encodeStateAsUpdate } from 'yjs'
/** /**
* A mocked connection that doesn't send or receive any data and is instantly ready. * A mocked connection that doesn't send or receive any data and is instantly ready.
* The only exception is the note content state request that is answered with the given initial content. * The only exception is the note content state request that is answered with the given initial content.
*/ */
export class MockedBackendMessageTransporter extends MessageTransporter { export class MockedBackendMessageTransporter extends MessageTransporter {
private readonly doc: Doc private readonly doc: RealtimeDoc
private connected = true private connected = true
@ -41,10 +40,10 @@ export class MockedBackendMessageTransporter extends MessageTransporter {
sendMessage<M extends MessageType>(content: Message<M>) { sendMessage<M extends MessageType>(content: Message<M>) {
if (content.type === MessageType.NOTE_CONTENT_STATE_REQUEST) { if (content.type === MessageType.NOTE_CONTENT_STATE_REQUEST) {
setTimeout(() => { setTimeout(() => {
const payload = Array.from( this.receiveMessage({
encodeStateAsUpdate(this.doc, new Uint8Array(content.payload)) type: MessageType.NOTE_CONTENT_UPDATE,
) payload: this.doc.encodeStateAsUpdate(content.payload)
this.receiveMessage({ type: MessageType.NOTE_CONTENT_UPDATE, payload }) })
}, 10) }, 10)
} }
} }

View file

@ -6,11 +6,31 @@
import { RealtimeDoc } from './realtime-doc.js' import { RealtimeDoc } from './realtime-doc.js'
import { describe, expect, it } from '@jest/globals' import { describe, expect, it } from '@jest/globals'
describe('websocket-doc', () => { describe('realtime doc', () => {
it('saves the initial content', () => { it('saves an initial text content correctly', () => {
const textContent = 'textContent' const textContent = 'textContent'
const websocketDoc = new RealtimeDoc(textContent) const realtimeDoc = new RealtimeDoc(textContent)
expect(realtimeDoc.getCurrentContent()).toBe(textContent)
})
expect(websocketDoc.getCurrentContent()).toBe(textContent) it('will initialize an empty text if no initial content is given', () => {
const realtimeDoc = new RealtimeDoc()
expect(realtimeDoc.getCurrentContent()).toBe('')
})
it('restores a yjs state vector update correctly', () => {
const realtimeDoc = new RealtimeDoc(
'notTheVectorText',
[
1, 1, 221, 208, 165, 230, 3, 0, 4, 1, 15, 109, 97, 114, 107, 100, 111,
119, 110, 67, 111, 110, 116, 101, 110, 116, 32, 116, 101, 120, 116, 67,
111, 110, 116, 101, 110, 116, 70, 114, 111, 109, 83, 116, 97, 116, 101,
86, 101, 99, 116, 111, 114, 85, 112, 100, 97, 116, 101, 0
]
)
expect(realtimeDoc.getCurrentContent()).toBe(
'textContentFromStateVectorUpdate'
)
}) })
}) })

View file

@ -3,27 +3,52 @@
* *
* SPDX-License-Identifier: AGPL-3.0-only * SPDX-License-Identifier: AGPL-3.0-only
*/ */
import { Doc } from 'yjs' import { EventEmitter2 } from 'eventemitter2'
import { Text as YText } from 'yjs' import type { EventMap } from 'eventemitter2'
import {
applyUpdate,
Doc,
encodeStateAsUpdate,
encodeStateVector,
Text as YText
} from 'yjs'
const MARKDOWN_CONTENT_CHANNEL_NAME = 'markdownContent' const MARKDOWN_CONTENT_CHANNEL_NAME = 'markdownContent'
export interface RealtimeDocEvents extends EventMap {
update: (update: number[], origin: unknown) => void
}
/** /**
* This is the implementation of {@link Doc YDoc} which includes additional handlers for message sending and receiving. * This is the implementation of {@link Doc YDoc} which includes additional handlers for message sending and receiving.
*/ */
export class RealtimeDoc extends Doc { export class RealtimeDoc extends EventEmitter2<RealtimeDocEvents> {
private doc: Doc = new Doc()
private readonly docUpdateListener: (
update: Uint8Array,
origin: unknown
) => void
/** /**
* Creates a new instance. * Creates a new instance.
* *
* The new instance is filled with the given initial content. * The new instance is filled with the given initial content.
* *
* @param initialContent - the initial content of the {@link Doc YDoc} * @param initialTextContent the initial text content of the {@link Doc YDoc}
* @param initialYjsState the initial yjs state. If provided this will be used instead of the text content
*/ */
constructor(initialContent?: string) { constructor(initialTextContent?: string, initialYjsState?: number[]) {
super() super()
if (initialContent) { if (initialYjsState) {
this.getMarkdownContentChannel().insert(0, initialContent) this.applyUpdate(initialYjsState, this)
} else if (initialTextContent) {
this.getMarkdownContentChannel().insert(0, initialTextContent)
} }
this.docUpdateListener = (update, origin) => {
this.emit('update', Array.from(update), origin)
}
this.doc.on('update', this.docUpdateListener)
} }
/** /**
@ -32,7 +57,7 @@ export class RealtimeDoc extends Doc {
* @return The markdown channel * @return The markdown channel
*/ */
public getMarkdownContentChannel(): YText { public getMarkdownContentChannel(): YText {
return this.getText(MARKDOWN_CONTENT_CHANNEL_NAME) return this.doc.getText(MARKDOWN_CONTENT_CHANNEL_NAME)
} }
/** /**
@ -45,4 +70,35 @@ export class RealtimeDoc extends Doc {
public getCurrentContent(): string { public getCurrentContent(): string {
return this.getMarkdownContentChannel().toString() return this.getMarkdownContentChannel().toString()
} }
/**
* Encodes the current state of the doc as update so it can be applied to other y-docs.
*
* @param encodedTargetStateVector The current state vector of the other y-doc. If provided the update will contain only the differences.
*/
public encodeStateAsUpdate(encodedTargetStateVector?: number[]): number[] {
const update = encodedTargetStateVector
? new Uint8Array(encodedTargetStateVector)
: undefined
return Array.from(encodeStateAsUpdate(this.doc, update))
}
public destroy(): void {
this.doc.off('update', this.docUpdateListener)
this.doc.destroy()
}
/**
* Applies an update to the y-doc.
*
* @param payload The update to apply
* @param origin A reference that triggered the update
*/
public applyUpdate(payload: number[], origin: unknown): void {
applyUpdate(this.doc, new Uint8Array(payload), origin)
}
public encodeStateVector(): number[] {
return Array.from(encodeStateVector(this.doc))
}
} }

View file

@ -68,10 +68,10 @@ describe('message transporter', () => {
console.debug('s>2 is connected') console.debug('s>2 is connected')
) )
docServer.on('update', (update: Uint8Array, origin: unknown) => { docServer.on('update', (update: number[], origin: unknown) => {
const message: Message<MessageType.NOTE_CONTENT_UPDATE> = { const message: Message<MessageType.NOTE_CONTENT_UPDATE> = {
type: MessageType.NOTE_CONTENT_UPDATE, type: MessageType.NOTE_CONTENT_UPDATE,
payload: Array.from(update) payload: update
} }
if (origin !== transporterServerTo1) { if (origin !== transporterServerTo1) {
console.debug('YDoc on Server updated. Sending to Client 1') console.debug('YDoc on Server updated. Sending to Client 1')
@ -82,32 +82,35 @@ describe('message transporter', () => {
transporterServerTo2.sendMessage(message) transporterServerTo2.sendMessage(message)
} }
}) })
docClient1.on('update', (update: Uint8Array, origin: unknown) => { docClient1.on('update', (update: number[], origin: unknown) => {
if (origin !== transporterClient1) { if (origin !== transporterClient1) {
console.debug('YDoc on client 1 updated. Sending to Server') console.debug('YDoc on client 1 updated. Sending to Server')
} }
}) })
docClient2.on('update', (update: Uint8Array, origin: unknown) => { docClient2.on('update', (update: number[], origin: unknown) => {
if (origin !== transporterClient2) { if (origin !== transporterClient2) {
console.debug('YDoc on client 2 updated. Sending to Server') console.debug('YDoc on client 2 updated. Sending to Server')
} }
}) })
const yDocSyncAdapter1 = new YDocSyncClientAdapter(transporterClient1) const yDocSyncAdapter1 = new YDocSyncClientAdapter(
yDocSyncAdapter1.setYDoc(docClient1) transporterClient1,
docClient1
const yDocSyncAdapter2 = new YDocSyncClientAdapter(transporterClient2) )
yDocSyncAdapter2.setYDoc(docClient2) const yDocSyncAdapter2 = new YDocSyncClientAdapter(
transporterClient2,
docClient2
)
const yDocSyncAdapterServerTo1 = new YDocSyncServerAdapter( const yDocSyncAdapterServerTo1 = new YDocSyncServerAdapter(
transporterServerTo1 transporterServerTo1,
docServer
) )
yDocSyncAdapterServerTo1.setYDoc(docServer)
const yDocSyncAdapterServerTo2 = new YDocSyncServerAdapter( const yDocSyncAdapterServerTo2 = new YDocSyncServerAdapter(
transporterServerTo2 transporterServerTo2,
docServer
) )
yDocSyncAdapterServerTo2.setYDoc(docServer)
const waitForClient1Sync = new Promise<void>((resolve) => { const waitForClient1Sync = new Promise<void>((resolve) => {
yDocSyncAdapter1.doAsSoonAsSynced(() => { yDocSyncAdapter1.doAsSoonAsSynced(() => {

View file

@ -5,26 +5,37 @@
*/ */
import { MessageTransporter } from '../message-transporters/message-transporter.js' import { MessageTransporter } from '../message-transporters/message-transporter.js'
import { Message, MessageType } from '../message-transporters/message.js' import { Message, MessageType } from '../message-transporters/message.js'
import { RealtimeDoc } from './realtime-doc.js'
import { Listener } from 'eventemitter2' import { Listener } from 'eventemitter2'
import { EventEmitter2 } from 'eventemitter2' import { EventEmitter2 } from 'eventemitter2'
import { applyUpdate, Doc, encodeStateAsUpdate, encodeStateVector } from 'yjs'
type EventMap = Record<'synced' | 'desynced', () => void> type EventMap = Record<'synced' | 'desynced', () => void>
/** /**
* Sends and processes messages that are used to first-synchronize and update a {@link Doc y-doc}. * Sends and processes messages that are used to first-synchronize and update a {@link RealtimeDoc y-doc}.
*/ */
export abstract class YDocSyncAdapter { export abstract class YDocSyncAdapter {
public readonly eventEmitter = new EventEmitter2<EventMap>() public readonly eventEmitter = new EventEmitter2<EventMap>()
protected doc: Doc | undefined private readonly yDocUpdateListener: Listener
private readonly destroyEventListenerCallback: undefined | (() => void)
private destroyYDocUpdateCallback: undefined | (() => void)
private destroyEventListenerCallback: undefined | (() => void)
private synced = false private synced = false
constructor(protected readonly messageTransporter: MessageTransporter) { constructor(
this.bindDocumentSyncMessageEvents() protected readonly messageTransporter: MessageTransporter,
protected readonly doc: RealtimeDoc
) {
this.yDocUpdateListener = doc.on(
'update',
(update, origin) => {
this.processDocUpdate(update, origin)
},
{
objectify: true
}
) as Listener
this.destroyEventListenerCallback = this.bindDocumentSyncMessageEvents()
} }
/** /**
@ -51,39 +62,19 @@ export abstract class YDocSyncAdapter {
return this.synced return this.synced
} }
/**
* Sets the {@link Doc y-doc} that should be synchronized.
*
* @param doc the doc to synchronize.
*/
public setYDoc(doc: Doc | undefined): void {
this.doc = doc
this.destroyYDocUpdateCallback?.()
if (!doc) {
return
}
const yDocUpdateCallback = this.processDocUpdate.bind(this)
doc.on('update', yDocUpdateCallback)
this.destroyYDocUpdateCallback = () => doc.off('update', yDocUpdateCallback)
this.eventEmitter.emit('desynced')
}
public destroy(): void { public destroy(): void {
this.destroyYDocUpdateCallback?.() this.yDocUpdateListener.off()
this.destroyEventListenerCallback?.() this.destroyEventListenerCallback?.()
} }
protected bindDocumentSyncMessageEvents(): void { protected bindDocumentSyncMessageEvents(): () => void {
const stateRequestListener = this.messageTransporter.on( const stateRequestListener = this.messageTransporter.on(
MessageType.NOTE_CONTENT_STATE_REQUEST, MessageType.NOTE_CONTENT_STATE_REQUEST,
(payload) => { (payload) => {
if (this.doc) { if (this.doc) {
this.messageTransporter.sendMessage({ this.messageTransporter.sendMessage({
type: MessageType.NOTE_CONTENT_UPDATE, type: MessageType.NOTE_CONTENT_UPDATE,
payload: Array.from( payload: this.doc.encodeStateAsUpdate(payload.payload)
encodeStateAsUpdate(this.doc, new Uint8Array(payload.payload))
)
}) })
} }
}, },
@ -95,35 +86,30 @@ export abstract class YDocSyncAdapter {
() => { () => {
this.synced = false this.synced = false
this.eventEmitter.emit('desynced') this.eventEmitter.emit('desynced')
this.destroy()
}, },
{ objectify: true } { objectify: true }
) as Listener ) as Listener
const noteContentUpdateListener = this.messageTransporter.on( const noteContentUpdateListener = this.messageTransporter.on(
MessageType.NOTE_CONTENT_UPDATE, MessageType.NOTE_CONTENT_UPDATE,
(payload) => { (payload) => this.doc.applyUpdate(payload.payload, this),
if (this.doc) {
applyUpdate(this.doc, new Uint8Array(payload.payload), this)
}
},
{ objectify: true } { objectify: true }
) as Listener ) as Listener
this.destroyEventListenerCallback = () => { return () => {
stateRequestListener.off() stateRequestListener.off()
disconnectedListener.off() disconnectedListener.off()
noteContentUpdateListener.off() noteContentUpdateListener.off()
} }
} }
private processDocUpdate(update: Uint8Array, origin: unknown): void { private processDocUpdate(update: number[], origin: unknown): void {
if (!this.isSynced() || origin === this) { if (!this.isSynced() || origin === this) {
return return
} }
const message: Message<MessageType.NOTE_CONTENT_UPDATE> = { const message: Message<MessageType.NOTE_CONTENT_UPDATE> = {
type: MessageType.NOTE_CONTENT_UPDATE, type: MessageType.NOTE_CONTENT_UPDATE,
payload: Array.from(update) payload: update
} }
this.messageTransporter.sendMessage(message) this.messageTransporter.sendMessage(message)
@ -141,7 +127,7 @@ export abstract class YDocSyncAdapter {
if (this.doc) { if (this.doc) {
this.messageTransporter.sendMessage({ this.messageTransporter.sendMessage({
type: MessageType.NOTE_CONTENT_STATE_REQUEST, type: MessageType.NOTE_CONTENT_STATE_REQUEST,
payload: Array.from(encodeStateVector(this.doc)) payload: this.doc.encodeStateVector()
}) })
} }
} }

View file

@ -5,13 +5,23 @@
*/ */
import { MessageType } from '../message-transporters/message.js' import { MessageType } from '../message-transporters/message.js'
import { YDocSyncAdapter } from './y-doc-sync-adapter.js' import { YDocSyncAdapter } from './y-doc-sync-adapter.js'
import { Listener } from 'eventemitter2'
export class YDocSyncClientAdapter extends YDocSyncAdapter { export class YDocSyncClientAdapter extends YDocSyncAdapter {
protected bindDocumentSyncMessageEvents() { protected bindDocumentSyncMessageEvents() {
super.bindDocumentSyncMessageEvents() const destroyCallback = super.bindDocumentSyncMessageEvents()
this.messageTransporter.on(MessageType.NOTE_CONTENT_UPDATE, () => { const noteContentUpdateListener = this.messageTransporter.on(
MessageType.NOTE_CONTENT_UPDATE,
() => {
this.markAsSynced() this.markAsSynced()
}) },
{ objectify: true }
) as Listener
return () => {
destroyCallback()
noteContentUpdateListener.off()
}
} }
} }

View file

@ -4,11 +4,15 @@
* SPDX-License-Identifier: AGPL-3.0-only * SPDX-License-Identifier: AGPL-3.0-only
*/ */
import { MessageTransporter } from '../message-transporters/message-transporter.js' import { MessageTransporter } from '../message-transporters/message-transporter.js'
import { RealtimeDoc } from './realtime-doc.js'
import { YDocSyncAdapter } from './y-doc-sync-adapter.js' import { YDocSyncAdapter } from './y-doc-sync-adapter.js'
export class YDocSyncServerAdapter extends YDocSyncAdapter { export class YDocSyncServerAdapter extends YDocSyncAdapter {
constructor(readonly messageTransporter: MessageTransporter) { constructor(
super(messageTransporter) readonly messageTransporter: MessageTransporter,
readonly doc: RealtimeDoc
) {
super(messageTransporter, doc)
this.markAsSynced() this.markAsSynced()
} }
} }

View file

@ -17,7 +17,6 @@ const syncAnnotation = Annotation.define()
*/ */
export class YTextSyncViewPlugin implements PluginValue { export class YTextSyncViewPlugin implements PluginValue {
private readonly observer: YTextSyncViewPlugin['onYTextUpdate'] private readonly observer: YTextSyncViewPlugin['onYTextUpdate']
private firstUpdate = true
constructor(private view: EditorView, private readonly yText: YText, pluginLoaded: () => void) { constructor(private view: EditorView, private readonly yText: YText, pluginLoaded: () => void) {
this.observer = this.onYTextUpdate.bind(this) this.observer = this.onYTextUpdate.bind(this)
@ -47,17 +46,8 @@ export class YTextSyncViewPlugin implements PluginValue {
}, },
[[], 0] as [ChangeSpec[], number] [[], 0] as [ChangeSpec[], number]
) )
return this.addDeleteAllChanges(changes)
}
private addDeleteAllChanges(changes: ChangeSpec[]): ChangeSpec[] {
if (this.firstUpdate) {
this.firstUpdate = false
return [{ from: 0, to: this.view.state.doc.length, insert: '' }, ...changes]
} else {
return changes return changes
} }
}
public update(update: ViewUpdate): void { public update(update: ViewUpdate): void {
if (!update.docChanged) { if (!update.docChanged) {

View file

@ -21,12 +21,11 @@ import { useCursorActivityCallback } from './hooks/use-cursor-activity-callback'
import { useUpdateCodeMirrorReference } from './hooks/use-update-code-mirror-reference' import { useUpdateCodeMirrorReference } from './hooks/use-update-code-mirror-reference'
import { useBindYTextToRedux } from './hooks/yjs/use-bind-y-text-to-redux' import { useBindYTextToRedux } from './hooks/yjs/use-bind-y-text-to-redux'
import { useCodeMirrorYjsExtension } from './hooks/yjs/use-code-mirror-yjs-extension' import { useCodeMirrorYjsExtension } from './hooks/yjs/use-code-mirror-yjs-extension'
import { useMarkdownContentYText } from './hooks/yjs/use-markdown-content-y-text'
import { useOnMetadataUpdated } from './hooks/yjs/use-on-metadata-updated' import { useOnMetadataUpdated } from './hooks/yjs/use-on-metadata-updated'
import { useOnNoteDeleted } from './hooks/yjs/use-on-note-deleted' import { useOnNoteDeleted } from './hooks/yjs/use-on-note-deleted'
import { useRealtimeConnection } from './hooks/yjs/use-realtime-connection' import { useRealtimeConnection } from './hooks/yjs/use-realtime-connection'
import { useRealtimeDoc } from './hooks/yjs/use-realtime-doc'
import { useReceiveRealtimeUsers } from './hooks/yjs/use-receive-realtime-users' import { useReceiveRealtimeUsers } from './hooks/yjs/use-receive-realtime-users'
import { useYDoc } from './hooks/yjs/use-y-doc'
import { useYDocSyncClientAdapter } from './hooks/yjs/use-y-doc-sync-client-adapter' import { useYDocSyncClientAdapter } from './hooks/yjs/use-y-doc-sync-client-adapter'
import { useLinter } from './linter/linter' import { useLinter } from './linter/linter'
import { MaxLengthWarning } from './max-length-warning/max-length-warning' import { MaxLengthWarning } from './max-length-warning/max-length-warning'
@ -57,8 +56,7 @@ export const EditorPane: React.FC<EditorPaneProps> = ({ scrollState, onScroll, o
useApplyScrollState(scrollState) useApplyScrollState(scrollState)
const messageTransporter = useRealtimeConnection() const messageTransporter = useRealtimeConnection()
const yDoc = useYDoc(messageTransporter) const realtimeDoc = useRealtimeDoc()
const yText = useMarkdownContentYText(yDoc)
const editorScrollExtension = useCodeMirrorScrollWatchExtension(onScroll) const editorScrollExtension = useCodeMirrorScrollWatchExtension(onScroll)
const tablePasteExtensions = useCodeMirrorTablePasteExtension() const tablePasteExtensions = useCodeMirrorTablePasteExtension()
const fileInsertExtension = useCodeMirrorFileInsertExtension() const fileInsertExtension = useCodeMirrorFileInsertExtension()
@ -70,13 +68,13 @@ export const EditorPane: React.FC<EditorPaneProps> = ({ scrollState, onScroll, o
const linterExtension = useLinter() const linterExtension = useLinter()
const syncAdapter = useYDocSyncClientAdapter(messageTransporter, yDoc) const syncAdapter = useYDocSyncClientAdapter(messageTransporter, realtimeDoc)
const yjsExtension = useCodeMirrorYjsExtension(yText, syncAdapter) const yjsExtension = useCodeMirrorYjsExtension(realtimeDoc, syncAdapter)
useOnMetadataUpdated(messageTransporter) useOnMetadataUpdated(messageTransporter)
useOnNoteDeleted(messageTransporter) useOnNoteDeleted(messageTransporter)
useBindYTextToRedux(yText) useBindYTextToRedux(realtimeDoc)
useReceiveRealtimeUsers(messageTransporter) useReceiveRealtimeUsers(messageTransporter)
const extensions = useMemo( const extensions = useMemo(

View file

@ -4,21 +4,19 @@
* SPDX-License-Identifier: AGPL-3.0-only * SPDX-License-Identifier: AGPL-3.0-only
*/ */
import { setNoteContent } from '../../../../../redux/note-details/methods' import { setNoteContent } from '../../../../../redux/note-details/methods'
import type { RealtimeDoc } from '@hedgedoc/commons'
import { useEffect } from 'react' import { useEffect } from 'react'
import type { YText } from 'yjs/dist/src/types/YText'
/** /**
* One-Way-synchronizes the text of the given {@link YText y-text} into the global application state. * One-Way-synchronizes the text of the markdown content channel from the given {@link RealtimeDoc realtime doc} into the global application state.
* *
* @param yText The source text * @param realtimeDoc The {@link RealtimeDoc realtime doc} that contains the markdown content
*/ */
export const useBindYTextToRedux = (yText: YText | undefined): void => { export const useBindYTextToRedux = (realtimeDoc: RealtimeDoc): void => {
useEffect(() => { useEffect(() => {
if (!yText) { const yText = realtimeDoc.getMarkdownContentChannel()
return
}
const yTextCallback = () => setNoteContent(yText.toString()) const yTextCallback = () => setNoteContent(yText.toString())
yText.observe(yTextCallback) yText.observe(yTextCallback)
return () => yText.unobserve(yTextCallback) return () => yText.unobserve(yTextCallback)
}, [yText]) }, [realtimeDoc])
} }

View file

@ -8,30 +8,33 @@ import { YTextSyncViewPlugin } from '../../codemirror-extensions/document-sync/y
import type { Extension } from '@codemirror/state' import type { Extension } from '@codemirror/state'
import { ViewPlugin } from '@codemirror/view' import { ViewPlugin } from '@codemirror/view'
import type { YDocSyncClientAdapter } from '@hedgedoc/commons' import type { YDocSyncClientAdapter } from '@hedgedoc/commons'
import type { RealtimeDoc } from '@hedgedoc/commons'
import { useEffect, useMemo, useState } from 'react' import { useEffect, useMemo, useState } from 'react'
import type { Text as YText } from 'yjs'
/** /**
* Creates a {@link Extension code mirror extension} that synchronizes an editor with the given {@link YText ytext}. * Creates a {@link Extension code mirror extension} that synchronizes an editor with the given {@link YText ytext}.
* *
* @param yText The source and target for the editor content * @param doc The {@link RealtimeDoc realtime doc} that contains the markdown content text channel
* @param syncAdapter The sync adapter that processes the communication for content synchronisation. * @param syncAdapter The sync adapter that processes the communication for content synchronisation.
* @return the created extension * @return the created extension
*/ */
export const useCodeMirrorYjsExtension = (yText: YText | undefined, syncAdapter: YDocSyncClientAdapter): Extension => { export const useCodeMirrorYjsExtension = (doc: RealtimeDoc, syncAdapter: YDocSyncClientAdapter): Extension => {
const [editorReady, setEditorReady] = useState(false) const [editorReady, setEditorReady] = useState(false)
const synchronized = useApplicationState((state) => state.realtimeStatus.isSynced) const synchronized = useApplicationState((state) => state.realtimeStatus.isSynced)
const connected = useApplicationState((state) => state.realtimeStatus.isConnected) const connected = useApplicationState((state) => state.realtimeStatus.isConnected)
useEffect(() => { useEffect(() => {
if (editorReady && connected && !synchronized && yText) { if (editorReady && connected && !synchronized) {
syncAdapter.requestDocumentState() syncAdapter.requestDocumentState()
} }
}, [connected, editorReady, syncAdapter, synchronized, yText]) }, [connected, editorReady, syncAdapter, synchronized])
return useMemo( return useMemo(
() => () => [
yText ? [ViewPlugin.define((view) => new YTextSyncViewPlugin(view, yText, () => setEditorReady(true)))] : [], ViewPlugin.define(
[yText] (view) => new YTextSyncViewPlugin(view, doc.getMarkdownContentChannel(), () => setEditorReady(true))
)
],
[doc]
) )
} }

View file

@ -1,18 +0,0 @@
/*
* SPDX-FileCopyrightText: 2022 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
import type { RealtimeDoc } from '@hedgedoc/commons'
import { useMemo } from 'react'
import type { Text as YText } from 'yjs'
/**
* Extracts the y-text channel that saves the markdown content from the given yDoc.
*
* @param yDoc The yjs document from which the yText should be extracted
* @return the extracted yText channel
*/
export const useMarkdownContentYText = (yDoc: RealtimeDoc | undefined): YText | undefined => {
return useMemo(() => yDoc?.getMarkdownContentChannel(), [yDoc])
}

View file

@ -95,7 +95,7 @@ export const useRealtimeConnection = (): MessageTransporter => {
}, [messageTransporter]) }, [messageTransporter])
useEffect(() => { useEffect(() => {
const connectedListener = messageTransporter.doAsSoonAsConnected(() => setRealtimeConnectionState(true)) const connectedListener = messageTransporter.doAsSoonAsReady(() => setRealtimeConnectionState(true))
const disconnectedListener = messageTransporter.on('disconnected', () => setRealtimeConnectionState(false), { const disconnectedListener = messageTransporter.on('disconnected', () => setRealtimeConnectionState(false), {
objectify: true objectify: true
}) as Listener }) as Listener

View file

@ -0,0 +1,20 @@
/*
* SPDX-FileCopyrightText: 2022 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { RealtimeDoc } from '@hedgedoc/commons'
import { useEffect, useMemo } from 'react'
/**
* Creates a new {@link RealtimeDoc y-doc}.
*
* @return The created {@link RealtimeDoc y-doc}
*/
export const useRealtimeDoc = (): RealtimeDoc => {
const doc = useMemo(() => new RealtimeDoc(), [])
useEffect(() => () => doc.destroy(), [doc])
return doc
}

View file

@ -5,11 +5,10 @@
*/ */
import { setRealtimeSyncedState } from '../../../../../redux/realtime/methods' import { setRealtimeSyncedState } from '../../../../../redux/realtime/methods'
import { Logger } from '../../../../../utils/logger' import { Logger } from '../../../../../utils/logger'
import type { MessageTransporter } from '@hedgedoc/commons' import type { MessageTransporter, RealtimeDoc } from '@hedgedoc/commons'
import { YDocSyncClientAdapter } from '@hedgedoc/commons' import { YDocSyncClientAdapter } from '@hedgedoc/commons'
import type { Listener } from 'eventemitter2' import type { Listener } from 'eventemitter2'
import { useEffect, useMemo } from 'react' import { useEffect, useMemo } from 'react'
import type { Doc } from 'yjs'
const logger = new Logger('useYDocSyncClient') const logger = new Logger('useYDocSyncClient')
@ -17,18 +16,14 @@ const logger = new Logger('useYDocSyncClient')
* Creates a {@link YDocSyncClientAdapter} and mirrors its sync state to the global application state. * Creates a {@link YDocSyncClientAdapter} and mirrors its sync state to the global application state.
* *
* @param messageTransporter The {@link MessageTransporter message transporter} that sends and receives messages for the synchronisation * @param messageTransporter The {@link MessageTransporter message transporter} that sends and receives messages for the synchronisation
* @param yDoc The {@link Doc y-doc} that should be synchronized * @param doc The {@link RealtimeDoc realtime doc} that should be synchronized
* @return the created adapter * @return the created adapter
*/ */
export const useYDocSyncClientAdapter = ( export const useYDocSyncClientAdapter = (
messageTransporter: MessageTransporter, messageTransporter: MessageTransporter,
yDoc: Doc | undefined doc: RealtimeDoc
): YDocSyncClientAdapter => { ): YDocSyncClientAdapter => {
const syncAdapter = useMemo(() => new YDocSyncClientAdapter(messageTransporter), [messageTransporter]) const syncAdapter = useMemo(() => new YDocSyncClientAdapter(messageTransporter, doc), [doc, messageTransporter])
useEffect(() => {
syncAdapter.setYDoc(yDoc)
}, [syncAdapter, yDoc])
useEffect(() => { useEffect(() => {
const onceSyncedListener = syncAdapter.doAsSoonAsSynced(() => { const onceSyncedListener = syncAdapter.doAsSoonAsSynced(() => {

View file

@ -1,30 +0,0 @@
/*
* SPDX-FileCopyrightText: 2022 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
import type { MessageTransporter } from '@hedgedoc/commons'
import { RealtimeDoc } from '@hedgedoc/commons'
import { useEffect, useState } from 'react'
/**
* Creates a new {@link RealtimeDoc y-doc}.
*
* @return The created {@link RealtimeDoc y-doc}
*/
export const useYDoc = (messageTransporter: MessageTransporter): RealtimeDoc | undefined => {
const [yDoc, setYDoc] = useState<RealtimeDoc>()
useEffect(() => {
messageTransporter.doAsSoonAsConnected(() => {
setYDoc(new RealtimeDoc())
})
messageTransporter.on('disconnected', () => {
setYDoc(undefined)
})
}, [messageTransporter])
useEffect(() => () => yDoc?.destroy(), [yDoc])
return yDoc
}