feat(backend revision): add clean-up note revisions job (#5349)

backend/src/config/mock/note.config.mock.ts

@@ -21,6 +21,7 @@ export function createDefaultMockNoteConfig(): NoteConfig {
       },
     },
     guestAccess: GuestAccess.CREATE,
+    revisionRetentionDays: 0,
   };
 }
 

backend/src/config/note.config.spec.ts

@@ -19,6 +19,7 @@ describe('noteConfig', () => {
   const invalidMaxDocumentLength = 'not-a-max-document-length';
   const guestAccess = GuestAccess.CREATE;
   const wrongDefaultPermission = 'wrong';
+  const retentionDays = 30;
 
   describe('correctly parses config', () => {
     it('when given correct and complete environment variables', () => {
@@ -30,6 +31,7 @@ describe('noteConfig', () => {
           HD_PERMISSION_DEFAULT_EVERYONE: DefaultAccessLevel.READ,
           HD_PERMISSION_DEFAULT_LOGGED_IN: DefaultAccessLevel.READ,
           HD_GUEST_ACCESS: guestAccess,
+          HD_REVISION_RETENTION_DAYS: retentionDays.toString(),
           /* eslint-enable @typescript-eslint/naming-convention */
         },
         {
@@ -47,6 +49,7 @@ describe('noteConfig', () => {
         DefaultAccessLevel.READ,
       );
       expect(config.guestAccess).toEqual(guestAccess);
+      expect(config.revisionRetentionDays).toEqual(retentionDays);
       restore();
     });
 
@@ -221,6 +224,36 @@ describe('noteConfig', () => {
       expect(config.guestAccess).toEqual(GuestAccess.WRITE);
       restore();
     });
+
+    it('when no HD_REVISION_RETENTION_DAYS is set', () => {
+      const restore = mockedEnv(
+        {
+          /* eslint-disable @typescript-eslint/naming-convention */
+          HD_FORBIDDEN_NOTE_IDS: forbiddenNoteIds.join(' , '),
+          HD_MAX_DOCUMENT_LENGTH: maxDocumentLength.toString(),
+          HD_PERMISSION_DEFAULT_EVERYONE: DefaultAccessLevel.READ,
+          HD_PERMISSION_DEFAULT_LOGGED_IN: DefaultAccessLevel.READ,
+          HD_GUEST_ACCESS: guestAccess,
+          /* eslint-enable @typescript-eslint/naming-convention */
+        },
+        {
+          clear: true,
+        },
+      );
+      const config = noteConfig();
+      expect(config.forbiddenNoteIds).toHaveLength(forbiddenNoteIds.length);
+      expect(config.forbiddenNoteIds).toEqual(forbiddenNoteIds);
+      expect(config.maxDocumentLength).toEqual(maxDocumentLength);
+      expect(config.permissions.default.everyone).toEqual(
+        DefaultAccessLevel.READ,
+      );
+      expect(config.permissions.default.loggedIn).toEqual(
+        DefaultAccessLevel.READ,
+      );
+      expect(config.guestAccess).toEqual(guestAccess);
+      expect(config.revisionRetentionDays).toEqual(0);
+      restore();
+    });
   });
 
   describe('throws error', () => {
@@ -454,5 +487,27 @@ describe('noteConfig', () => {
       );
       restore();
     });
+
+    it('when given a negative retention days', async () => {
+      const restore = mockedEnv(
+        {
+          /* eslint-disable @typescript-eslint/naming-convention */
+          HD_FORBIDDEN_NOTE_IDS: forbiddenNoteIds.join(' , '),
+          HD_MAX_DOCUMENT_LENGTH: maxDocumentLength.toString(),
+          HD_PERMISSION_DEFAULT_EVERYONE: DefaultAccessLevel.READ,
+          HD_PERMISSION_DEFAULT_LOGGED_IN: DefaultAccessLevel.READ,
+          HD_GUEST_ACCESS: guestAccess,
+          HD_REVISION_RETENTION_DAYS: (-1).toString(),
+          /* eslint-enable @typescript-eslint/naming-convention */
+        },
+        {
+          clear: true,
+        },
+      );
+      expect(() => noteConfig()).toThrow(
+        '"HD_REVISION_RETENTION_DAYS" must be greater than or equal to 0',
+      );
+      restore();
+    });
   });
 });

backend/src/config/note.config.ts

@@ -23,6 +23,7 @@ export interface NoteConfig {
       loggedIn: DefaultAccessLevel;
     };
   };
+  revisionRetentionDays: number;
 }
 
 const schema = Joi.object<NoteConfig>({
@@ -56,6 +57,12 @@ const schema = Joi.object<NoteConfig>({
         .label('HD_PERMISSION_DEFAULT_LOGGED_IN'),
     },
   },
+  revisionRetentionDays: Joi.number()
+    .integer()
+    .default(0)
+    .min(0)
+    .optional()
+    .label('HD_REVISION_RETENTION_DAYS'),
 });
 
 function checkEveryoneConfigIsConsistent(config: NoteConfig): void {
@@ -97,6 +104,9 @@ export default registerAs('noteConfig', () => {
           loggedIn: process.env.HD_PERMISSION_DEFAULT_LOGGED_IN,
         },
       },
+      revisionRetentionDays: parseOptionalNumber(
+        process.env.HD_REVISION_RETENTION_DAYS,
+      ),
     } as NoteConfig,
     {
       abortEarly: false,

backend/src/revisions/revisions.module.ts

@@ -9,6 +9,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
 
 import { AuthorsModule } from '../authors/authors.module';
 import { LoggerModule } from '../logger/logger.module';
+import { Note } from '../notes/note.entity';
 import { Edit } from './edit.entity';
 import { EditService } from './edit.service';
 import { Revision } from './revision.entity';
@@ -16,7 +17,7 @@ import { RevisionsService } from './revisions.service';
 
 @Module({
   imports: [
-    TypeOrmModule.forFeature([Revision, Edit]),
+    TypeOrmModule.forFeature([Revision, Edit, Note]),
     LoggerModule,
     ConfigModule,
     AuthorsModule,

backend/src/revisions/revisions.service.spec.ts

@@ -7,8 +7,9 @@ import { ConfigModule } from '@nestjs/config';
 import { EventEmitterModule } from '@nestjs/event-emitter';
 import { Test, TestingModule } from '@nestjs/testing';
 import { getRepositoryToken } from '@nestjs/typeorm';
+import { createPatch } from 'diff';
 import { Mock } from 'ts-mockery';
-import { Repository } from 'typeorm';
+import { DataSource, EntityManager, Repository } from 'typeorm';
 
 import { ApiToken } from '../api-token/api-token.entity';
 import { Author } from '../authors/author.entity';
@@ -16,6 +17,11 @@ import appConfigMock from '../config/mock/app.config.mock';
 import authConfigMock from '../config/mock/auth.config.mock';
 import databaseConfigMock from '../config/mock/database.config.mock';
 import noteConfigMock from '../config/mock/note.config.mock';
+import {
+  createDefaultMockNoteConfig,
+  registerNoteConfig,
+} from '../config/mock/note.config.mock';
+import { NoteConfig } from '../config/note.config';
 import { NotInDBError } from '../errors/errors';
 import { eventModuleConfig } from '../events';
 import { Group } from '../groups/group.entity';
@@ -37,8 +43,21 @@ import { RevisionsService } from './revisions.service';
 describe('RevisionsService', () => {
   let service: RevisionsService;
   let revisionRepo: Repository<Revision>;
+  let noteRepo: Repository<Note>;
+  const noteConfig: NoteConfig = createDefaultMockNoteConfig();
 
   beforeEach(async () => {
+    noteRepo = new Repository<Note>(
+      '',
+      new EntityManager(
+        new DataSource({
+          type: 'sqlite',
+          database: ':memory:',
+        }),
+      ),
+      undefined,
+    );
+
     const module: TestingModule = await Test.createTestingModule({
       providers: [
         RevisionsService,
@@ -47,6 +66,10 @@ describe('RevisionsService', () => {
           provide: getRepositoryToken(Revision),
           useClass: Repository,
         },
+        {
+          provide: getRepositoryToken(Note),
+          useClass: Repository,
+        },
       ],
       imports: [
         NotesModule,
@@ -58,6 +81,7 @@ describe('RevisionsService', () => {
             databaseConfigMock,
             authConfigMock,
             noteConfigMock,
+            registerNoteConfig(noteConfig),
           ],
         }),
         EventEmitterModule.forRoot(eventModuleConfig),
@@ -72,7 +96,7 @@ describe('RevisionsService', () => {
       .overrideProvider(getRepositoryToken(Identity))
       .useValue({})
       .overrideProvider(getRepositoryToken(Note))
-      .useValue({})
+      .useValue(noteRepo)
       .overrideProvider(getRepositoryToken(Revision))
       .useClass(Repository)
       .overrideProvider(getRepositoryToken(Tag))
@@ -95,6 +119,7 @@ describe('RevisionsService', () => {
     revisionRepo = module.get<Repository<Revision>>(
       getRepositoryToken(Revision),
     );
+    noteRepo = module.get<Repository<Note>>(getRepositoryToken(Note));
   });
 
   it('should be defined', () => {
@@ -423,4 +448,163 @@ describe('RevisionsService', () => {
       expect(repoSaveSpy).not.toHaveBeenCalled();
     });
   });
+
+  describe('auto remove old revisions', () => {
+    beforeEach(() => {
+      jest.spyOn(service, 'removeOldRevisions');
+    });
+
+    it('handleCron should call removeOldRevisions', async () => {
+      await service.handleCron();
+      expect(service.removeOldRevisions).toHaveBeenCalledTimes(1);
+    });
+
+    it('handleTimeout should call removeOldRevisions', async () => {
+      await service.handleTimeout();
+      expect(service.removeOldRevisions).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('removeOldRevisions', () => {
+    let note: Note;
+    let notes: Note[];
+    let revisions: Revision[];
+    let oldRevisions: Revision[];
+    const retentionDays = 30;
+
+    beforeEach(() => {
+      noteConfig.revisionRetentionDays = retentionDays;
+
+      note = Mock.of<Note>({ publicId: 'test-note', id: 1 });
+      notes = [note];
+    });
+
+    afterEach(() => {
+      jest.clearAllMocks();
+    });
+
+    it('remove all revisions except latest revision', async () => {
+      const date1 = new Date();
+      const date2 = new Date();
+      const date3 = new Date();
+      date1.setDate(date1.getDate() - retentionDays - 2);
+      date2.setDate(date2.getDate() - retentionDays - 1);
+
+      const revision1 = Mock.of<Revision>({
+        id: 1,
+        createdAt: date1,
+        note: Promise.resolve(note),
+      });
+      const revision2 = Mock.of<Revision>({
+        id: 2,
+        createdAt: date2,
+        note: Promise.resolve(note),
+        content: 'old content\n',
+      });
+      const revision3 = Mock.of<Revision>({
+        id: 3,
+        createdAt: date3,
+        note: Promise.resolve(note),
+        content:
+          '---\ntitle: new title\ndescription: new description\ntags: [ "tag1" ]\n---\nnew content\n',
+      });
+      revision3.patch = createPatch(
+        note.publicId,
+        revision2.content,
+        revision3.content,
+      );
+
+      revisions = [revision1, revision2, revision3];
+      oldRevisions = [revision1, revision2];
+
+      jest.spyOn(noteRepo, 'find').mockResolvedValueOnce(notes);
+      jest.spyOn(revisionRepo, 'find').mockResolvedValueOnce(revisions);
+      jest
+        .spyOn(revisionRepo, 'remove')
+        .mockImplementationOnce(async (entry, _) => {
+          expect(entry).toEqual(oldRevisions);
+          return entry;
+        });
+      jest.spyOn(revisionRepo, 'save').mockResolvedValue(revision3);
+
+      await service.removeOldRevisions();
+      expect(revision3.patch).toMatchSnapshot;
+    });
+
+    it('remove a part of old revisions', async () => {
+      const date1 = new Date();
+      const date2 = new Date();
+      const date3 = new Date();
+      date1.setDate(date1.getDate() - retentionDays);
+      date2.setDate(date2.getDate() - retentionDays + 1);
+
+      const revision1 = Mock.of<Revision>({
+        id: 1,
+        createdAt: date1,
+        note: Promise.resolve(note),
+        content: 'old content\n',
+      });
+      const revision2 = Mock.of<Revision>({
+        id: 2,
+        createdAt: date2,
+        note: Promise.resolve(note),
+        content:
+          '---\ntitle: new title\ndescription: new description\ntags: [ "tag1" ]\n---\nnew content\n',
+      });
+      const revision3 = Mock.of<Revision>({
+        id: 3,
+        createdAt: date3,
+        note: Promise.resolve(note),
+      });
+      revision2.patch = createPatch(
+        note.publicId,
+        revision1.content,
+        revision2.content,
+      );
+
+      revisions = [revision1, revision2, revision3];
+      oldRevisions = [revision1];
+
+      jest.spyOn(noteRepo, 'find').mockResolvedValueOnce(notes);
+      jest.spyOn(revisionRepo, 'find').mockResolvedValueOnce(revisions);
+      jest
+        .spyOn(revisionRepo, 'remove')
+        .mockImplementationOnce(async (entry, _) => {
+          expect(entry).toEqual(oldRevisions);
+          return entry;
+        });
+      jest.spyOn(revisionRepo, 'save').mockResolvedValue(revision2);
+
+      await service.removeOldRevisions();
+      expect(revision2.patch).toMatchSnapshot;
+    });
+
+    it('do nothing when only one revision', async () => {
+      const date = new Date();
+      date.setDate(date.getDate() - retentionDays * 2);
+
+      const revision1 = Mock.of<Revision>({
+        id: 1,
+        createdAt: date,
+        note: Promise.resolve(note),
+      });
+      revisions = [revision1];
+      oldRevisions = [];
+
+      jest.spyOn(noteRepo, 'find').mockResolvedValueOnce(notes);
+      jest.spyOn(revisionRepo, 'find').mockResolvedValueOnce(revisions);
+      const spyOnRemove = jest.spyOn(revisionRepo, 'remove');
+
+      await service.removeOldRevisions();
+      expect(spyOnRemove).toHaveBeenCalledTimes(0);
+    });
+
+    it('do nothing when retention days config is zero', async () => {
+      noteConfig.revisionRetentionDays = 0;
+      const spyOnRemove = jest.spyOn(revisionRepo, 'remove');
+
+      await service.removeOldRevisions();
+      expect(spyOnRemove).toHaveBeenCalledTimes(0);
+    });
+  });
 });

backend/src/revisions/revisions.service.ts

@@ -3,11 +3,13 @@
  *
  * SPDX-License-Identifier: AGPL-3.0-only
  */
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
+import { Cron, Timeout } from '@nestjs/schedule';
 import { InjectRepository } from '@nestjs/typeorm';
 import { createPatch } from 'diff';
 import { Repository } from 'typeorm';
 
+import noteConfiguration, { NoteConfig } from '../config/note.config';
 import { NotInDBError } from '../errors/errors';
 import { ConsoleLoggerService } from '../logger/console-logger.service';
 import { Note } from '../notes/note.entity';
@@ -29,6 +31,9 @@ export class RevisionsService {
     private readonly logger: ConsoleLoggerService,
     @InjectRepository(Revision)
     private revisionRepository: Repository<Revision>,
+    @InjectRepository(Note)
+    private noteRepository: Repository<Note>,
+    @Inject(noteConfiguration.KEY) private noteConfig: NoteConfig,
     private editService: EditService,
   ) {
     this.logger.setContext(RevisionsService.name);
@@ -230,4 +235,80 @@ export class RevisionsService {
       await this.revisionRepository.save(revision);
     }
   }
+
+  // Delete all old revisions everyday on 0:00 AM
+  @Cron('0 0 * * *')
+  async handleRevisionCleanup(): Promise<void> {
+    return await this.removeOldRevisions();
+  }
+
+  // Delete all old revisions 5 sec after startup
+  @Timeout(5000)
+  async handleRevisionCleanupTimeout(): Promise<void> {
+    return await this.removeOldRevisions();
+  }
+
+  /**
+   * Delete old {@link Revision}s except the latest one.
+   *
+   * @async
+   */
+  async removeOldRevisions(): Promise<void> {
+    const currentTime = new Date().getTime();
+    const revisionRetentionDays: number = this.noteConfig.revisionRetentionDays;
+    if (revisionRetentionDays <= 0) {
+      return;
+    }
+    const revisionRetentionSeconds =
+      revisionRetentionDays * 24 * 60 * 60 * 1000;
+
+    const notes: Note[] = await this.noteRepository.find();
+    for (const note of notes) {
+      const revisions: Revision[] = await this.revisionRepository.find({
+        where: {
+          note: { id: note.id },
+        },
+        order: {
+          createdAt: 'ASC',
+        },
+      });
+
+      const oldRevisions = revisions
+        .slice(0, -1) // always keep the latest revision
+        .filter(
+          (revision) =>
+            new Date(revision.createdAt).getTime() <=
+            currentTime - revisionRetentionSeconds,
+        );
+      const remainedRevisions = revisions.filter(
+        (val) => !oldRevisions.includes(val),
+      );
+
+      if (!oldRevisions.length) {
+        continue;
+      } else if (oldRevisions.length === revisions.length - 1) {
+        const beUpdatedRevision = revisions.slice(-1)[0];
+        beUpdatedRevision.patch = createPatch(
+          note.publicId,
+          '', // there is no older revision
+          beUpdatedRevision.content,
+        );
+        await this.revisionRepository.save(beUpdatedRevision);
+      } else {
+        const beUpdatedRevision = remainedRevisions.slice(0)[0];
+        beUpdatedRevision.patch = createPatch(
+          note.publicId,
+          oldRevisions.slice(-1)[0].content,
+          beUpdatedRevision.content,
+        );
+        await this.revisionRepository.save(beUpdatedRevision);
+      }
+
+      await this.revisionRepository.remove(oldRevisions);
+      this.logger.log(
+        `${oldRevisions.length} old revisions of the note '${note.id}' were removed from the DB`,
+        'removeOldRevisions',
+      );
+    }
+  }
 }

docs/content/references/config/notes.md

@@ -8,3 +8,4 @@
 | `HD_PERMISSION_DEFAULT_LOGGED_IN` | `write` | `none`, `read`, `write`           | The default permission for the "logged-in" group that is set on new notes.                                                                                                           |
 | `HD_PERMISSION_DEFAULT_EVERYONE`  | `read`  | `none`, `read`, `write`           | The default permission for the "everyone" group (logged-in & guest users), that is set on new notes created by logged-in users. Notes created by guests always set this to "write".  |
 | `HD_PERSIST_INTERVAL`             | 10      | `0`, `5`, `10`, `20`              | The time interval in **minutes** for the periodic note revision creation during realtime editing. `0` deactivates the periodic note revision creation.                               |
+| `HD_REVISION_RETENTION_DAYS`      | 0       |                                   | The number of days a revision should be kept. If the config option is not set or set to 0, all revisions will be kept forever.                                                       |