github/hedgedoc
1
0
Fork 0
mirror of https://github.com/hedgedoc/hedgedoc.git synced 2024-12-03 12:03:34 -05:00
Code Issues Projects Releases Packages Wiki Activity

feat(backend revision): add clean-up note revisions job (#5349)

Browse source
This commit is contained in:
yamashu 2024-09-27 00:24:24 +09:00 committed by GitHub
parent b80552bb29
commit 4fce422bdb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 337 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
backend/src/config/mock/note.config.mock.ts
View file

@ -21,6 +21,7 @@ export function createDefaultMockNoteConfig(): NoteConfig {
},
},
guestAccess: GuestAccess.CREATE,
revisionRetentionDays: 0,
};
}

55
backend/src/config/note.config.spec.ts
View file

@ -19,6 +19,7 @@ describe('noteConfig', () => {
const invalidMaxDocumentLength = 'not-a-max-document-length';
const guestAccess = GuestAccess.CREATE;
const wrongDefaultPermission = 'wrong';
const retentionDays = 30;
describe('correctly parses config', () => {
it('when given correct and complete environment variables', () => {
@ -30,6 +31,7 @@ describe('noteConfig', () => {
HD_PERMISSION_DEFAULT_EVERYONE: DefaultAccessLevel.READ,
HD_PERMISSION_DEFAULT_LOGGED_IN: DefaultAccessLevel.READ,
HD_GUEST_ACCESS: guestAccess,
HD_REVISION_RETENTION_DAYS: retentionDays.toString(),
/* eslint-enable @typescript-eslint/naming-convention */
},
{
@ -47,6 +49,7 @@ describe('noteConfig', () => {
DefaultAccessLevel.READ,
);
expect(config.guestAccess).toEqual(guestAccess);
expect(config.revisionRetentionDays).toEqual(retentionDays);
restore();
});
@ -221,6 +224,36 @@ describe('noteConfig', () => {
expect(config.guestAccess).toEqual(GuestAccess.WRITE);
restore();
});
it('when no HD_REVISION_RETENTION_DAYS is set', () => {
const restore = mockedEnv(
{
/* eslint-disable @typescript-eslint/naming-convention */
HD_FORBIDDEN_NOTE_IDS: forbiddenNoteIds.join(' , '),
HD_MAX_DOCUMENT_LENGTH: maxDocumentLength.toString(),
HD_PERMISSION_DEFAULT_EVERYONE: DefaultAccessLevel.READ,
HD_PERMISSION_DEFAULT_LOGGED_IN: DefaultAccessLevel.READ,
HD_GUEST_ACCESS: guestAccess,
/* eslint-enable @typescript-eslint/naming-convention */
},
{
clear: true,
},
);
const config = noteConfig();
expect(config.forbiddenNoteIds).toHaveLength(forbiddenNoteIds.length);
expect(config.forbiddenNoteIds).toEqual(forbiddenNoteIds);
expect(config.maxDocumentLength).toEqual(maxDocumentLength);
expect(config.permissions.default.everyone).toEqual(
DefaultAccessLevel.READ,
);
expect(config.permissions.default.loggedIn).toEqual(
DefaultAccessLevel.READ,
);
expect(config.guestAccess).toEqual(guestAccess);
expect(config.revisionRetentionDays).toEqual(0);
restore();
});
});
describe('throws error', () => {
@ -454,5 +487,27 @@ describe('noteConfig', () => {
);
restore();
});
it('when given a negative retention days', async () => {
const restore = mockedEnv(
{
/* eslint-disable @typescript-eslint/naming-convention */
HD_FORBIDDEN_NOTE_IDS: forbiddenNoteIds.join(' , '),
HD_MAX_DOCUMENT_LENGTH: maxDocumentLength.toString(),
HD_PERMISSION_DEFAULT_EVERYONE: DefaultAccessLevel.READ,
HD_PERMISSION_DEFAULT_LOGGED_IN: DefaultAccessLevel.READ,
HD_GUEST_ACCESS: guestAccess,
HD_REVISION_RETENTION_DAYS: (-1).toString(),
/* eslint-enable @typescript-eslint/naming-convention */
},
{
clear: true,
},
);
expect(() => noteConfig()).toThrow(
'"HD_REVISION_RETENTION_DAYS" must be greater than or equal to 0',
);
restore();
});
});
});

10
backend/src/config/note.config.ts
View file

@ -23,6 +23,7 @@ export interface NoteConfig {
loggedIn: DefaultAccessLevel;
};
};
revisionRetentionDays: number;
}
const schema = Joi.object<NoteConfig>({
@ -56,6 +57,12 @@ const schema = Joi.object<NoteConfig>({
.label('HD_PERMISSION_DEFAULT_LOGGED_IN'),
},
},
revisionRetentionDays: Joi.number()
.integer()
.default(0)
.min(0)
.optional()
.label('HD_REVISION_RETENTION_DAYS'),
});
function checkEveryoneConfigIsConsistent(config: NoteConfig): void {
@ -97,6 +104,9 @@ export default registerAs('noteConfig', () => {
loggedIn: process.env.HD_PERMISSION_DEFAULT_LOGGED_IN,
},
},
revisionRetentionDays: parseOptionalNumber(
process.env.HD_REVISION_RETENTION_DAYS,
),
} as NoteConfig,
{
abortEarly: false,

3
backend/src/revisions/revisions.module.ts
View file

@ -9,6 +9,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { AuthorsModule } from '../authors/authors.module';
import { LoggerModule } from '../logger/logger.module';
import { Note } from '../notes/note.entity';
import { Edit } from './edit.entity';
import { EditService } from './edit.service';
import { Revision } from './revision.entity';
@ -16,7 +17,7 @@ import { RevisionsService } from './revisions.service';
@Module({
imports: [
TypeOrmModule.forFeature([Revision, Edit]),
TypeOrmModule.forFeature([Revision, Edit, Note]),
LoggerModule,
ConfigModule,
AuthorsModule,

188
backend/src/revisions/revisions.service.spec.ts
View file

@ -7,8 +7,9 @@ import { ConfigModule } from '@nestjs/config';
import { EventEmitterModule } from '@nestjs/event-emitter';
import { Test, TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import { createPatch } from 'diff';
import { Mock } from 'ts-mockery';
import { Repository } from 'typeorm';
import { DataSource, EntityManager, Repository } from 'typeorm';
import { ApiToken } from '../api-token/api-token.entity';
import { Author } from '../authors/author.entity';
@ -16,6 +17,11 @@ import appConfigMock from '../config/mock/app.config.mock';
import authConfigMock from '../config/mock/auth.config.mock';
import databaseConfigMock from '../config/mock/database.config.mock';
import noteConfigMock from '../config/mock/note.config.mock';
import {
createDefaultMockNoteConfig,
registerNoteConfig,
} from '../config/mock/note.config.mock';
import { NoteConfig } from '../config/note.config';
import { NotInDBError } from '../errors/errors';
import { eventModuleConfig } from '../events';
import { Group } from '../groups/group.entity';
@ -37,8 +43,21 @@ import { RevisionsService } from './revisions.service';
describe('RevisionsService', () => {
let service: RevisionsService;
let revisionRepo: Repository<Revision>;
let noteRepo: Repository<Note>;
const noteConfig: NoteConfig = createDefaultMockNoteConfig();
beforeEach(async () => {
noteRepo = new Repository<Note>(
'',
new EntityManager(
new DataSource({
type: 'sqlite',
database: ':memory:',
}),
),
undefined,
);
const module: TestingModule = await Test.createTestingModule({
providers: [
RevisionsService,
@ -47,6 +66,10 @@ describe('RevisionsService', () => {
provide: getRepositoryToken(Revision),
useClass: Repository,
},
{
provide: getRepositoryToken(Note),
useClass: Repository,
},
],
imports: [
NotesModule,
@ -58,6 +81,7 @@ describe('RevisionsService', () => {
databaseConfigMock,
authConfigMock,
noteConfigMock,
registerNoteConfig(noteConfig),
],
}),
EventEmitterModule.forRoot(eventModuleConfig),
@ -72,7 +96,7 @@ describe('RevisionsService', () => {
.overrideProvider(getRepositoryToken(Identity))
.useValue({})
.overrideProvider(getRepositoryToken(Note))
.useValue({})
.useValue(noteRepo)
.overrideProvider(getRepositoryToken(Revision))
.useClass(Repository)
.overrideProvider(getRepositoryToken(Tag))
@ -95,6 +119,7 @@ describe('RevisionsService', () => {
revisionRepo = module.get<Repository<Revision>>(
getRepositoryToken(Revision),
);
noteRepo = module.get<Repository<Note>>(getRepositoryToken(Note));
});
it('should be defined', () => {
@ -423,4 +448,163 @@ describe('RevisionsService', () => {
expect(repoSaveSpy).not.toHaveBeenCalled();
});
});
describe('auto remove old revisions', () => {
beforeEach(() => {
jest.spyOn(service, 'removeOldRevisions');
});
it('handleCron should call removeOldRevisions', async () => {
await service.handleCron();
expect(service.removeOldRevisions).toHaveBeenCalledTimes(1);
});
it('handleTimeout should call removeOldRevisions', async () => {
await service.handleTimeout();
expect(service.removeOldRevisions).toHaveBeenCalledTimes(1);
});
});
describe('removeOldRevisions', () => {
let note: Note;
let notes: Note[];
let revisions: Revision[];
let oldRevisions: Revision[];
const retentionDays = 30;
beforeEach(() => {
noteConfig.revisionRetentionDays = retentionDays;
note = Mock.of<Note>({ publicId: 'test-note', id: 1 });
notes = [note];
});
afterEach(() => {
jest.clearAllMocks();
});
it('remove all revisions except latest revision', async () => {
const date1 = new Date();
const date2 = new Date();
const date3 = new Date();
date1.setDate(date1.getDate() - retentionDays - 2);
date2.setDate(date2.getDate() - retentionDays - 1);
const revision1 = Mock.of<Revision>({
id: 1,
createdAt: date1,
note: Promise.resolve(note),
});
const revision2 = Mock.of<Revision>({
id: 2,
createdAt: date2,
note: Promise.resolve(note),
content: 'old content\n',
});
const revision3 = Mock.of<Revision>({
id: 3,
createdAt: date3,
note: Promise.resolve(note),
content:
'---\ntitle: new title\ndescription: new description\ntags: [ "tag1" ]\n---\nnew content\n',
});
revision3.patch = createPatch(
note.publicId,
revision2.content,
revision3.content,
);
revisions = [revision1, revision2, revision3];
oldRevisions = [revision1, revision2];
jest.spyOn(noteRepo, 'find').mockResolvedValueOnce(notes);
jest.spyOn(revisionRepo, 'find').mockResolvedValueOnce(revisions);
jest
.spyOn(revisionRepo, 'remove')
.mockImplementationOnce(async (entry, _) => {
expect(entry).toEqual(oldRevisions);
return entry;
});
jest.spyOn(revisionRepo, 'save').mockResolvedValue(revision3);
await service.removeOldRevisions();
expect(revision3.patch).toMatchSnapshot;
});
it('remove a part of old revisions', async () => {
const date1 = new Date();
const date2 = new Date();
const date3 = new Date();
date1.setDate(date1.getDate() - retentionDays);
date2.setDate(date2.getDate() - retentionDays + 1);
const revision1 = Mock.of<Revision>({
id: 1,
createdAt: date1,
note: Promise.resolve(note),
content: 'old content\n',
});
const revision2 = Mock.of<Revision>({
id: 2,
createdAt: date2,
note: Promise.resolve(note),
content:
'---\ntitle: new title\ndescription: new description\ntags: [ "tag1" ]\n---\nnew content\n',
});
const revision3 = Mock.of<Revision>({
id: 3,
createdAt: date3,
note: Promise.resolve(note),
});
revision2.patch = createPatch(
note.publicId,
revision1.content,
revision2.content,
);
revisions = [revision1, revision2, revision3];
oldRevisions = [revision1];
jest.spyOn(noteRepo, 'find').mockResolvedValueOnce(notes);
jest.spyOn(revisionRepo, 'find').mockResolvedValueOnce(revisions);
jest
.spyOn(revisionRepo, 'remove')
.mockImplementationOnce(async (entry, _) => {
expect(entry).toEqual(oldRevisions);
return entry;
});
jest.spyOn(revisionRepo, 'save').mockResolvedValue(revision2);
await service.removeOldRevisions();
expect(revision2.patch).toMatchSnapshot;
});
it('do nothing when only one revision', async () => {
const date = new Date();
date.setDate(date.getDate() - retentionDays * 2);
const revision1 = Mock.of<Revision>({
id: 1,
createdAt: date,
note: Promise.resolve(note),
});
revisions = [revision1];
oldRevisions = [];
jest.spyOn(noteRepo, 'find').mockResolvedValueOnce(notes);
jest.spyOn(revisionRepo, 'find').mockResolvedValueOnce(revisions);
const spyOnRemove = jest.spyOn(revisionRepo, 'remove');
await service.removeOldRevisions();
expect(spyOnRemove).toHaveBeenCalledTimes(0);
});
it('do nothing when retention days config is zero', async () => {
noteConfig.revisionRetentionDays = 0;
const spyOnRemove = jest.spyOn(revisionRepo, 'remove');
await service.removeOldRevisions();
expect(spyOnRemove).toHaveBeenCalledTimes(0);
});
});
});

83
backend/src/revisions/revisions.service.ts
View file

@ -3,11 +3,13 @@
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Injectable } from '@nestjs/common';
import { Inject, Injectable } from '@nestjs/common';
import { Cron, Timeout } from '@nestjs/schedule';
import { InjectRepository } from '@nestjs/typeorm';
import { createPatch } from 'diff';
import { Repository } from 'typeorm';
import noteConfiguration, { NoteConfig } from '../config/note.config';
import { NotInDBError } from '../errors/errors';
import { ConsoleLoggerService } from '../logger/console-logger.service';
import { Note } from '../notes/note.entity';
@ -29,6 +31,9 @@ export class RevisionsService {
private readonly logger: ConsoleLoggerService,
@InjectRepository(Revision)
private revisionRepository: Repository<Revision>,
@InjectRepository(Note)
private noteRepository: Repository<Note>,
@Inject(noteConfiguration.KEY) private noteConfig: NoteConfig,
private editService: EditService,
) {
this.logger.setContext(RevisionsService.name);
@ -230,4 +235,80 @@ export class RevisionsService {
await this.revisionRepository.save(revision);
}
}
// Delete all old revisions everyday on 0:00 AM
@Cron('0 0 * * *')
async handleRevisionCleanup(): Promise<void> {
return await this.removeOldRevisions();
}
// Delete all old revisions 5 sec after startup
@Timeout(5000)
async handleRevisionCleanupTimeout(): Promise<void> {
return await this.removeOldRevisions();
}
/**
* Delete old {@link Revision}s except the latest one.
*
* @async
*/
async removeOldRevisions(): Promise<void> {
const currentTime = new Date().getTime();
const revisionRetentionDays: number = this.noteConfig.revisionRetentionDays;
if (revisionRetentionDays <= 0) {
return;
}
const revisionRetentionSeconds =
revisionRetentionDays * 24 * 60 * 60 * 1000;
const notes: Note[] = await this.noteRepository.find();
for (const note of notes) {
const revisions: Revision[] = await this.revisionRepository.find({
where: {
note: { id: note.id },
},
order: {
createdAt: 'ASC',
},
});
const oldRevisions = revisions
.slice(0, -1) // always keep the latest revision
.filter(
(revision) =>
new Date(revision.createdAt).getTime() <=
currentTime - revisionRetentionSeconds,
);
const remainedRevisions = revisions.filter(
(val) => !oldRevisions.includes(val),
);
if (!oldRevisions.length) {
continue;
} else if (oldRevisions.length === revisions.length - 1) {
const beUpdatedRevision = revisions.slice(-1)[0];
beUpdatedRevision.patch = createPatch(
note.publicId,
'', // there is no older revision
beUpdatedRevision.content,
);
await this.revisionRepository.save(beUpdatedRevision);
} else {
const beUpdatedRevision = remainedRevisions.slice(0)[0];
beUpdatedRevision.patch = createPatch(
note.publicId,
oldRevisions.slice(-1)[0].content,
beUpdatedRevision.content,
);
await this.revisionRepository.save(beUpdatedRevision);
}
await this.revisionRepository.remove(oldRevisions);
this.logger.log(
`${oldRevisions.length} old revisions of the note '${note.id}' were removed from the DB`,
'removeOldRevisions',
);
}
}
}

1
docs/content/references/config/notes.md
View file

@ -8,3 +8,4 @@
| `HD_PERMISSION_DEFAULT_LOGGED_IN` | `write` | `none`, `read`, `write` | The default permission for the "logged-in" group that is set on new notes. |
| `HD_PERMISSION_DEFAULT_EVERYONE` | `read` | `none`, `read`, `write` | The default permission for the "everyone" group (logged-in & guest users), that is set on new notes created by logged-in users. Notes created by guests always set this to "write". |
| `HD_PERSIST_INTERVAL` | 10 | `0`, `5`, `10`, `20` | The time interval in **minutes** for the periodic note revision creation during realtime editing. `0` deactivates the periodic note revision creation. |
| `HD_REVISION_RETENTION_DAYS` | 0 | | The number of days a revision should be kept. If the config option is not set or set to 0, all revisions will be kept forever. |