diff --git a/package.json b/package.json index d3d165d5e..d2abc9dcd 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "@types/bcrypt": "3.0.0", "@types/cron": "1.7.2", "@types/node-fetch": "^2.5.8", + "@types/minio": "^7.0.7", "@types/passport-http-bearer": "1.0.36", "bcrypt": "5.0.1", "class-transformer": "0.4.0", @@ -44,6 +45,7 @@ "connect-typeorm": "1.1.4", "file-type": "16.2.0", "joi": "17.4.0", + "minio": "^7.0.18", "nest-router": "1.0.9", "node-fetch": "^2.6.1", "passport": "0.4.1", diff --git a/src/media/backends/s3-backend.ts b/src/media/backends/s3-backend.ts new file mode 100644 index 000000000..df048d7d8 --- /dev/null +++ b/src/media/backends/s3-backend.ts @@ -0,0 +1,79 @@ +/* + * SPDX-FileCopyrightText: 2021 The HedgeDoc developers (see AUTHORS file) + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import mediaConfiguration from '../../config/media.config'; +import { ConsoleLoggerService } from '../../logger/console-logger.service'; +import { MediaBackend } from '../media-backend.interface'; +import { BackendData } from '../media-upload.entity'; +import { MediaConfig } from '../../config/media.config'; +import { Client } from 'minio'; +import { BackendType } from './backend-type.enum'; +import { MediaBackendError } from '../../errors/errors'; + +@Injectable() +export class S3Backend implements MediaBackend { + private config: MediaConfig['backend']['s3']; + private client: Client; + + constructor( + private readonly logger: ConsoleLoggerService, + @Inject(mediaConfiguration.KEY) + private mediaConfig: MediaConfig, + ) { + this.logger.setContext(S3Backend.name); + if (mediaConfig.backend.use === BackendType.S3) { + this.config = mediaConfig.backend.s3; + const url = new URL(this.config.endPoint); + const secure = url.protocol === 'https:'; // url.protocol contains a trailing ':' + const endpoint = `${url.hostname}${url.pathname}`; + let port = parseInt(url.port); + if (isNaN(port)) { + port = secure ? 443 : 80; + } + this.client = new Client({ + endPoint: endpoint.substr(0, endpoint.length - 1), // remove trailing '/' + port: port, + useSSL: secure, + accessKey: this.config.accessKeyId, + secretKey: this.config.secretAccessKey, + }); + } + } + + async saveFile( + buffer: Buffer, + fileName: string, + ): Promise<[string, BackendData]> { + try { + await this.client.putObject(this.config.bucket, fileName, buffer); + this.logger.log(`Uploaded file ${fileName}`, 'saveFile'); + return [this.getUrl(fileName), null]; + } catch (e) { + this.logger.error((e as Error).message, (e as Error).stack, 'saveFile'); + throw new MediaBackendError(`Could not save '${fileName}' on S3`); + } + } + + async deleteFile(fileName: string, _: BackendData): Promise { + try { + await this.client.removeObject(this.config.bucket, fileName); + const url = this.getUrl(fileName); + this.logger.log(`Deleted ${url}`, 'saveFile'); + return; + } catch (e) { + this.logger.error((e as Error).message, (e as Error).stack, 'saveFile'); + throw new MediaBackendError(`Could not delete '${fileName}' on S3`); + } + } + + private getUrl(fileName: string): string { + const url = new URL(this.config.endPoint); + const port = url.port !== '' ? `:${url.port}` : ''; + const bucket = this.config.bucket; + return `${url.protocol}//${url.hostname}${port}${url.pathname}${bucket}/${fileName}`; + } +} diff --git a/src/media/media.module.ts b/src/media/media.module.ts index 901f56e6d..eed3262da 100644 --- a/src/media/media.module.ts +++ b/src/media/media.module.ts @@ -13,6 +13,7 @@ import { UsersModule } from '../users/users.module'; import { FilesystemBackend } from './backends/filesystem-backend'; import { MediaUpload } from './media-upload.entity'; import { MediaService } from './media.service'; +import { S3Backend } from './backends/s3-backend'; import { ImgurBackend } from './backends/imgur-backend'; import { AzureBackend } from './backends/azure-backend'; @@ -24,7 +25,13 @@ import { AzureBackend } from './backends/azure-backend'; LoggerModule, ConfigModule, ], - providers: [MediaService, FilesystemBackend, AzureBackend, ImgurBackend], + providers: [ + MediaService, + FilesystemBackend, + AzureBackend, + ImgurBackend, + S3Backend, + ], exports: [MediaService], }) export class MediaModule {} diff --git a/src/media/media.service.ts b/src/media/media.service.ts index 58fe1c2c2..910d4211d 100644 --- a/src/media/media.service.ts +++ b/src/media/media.service.ts @@ -19,6 +19,7 @@ import { FilesystemBackend } from './backends/filesystem-backend'; import { MediaBackend } from './media-backend.interface'; import { MediaUpload } from './media-upload.entity'; import { MediaUploadUrlDto } from './media-upload-url.dto'; +import { S3Backend } from './backends/s3-backend'; import { AzureBackend } from './backends/azure-backend'; import { ImgurBackend } from './backends/imgur-backend'; @@ -164,6 +165,8 @@ export class MediaService { return BackendType.AZURE; case 'imgur': return BackendType.IMGUR; + case 's3': + return BackendType.S3; } } @@ -171,6 +174,8 @@ export class MediaService { switch (type) { case BackendType.FILESYSTEM: return this.moduleRef.get(FilesystemBackend); + case BackendType.S3: + return this.moduleRef.get(S3Backend); case BackendType.AZURE: return this.moduleRef.get(AzureBackend); case BackendType.IMGUR: diff --git a/yarn.lock b/yarn.lock index 93d486e0f..30d53e723 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1139,6 +1139,13 @@ resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw== +"@types/minio@^7.0.7": + version "7.0.7" + resolved "https://registry.yarnpkg.com/@types/minio/-/minio-7.0.7.tgz#5d7c9a834f55e79dbd83f88c466ad8ff45de7b28" + integrity sha512-cK0VuWZ5zvFmkxQzK46RtJ4qt2Gtrv34hsXDr0D3vqesjrnEKT+0rw0zWk40JtCPkoEFgLdIUbXnePsKjbb80Q== + dependencies: + "@types/node" "*" + "@types/node-fetch@^2.5.0", "@types/node-fetch@^2.5.8": version "2.5.8" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.8.tgz#e199c835d234c7eb0846f6618012e558544ee2fb" @@ -1778,6 +1785,11 @@ astral-regex@^2.0.0: resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31" integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ== +async@^3.1.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720" + integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw== + asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -1928,6 +1940,13 @@ bl@^4.0.3: inherits "^2.0.4" readable-stream "^3.4.0" +block-stream2@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/block-stream2/-/block-stream2-2.1.0.tgz#ac0c5ef4298b3857796e05be8ebed72196fa054b" + integrity sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg== + dependencies: + readable-stream "^3.4.0" + block-stream@*: version "0.0.9" resolved "https://registry.yarnpkg.com/block-stream/-/block-stream-0.0.9.tgz#13ebfe778a03205cfe03751481ebb4b3300c126a" @@ -2834,6 +2853,11 @@ es5-ext@^0.10.35, es5-ext@^0.10.46, es5-ext@^0.10.50, es5-ext@^0.10.51, es5-ext@ es6-symbol "~3.1.3" next-tick "~1.0.0" +es6-error@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.1.1.tgz#9e3af407459deed47e9a91f9b885a84eb05c561d" + integrity sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg== + es6-iterator@^2.0.3, es6-iterator@~2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/es6-iterator/-/es6-iterator-2.0.3.tgz#a7de889141a05a94b0854403b2d0a0fbfa98f3b7" @@ -3267,6 +3291,11 @@ fast-safe-stringify@2.0.7, fast-safe-stringify@^2.0.7: resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz#124aa885899261f68aedb42a7c080de9da608743" integrity sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA== +fast-xml-parser@^3.17.5: + version "3.18.0" + resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-3.18.0.tgz#b77f4a494cd64e6f44aadfa68fbde30cd922b2df" + integrity sha512-tRrwShhppv0K5GKEtuVs92W0VGDaVltZAwtHbpjNF+JOT7cjIFySBGTEOmdBslXYyWYaZwEX/g4Su8ZeKg0LKQ== + fastq@^1.6.0: version "1.10.1" resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.10.1.tgz#8b8f2ac8bf3632d67afcd65dac248d5fdc45385e" @@ -4653,6 +4682,11 @@ json-stable-stringify-without-jsonify@^1.0.1: resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" integrity sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE= +json-stream@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/json-stream/-/json-stream-1.0.0.tgz#1a3854e28d2bbeeab31cc7ddf683d2ddc5652708" + integrity sha1-GjhU4o0rvuqzHMfd9oPS3cVlJwg= + json-stringify-safe@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" @@ -4977,6 +5011,11 @@ mime-db@1.45.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.45.0.tgz#cceeda21ccd7c3a745eba2decd55d4b73e7879ea" integrity sha512-CkqLUxUk15hofLoLyljJSrukZi8mAtgd+yE5uO4tqRZsdsAJKv0O+rFMhVDRJgozy+yG6md5KwuXhD4ocIoP+w== +mime-db@1.46.0: + version "1.46.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.46.0.tgz#6267748a7f799594de3cbc8cde91def349661cee" + integrity sha512-svXaP8UQRZ5K7or+ZmfNhg2xX3yKDMUzqadsSqi4NCH/KomcH75MAMYAGVlvXn4+b/xOPhS3I2uHKRUzvjY7BQ== + mime-types@^2.1.12, mime-types@^2.1.27, mime-types@~2.1.19, mime-types@~2.1.24: version "2.1.28" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.28.tgz#1160c4757eab2c5363888e005273ecf79d2a0ecd" @@ -4984,6 +5023,13 @@ mime-types@^2.1.12, mime-types@^2.1.27, mime-types@~2.1.19, mime-types@~2.1.24: dependencies: mime-db "1.45.0" +mime-types@^2.1.14: + version "2.1.29" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.29.tgz#1d4ab77da64b91f5f72489df29236563754bb1b2" + integrity sha512-Y/jMt/S5sR9OaqteJtslsFZKWOIIqMACsJSiHghlCAyhf7jfVYjKBmLiX8OgpWeW+fjJ2b+Az69aPFPkUOY6xQ== + dependencies: + mime-db "1.46.0" + mime@1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" @@ -5011,6 +5057,24 @@ minimist@1.2.5, minimist@^1.1.1, minimist@^1.2.0, minimist@^1.2.5: resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== +minio@^7.0.18: + version "7.0.18" + resolved "https://registry.yarnpkg.com/minio/-/minio-7.0.18.tgz#a2a6dae52a4dde9e35ed47cdf2accc21df4a512d" + integrity sha512-jVRjkw8A5Spf+ETY5OXQUcQckHriuUA3u2+MAcX36btLT8EytlOVivxIseXvyFf9cNn3dy5w1F1UyjMvHU+nqg== + dependencies: + async "^3.1.0" + block-stream2 "^2.0.0" + es6-error "^4.1.1" + fast-xml-parser "^3.17.5" + json-stream "^1.0.0" + lodash "^4.17.20" + mime-types "^2.1.14" + mkdirp "^0.5.1" + querystring "0.2.0" + through2 "^3.0.1" + xml "^1.0.0" + xml2js "^0.4.15" + minipass@^2.6.0, minipass@^2.8.6, minipass@^2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/minipass/-/minipass-2.9.0.tgz#e713762e7d3e32fed803115cf93e04bca9fcc9a6" @@ -5888,6 +5952,11 @@ qs@~6.5.2: resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA== +querystring@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" + integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= + queue-microtask@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.2.tgz#abf64491e6ecf0f38a6502403d4cda04f372dfd3" @@ -5991,6 +6060,15 @@ readable-stream@1.1.x: isarray "0.0.1" string_decoder "~0.10.x" +"readable-stream@2 || 3", readable-stream@^3.4.0, readable-stream@^3.6.0: + version "3.6.0" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" + integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== + dependencies: + inherits "^2.0.3" + string_decoder "^1.1.1" + util-deprecate "^1.0.1" + readable-stream@^2.0.1, readable-stream@^2.0.6, readable-stream@^2.2.2: version "2.3.7" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" @@ -6004,15 +6082,6 @@ readable-stream@^2.0.1, readable-stream@^2.0.6, readable-stream@^2.2.2: string_decoder "~1.1.1" util-deprecate "~1.0.1" -readable-stream@^3.4.0, readable-stream@^3.6.0: - version "3.6.0" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" - integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== - dependencies: - inherits "^2.0.3" - string_decoder "^1.1.1" - util-deprecate "^1.0.1" - readable-web-to-node-stream@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.1.tgz#3f619b1bc5dd73a4cfe5c5f9b4f6faba55dff845" @@ -6936,6 +7005,14 @@ throat@^5.0.0: resolved "https://registry.yarnpkg.com/throat/-/throat-5.0.0.tgz#c5199235803aad18754a667d659b5e72ce16764b" integrity sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA== +through2@^3.0.1: + version "3.0.2" + resolved "https://registry.yarnpkg.com/through2/-/through2-3.0.2.tgz#99f88931cfc761ec7678b41d5d7336b5b6a07bf4" + integrity sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ== + dependencies: + inherits "^2.0.4" + readable-stream "2 || 3" + through@^2.3.6: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" @@ -7552,7 +7629,7 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== -xml2js@^0.4.19, xml2js@^0.4.23: +xml2js@^0.4.15, xml2js@^0.4.19, xml2js@^0.4.23: version "0.4.23" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== @@ -7560,6 +7637,11 @@ xml2js@^0.4.19, xml2js@^0.4.23: sax ">=0.6.0" xmlbuilder "~11.0.0" +xml@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" + integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU= + xmlbuilder@~11.0.0: version "11.0.1" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"