S3Backend: Add S3 MediaBackend

Add minio dependency Signed-off-by: Philip Molares <philip.molares@udo.edu>
2024-10-18 13:20:16 -04:00 · 2021-02-23 16:14:14 +01:00 · 2021-02-23 16:14:14 +01:00 · d6e8d9a592
commit d6e8d9a592
parent d2bce6e091
5 changed files with 186 additions and 11 deletions
--- a/package.json
+++ b/package.json
@ -36,6 +36,7 @@
    "@types/bcrypt": "3.0.0",
    "@types/cron": "1.7.2",
    "@types/node-fetch": "^2.5.8",
+    "@types/minio": "^7.0.7",
    "@types/passport-http-bearer": "1.0.36",
    "bcrypt": "5.0.1",
    "class-transformer": "0.4.0",
@ -44,6 +45,7 @@
    "connect-typeorm": "1.1.4",
    "file-type": "16.2.0",
    "joi": "17.4.0",
+    "minio": "^7.0.18",
    "nest-router": "1.0.9",
    "node-fetch": "^2.6.1",
    "passport": "0.4.1",
--- a/src/media/backends/s3-backend.ts
+++ b/src/media/backends/s3-backend.ts
@ -0,0 +1,79 @@
+/*
+ * SPDX-FileCopyrightText: 2021 The HedgeDoc developers (see AUTHORS file)
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import { Inject, Injectable } from '@nestjs/common';
+import mediaConfiguration from '../../config/media.config';
+import { ConsoleLoggerService } from '../../logger/console-logger.service';
+import { MediaBackend } from '../media-backend.interface';
+import { BackendData } from '../media-upload.entity';
+import { MediaConfig } from '../../config/media.config';
+import { Client } from 'minio';
+import { BackendType } from './backend-type.enum';
+import { MediaBackendError } from '../../errors/errors';
+
+@Injectable()
+export class S3Backend implements MediaBackend {
+  private config: MediaConfig['backend']['s3'];
+  private client: Client;
+
+  constructor(
+    private readonly logger: ConsoleLoggerService,
+    @Inject(mediaConfiguration.KEY)
+    private mediaConfig: MediaConfig,
+  ) {
+    this.logger.setContext(S3Backend.name);
+    if (mediaConfig.backend.use === BackendType.S3) {
+      this.config = mediaConfig.backend.s3;
+      const url = new URL(this.config.endPoint);
+      const secure = url.protocol === 'https:'; // url.protocol contains a trailing ':'
+      const endpoint = `${url.hostname}${url.pathname}`;
+      let port = parseInt(url.port);
+      if (isNaN(port)) {
+        port = secure ? 443 : 80;
+      }
+      this.client = new Client({
+        endPoint: endpoint.substr(0, endpoint.length - 1), // remove trailing '/'
+        port: port,
+        useSSL: secure,
+        accessKey: this.config.accessKeyId,
+        secretKey: this.config.secretAccessKey,
+      });
+    }
+  }
+
+  async saveFile(
+    buffer: Buffer,
+    fileName: string,
+  ): Promise<[string, BackendData]> {
+    try {
+      await this.client.putObject(this.config.bucket, fileName, buffer);
+      this.logger.log(`Uploaded file ${fileName}`, 'saveFile');
+      return [this.getUrl(fileName), null];
+    } catch (e) {
+      this.logger.error((e as Error).message, (e as Error).stack, 'saveFile');
+      throw new MediaBackendError(`Could not save '${fileName}' on S3`);
+    }
+  }
+
+  async deleteFile(fileName: string, _: BackendData): Promise<void> {
+    try {
+      await this.client.removeObject(this.config.bucket, fileName);
+      const url = this.getUrl(fileName);
+      this.logger.log(`Deleted ${url}`, 'saveFile');
+      return;
+    } catch (e) {
+      this.logger.error((e as Error).message, (e as Error).stack, 'saveFile');
+      throw new MediaBackendError(`Could not delete '${fileName}' on S3`);
+    }
+  }
+
+  private getUrl(fileName: string): string {
+    const url = new URL(this.config.endPoint);
+    const port = url.port !== '' ? `:${url.port}` : '';
+    const bucket = this.config.bucket;
+    return `${url.protocol}//${url.hostname}${port}${url.pathname}${bucket}/${fileName}`;
+  }
+}
--- a/src/media/media.module.ts
+++ b/src/media/media.module.ts
@ -13,6 +13,7 @@ import { UsersModule } from '../users/users.module';
 import { FilesystemBackend } from './backends/filesystem-backend';
 import { MediaUpload } from './media-upload.entity';
 import { MediaService } from './media.service';
+import { S3Backend } from './backends/s3-backend';
 import { ImgurBackend } from './backends/imgur-backend';
 import { AzureBackend } from './backends/azure-backend';

@ -24,7 +25,13 @@ import { AzureBackend } from './backends/azure-backend';
    LoggerModule,
    ConfigModule,
  ],
-  providers: [MediaService, FilesystemBackend, AzureBackend, ImgurBackend],
+  providers: [
+    MediaService,
+    FilesystemBackend,
+    AzureBackend,
+    ImgurBackend,
+    S3Backend,
+  ],
  exports: [MediaService],
 })
 export class MediaModule {}
--- a/src/media/media.service.ts
+++ b/src/media/media.service.ts
@ -19,6 +19,7 @@ import { FilesystemBackend } from './backends/filesystem-backend';
 import { MediaBackend } from './media-backend.interface';
 import { MediaUpload } from './media-upload.entity';
 import { MediaUploadUrlDto } from './media-upload-url.dto';
+import { S3Backend } from './backends/s3-backend';
 import { AzureBackend } from './backends/azure-backend';
 import { ImgurBackend } from './backends/imgur-backend';

@ -164,6 +165,8 @@ export class MediaService {
        return BackendType.AZURE;
      case 'imgur':
        return BackendType.IMGUR;
+      case 's3':
+        return BackendType.S3;
    }
  }

@ -171,6 +174,8 @@ export class MediaService {
    switch (type) {
      case BackendType.FILESYSTEM:
        return this.moduleRef.get(FilesystemBackend);
+      case BackendType.S3:
+        return this.moduleRef.get(S3Backend);
      case BackendType.AZURE:
        return this.moduleRef.get(AzureBackend);
      case BackendType.IMGUR:
--- a/yarn.lock
+++ b/yarn.lock
@ -1139,6 +1139,13 @@
  resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a"
  integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==

+"@types/minio@^7.0.7":
+  version "7.0.7"
+  resolved "https://registry.yarnpkg.com/@types/minio/-/minio-7.0.7.tgz#5d7c9a834f55e79dbd83f88c466ad8ff45de7b28"
+  integrity sha512-cK0VuWZ5zvFmkxQzK46RtJ4qt2Gtrv34hsXDr0D3vqesjrnEKT+0rw0zWk40JtCPkoEFgLdIUbXnePsKjbb80Q==
+  dependencies:
+    "@types/node" "*"
+
 "@types/node-fetch@^2.5.0", "@types/node-fetch@^2.5.8":
  version "2.5.8"
  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.8.tgz#e199c835d234c7eb0846f6618012e558544ee2fb"
@ -1778,6 +1785,11 @@ astral-regex@^2.0.0:
  resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31"
  integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==

+async@^3.1.0:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720"
+  integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==
+
 asynckit@^0.4.0:
  version "0.4.0"
  resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
@ -1928,6 +1940,13 @@ bl@^4.0.3:
    inherits "^2.0.4"
    readable-stream "^3.4.0"

+block-stream2@^2.0.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/block-stream2/-/block-stream2-2.1.0.tgz#ac0c5ef4298b3857796e05be8ebed72196fa054b"
+  integrity sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg==
+  dependencies:
+    readable-stream "^3.4.0"
+
 block-stream@*:
  version "0.0.9"
  resolved "https://registry.yarnpkg.com/block-stream/-/block-stream-0.0.9.tgz#13ebfe778a03205cfe03751481ebb4b3300c126a"
@ -2834,6 +2853,11 @@ es5-ext@^0.10.35, es5-ext@^0.10.46, es5-ext@^0.10.50, es5-ext@^0.10.51, es5-ext@
    es6-symbol "~3.1.3"
    next-tick "~1.0.0"

+es6-error@^4.1.1:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.1.1.tgz#9e3af407459deed47e9a91f9b885a84eb05c561d"
+  integrity sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==
+
 es6-iterator@^2.0.3, es6-iterator@~2.0.3:
  version "2.0.3"
  resolved "https://registry.yarnpkg.com/es6-iterator/-/es6-iterator-2.0.3.tgz#a7de889141a05a94b0854403b2d0a0fbfa98f3b7"
@ -3267,6 +3291,11 @@ fast-safe-stringify@2.0.7, fast-safe-stringify@^2.0.7:
  resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz#124aa885899261f68aedb42a7c080de9da608743"
  integrity sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==

+fast-xml-parser@^3.17.5:
+  version "3.18.0"
+  resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-3.18.0.tgz#b77f4a494cd64e6f44aadfa68fbde30cd922b2df"
+  integrity sha512-tRrwShhppv0K5GKEtuVs92W0VGDaVltZAwtHbpjNF+JOT7cjIFySBGTEOmdBslXYyWYaZwEX/g4Su8ZeKg0LKQ==
+
 fastq@^1.6.0:
  version "1.10.1"
  resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.10.1.tgz#8b8f2ac8bf3632d67afcd65dac248d5fdc45385e"
@ -4653,6 +4682,11 @@ json-stable-stringify-without-jsonify@^1.0.1:
  resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651"
  integrity sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=

+json-stream@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/json-stream/-/json-stream-1.0.0.tgz#1a3854e28d2bbeeab31cc7ddf683d2ddc5652708"
+  integrity sha1-GjhU4o0rvuqzHMfd9oPS3cVlJwg=
+
 json-stringify-safe@~5.0.1:
  version "5.0.1"
  resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
@ -4977,6 +5011,11 @@ mime-db@1.45.0:
  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.45.0.tgz#cceeda21ccd7c3a745eba2decd55d4b73e7879ea"
  integrity sha512-CkqLUxUk15hofLoLyljJSrukZi8mAtgd+yE5uO4tqRZsdsAJKv0O+rFMhVDRJgozy+yG6md5KwuXhD4ocIoP+w==

+mime-db@1.46.0:
+  version "1.46.0"
+  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.46.0.tgz#6267748a7f799594de3cbc8cde91def349661cee"
+  integrity sha512-svXaP8UQRZ5K7or+ZmfNhg2xX3yKDMUzqadsSqi4NCH/KomcH75MAMYAGVlvXn4+b/xOPhS3I2uHKRUzvjY7BQ==
+
 mime-types@^2.1.12, mime-types@^2.1.27, mime-types@~2.1.19, mime-types@~2.1.24:
  version "2.1.28"
  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.28.tgz#1160c4757eab2c5363888e005273ecf79d2a0ecd"
@ -4984,6 +5023,13 @@ mime-types@^2.1.12, mime-types@^2.1.27, mime-types@~2.1.19, mime-types@~2.1.24:
  dependencies:
    mime-db "1.45.0"

+mime-types@^2.1.14:
+  version "2.1.29"
+  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.29.tgz#1d4ab77da64b91f5f72489df29236563754bb1b2"
+  integrity sha512-Y/jMt/S5sR9OaqteJtslsFZKWOIIqMACsJSiHghlCAyhf7jfVYjKBmLiX8OgpWeW+fjJ2b+Az69aPFPkUOY6xQ==
+  dependencies:
+    mime-db "1.46.0"
+
 mime@1.6.0:
  version "1.6.0"
  resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
@ -5011,6 +5057,24 @@ minimist@1.2.5, minimist@^1.1.1, minimist@^1.2.0, minimist@^1.2.5:
  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"
  integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==

+minio@^7.0.18:
+  version "7.0.18"
+  resolved "https://registry.yarnpkg.com/minio/-/minio-7.0.18.tgz#a2a6dae52a4dde9e35ed47cdf2accc21df4a512d"
+  integrity sha512-jVRjkw8A5Spf+ETY5OXQUcQckHriuUA3u2+MAcX36btLT8EytlOVivxIseXvyFf9cNn3dy5w1F1UyjMvHU+nqg==
+  dependencies:
+    async "^3.1.0"
+    block-stream2 "^2.0.0"
+    es6-error "^4.1.1"
+    fast-xml-parser "^3.17.5"
+    json-stream "^1.0.0"
+    lodash "^4.17.20"
+    mime-types "^2.1.14"
+    mkdirp "^0.5.1"
+    querystring "0.2.0"
+    through2 "^3.0.1"
+    xml "^1.0.0"
+    xml2js "^0.4.15"
+
 minipass@^2.6.0, minipass@^2.8.6, minipass@^2.9.0:
  version "2.9.0"
  resolved "https://registry.yarnpkg.com/minipass/-/minipass-2.9.0.tgz#e713762e7d3e32fed803115cf93e04bca9fcc9a6"
@ -5888,6 +5952,11 @@ qs@~6.5.2:
  resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
  integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==

+querystring@0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
+  integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=
+
 queue-microtask@^1.2.2:
  version "1.2.2"
  resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.2.tgz#abf64491e6ecf0f38a6502403d4cda04f372dfd3"
@ -5991,6 +6060,15 @@ readable-stream@1.1.x:
    isarray "0.0.1"
    string_decoder "~0.10.x"

+"readable-stream@2 || 3", readable-stream@^3.4.0, readable-stream@^3.6.0:
+  version "3.6.0"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
+  integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
+  dependencies:
+    inherits "^2.0.3"
+    string_decoder "^1.1.1"
+    util-deprecate "^1.0.1"
+
 readable-stream@^2.0.1, readable-stream@^2.0.6, readable-stream@^2.2.2:
  version "2.3.7"
  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
@ -6004,15 +6082,6 @@ readable-stream@^2.0.1, readable-stream@^2.0.6, readable-stream@^2.2.2:
    string_decoder "~1.1.1"
    util-deprecate "~1.0.1"

-readable-stream@^3.4.0, readable-stream@^3.6.0:
-  version "3.6.0"
-  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
-  integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
-  dependencies:
-    inherits "^2.0.3"
-    string_decoder "^1.1.1"
-    util-deprecate "^1.0.1"
-
 readable-web-to-node-stream@^3.0.0:
  version "3.0.1"
  resolved "https://registry.yarnpkg.com/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.1.tgz#3f619b1bc5dd73a4cfe5c5f9b4f6faba55dff845"
@ -6936,6 +7005,14 @@ throat@^5.0.0:
  resolved "https://registry.yarnpkg.com/throat/-/throat-5.0.0.tgz#c5199235803aad18754a667d659b5e72ce16764b"
  integrity sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA==

+through2@^3.0.1:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/through2/-/through2-3.0.2.tgz#99f88931cfc761ec7678b41d5d7336b5b6a07bf4"
+  integrity sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==
+  dependencies:
+    inherits "^2.0.4"
+    readable-stream "2 || 3"
+
 through@^2.3.6:
  version "2.3.8"
  resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
@ -7552,7 +7629,7 @@ xml-name-validator@^3.0.0:
  resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"
  integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==

-xml2js@^0.4.19, xml2js@^0.4.23:
+xml2js@^0.4.15, xml2js@^0.4.19, xml2js@^0.4.23:
  version "0.4.23"
  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
  integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==
@ -7560,6 +7637,11 @@ xml2js@^0.4.19, xml2js@^0.4.23:
    sax ">=0.6.0"
    xmlbuilder "~11.0.0"

+xml@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"
+  integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=
+
 xmlbuilder@~11.0.0:
  version "11.0.1"
  resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"