From dde74f37ffead476dc4b8733c79bef8735d73dfe Mon Sep 17 00:00:00 2001 From: Philip Molares Date: Mon, 22 Feb 2021 22:33:56 +0100 Subject: [PATCH] AzureBackend: Add Azure MediaBackend Add @azure/storage-blob dependency This is the relevant part of the official azure sdk. Signed-off-by: Philip Molares --- package.json | 1 + src/media/backends/azure-backend.ts | 85 ++++++++++++++++ src/media/media.module.ts | 3 +- src/media/media.service.ts | 5 + yarn.lock | 149 ++++++++++++++++++++++++++-- 5 files changed, 236 insertions(+), 7 deletions(-) create mode 100644 src/media/backends/azure-backend.ts diff --git a/package.json b/package.json index a7501a690..d3d165d5e 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "test:e2e:cov": "jest --config jest-e2e.json --coverage" }, "dependencies": { + "@azure/storage-blob": "^12.4.1", "@nestjs/common": "7.6.13", "@nestjs/config": "0.6.3", "@nestjs/core": "7.6.13", diff --git a/src/media/backends/azure-backend.ts b/src/media/backends/azure-backend.ts new file mode 100644 index 000000000..2ed8f0986 --- /dev/null +++ b/src/media/backends/azure-backend.ts @@ -0,0 +1,85 @@ +/* + * SPDX-FileCopyrightText: 2021 The HedgeDoc developers (see AUTHORS file) + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import mediaConfiguration, { MediaConfig } from '../../config/media.config'; +import { ConsoleLoggerService } from '../../logger/console-logger.service'; +import { MediaBackend } from '../media-backend.interface'; +import { BackendData } from '../media-upload.entity'; +import { + BlobServiceClient, + BlockBlobClient, + ContainerClient, +} from '@azure/storage-blob'; +import { BackendType } from './backend-type.enum'; +import { MediaBackendError } from '../../errors/errors'; + +@Injectable() +export class AzureBackend implements MediaBackend { + private config: MediaConfig['backend']['azure']; + private client: ContainerClient; + + constructor( + private readonly logger: ConsoleLoggerService, + @Inject(mediaConfiguration.KEY) + private mediaConfig: MediaConfig, + ) { + this.logger.setContext(AzureBackend.name); + this.config = mediaConfig.backend.azure; + if (mediaConfig.backend.use === BackendType.AZURE) { + // only create the client if the backend is configured to azure + const blobServiceClient = BlobServiceClient.fromConnectionString( + this.config.connectionString, + ); + this.client = blobServiceClient.getContainerClient(this.config.container); + } + } + + async saveFile( + buffer: Buffer, + fileName: string, + ): Promise<[string, BackendData]> { + const blockBlobClient: BlockBlobClient = this.client.getBlockBlobClient( + fileName, + ); + try { + await blockBlobClient.upload(buffer, buffer.length); + const url = this.getUrl(fileName); + this.logger.log(`Uploaded ${url}`, 'saveFile'); + return [url, null]; + } catch (e) { + this.logger.error( + `error: ${(e as Error).message}`, + (e as Error).stack, + 'saveFile', + ); + throw new MediaBackendError(`Could not save '${fileName}' on Azure`); + } + } + + async deleteFile(fileName: string, _: BackendData): Promise { + const blockBlobClient: BlockBlobClient = this.client.getBlockBlobClient( + fileName, + ); + try { + await blockBlobClient.delete(); + const url = this.getUrl(fileName); + this.logger.log(`Deleted ${url}`, 'deleteFile'); + return; + } catch (e) { + this.logger.error( + `error: ${(e as Error).message}`, + (e as Error).stack, + 'deleteFile', + ); + throw new MediaBackendError(`Could not delete '${fileName}' on Azure`); + } + } + + private getUrl(fileName: string): string { + return `${this.client.url}/${fileName}`; + } +} diff --git a/src/media/media.module.ts b/src/media/media.module.ts index 26f186814..901f56e6d 100644 --- a/src/media/media.module.ts +++ b/src/media/media.module.ts @@ -14,6 +14,7 @@ import { FilesystemBackend } from './backends/filesystem-backend'; import { MediaUpload } from './media-upload.entity'; import { MediaService } from './media.service'; import { ImgurBackend } from './backends/imgur-backend'; +import { AzureBackend } from './backends/azure-backend'; @Module({ imports: [ @@ -23,7 +24,7 @@ import { ImgurBackend } from './backends/imgur-backend'; LoggerModule, ConfigModule, ], - providers: [MediaService, FilesystemBackend, ImgurBackend], + providers: [MediaService, FilesystemBackend, AzureBackend, ImgurBackend], exports: [MediaService], }) export class MediaModule {} diff --git a/src/media/media.service.ts b/src/media/media.service.ts index e24fae0c8..58fe1c2c2 100644 --- a/src/media/media.service.ts +++ b/src/media/media.service.ts @@ -19,6 +19,7 @@ import { FilesystemBackend } from './backends/filesystem-backend'; import { MediaBackend } from './media-backend.interface'; import { MediaUpload } from './media-upload.entity'; import { MediaUploadUrlDto } from './media-upload-url.dto'; +import { AzureBackend } from './backends/azure-backend'; import { ImgurBackend } from './backends/imgur-backend'; @Injectable() @@ -159,6 +160,8 @@ export class MediaService { switch (this.mediaConfig.backend.use) { case 'filesystem': return BackendType.FILESYSTEM; + case 'azure': + return BackendType.AZURE; case 'imgur': return BackendType.IMGUR; } @@ -168,6 +171,8 @@ export class MediaService { switch (type) { case BackendType.FILESYSTEM: return this.moduleRef.get(FilesystemBackend); + case BackendType.AZURE: + return this.moduleRef.get(AzureBackend); case BackendType.IMGUR: return this.moduleRef.get(ImgurBackend); } diff --git a/yarn.lock b/yarn.lock index 5ace5a1f2..93d486e0f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -55,6 +55,95 @@ ora "5.3.0" rxjs "6.6.3" +"@azure/abort-controller@^1.0.0": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@azure/abort-controller/-/abort-controller-1.0.2.tgz#822405c966b2aec16fb62c1b19d37eaccf231995" + integrity sha512-XUyTo+bcyxHEf+jlN2MXA7YU9nxVehaubngHV1MIZZaqYmZqykkoeAz/JMMEeR7t3TcyDwbFa3Zw8BZywmIx4g== + dependencies: + tslib "^2.0.0" + +"@azure/core-asynciterator-polyfill@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@azure/core-asynciterator-polyfill/-/core-asynciterator-polyfill-1.0.0.tgz#dcccebb88406e5c76e0e1d52e8cc4c43a68b3ee7" + integrity sha512-kmv8CGrPfN9SwMwrkiBK9VTQYxdFQEGe0BmQk+M8io56P9KNzpAxcWE/1fxJj7uouwN4kXF0BHW8DNlgx+wtCg== + +"@azure/core-auth@^1.1.3": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@azure/core-auth/-/core-auth-1.2.0.tgz#a5a181164e99f8446a3ccf9039345ddc9bb63bb9" + integrity sha512-KUl+Nwn/Sm6Lw5d3U90m1jZfNSL087SPcqHLxwn2T6PupNKmcgsEbDjHB25gDvHO4h7pBsTlrdJAY7dz+Qk8GA== + dependencies: + "@azure/abort-controller" "^1.0.0" + tslib "^2.0.0" + +"@azure/core-http@^1.2.0": + version "1.2.3" + resolved "https://registry.yarnpkg.com/@azure/core-http/-/core-http-1.2.3.tgz#b1e459f6705df1f8d09bf6582292891c04bcace1" + integrity sha512-g5C1zUJO5dehP2Riv+vy9iCYoS1UwKnZsBVCzanScz9A83LbnXKpZDa9wie26G9dfXUhQoFZoFT8LYWhPKmwcg== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-auth" "^1.1.3" + "@azure/core-tracing" "1.0.0-preview.9" + "@azure/logger" "^1.0.0" + "@opentelemetry/api" "^0.10.2" + "@types/node-fetch" "^2.5.0" + "@types/tunnel" "^0.0.1" + form-data "^3.0.0" + node-fetch "^2.6.0" + process "^0.11.10" + tough-cookie "^4.0.0" + tslib "^2.0.0" + tunnel "^0.0.6" + uuid "^8.3.0" + xml2js "^0.4.19" + +"@azure/core-lro@^1.0.2": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@azure/core-lro/-/core-lro-1.0.3.tgz#1ddfb4ecdb81ce87b5f5d972ffe2acbbc46e524e" + integrity sha512-Py2crJ84qx1rXkzIwfKw5Ni4WJuzVU7KAF6i1yP3ce8fbynUeu8eEWS4JGtSQgU7xv02G55iPDROifmSDbxeHA== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-http" "^1.2.0" + events "^3.0.0" + tslib "^2.0.0" + +"@azure/core-paging@^1.1.1": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@azure/core-paging/-/core-paging-1.1.3.tgz#3587c9898a0530cacb64bab216d7318468aa5efc" + integrity sha512-his7Ah40ThEYORSpIAwuh6B8wkGwO/zG7gqVtmSE4WAJ46e36zUDXTKReUCLBDc6HmjjApQQxxcRFy5FruG79A== + dependencies: + "@azure/core-asynciterator-polyfill" "^1.0.0" + +"@azure/core-tracing@1.0.0-preview.9": + version "1.0.0-preview.9" + resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.0.0-preview.9.tgz#84f3b85572013f9d9b85e1e5d89787aa180787eb" + integrity sha512-zczolCLJ5QG42AEPQ+Qg9SRYNUyB+yZ5dzof4YEc+dyWczO9G2sBqbAjLB7IqrsdHN2apkiB2oXeDKCsq48jug== + dependencies: + "@opencensus/web-types" "0.0.7" + "@opentelemetry/api" "^0.10.2" + tslib "^2.0.0" + +"@azure/logger@^1.0.0": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@azure/logger/-/logger-1.0.1.tgz#19b333203d1b2931353d8879e814b64a7274837a" + integrity sha512-QYQeaJ+A5x6aMNu8BG5qdsVBnYBop9UMwgUvGihSjf1PdZZXB+c/oMdM2ajKwzobLBh9e9QuMQkN9iL+IxLBLA== + dependencies: + tslib "^2.0.0" + +"@azure/storage-blob@^12.4.1": + version "12.4.1" + resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.4.1.tgz#f2cc4a36a0df770b7b918ba89e72c02d72afbf2f" + integrity sha512-RH6ru8LbnCC+m1rlVLon6mYUXdHsTcyUXFCJAWRQQM7p0XOwVKPS+UiVk2tZXfvMWd3q/qT/meOrEbHEcp/c4g== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-http" "^1.2.0" + "@azure/core-lro" "^1.0.2" + "@azure/core-paging" "^1.1.1" + "@azure/core-tracing" "1.0.0-preview.9" + "@azure/logger" "^1.0.0" + "@opentelemetry/api" "^0.10.2" + events "^3.0.0" + tslib "^2.0.0" + "@babel/code-frame@7.12.11": version "7.12.11" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.11.tgz#f4ad435aa263db935b8f10f2c552d23fb716a63f" @@ -734,6 +823,23 @@ consola "^2.15.0" node-fetch "^2.6.1" +"@opencensus/web-types@0.0.7": + version "0.0.7" + resolved "https://registry.yarnpkg.com/@opencensus/web-types/-/web-types-0.0.7.tgz#4426de1fe5aa8f624db395d2152b902874f0570a" + integrity sha512-xB+w7ZDAu3YBzqH44rCmG9/RlrOmFuDPt/bpf17eJr8eZSrLt7nc7LnWdxM9Mmoj/YKMHpxRg28txu3TcpiL+g== + +"@opentelemetry/api@^0.10.2": + version "0.10.2" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-0.10.2.tgz#9647b881f3e1654089ff7ea59d587b2d35060654" + integrity sha512-GtpMGd6vkzDMYcpu2t9LlhEgMy/SzBwRnz48EejlRArYqZzqSzAsKmegUK7zHgl+EOIaK9mKHhnRaQu3qw20cA== + dependencies: + "@opentelemetry/context-base" "^0.10.2" + +"@opentelemetry/context-base@^0.10.2": + version "0.10.2" + resolved "https://registry.yarnpkg.com/@opentelemetry/context-base/-/context-base-0.10.2.tgz#55bea904b2b91aa8a8675df9eaba5961bddb1def" + integrity sha512-hZNKjKOYsckoOEgBziGMnBcX0M7EtstnCmwz5jZUOUYwlZ+/xxX6z3jPu1XVO2Jivk0eLfuP9GP+vFD49CMetw== + "@schematics/schematics@0.1102.0": version "0.1102.0" resolved "https://registry.yarnpkg.com/@schematics/schematics/-/schematics-0.1102.0.tgz#f47d8fb6d9029ffe646ab9680ac5b2833322df1e" @@ -1033,7 +1139,7 @@ resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw== -"@types/node-fetch@^2.5.8": +"@types/node-fetch@^2.5.0", "@types/node-fetch@^2.5.8": version "2.5.8" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.8.tgz#e199c835d234c7eb0846f6618012e558544ee2fb" integrity sha512-fbjI6ja0N5ZA8TV53RUqzsKNkl9fv8Oj3T7zxW7FGv1GSH7gwJaNF8dzCjrqKaxKeUpTz4yT1DaJFq/omNpGfw== @@ -1143,6 +1249,13 @@ resolved "https://registry.yarnpkg.com/@types/tapable/-/tapable-1.0.6.tgz#a9ca4b70a18b270ccb2bc0aaafefd1d486b7ea74" integrity sha512-W+bw9ds02rAQaMvaLYxAbJ6cvguW/iJXNT6lTssS1ps6QdrMKttqEAMEG/b5CR8TZl3/L7/lH0ZV5nNR1LXikA== +"@types/tunnel@^0.0.1": + version "0.0.1" + resolved "https://registry.yarnpkg.com/@types/tunnel/-/tunnel-0.0.1.tgz#0d72774768b73df26f25df9184273a42da72b19c" + integrity sha512-AOqu6bQu5MSWwYvehMXLukFHnupHrpZ8nvgae5Ggie9UwzDR1CCwoXgSSWNZJuyOlCdfdsWMA5F2LlmvyoTv8A== + dependencies: + "@types/node" "*" + "@types/uglify-js@*": version "3.12.0" resolved "https://registry.yarnpkg.com/@types/uglify-js/-/uglify-js-3.12.0.tgz#2bb061c269441620d46b946350c8f16d52ef37c5" @@ -2944,7 +3057,7 @@ event-emitter@^0.3.5: d "1" es5-ext "~0.10.14" -events@^3.2.0: +events@^3.0.0, events@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/events/-/events-3.2.0.tgz#93b87c18f8efcd4202a461aec4dfc0556b639379" integrity sha512-/46HWwbfCX2xTawVfkKLGxMifJYQBWMwY1mjywRtb4c9x8l5NP3KoJtnIOiL1hfdRkIuYhETxQlo62IF8tcnlg== @@ -5086,7 +5199,7 @@ node-emoji@1.10.0: dependencies: lodash.toarray "^4.4.0" -node-fetch@^2.6.1: +node-fetch@^2.6.0, node-fetch@^2.6.1: version "2.6.1" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== @@ -5711,6 +5824,11 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== +process@^0.11.10: + version "0.11.10" + resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" + integrity sha1-czIwDoQBYb2j5podHZGn1LwW8YI= + progress@^2.0.0: version "2.0.3" resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" @@ -5737,7 +5855,7 @@ prr@~1.0.1: resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" integrity sha1-0/wRS6BplaRexok/SEzrHXj19HY= -psl@^1.1.28: +psl@^1.1.28, psl@^1.1.33: version "1.8.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== @@ -6910,6 +7028,15 @@ tough-cookie@^3.0.1: psl "^1.1.28" punycode "^2.1.1" +tough-cookie@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.0.0.tgz#d822234eeca882f991f0f908824ad2622ddbece4" + integrity sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg== + dependencies: + psl "^1.1.33" + punycode "^2.1.1" + universalify "^0.1.2" + tr46@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.0.2.tgz#03273586def1595ae08fedb38d7733cee91d2479" @@ -6981,7 +7108,7 @@ tsconfig-paths@3.9.0, tsconfig-paths@^3.4.0, tsconfig-paths@^3.9.0: minimist "^1.2.0" strip-bom "^3.0.0" -tslib@2.1.0: +tslib@2.1.0, tslib@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.1.0.tgz#da60860f1c2ecaa5703ab7d39bc05b6bf988b97a" integrity sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A== @@ -7005,6 +7132,11 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" +tunnel@^0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" + integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg== + tweetnacl@^0.14.3, tweetnacl@~0.14.0: version "0.14.5" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" @@ -7123,6 +7255,11 @@ union-value@^1.0.0: is-extendable "^0.1.1" set-value "^2.0.1" +universalify@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" + integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== + universalify@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717" @@ -7415,7 +7552,7 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== -xml2js@^0.4.23: +xml2js@^0.4.19, xml2js@^0.4.23: version "0.4.23" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==