Delete files from a directory in parallel

This commit is contained in:
Simon Detheridge 2020-03-14 14:02:58 +00:00
parent 183cb0179a
commit edf1ce1f7e
4 changed files with 28 additions and 3 deletions

View file

@ -5,6 +5,7 @@ const Stream = require('stream')
const { Storage } = require('@google-cloud/storage')
const { callbackify } = require('util')
const { WriteError, ReadError, NotFoundError } = require('./Errors')
const asyncPool = require('tiny-async-pool')
const PersistorHelper = require('./PersistorHelper')
const pipeline = promisify(Stream.pipeline)
@ -215,9 +216,9 @@ async function deleteDirectory(bucketName, key) {
.bucket(bucketName)
.getFiles({ directory: key })
for (const file of files) {
await asyncPool(50, files, async file => {
await deleteFile(bucketName, file.name)
}
})
} catch (err) {
const error = PersistorHelper.wrapError(
err,

View file

@ -5582,6 +5582,22 @@
"readable-stream": "2 || 3"
}
},
"tiny-async-pool": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-1.1.0.tgz",
"integrity": "sha512-jIglyHF/9QdCC3662m/UMVADE6SlocBDpXdFLMZyiAfrw8MSG1pml7lwRtBMT6L/z4dddAxfzw2lpW2Vm42fyQ==",
"requires": {
"semver": "^5.5.0",
"yaassertion": "^1.0.0"
},
"dependencies": {
"semver": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
"integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
}
}
},
"tmp": {
"version": "0.0.33",
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
@ -5980,6 +5996,11 @@
"integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
"dev": true
},
"yaassertion": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/yaassertion/-/yaassertion-1.0.2.tgz",
"integrity": "sha512-sBoJBg5vTr3lOpRX0yFD+tz7wv/l2UPMFthag4HGTMPrypBRKerjjS8jiEnNMjcAEtPXjbHiKE0UwRR1W1GXBg=="
},
"yallist": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",

View file

@ -36,7 +36,8 @@
"rimraf": "2.2.8",
"settings-sharelatex": "^1.1.0",
"stream-buffers": "~0.2.5",
"stream-meter": "^1.0.4"
"stream-meter": "^1.0.4",
"tiny-async-pool": "^1.1.0"
},
"devDependencies": {
"babel-eslint": "^10.0.3",

View file

@ -4,6 +4,7 @@ const { expect } = chai
const modulePath = '../../../app/js/GcsPersistor.js'
const SandboxedModule = require('sandboxed-module')
const { ObjectId } = require('mongodb')
const asyncPool = require('tiny-async-pool')
const Errors = require('../../../app/js/Errors')
@ -135,6 +136,7 @@ describe('GcsPersistorTests', function() {
'@google-cloud/storage': { Storage },
'settings-sharelatex': settings,
'logger-sharelatex': Logger,
'tiny-async-pool': asyncPool,
'./Errors': Errors,
fs: Fs,
'stream-meter': Meter,