mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-21 20:47:08 -05:00
Delete files from a directory in parallel
This commit is contained in:
parent
183cb0179a
commit
edf1ce1f7e
4 changed files with 28 additions and 3 deletions
|
@ -5,6 +5,7 @@ const Stream = require('stream')
|
|||
const { Storage } = require('@google-cloud/storage')
|
||||
const { callbackify } = require('util')
|
||||
const { WriteError, ReadError, NotFoundError } = require('./Errors')
|
||||
const asyncPool = require('tiny-async-pool')
|
||||
const PersistorHelper = require('./PersistorHelper')
|
||||
|
||||
const pipeline = promisify(Stream.pipeline)
|
||||
|
@ -215,9 +216,9 @@ async function deleteDirectory(bucketName, key) {
|
|||
.bucket(bucketName)
|
||||
.getFiles({ directory: key })
|
||||
|
||||
for (const file of files) {
|
||||
await asyncPool(50, files, async file => {
|
||||
await deleteFile(bucketName, file.name)
|
||||
}
|
||||
})
|
||||
} catch (err) {
|
||||
const error = PersistorHelper.wrapError(
|
||||
err,
|
||||
|
|
21
services/filestore/package-lock.json
generated
21
services/filestore/package-lock.json
generated
|
@ -5582,6 +5582,22 @@
|
|||
"readable-stream": "2 || 3"
|
||||
}
|
||||
},
|
||||
"tiny-async-pool": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-1.1.0.tgz",
|
||||
"integrity": "sha512-jIglyHF/9QdCC3662m/UMVADE6SlocBDpXdFLMZyiAfrw8MSG1pml7lwRtBMT6L/z4dddAxfzw2lpW2Vm42fyQ==",
|
||||
"requires": {
|
||||
"semver": "^5.5.0",
|
||||
"yaassertion": "^1.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"semver": {
|
||||
"version": "5.7.1",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
|
||||
"integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"tmp": {
|
||||
"version": "0.0.33",
|
||||
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
|
||||
|
@ -5980,6 +5996,11 @@
|
|||
"integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
|
||||
"dev": true
|
||||
},
|
||||
"yaassertion": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/yaassertion/-/yaassertion-1.0.2.tgz",
|
||||
"integrity": "sha512-sBoJBg5vTr3lOpRX0yFD+tz7wv/l2UPMFthag4HGTMPrypBRKerjjS8jiEnNMjcAEtPXjbHiKE0UwRR1W1GXBg=="
|
||||
},
|
||||
"yallist": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
|
||||
|
|
|
@ -36,7 +36,8 @@
|
|||
"rimraf": "2.2.8",
|
||||
"settings-sharelatex": "^1.1.0",
|
||||
"stream-buffers": "~0.2.5",
|
||||
"stream-meter": "^1.0.4"
|
||||
"stream-meter": "^1.0.4",
|
||||
"tiny-async-pool": "^1.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"babel-eslint": "^10.0.3",
|
||||
|
|
|
@ -4,6 +4,7 @@ const { expect } = chai
|
|||
const modulePath = '../../../app/js/GcsPersistor.js'
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
const { ObjectId } = require('mongodb')
|
||||
const asyncPool = require('tiny-async-pool')
|
||||
|
||||
const Errors = require('../../../app/js/Errors')
|
||||
|
||||
|
@ -135,6 +136,7 @@ describe('GcsPersistorTests', function() {
|
|||
'@google-cloud/storage': { Storage },
|
||||
'settings-sharelatex': settings,
|
||||
'logger-sharelatex': Logger,
|
||||
'tiny-async-pool': asyncPool,
|
||||
'./Errors': Errors,
|
||||
fs: Fs,
|
||||
'stream-meter': Meter,
|
||||
|
|
Loading…
Reference in a new issue