github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Delete files from a directory in parallel

Browse source
This commit is contained in:
Simon Detheridge 2020-03-14 14:02:58 +00:00
parent 183cb0179a
commit edf1ce1f7e
4 changed files with 28 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
services/filestore/app/js/GcsPersistor.js
View file

@ -5,6 +5,7 @@ const Stream = require('stream')
const { Storage } = require('@google-cloud/storage') const { Storage } = require('@google-cloud/storage')
const { callbackify } = require('util') const { callbackify } = require('util')
const { WriteError, ReadError, NotFoundError } = require('./Errors') const { WriteError, ReadError, NotFoundError } = require('./Errors')
const asyncPool = require('tiny-async-pool')
const PersistorHelper = require('./PersistorHelper') const PersistorHelper = require('./PersistorHelper')
const pipeline = promisify(Stream.pipeline) const pipeline = promisify(Stream.pipeline)
@ -215,9 +216,9 @@ async function deleteDirectory(bucketName, key) {
.bucket(bucketName) .bucket(bucketName)
.getFiles({ directory: key }) .getFiles({ directory: key })
for (const file of files) { await asyncPool(50, files, async file => {
await deleteFile(bucketName, file.name) await deleteFile(bucketName, file.name)
} })
} catch (err) { } catch (err) {
const error = PersistorHelper.wrapError( const error = PersistorHelper.wrapError(
err, err,

21
services/filestore/package-lock.json generated
View file

@ -5582,6 +5582,22 @@
"readable-stream": "2 || 3" "readable-stream": "2 || 3"
} }
}, },
"tiny-async-pool": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-1.1.0.tgz",
"integrity": "sha512-jIglyHF/9QdCC3662m/UMVADE6SlocBDpXdFLMZyiAfrw8MSG1pml7lwRtBMT6L/z4dddAxfzw2lpW2Vm42fyQ==",
"requires": {
"semver": "^5.5.0",
"yaassertion": "^1.0.0"
},
"dependencies": {
"semver": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
"integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
}
}
},
"tmp": { "tmp": {
"version": "0.0.33", "version": "0.0.33",
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
@ -5980,6 +5996,11 @@
"integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==", "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
"dev": true "dev": true
}, },
"yaassertion": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/yaassertion/-/yaassertion-1.0.2.tgz",
"integrity": "sha512-sBoJBg5vTr3lOpRX0yFD+tz7wv/l2UPMFthag4HGTMPrypBRKerjjS8jiEnNMjcAEtPXjbHiKE0UwRR1W1GXBg=="
},
"yallist": { "yallist": {
"version": "3.1.1", "version": "3.1.1",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",

3
services/filestore/package.json
View file

@ -36,7 +36,8 @@
"rimraf": "2.2.8", "rimraf": "2.2.8",
"settings-sharelatex": "^1.1.0", "settings-sharelatex": "^1.1.0",
"stream-buffers": "~0.2.5", "stream-buffers": "~0.2.5",
"stream-meter": "^1.0.4" "stream-meter": "^1.0.4",
"tiny-async-pool": "^1.1.0"
}, },
"devDependencies": { "devDependencies": {
"babel-eslint": "^10.0.3", "babel-eslint": "^10.0.3",

2
services/filestore/test/unit/js/GcsPersistorTests.js
View file

@ -4,6 +4,7 @@ const { expect } = chai
const modulePath = '../../../app/js/GcsPersistor.js' const modulePath = '../../../app/js/GcsPersistor.js'
const SandboxedModule = require('sandboxed-module') const SandboxedModule = require('sandboxed-module')
const { ObjectId } = require('mongodb') const { ObjectId } = require('mongodb')
const asyncPool = require('tiny-async-pool')
const Errors = require('../../../app/js/Errors') const Errors = require('../../../app/js/Errors')
@ -135,6 +136,7 @@ describe('GcsPersistorTests', function() {
'@google-cloud/storage': { Storage }, '@google-cloud/storage': { Storage },
'settings-sharelatex': settings, 'settings-sharelatex': settings,
'logger-sharelatex': Logger, 'logger-sharelatex': Logger,
'tiny-async-pool': asyncPool,
'./Errors': Errors, './Errors': Errors,
fs: Fs, fs: Fs,
'stream-meter': Meter, 'stream-meter': Meter,