github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

[misc] bump the dev-env to 3.3.2

Browse source
This commit is contained in:
Jakob Ackermann 2020-08-10 17:01:12 +01:00
parent de81ab2f70
commit f0551307d2
28 changed files with 383 additions and 497 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
services/filestore/.github/dependabot.yml vendored Normal file
View file

@ -0,0 +1,17 @@
version: 2
updates:
- package-ecosystem: "npm"
directory: "/"
schedule:
interval: "daily"
pull-request-branch-name:
# Separate sections of the branch name with a hyphen
# Docker images use the branch name and do not support slashes in tags
# https://github.com/overleaf/google-ops/issues/822
# https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#pull-request-branch-nameseparator
separator: "-"
# Block informal upgrades -- security upgrades use a separate queue.
# https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#open-pull-requests-limit
open-pull-requests-limit: 0

3
services/filestore/.gitignore vendored
View file

@ -49,3 +49,6 @@ template_files/*
/log.json /log.json
hash_folder hash_folder
# managed by dev-environment$ bin/update_build_scripts
.npmrc

2
services/filestore/Dockerfile
View file

@ -17,8 +17,6 @@ RUN npm ci --quiet
COPY . /app COPY . /app
FROM base FROM base
COPY --from=app /app /app COPY --from=app /app /app

131
services/filestore/Jenkinsfile vendored
View file

@ -1,131 +0,0 @@
String cron_string = BRANCH_NAME == "master" ? "@daily" : ""
pipeline {
agent any
environment {
GIT_PROJECT = "filestore"
JENKINS_WORKFLOW = "filestore-sharelatex"
TARGET_URL = "${env.JENKINS_URL}blue/organizations/jenkins/${JENKINS_WORKFLOW}/detail/$BRANCH_NAME/$BUILD_NUMBER/pipeline"
GIT_API_URL = "https://api.github.com/repos/overleaf/${GIT_PROJECT}/statuses/$GIT_COMMIT"
}
triggers {
pollSCM('* * * * *')
cron(cron_string)
}
stages {
stage('Install') {
steps {
withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
sh "curl $GIT_API_URL \
--data '{ \
\"state\" : \"pending\", \
\"target_url\": \"$TARGET_URL\", \
\"description\": \"Your build is underway\", \
\"context\": \"ci/jenkins\" }' \
-u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
}
}
}
stage('Build') {
steps {
sh 'make build'
}
}
stage('Linting') {
steps {
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make format'
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make lint'
}
}
stage('Unit Tests') {
steps {
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'
}
}
stage('Acceptance Tests') {
steps {
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance'
}
}
stage('Package and docker push') {
steps {
sh 'echo ${BUILD_NUMBER} > build_number.txt'
sh 'touch build.tar.gz' // Avoid tar warning about files changing during read
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make tar'
withCredentials([file(credentialsId: 'gcr.io_overleaf-ops', variable: 'DOCKER_REPO_KEY_PATH')]) {
sh 'docker login -u _json_key --password-stdin https://gcr.io/overleaf-ops < ${DOCKER_REPO_KEY_PATH}'
}
sh 'DOCKER_REPO=gcr.io/overleaf-ops make publish'
sh 'docker logout https://gcr.io/overleaf-ops'
}
}
stage('Publish to s3') {
steps {
sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt'
withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
}
withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
// The deployment process uses this file to figure out the latest build
s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest")
}
}
}
}
post {
always {
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean'
sh 'make clean'
}
success {
withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
sh "curl $GIT_API_URL \
--data '{ \
\"state\" : \"success\", \
\"target_url\": \"$TARGET_URL\", \
\"description\": \"Your build succeeded!\", \
\"context\": \"ci/jenkins\" }' \
-u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
}
}
failure {
mail(from: "${EMAIL_ALERT_FROM}",
to: "${EMAIL_ALERT_TO}",
subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}",
body: "Build: ${BUILD_URL}")
withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
sh "curl $GIT_API_URL \
--data '{ \
\"state\" : \"failure\", \
\"target_url\": \"$TARGET_URL\", \
\"description\": \"Your build failed\", \
\"context\": \"ci/jenkins\" }' \
-u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
}
}
}
// The options directive is for configuration that applies to the whole job.
options {
// we'd like to make sure remove old builds, so we don't fill up our storage!
buildDiscarder(logRotator(numToKeepStr:'50'))
// And we'd really like to be sure that this build doesn't hang forever, so let's time it out after:
timeout(time: 30, unit: 'MINUTES')
}
}

6
services/filestore/Makefile
View file

@ -25,13 +25,13 @@ clean:
docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
format: format:
$(DOCKER_COMPOSE) run --rm test_unit npm run format $(DOCKER_COMPOSE) run --rm test_unit npm run --silent format
format_fix: format_fix:
$(DOCKER_COMPOSE) run --rm test_unit npm run format:fix $(DOCKER_COMPOSE) run --rm test_unit npm run --silent format:fix
lint: lint:
$(DOCKER_COMPOSE) run --rm test_unit npm run lint $(DOCKER_COMPOSE) run --rm test_unit npm run --silent lint
test: format lint test_unit test_acceptance test: format lint test_unit test_acceptance

8
services/filestore/app.js
View file

@ -28,7 +28,7 @@ if (Metrics.event_loop) {
Metrics.event_loop.monitor(logger) Metrics.event_loop.monitor(logger)
} }
app.use(function(req, res, next) { app.use(function (req, res, next) {
Metrics.inc('http-request') Metrics.inc('http-request')
next() next()
}) })
@ -127,7 +127,7 @@ app.get(
fileController.getFile fileController.getFile
) )
app.get('/status', function(req, res) { app.get('/status', function (req, res) {
res.send('filestore sharelatex up') res.send('filestore sharelatex up')
}) })
@ -140,7 +140,7 @@ const host = '0.0.0.0'
if (!module.parent) { if (!module.parent) {
// Called directly // Called directly
app.listen(port, host, error => { app.listen(port, host, (error) => {
if (error) { if (error) {
logger.error('Error starting Filestore', error) logger.error('Error starting Filestore', error)
throw error throw error
@ -153,7 +153,7 @@ process
.on('unhandledRejection', (reason, p) => { .on('unhandledRejection', (reason, p) => {
logger.err(reason, 'Unhandled Rejection at Promise', p) logger.err(reason, 'Unhandled Rejection at Promise', p)
}) })
.on('uncaughtException', err => { .on('uncaughtException', (err) => {
logger.err(err, 'Uncaught Exception thrown') logger.err(err, 'Uncaught Exception thrown')
process.exit(1) process.exit(1)
}) })

18
services/filestore/app/js/FileController.js
View file

@ -46,7 +46,7 @@ function getFile(req, res, next) {
} }
} }
FileHandler.getRedirectUrl(bucket, key, options, function(err, redirectUrl) { FileHandler.getRedirectUrl(bucket, key, options, function (err, redirectUrl) {
if (err) { if (err) {
metrics.inc('file_redirect_error') metrics.inc('file_redirect_error')
} }
@ -56,7 +56,7 @@ function getFile(req, res, next) {
return res.redirect(redirectUrl) return res.redirect(redirectUrl)
} }
FileHandler.getFile(bucket, key, options, function(err, fileStream) { FileHandler.getFile(bucket, key, options, function (err, fileStream) {
if (err) { if (err) {
if (err instanceof Errors.NotFoundError) { if (err instanceof Errors.NotFoundError) {
res.sendStatus(404) res.sendStatus(404)
@ -70,7 +70,7 @@ function getFile(req, res, next) {
return res.sendStatus(200).end() return res.sendStatus(200).end()
} }
pipeline(fileStream, res, err => { pipeline(fileStream, res, (err) => {
if (err && err.code === 'ERR_STREAM_PREMATURE_CLOSE') { if (err && err.code === 'ERR_STREAM_PREMATURE_CLOSE') {
res.end() res.end()
} else if (err) { } else if (err) {
@ -94,7 +94,7 @@ function getFileHead(req, res, next) {
req.requestLogger.setMessage('getting file size') req.requestLogger.setMessage('getting file size')
req.requestLogger.addFields({ key, bucket }) req.requestLogger.addFields({ key, bucket })
FileHandler.getFileSize(bucket, key, function(err, fileSize) { FileHandler.getFileSize(bucket, key, function (err, fileSize) {
if (err) { if (err) {
if (err instanceof Errors.NotFoundError) { if (err instanceof Errors.NotFoundError) {
res.sendStatus(404) res.sendStatus(404)
@ -115,7 +115,7 @@ function insertFile(req, res, next) {
req.requestLogger.setMessage('inserting file') req.requestLogger.setMessage('inserting file')
req.requestLogger.addFields({ key, bucket }) req.requestLogger.addFields({ key, bucket })
FileHandler.insertFile(bucket, key, req, function(err) { FileHandler.insertFile(bucket, key, req, function (err) {
if (err) { if (err) {
next(err) next(err)
} else { } else {
@ -140,7 +140,7 @@ function copyFile(req, res, next) {
PersistorManager.copyObject(bucket, `${oldProjectId}/${oldFileId}`, key) PersistorManager.copyObject(bucket, `${oldProjectId}/${oldFileId}`, key)
.then(() => res.sendStatus(200)) .then(() => res.sendStatus(200))
.catch(err => { .catch((err) => {
if (err) { if (err) {
if (err instanceof Errors.NotFoundError) { if (err instanceof Errors.NotFoundError) {
res.sendStatus(404) res.sendStatus(404)
@ -158,7 +158,7 @@ function deleteFile(req, res, next) {
req.requestLogger.addFields({ key, bucket }) req.requestLogger.addFields({ key, bucket })
req.requestLogger.setMessage('deleting file') req.requestLogger.setMessage('deleting file')
FileHandler.deleteFile(bucket, key, function(err) { FileHandler.deleteFile(bucket, key, function (err) {
if (err) { if (err) {
next(err) next(err)
} else { } else {
@ -174,7 +174,7 @@ function deleteProject(req, res, next) {
req.requestLogger.setMessage('deleting project') req.requestLogger.setMessage('deleting project')
req.requestLogger.addFields({ key, bucket }) req.requestLogger.addFields({ key, bucket })
FileHandler.deleteProject(bucket, key, function(err) { FileHandler.deleteProject(bucket, key, function (err) {
if (err) { if (err) {
if (err instanceof Errors.InvalidParametersError) { if (err instanceof Errors.InvalidParametersError) {
return res.sendStatus(400) return res.sendStatus(400)
@ -193,7 +193,7 @@ function directorySize(req, res, next) {
req.requestLogger.setMessage('getting project size') req.requestLogger.setMessage('getting project size')
req.requestLogger.addFields({ projectId, bucket }) req.requestLogger.addFields({ projectId, bucket })
FileHandler.getDirectorySize(bucket, projectId, function(err, size) { FileHandler.getDirectorySize(bucket, projectId, function (err, size) {
if (err) { if (err) {
return next(err) return next(err)
} }

6
services/filestore/app/js/FileHandler.js
View file

@ -143,8 +143,8 @@ async function _getConvertedFileAndCache(bucket, key, convertedKey, opts) {
// S3 provides eventual consistency for read-after-write."" // S3 provides eventual consistency for read-after-write.""
// https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel // https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel
const readStream = fs.createReadStream(convertedFsPath) const readStream = fs.createReadStream(convertedFsPath)
readStream.on('end', function() { readStream.on('end', function () {
LocalFileWriter.deleteFile(convertedFsPath, function() {}) LocalFileWriter.deleteFile(convertedFsPath, function () {})
}) })
return readStream return readStream
} }
@ -185,7 +185,7 @@ async function _convertFile(bucket, originalKey, opts) {
err err
) )
} }
LocalFileWriter.deleteFile(originalFsPath, function() {}) LocalFileWriter.deleteFile(originalFsPath, function () {})
return destPath return destPath
} }

2
services/filestore/app/js/HealthCheckController.js
View file

@ -62,7 +62,7 @@ module.exports = {
check(req, res, next) { check(req, res, next) {
Promise.all([checkCanGetFiles(), checkFileConvert()]) Promise.all([checkCanGetFiles(), checkFileConvert()])
.then(() => res.sendStatus(200)) .then(() => res.sendStatus(200))
.catch(err => { .catch((err) => {
next(err) next(err)
}) })
} }

7
services/filestore/app/js/RequestLogger.js
View file

@ -26,7 +26,7 @@ class RequestLogger {
// override the 'end' method to log and record metrics // override the 'end' method to log and record metrics
const end = res.end const end = res.end
res.end = function() { res.end = function () {
// apply the standard request 'end' method before logging and metrics // apply the standard request 'end' method before logging and metrics
end.apply(this, arguments) end.apply(this, arguments)
@ -38,10 +38,7 @@ class RequestLogger {
metrics.timing('http_request', responseTime, null, { metrics.timing('http_request', responseTime, null, {
method: req.method, method: req.method,
status_code: res.statusCode, status_code: res.statusCode,
path: routePath path: routePath.replace(/\//g, '_').replace(/:/g, '').slice(1)
.replace(/\//g, '_')
.replace(/:/g, '')
.slice(1)
}) })
} }

12
services/filestore/app/js/SafeExec.js
View file

@ -28,7 +28,7 @@ function safeExec(command, options, callback) {
let killTimer let killTimer
const cleanup = lodashOnce(function(err) { const cleanup = lodashOnce(function (err) {
if (killTimer) { if (killTimer) {
clearTimeout(killTimer) clearTimeout(killTimer)
} }
@ -36,7 +36,7 @@ function safeExec(command, options, callback) {
}) })
if (options.timeout) { if (options.timeout) {
killTimer = setTimeout(function() { killTimer = setTimeout(function () {
try { try {
// use negative process id to kill process group // use negative process id to kill process group
process.kill(-child.pid, options.killSignal || 'SIGTERM') process.kill(-child.pid, options.killSignal || 'SIGTERM')
@ -52,7 +52,7 @@ function safeExec(command, options, callback) {
}, options.timeout) }, options.timeout)
} }
child.on('close', function(code, signal) { child.on('close', function (code, signal) {
if (code || signal) { if (code || signal) {
return cleanup( return cleanup(
new FailedCommandError(command, code || signal, stdout, stderr) new FailedCommandError(command, code || signal, stdout, stderr)
@ -62,13 +62,13 @@ function safeExec(command, options, callback) {
cleanup() cleanup()
}) })
child.on('error', err => { child.on('error', (err) => {
cleanup(err) cleanup(err)
}) })
child.stdout.on('data', chunk => { child.stdout.on('data', (chunk) => {
stdout += chunk stdout += chunk
}) })
child.stderr.on('data', chunk => { child.stderr.on('data', (chunk) => {
stderr += chunk stderr += chunk
}) })
} }

4
services/filestore/buildscript.txt
View file

@ -1,11 +1,9 @@
filestore filestore
--acceptance-creds=
--data-dirs=uploads,user_files,template_files --data-dirs=uploads,user_files,template_files
--dependencies=s3,gcs --dependencies=s3,gcs
--docker-repos=gcr.io/overleaf-ops --docker-repos=gcr.io/overleaf-ops
--env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_BUCKET_NAME=fake_user_files,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake_template_files,AWS_S3_PUBLIC_FILES_BUCKET_NAME=fake_public_files --env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_BUCKET_NAME=fake_user_files,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake_template_files,AWS_S3_PUBLIC_FILES_BUCKET_NAME=fake_public_files
--env-pass-through= --env-pass-through=
--language=es
--node-version=12.18.0 --node-version=12.18.0
--public-repo=True --public-repo=True
--script-version=2.2.0 --script-version=3.3.2

23
services/filestore/docker-compose.ci.yml
View file

@ -11,6 +11,7 @@ services:
command: npm run test:unit:_run command: npm run test:unit:_run
environment: environment:
NODE_ENV: test NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
test_acceptance: test_acceptance:
@ -21,23 +22,22 @@ services:
REDIS_HOST: redis REDIS_HOST: redis
MONGO_HOST: mongo MONGO_HOST: mongo
POSTGRES_HOST: postgres POSTGRES_HOST: postgres
AWS_S3_ENDPOINT: http://s3:9090
AWS_S3_PATH_STYLE: 'true'
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake
GCS_API_ENDPOINT: gcs:9090
GCS_API_SCHEME: http
GCS_PROJECT_ID: fake
STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
MOCHA_GREP: ${MOCHA_GREP} MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
ENABLE_CONVERSIONS: "true" ENABLE_CONVERSIONS: "true"
USE_PROM_METRICS: "true" USE_PROM_METRICS: "true"
AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files
AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files
AWS_S3_PUBLIC_FILES_BUCKET_NAME: fake_public_files AWS_S3_PUBLIC_FILES_BUCKET_NAME: fake_public_files
AWS_S3_ENDPOINT: http://s3:9090
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake
AWS_S3_PATH_STYLE: 'true'
GCS_API_ENDPOINT: gcs:9090
GCS_API_SCHEME: http
GCS_USER_FILES_BUCKET_NAME: fake_userfiles
GCS_TEMPLATE_FILES_BUCKET_NAME: fake_templatefiles
GCS_PUBLIC_FILES_BUCKET_NAME: fake_publicfiles
STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
depends_on: depends_on:
s3: s3:
condition: service_healthy condition: service_healthy
@ -59,8 +59,7 @@ services:
context: test/acceptance/deps context: test/acceptance/deps
dockerfile: Dockerfile.s3mock dockerfile: Dockerfile.s3mock
environment: environment:
- initialBuckets=fake_user_files,fake_template_files,fake_public_files - initialBuckets=fake_user_files,fake_template_files,fake_public_files,bucket
gcs: gcs:
build: build:
context: test/acceptance/deps context: test/acceptance/deps

29
services/filestore/docker-compose.yml
View file

@ -15,7 +15,8 @@ services:
environment: environment:
MOCHA_GREP: ${MOCHA_GREP} MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test NODE_ENV: test
command: npm run test:unit NODE_OPTIONS: "--unhandled-rejections=strict"
command: npm run --silent test:unit
user: node user: node
test_acceptance: test_acceptance:
@ -30,39 +31,37 @@ services:
REDIS_HOST: redis REDIS_HOST: redis
MONGO_HOST: mongo MONGO_HOST: mongo
POSTGRES_HOST: postgres POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
LOG_LEVEL: ERROR
NODE_ENV: test
ENABLE_CONVERSIONS: "true"
USE_PROM_METRICS: "true"
AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files
AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files
AWS_S3_PUBLIC_FILES_BUCKET_NAME: fake_public_files
AWS_S3_ENDPOINT: http://s3:9090 AWS_S3_ENDPOINT: http://s3:9090
AWS_S3_PATH_STYLE: 'true' AWS_S3_PATH_STYLE: 'true'
AWS_ACCESS_KEY_ID: fake AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake AWS_SECRET_ACCESS_KEY: fake
GCS_API_ENDPOINT: gcs:9090 GCS_API_ENDPOINT: gcs:9090
GCS_API_SCHEME: http GCS_API_SCHEME: http
GCS_USER_FILES_BUCKET_NAME: fake_userfiles GCS_PROJECT_ID: fake
GCS_TEMPLATE_FILES_BUCKET_NAME: fake_templatefiles
GCS_PUBLIC_FILES_BUCKET_NAME: fake_publicfiles
STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1 STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
MOCHA_GREP: ${MOCHA_GREP}
LOG_LEVEL: ERROR
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
ENABLE_CONVERSIONS: "true"
USE_PROM_METRICS: "true"
AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files
AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files
AWS_S3_PUBLIC_FILES_BUCKET_NAME: fake_public_files
user: node user: node
depends_on: depends_on:
s3: s3:
condition: service_healthy condition: service_healthy
gcs: gcs:
condition: service_healthy condition: service_healthy
command: npm run test:acceptance command: npm run --silent test:acceptance
s3: s3:
build: build:
context: test/acceptance/deps context: test/acceptance/deps
dockerfile: Dockerfile.s3mock dockerfile: Dockerfile.s3mock
environment: environment:
- initialBuckets=fake_user_files,fake_template_files,fake_public_files - initialBuckets=fake_user_files,fake_template_files,fake_public_files,bucket
gcs: gcs:
build: build:
context: test/acceptance/deps context: test/acceptance/deps

1
services/filestore/nodemon.json
View file

@ -8,7 +8,6 @@
"execMap": { "execMap": {
"js": "npm run start" "js": "npm run start"
}, },
"watch": [ "watch": [
"app/js/", "app/js/",
"app.js", "app.js",

18
services/filestore/package-lock.json generated
View file

@ -2521,9 +2521,9 @@
"dev": true "dev": true
}, },
"eslint-plugin-chai-friendly": { "eslint-plugin-chai-friendly": {
"version": "0.6.0", "version": "0.5.0",
"resolved": "https://registry.npmjs.org/eslint-plugin-chai-friendly/-/eslint-plugin-chai-friendly-0.6.0.tgz", "resolved": "https://registry.npmjs.org/eslint-plugin-chai-friendly/-/eslint-plugin-chai-friendly-0.5.0.tgz",
"integrity": "sha512-Uvvv1gkbRGp/qfN15B0kQyQWg+oFA8buDSqrwmW3egNSk/FpqH2MjQqKOuKwmEL6w4QIQrIjDp+gg6kGGmD3oQ==", "integrity": "sha512-Pxe6z8C9fP0pn2X2nGFU/b3GBOCM/5FVus1hsMwJsXP3R7RiXFl7g0ksJbsc0GxiLyidTW4mEFk77qsNn7Tk7g==",
"dev": true "dev": true
}, },
"eslint-plugin-es": { "eslint-plugin-es": {
@ -4737,9 +4737,9 @@
"dev": true "dev": true
}, },
"prettier": { "prettier": {
"version": "1.19.1", "version": "2.0.5",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-1.19.1.tgz", "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.0.5.tgz",
"integrity": "sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew==", "integrity": "sha512-7PtVymN48hGcO4fGjybyBSIWDsLU4H4XlvOHfq91pz9kkGlonzwTfYkaIEwiRg/dAJF9YlbsduBAgtYLi+8cFg==",
"dev": true "dev": true
}, },
"prettier-eslint": { "prettier-eslint": {
@ -4938,6 +4938,12 @@
"mimic-fn": "^1.0.0" "mimic-fn": "^1.0.0"
} }
}, },
"prettier": {
"version": "1.19.1",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-1.19.1.tgz",
"integrity": "sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew==",
"dev": true
},
"restore-cursor": { "restore-cursor": {
"version": "2.0.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz",

15
services/filestore/package.json
View file

@ -13,7 +13,7 @@
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"start": "node $NODE_APP_OPTIONS app.js", "start": "node $NODE_APP_OPTIONS app.js",
"nodemon": "nodemon --config nodemon.json", "nodemon": "nodemon --config nodemon.json",
"lint": "node_modules/.bin/eslint app test *.js", "lint": "node_modules/.bin/eslint --max-warnings 0 .",
"format": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --list-different", "format": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --list-different",
"format:fix": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --write", "format:fix": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --write",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
@ -46,18 +46,19 @@
"chai-as-promised": "^7.1.1", "chai-as-promised": "^7.1.1",
"disrequire": "^1.1.0", "disrequire": "^1.1.0",
"eslint": "^6.8.0", "eslint": "^6.8.0",
"eslint-config-prettier": "^6.11.0", "eslint-config-prettier": "^6.10.0",
"eslint-config-standard": "^14.1.1", "eslint-config-standard": "^14.1.0",
"eslint-plugin-chai-expect": "^2.1.0", "eslint-plugin-chai-expect": "^2.1.0",
"eslint-plugin-chai-friendly": "^0.6.0", "eslint-plugin-chai-friendly": "^0.5.0",
"eslint-plugin-import": "^2.22.0", "eslint-plugin-import": "^2.20.1",
"eslint-plugin-mocha": "^6.3.0", "eslint-plugin-mocha": "^6.3.0",
"eslint-plugin-node": "^11.1.0", "eslint-plugin-node": "^11.0.0",
"eslint-plugin-prettier": "^3.1.4", "eslint-plugin-prettier": "^3.1.2",
"eslint-plugin-promise": "^4.2.1", "eslint-plugin-promise": "^4.2.1",
"eslint-plugin-standard": "^4.0.1", "eslint-plugin-standard": "^4.0.1",
"mocha": "7.2.0", "mocha": "7.2.0",
"mongodb": "^3.5.9", "mongodb": "^3.5.9",
"prettier": "^2.0.0",
"prettier-eslint": "^9.0.2", "prettier-eslint": "^9.0.2",
"prettier-eslint-cli": "^5.0.0", "prettier-eslint-cli": "^5.0.0",
"sandboxed-module": "2.0.4", "sandboxed-module": "2.0.4",

4
services/filestore/test/acceptance/js/FilestoreApp.js
View file

@ -33,7 +33,7 @@ class FilestoreApp {
this.server = this.app.listen( this.server = this.app.listen(
Settings.internal.filestore.port, Settings.internal.filestore.port,
'localhost', 'localhost',
err => { (err) => {
if (err) { if (err) {
return reject(err) return reject(err)
} }
@ -110,7 +110,7 @@ class FilestoreApp {
// unload the app, as we may be doing this on multiple runs with // unload the app, as we may be doing this on multiple runs with
// different settings, which affect startup in some cases // different settings, which affect startup in some cases
const files = await fsReaddir(Path.resolve(__dirname, '../../../app/js')) const files = await fsReaddir(Path.resolve(__dirname, '../../../app/js'))
files.forEach(file => { files.forEach((file) => {
disrequire(Path.resolve(__dirname, '../../../app/js', file)) disrequire(Path.resolve(__dirname, '../../../app/js', file))
}) })
disrequire(Path.resolve(__dirname, '../../../app')) disrequire(Path.resolve(__dirname, '../../../app'))

208
services/filestore/test/acceptance/js/FilestoreTests.js
View file

@ -29,7 +29,7 @@ if (!process.env.AWS_ACCESS_KEY_ID) {
throw new Error('please provide credentials for the AWS S3 test server') throw new Error('please provide credentials for the AWS S3 test server')
} }
process.on('unhandledRejection', e => { process.on('unhandledRejection', (e) => {
// eslint-disable-next-line no-console // eslint-disable-next-line no-console
console.log('** Unhandled Promise Rejection **\n', e) console.log('** Unhandled Promise Rejection **\n', e)
throw e throw e
@ -39,7 +39,7 @@ process.on('unhandledRejection', e => {
// fs will always be available - add others if they are configured // fs will always be available - add others if they are configured
const BackendSettings = require('./TestConfig') const BackendSettings = require('./TestConfig')
describe('Filestore', function() { describe('Filestore', function () {
this.timeout(1000 * 10) this.timeout(1000 * 10)
const filestoreUrl = `http://localhost:${Settings.internal.filestore.port}` const filestoreUrl = `http://localhost:${Settings.internal.filestore.port}`
@ -51,7 +51,7 @@ describe('Filestore', function() {
const badSockets = [] const badSockets = []
for (const socket of stdout.split('\n')) { for (const socket of stdout.split('\n')) {
const fields = socket.split(' ').filter(part => part !== '') const fields = socket.split(' ').filter((part) => part !== '')
if ( if (
fields.length > 2 && fields.length > 2 &&
parseInt(fields[1]) && parseInt(fields[1]) &&
@ -79,11 +79,11 @@ describe('Filestore', function() {
} }
// redefine the test suite for every available backend // redefine the test suite for every available backend
Object.keys(BackendSettings).forEach(backend => { Object.keys(BackendSettings).forEach((backend) => {
describe(backend, function() { describe(backend, function () {
let app, previousEgress, previousIngress, metricPrefix, projectId let app, previousEgress, previousIngress, metricPrefix, projectId
before(async function() { before(async function () {
// create the app with the relevant filestore settings // create the app with the relevant filestore settings
Settings.filestore = BackendSettings[backend] Settings.filestore = BackendSettings[backend]
app = new FilestoreApp() app = new FilestoreApp()
@ -91,7 +91,7 @@ describe('Filestore', function() {
}) })
if (BackendSettings[backend].gcs) { if (BackendSettings[backend].gcs) {
before(async function() { before(async function () {
const storage = new Storage(Settings.filestore.gcs.endpoint) const storage = new Storage(Settings.filestore.gcs.endpoint)
await storage.createBucket(process.env.GCS_USER_FILES_BUCKET_NAME) await storage.createBucket(process.env.GCS_USER_FILES_BUCKET_NAME)
await storage.createBucket(process.env.GCS_PUBLIC_FILES_BUCKET_NAME) await storage.createBucket(process.env.GCS_PUBLIC_FILES_BUCKET_NAME)
@ -108,12 +108,12 @@ describe('Filestore', function() {
}) })
} }
after(async function() { after(async function () {
await msleep(3000) await msleep(3000)
await app.stop() await app.stop()
}) })
beforeEach(async function() { beforeEach(async function () {
// retrieve previous metrics from the app // retrieve previous metrics from the app
if (['s3', 'gcs'].includes(Settings.filestore.backend)) { if (['s3', 'gcs'].includes(Settings.filestore.backend)) {
metricPrefix = Settings.filestore.backend metricPrefix = Settings.filestore.backend
@ -125,26 +125,26 @@ describe('Filestore', function() {
projectId = ObjectId().toString() projectId = ObjectId().toString()
}) })
it('should send a 200 for the status endpoint', async function() { it('should send a 200 for the status endpoint', async function () {
const response = await rp(`${filestoreUrl}/status`) const response = await rp(`${filestoreUrl}/status`)
expect(response.statusCode).to.equal(200) expect(response.statusCode).to.equal(200)
expect(response.body).to.contain('filestore') expect(response.body).to.contain('filestore')
expect(response.body).to.contain('up') expect(response.body).to.contain('up')
}) })
it('should send a 200 for the health-check endpoint', async function() { it('should send a 200 for the health-check endpoint', async function () {
const response = await rp(`${filestoreUrl}/health_check`) const response = await rp(`${filestoreUrl}/health_check`)
expect(response.statusCode).to.equal(200) expect(response.statusCode).to.equal(200)
expect(response.body).to.equal('OK') expect(response.body).to.equal('OK')
}) })
describe('with a file on the server', function() { describe('with a file on the server', function () {
let fileId, fileUrl, constantFileContent let fileId, fileUrl, constantFileContent
const localFileReadPath = const localFileReadPath =
'/tmp/filestore_acceptance_tests_file_read.txt' '/tmp/filestore_acceptance_tests_file_read.txt'
beforeEach(async function() { beforeEach(async function () {
fileId = ObjectId().toString() fileId = ObjectId().toString()
fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}` fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
constantFileContent = [ constantFileContent = [
@ -174,14 +174,14 @@ describe('Filestore', function() {
} }
}) })
it('should return 404 for a non-existant id', async function() { it('should return 404 for a non-existant id', async function () {
const options = { uri: fileUrl + '___this_is_clearly_wrong___' } const options = { uri: fileUrl + '___this_is_clearly_wrong___' }
await expect( await expect(
rp.get(options) rp.get(options)
).to.eventually.be.rejected.and.have.property('statusCode', 404) ).to.eventually.be.rejected.and.have.property('statusCode', 404)
}) })
it('should return the file size on a HEAD request', async function() { it('should return the file size on a HEAD request', async function () {
const expectedLength = Buffer.byteLength(constantFileContent) const expectedLength = Buffer.byteLength(constantFileContent)
const res = await rp.head(fileUrl) const res = await rp.head(fileUrl)
expect(res.statusCode).to.equal(200) expect(res.statusCode).to.equal(200)
@ -190,17 +190,17 @@ describe('Filestore', function() {
) )
}) })
it('should be able get the file back', async function() { it('should be able get the file back', async function () {
const res = await rp.get(fileUrl) const res = await rp.get(fileUrl)
expect(res.body).to.equal(constantFileContent) expect(res.body).to.equal(constantFileContent)
}) })
it('should not leak a socket', async function() { it('should not leak a socket', async function () {
await rp.get(fileUrl) await rp.get(fileUrl)
await expectNoSockets() await expectNoSockets()
}) })
it('should be able to get back the first 9 bytes of the file', async function() { it('should be able to get back the first 9 bytes of the file', async function () {
const options = { const options = {
uri: fileUrl, uri: fileUrl,
headers: { headers: {
@ -211,7 +211,7 @@ describe('Filestore', function() {
expect(res.body).to.equal('hello wor') expect(res.body).to.equal('hello wor')
}) })
it('should be able to get back bytes 4 through 10 of the file', async function() { it('should be able to get back bytes 4 through 10 of the file', async function () {
const options = { const options = {
uri: fileUrl, uri: fileUrl,
headers: { headers: {
@ -222,7 +222,7 @@ describe('Filestore', function() {
expect(res.body).to.equal('o world') expect(res.body).to.equal('o world')
}) })
it('should be able to delete the file', async function() { it('should be able to delete the file', async function () {
const response = await rp.del(fileUrl) const response = await rp.del(fileUrl)
expect(response.statusCode).to.equal(204) expect(response.statusCode).to.equal(204)
await expect( await expect(
@ -230,7 +230,7 @@ describe('Filestore', function() {
).to.eventually.be.rejected.and.have.property('statusCode', 404) ).to.eventually.be.rejected.and.have.property('statusCode', 404)
}) })
it('should be able to copy files', async function() { it('should be able to copy files', async function () {
const newProjectID = ObjectId().toString() const newProjectID = ObjectId().toString()
const newFileId = ObjectId().toString() const newFileId = ObjectId().toString()
const newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${newFileId}` const newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${newFileId}`
@ -252,7 +252,7 @@ describe('Filestore', function() {
expect(response.body).to.equal(constantFileContent) expect(response.body).to.equal(constantFileContent)
}) })
it('should be able to overwrite the file', async function() { it('should be able to overwrite the file', async function () {
const newContent = `here is some different content, ${Math.random()}` const newContent = `here is some different content, ${Math.random()}`
const writeStream = request.post(fileUrl) const writeStream = request.post(fileUrl)
const readStream = streamifier.createReadStream(newContent) const readStream = streamifier.createReadStream(newContent)
@ -265,7 +265,7 @@ describe('Filestore', function() {
}) })
if (['S3Persistor', 'GcsPersistor'].includes(backend)) { if (['S3Persistor', 'GcsPersistor'].includes(backend)) {
it('should record an egress metric for the upload', async function() { it('should record an egress metric for the upload', async function () {
const metric = await TestHelper.getMetric( const metric = await TestHelper.getMetric(
filestoreUrl, filestoreUrl,
`${metricPrefix}_egress` `${metricPrefix}_egress`
@ -273,7 +273,7 @@ describe('Filestore', function() {
expect(metric - previousEgress).to.equal(constantFileContent.length) expect(metric - previousEgress).to.equal(constantFileContent.length)
}) })
it('should record an ingress metric when downloading the file', async function() { it('should record an ingress metric when downloading the file', async function () {
await rp.get(fileUrl) await rp.get(fileUrl)
const metric = await TestHelper.getMetric( const metric = await TestHelper.getMetric(
filestoreUrl, filestoreUrl,
@ -284,7 +284,7 @@ describe('Filestore', function() {
) )
}) })
it('should record an ingress metric for a partial download', async function() { it('should record an ingress metric for a partial download', async function () {
const options = { const options = {
uri: fileUrl, uri: fileUrl,
headers: { headers: {
@ -301,7 +301,7 @@ describe('Filestore', function() {
} }
}) })
describe('with multiple files', function() { describe('with multiple files', function () {
let fileIds, fileUrls, projectUrl let fileIds, fileUrls, projectUrl
const localFileReadPaths = [ const localFileReadPaths = [
'/tmp/filestore_acceptance_tests_file_read_1.txt', '/tmp/filestore_acceptance_tests_file_read_1.txt',
@ -320,14 +320,14 @@ describe('Filestore', function() {
].join('\n') ].join('\n')
] ]
before(async function() { before(async function () {
return Promise.all([ return Promise.all([
fsWriteFile(localFileReadPaths[0], constantFileContents[0]), fsWriteFile(localFileReadPaths[0], constantFileContents[0]),
fsWriteFile(localFileReadPaths[1], constantFileContents[1]) fsWriteFile(localFileReadPaths[1], constantFileContents[1])
]) ])
}) })
beforeEach(async function() { beforeEach(async function () {
projectUrl = `${filestoreUrl}/project/${projectId}` projectUrl = `${filestoreUrl}/project/${projectId}`
fileIds = [ObjectId().toString(), ObjectId().toString()] fileIds = [ObjectId().toString(), ObjectId().toString()]
fileUrls = [ fileUrls = [
@ -354,7 +354,7 @@ describe('Filestore', function() {
]) ])
}) })
it('should get the directory size', async function() { it('should get the directory size', async function () {
const response = await rp.get( const response = await rp.get(
`${filestoreUrl}/project/${projectId}/size` `${filestoreUrl}/project/${projectId}/size`
) )
@ -363,7 +363,7 @@ describe('Filestore', function() {
) )
}) })
it('should store the files', async function() { it('should store the files', async function () {
for (const index in fileUrls) { for (const index in fileUrls) {
await expect(rp.get(fileUrls[index])).to.eventually.have.property( await expect(rp.get(fileUrls[index])).to.eventually.have.property(
'body', 'body',
@ -372,7 +372,7 @@ describe('Filestore', function() {
} }
}) })
it('should be able to delete the project', async function() { it('should be able to delete the project', async function () {
await expect(rp.delete(projectUrl)).to.eventually.have.property( await expect(rp.delete(projectUrl)).to.eventually.have.property(
'statusCode', 'statusCode',
204 204
@ -385,17 +385,17 @@ describe('Filestore', function() {
} }
}) })
it('should not delete a partial project id', async function() { it('should not delete a partial project id', async function () {
await expect( await expect(
rp.delete(`${filestoreUrl}/project/5`) rp.delete(`${filestoreUrl}/project/5`)
).to.eventually.be.rejected.and.have.property('statusCode', 400) ).to.eventually.be.rejected.and.have.property('statusCode', 400)
}) })
}) })
describe('with a large file', function() { describe('with a large file', function () {
let fileId, fileUrl, largeFileContent, error let fileId, fileUrl, largeFileContent, error
beforeEach(async function() { beforeEach(async function () {
fileId = ObjectId().toString() fileId = ObjectId().toString()
fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}` fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
@ -414,26 +414,26 @@ describe('Filestore', function() {
} }
}) })
it('should be able to get the file back', async function() { it('should be able to get the file back', async function () {
const response = await rp.get(fileUrl) const response = await rp.get(fileUrl)
expect(response.body).to.equal(largeFileContent) expect(response.body).to.equal(largeFileContent)
}) })
it('should not throw an error', function() { it('should not throw an error', function () {
expect(error).not.to.exist expect(error).not.to.exist
}) })
it('should not leak a socket', async function() { it('should not leak a socket', async function () {
await rp.get(fileUrl) await rp.get(fileUrl)
await expectNoSockets() await expectNoSockets()
}) })
it('should not leak a socket if the connection is aborted', async function() { it('should not leak a socket if the connection is aborted', async function () {
this.timeout(20000) this.timeout(20000)
for (let i = 0; i < 5; i++) { for (let i = 0; i < 5; i++) {
// test is not 100% reliable, so repeat // test is not 100% reliable, so repeat
// create a new connection and have it time out before reading any data // create a new connection and have it time out before reading any data
await new Promise(resolve => { await new Promise((resolve) => {
const streamThatHangs = new Stream.PassThrough() const streamThatHangs = new Stream.PassThrough()
const stream = request({ url: fileUrl, timeout: 1000 }) const stream = request({ url: fileUrl, timeout: 1000 })
stream.pipe(streamThatHangs) stream.pipe(streamThatHangs)
@ -449,10 +449,10 @@ describe('Filestore', function() {
}) })
if (backend === 'S3Persistor' || backend === 'FallbackGcsToS3Persistor') { if (backend === 'S3Persistor' || backend === 'FallbackGcsToS3Persistor') {
describe('with a file in a specific bucket', function() { describe('with a file in a specific bucket', function () {
let constantFileContent, fileId, fileUrl, bucketName let constantFileContent, fileId, fileUrl, bucketName
beforeEach(async function() { beforeEach(async function () {
constantFileContent = `This is a file in a different S3 bucket ${Math.random()}` constantFileContent = `This is a file in a different S3 bucket ${Math.random()}`
fileId = ObjectId().toString() fileId = ObjectId().toString()
bucketName = ObjectId().toString() bucketName = ObjectId().toString()
@ -483,7 +483,7 @@ describe('Filestore', function() {
.promise() .promise()
}) })
it('should get the file from the specified bucket', async function() { it('should get the file from the specified bucket', async function () {
const response = await rp.get(fileUrl) const response = await rp.get(fileUrl)
expect(response.body).to.equal(constantFileContent) expect(response.body).to.equal(constantFileContent)
}) })
@ -491,10 +491,10 @@ describe('Filestore', function() {
} }
if (backend === 'GcsPersistor') { if (backend === 'GcsPersistor') {
describe('when deleting a file in GCS', function() { describe('when deleting a file in GCS', function () {
let fileId, fileUrl, content, error, date let fileId, fileUrl, content, error, date
beforeEach(async function() { beforeEach(async function () {
date = new Date() date = new Date()
tk.freeze(date) tk.freeze(date)
fileId = ObjectId() fileId = ObjectId()
@ -515,15 +515,15 @@ describe('Filestore', function() {
} }
}) })
afterEach(function() { afterEach(function () {
tk.reset() tk.reset()
}) })
it('should not throw an error', function() { it('should not throw an error', function () {
expect(error).not.to.exist expect(error).not.to.exist
}) })
it('should copy the file to the deleted-files bucket', async function() { it('should copy the file to the deleted-files bucket', async function () {
await TestHelper.expectPersistorToHaveFile( await TestHelper.expectPersistorToHaveFile(
app.persistor, app.persistor,
`${Settings.filestore.stores.user_files}-deleted`, `${Settings.filestore.stores.user_files}-deleted`,
@ -532,7 +532,7 @@ describe('Filestore', function() {
) )
}) })
it('should remove the file from the original bucket', async function() { it('should remove the file from the original bucket', async function () {
await TestHelper.expectPersistorNotToHaveFile( await TestHelper.expectPersistorNotToHaveFile(
app.persistor, app.persistor,
Settings.filestore.stores.user_files, Settings.filestore.stores.user_files,
@ -543,7 +543,7 @@ describe('Filestore', function() {
} }
if (BackendSettings[backend].fallback) { if (BackendSettings[backend].fallback) {
describe('with a fallback', function() { describe('with a fallback', function () {
let constantFileContent, let constantFileContent,
fileId, fileId,
fileKey, fileKey,
@ -551,7 +551,7 @@ describe('Filestore', function() {
bucket, bucket,
fallbackBucket fallbackBucket
beforeEach(function() { beforeEach(function () {
constantFileContent = `This is yet more file content ${Math.random()}` constantFileContent = `This is yet more file content ${Math.random()}`
fileId = ObjectId().toString() fileId = ObjectId().toString()
fileKey = `${projectId}/${fileId}` fileKey = `${projectId}/${fileId}`
@ -561,8 +561,8 @@ describe('Filestore', function() {
fallbackBucket = Settings.filestore.fallback.buckets[bucket] fallbackBucket = Settings.filestore.fallback.buckets[bucket]
}) })
describe('with a file in the fallback bucket', function() { describe('with a file in the fallback bucket', function () {
beforeEach(async function() { beforeEach(async function () {
await TestHelper.uploadStringToPersistor( await TestHelper.uploadStringToPersistor(
app.persistor.fallbackPersistor, app.persistor.fallbackPersistor,
fallbackBucket, fallbackBucket,
@ -571,7 +571,7 @@ describe('Filestore', function() {
) )
}) })
it('should not find file in the primary', async function() { it('should not find file in the primary', async function () {
await TestHelper.expectPersistorNotToHaveFile( await TestHelper.expectPersistorNotToHaveFile(
app.persistor.primaryPersistor, app.persistor.primaryPersistor,
bucket, bucket,
@ -579,7 +579,7 @@ describe('Filestore', function() {
) )
}) })
it('should find the file in the fallback', async function() { it('should find the file in the fallback', async function () {
await TestHelper.expectPersistorToHaveFile( await TestHelper.expectPersistorToHaveFile(
app.persistor.fallbackPersistor, app.persistor.fallbackPersistor,
fallbackBucket, fallbackBucket,
@ -588,17 +588,17 @@ describe('Filestore', function() {
) )
}) })
describe('when copyOnMiss is disabled', function() { describe('when copyOnMiss is disabled', function () {
beforeEach(function() { beforeEach(function () {
app.persistor.settings.copyOnMiss = false app.persistor.settings.copyOnMiss = false
}) })
it('should fetch the file', async function() { it('should fetch the file', async function () {
const res = await rp.get(fileUrl) const res = await rp.get(fileUrl)
expect(res.body).to.equal(constantFileContent) expect(res.body).to.equal(constantFileContent)
}) })
it('should not copy the file to the primary', async function() { it('should not copy the file to the primary', async function () {
await rp.get(fileUrl) await rp.get(fileUrl)
await TestHelper.expectPersistorNotToHaveFile( await TestHelper.expectPersistorNotToHaveFile(
@ -609,17 +609,17 @@ describe('Filestore', function() {
}) })
}) })
describe('when copyOnMiss is enabled', function() { describe('when copyOnMiss is enabled', function () {
beforeEach(function() { beforeEach(function () {
app.persistor.settings.copyOnMiss = true app.persistor.settings.copyOnMiss = true
}) })
it('should fetch the file', async function() { it('should fetch the file', async function () {
const res = await rp.get(fileUrl) const res = await rp.get(fileUrl)
expect(res.body).to.equal(constantFileContent) expect(res.body).to.equal(constantFileContent)
}) })
it('copies the file to the primary', async function() { it('copies the file to the primary', async function () {
await rp.get(fileUrl) await rp.get(fileUrl)
// wait for the file to copy in the background // wait for the file to copy in the background
await msleep(1000) await msleep(1000)
@ -633,10 +633,10 @@ describe('Filestore', function() {
}) })
}) })
describe('when copying a file', function() { describe('when copying a file', function () {
let newFileId, newFileUrl, newFileKey, opts let newFileId, newFileUrl, newFileKey, opts
beforeEach(function() { beforeEach(function () {
const newProjectID = ObjectId().toString() const newProjectID = ObjectId().toString()
newFileId = ObjectId().toString() newFileId = ObjectId().toString()
newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${newFileId}` newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${newFileId}`
@ -654,15 +654,15 @@ describe('Filestore', function() {
} }
}) })
describe('when copyOnMiss is false', function() { describe('when copyOnMiss is false', function () {
beforeEach(async function() { beforeEach(async function () {
app.persistor.settings.copyOnMiss = false app.persistor.settings.copyOnMiss = false
const response = await rp(opts) const response = await rp(opts)
expect(response.statusCode).to.equal(200) expect(response.statusCode).to.equal(200)
}) })
it('should leave the old file in the old bucket', async function() { it('should leave the old file in the old bucket', async function () {
await TestHelper.expectPersistorToHaveFile( await TestHelper.expectPersistorToHaveFile(
app.persistor.fallbackPersistor, app.persistor.fallbackPersistor,
fallbackBucket, fallbackBucket,
@ -671,7 +671,7 @@ describe('Filestore', function() {
) )
}) })
it('should not create a new file in the old bucket', async function() { it('should not create a new file in the old bucket', async function () {
await TestHelper.expectPersistorNotToHaveFile( await TestHelper.expectPersistorNotToHaveFile(
app.persistor.fallbackPersistor, app.persistor.fallbackPersistor,
fallbackBucket, fallbackBucket,
@ -679,7 +679,7 @@ describe('Filestore', function() {
) )
}) })
it('should create a new file in the new bucket', async function() { it('should create a new file in the new bucket', async function () {
await TestHelper.expectPersistorToHaveFile( await TestHelper.expectPersistorToHaveFile(
app.persistor.primaryPersistor, app.persistor.primaryPersistor,
bucket, bucket,
@ -688,7 +688,7 @@ describe('Filestore', function() {
) )
}) })
it('should not copy the old file to the primary with the old key', async function() { it('should not copy the old file to the primary with the old key', async function () {
// wait for the file to copy in the background // wait for the file to copy in the background
await msleep(1000) await msleep(1000)
@ -700,15 +700,15 @@ describe('Filestore', function() {
}) })
}) })
describe('when copyOnMiss is true', function() { describe('when copyOnMiss is true', function () {
beforeEach(async function() { beforeEach(async function () {
app.persistor.settings.copyOnMiss = true app.persistor.settings.copyOnMiss = true
const response = await rp(opts) const response = await rp(opts)
expect(response.statusCode).to.equal(200) expect(response.statusCode).to.equal(200)
}) })
it('should leave the old file in the old bucket', async function() { it('should leave the old file in the old bucket', async function () {
await TestHelper.expectPersistorToHaveFile( await TestHelper.expectPersistorToHaveFile(
app.persistor.fallbackPersistor, app.persistor.fallbackPersistor,
fallbackBucket, fallbackBucket,
@ -717,7 +717,7 @@ describe('Filestore', function() {
) )
}) })
it('should not create a new file in the old bucket', async function() { it('should not create a new file in the old bucket', async function () {
await TestHelper.expectPersistorNotToHaveFile( await TestHelper.expectPersistorNotToHaveFile(
app.persistor.fallbackPersistor, app.persistor.fallbackPersistor,
fallbackBucket, fallbackBucket,
@ -725,7 +725,7 @@ describe('Filestore', function() {
) )
}) })
it('should create a new file in the new bucket', async function() { it('should create a new file in the new bucket', async function () {
await TestHelper.expectPersistorToHaveFile( await TestHelper.expectPersistorToHaveFile(
app.persistor.primaryPersistor, app.persistor.primaryPersistor,
bucket, bucket,
@ -734,7 +734,7 @@ describe('Filestore', function() {
) )
}) })
it('should copy the old file to the primary with the old key', async function() { it('should copy the old file to the primary with the old key', async function () {
// wait for the file to copy in the background // wait for the file to copy in the background
await msleep(1000) await msleep(1000)
@ -749,8 +749,8 @@ describe('Filestore', function() {
}) })
}) })
describe('when sending a file', function() { describe('when sending a file', function () {
beforeEach(async function() { beforeEach(async function () {
const writeStream = request.post(fileUrl) const writeStream = request.post(fileUrl)
const readStream = streamifier.createReadStream( const readStream = streamifier.createReadStream(
constantFileContent constantFileContent
@ -760,7 +760,7 @@ describe('Filestore', function() {
await pipeline(readStream, writeStream, resultStream) await pipeline(readStream, writeStream, resultStream)
}) })
it('should store the file on the primary', async function() { it('should store the file on the primary', async function () {
await TestHelper.expectPersistorToHaveFile( await TestHelper.expectPersistorToHaveFile(
app.persistor.primaryPersistor, app.persistor.primaryPersistor,
bucket, bucket,
@ -769,7 +769,7 @@ describe('Filestore', function() {
) )
}) })
it('should not store the file on the fallback', async function() { it('should not store the file on the fallback', async function () {
await TestHelper.expectPersistorNotToHaveFile( await TestHelper.expectPersistorNotToHaveFile(
app.persistor.fallbackPersistor, app.persistor.fallbackPersistor,
fallbackBucket, fallbackBucket,
@ -778,9 +778,9 @@ describe('Filestore', function() {
}) })
}) })
describe('when deleting a file', function() { describe('when deleting a file', function () {
describe('when the file exists on the primary', function() { describe('when the file exists on the primary', function () {
beforeEach(async function() { beforeEach(async function () {
await TestHelper.uploadStringToPersistor( await TestHelper.uploadStringToPersistor(
app.persistor.primaryPersistor, app.persistor.primaryPersistor,
bucket, bucket,
@ -789,7 +789,7 @@ describe('Filestore', function() {
) )
}) })
it('should delete the file', async function() { it('should delete the file', async function () {
const response = await rp.del(fileUrl) const response = await rp.del(fileUrl)
expect(response.statusCode).to.equal(204) expect(response.statusCode).to.equal(204)
await expect( await expect(
@ -798,8 +798,8 @@ describe('Filestore', function() {
}) })
}) })
describe('when the file exists on the fallback', function() { describe('when the file exists on the fallback', function () {
beforeEach(async function() { beforeEach(async function () {
await TestHelper.uploadStringToPersistor( await TestHelper.uploadStringToPersistor(
app.persistor.fallbackPersistor, app.persistor.fallbackPersistor,
fallbackBucket, fallbackBucket,
@ -808,7 +808,7 @@ describe('Filestore', function() {
) )
}) })
it('should delete the file', async function() { it('should delete the file', async function () {
const response = await rp.del(fileUrl) const response = await rp.del(fileUrl)
expect(response.statusCode).to.equal(204) expect(response.statusCode).to.equal(204)
await expect( await expect(
@ -817,8 +817,8 @@ describe('Filestore', function() {
}) })
}) })
describe('when the file exists on both the primary and the fallback', function() { describe('when the file exists on both the primary and the fallback', function () {
beforeEach(async function() { beforeEach(async function () {
await TestHelper.uploadStringToPersistor( await TestHelper.uploadStringToPersistor(
app.persistor.primaryPersistor, app.persistor.primaryPersistor,
bucket, bucket,
@ -833,7 +833,7 @@ describe('Filestore', function() {
) )
}) })
it('should delete the files', async function() { it('should delete the files', async function () {
const response = await rp.del(fileUrl) const response = await rp.del(fileUrl)
expect(response.statusCode).to.equal(204) expect(response.statusCode).to.equal(204)
await expect( await expect(
@ -842,8 +842,8 @@ describe('Filestore', function() {
}) })
}) })
describe('when the file does not exist', function() { describe('when the file does not exist', function () {
it('should return return 204', async function() { it('should return return 204', async function () {
// S3 doesn't give us a 404 when the object doesn't exist, so to stay // S3 doesn't give us a 404 when the object doesn't exist, so to stay
// consistent we merrily return 204 ourselves here as well // consistent we merrily return 204 ourselves here as well
const response = await rp.del(fileUrl) const response = await rp.del(fileUrl)
@ -854,14 +854,14 @@ describe('Filestore', function() {
}) })
} }
describe('with a pdf file', function() { describe('with a pdf file', function () {
let fileId, fileUrl, localFileSize let fileId, fileUrl, localFileSize
const localFileReadPath = Path.resolve( const localFileReadPath = Path.resolve(
__dirname, __dirname,
'../../fixtures/test.pdf' '../../fixtures/test.pdf'
) )
beforeEach(async function() { beforeEach(async function () {
fileId = ObjectId().toString() fileId = ObjectId().toString()
fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}` fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
const stat = await fsStat(localFileReadPath) const stat = await fsStat(localFileReadPath)
@ -872,13 +872,13 @@ describe('Filestore', function() {
await pipeline(readStream, writeStream, endStream) await pipeline(readStream, writeStream, endStream)
}) })
it('should be able get the file back', async function() { it('should be able get the file back', async function () {
const response = await rp.get(fileUrl) const response = await rp.get(fileUrl)
expect(response.body.substring(0, 8)).to.equal('%PDF-1.5') expect(response.body.substring(0, 8)).to.equal('%PDF-1.5')
}) })
if (['S3Persistor', 'GcsPersistor'].includes(backend)) { if (['S3Persistor', 'GcsPersistor'].includes(backend)) {
it('should record an egress metric for the upload', async function() { it('should record an egress metric for the upload', async function () {
const metric = await TestHelper.getMetric( const metric = await TestHelper.getMetric(
filestoreUrl, filestoreUrl,
`${metricPrefix}_egress` `${metricPrefix}_egress`
@ -887,20 +887,20 @@ describe('Filestore', function() {
}) })
} }
describe('getting the preview image', function() { describe('getting the preview image', function () {
this.timeout(1000 * 20) this.timeout(1000 * 20)
let previewFileUrl let previewFileUrl
beforeEach(function() { beforeEach(function () {
previewFileUrl = `${fileUrl}?style=preview` previewFileUrl = `${fileUrl}?style=preview`
}) })
it('should not time out', async function() { it('should not time out', async function () {
const response = await rp.get(previewFileUrl) const response = await rp.get(previewFileUrl)
expect(response.statusCode).to.equal(200) expect(response.statusCode).to.equal(200)
}) })
it('should respond with image data', async function() { it('should respond with image data', async function () {
// note: this test relies of the imagemagick conversion working // note: this test relies of the imagemagick conversion working
const response = await rp.get(previewFileUrl) const response = await rp.get(previewFileUrl)
expect(response.body.length).to.be.greaterThan(400) expect(response.body.length).to.be.greaterThan(400)
@ -908,20 +908,20 @@ describe('Filestore', function() {
}) })
}) })
describe('warming the cache', function() { describe('warming the cache', function () {
this.timeout(1000 * 20) this.timeout(1000 * 20)
let previewFileUrl let previewFileUrl
beforeEach(function() { beforeEach(function () {
previewFileUrl = `${fileUrl}?style=preview&cacheWarm=true` previewFileUrl = `${fileUrl}?style=preview&cacheWarm=true`
}) })
it('should not time out', async function() { it('should not time out', async function () {
const response = await rp.get(previewFileUrl) const response = await rp.get(previewFileUrl)
expect(response.statusCode).to.equal(200) expect(response.statusCode).to.equal(200)
}) })
it("should respond with only an 'OK'", async function() { it("should respond with only an 'OK'", async function () {
// note: this test relies of the imagemagick conversion working // note: this test relies of the imagemagick conversion working
const response = await rp.get(previewFileUrl) const response = await rp.get(previewFileUrl)
expect(response.body).to.equal('OK') expect(response.body).to.equal('OK')

2
services/filestore/test/acceptance/js/TestHelper.js
View file

@ -25,7 +25,7 @@ async function getMetric(filestoreUrl, metric) {
function streamToString(stream) { function streamToString(stream) {
const chunks = [] const chunks = []
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
stream.on('data', chunk => chunks.push(chunk)) stream.on('data', (chunk) => chunks.push(chunk))
stream.on('error', reject) stream.on('error', reject)
stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))) stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')))
stream.resume() stream.resume()

100
services/filestore/test/unit/js/FileControllerTests.js
View file

@ -5,7 +5,7 @@ const SandboxedModule = require('sandboxed-module')
const Errors = require('../../../app/js/Errors') const Errors = require('../../../app/js/Errors')
const modulePath = '../../../app/js/FileController.js' const modulePath = '../../../app/js/FileController.js'
describe('FileController', function() { describe('FileController', function () {
let PersistorManager, let PersistorManager,
FileHandler, FileHandler,
LocalFileWriter, LocalFileWriter,
@ -29,7 +29,7 @@ describe('FileController', function() {
const key = `${projectId}/${fileId}` const key = `${projectId}/${fileId}`
const error = new Error('incorrect utensil') const error = new Error('incorrect utensil')
beforeEach(function() { beforeEach(function () {
PersistorManager = { PersistorManager = {
sendStream: sinon.stub().yields(), sendStream: sinon.stub().yields(),
copyObject: sinon.stub().resolves(), copyObject: sinon.stub().resolves(),
@ -91,76 +91,76 @@ describe('FileController', function() {
next = sinon.stub() next = sinon.stub()
}) })
describe('getFile', function() { describe('getFile', function () {
it('should try and get a redirect url first', function() { it('should try and get a redirect url first', function () {
FileController.getFile(req, res, next) FileController.getFile(req, res, next)
expect(FileHandler.getRedirectUrl).to.have.been.calledWith(bucket, key) expect(FileHandler.getRedirectUrl).to.have.been.calledWith(bucket, key)
}) })
it('should pipe the stream', function() { it('should pipe the stream', function () {
FileController.getFile(req, res, next) FileController.getFile(req, res, next)
expect(stream.pipeline).to.have.been.calledWith(fileStream, res) expect(stream.pipeline).to.have.been.calledWith(fileStream, res)
}) })
it('should send a 200 if the cacheWarm param is true', function(done) { it('should send a 200 if the cacheWarm param is true', function (done) {
req.query.cacheWarm = true req.query.cacheWarm = true
res.sendStatus = statusCode => { res.sendStatus = (statusCode) => {
statusCode.should.equal(200) statusCode.should.equal(200)
done() done()
} }
FileController.getFile(req, res, next) FileController.getFile(req, res, next)
}) })
it('should send an error if there is a problem', function() { it('should send an error if there is a problem', function () {
FileHandler.getFile.yields(error) FileHandler.getFile.yields(error)
FileController.getFile(req, res, next) FileController.getFile(req, res, next)
expect(next).to.have.been.calledWith(error) expect(next).to.have.been.calledWith(error)
}) })
describe('with a redirect url', function() { describe('with a redirect url', function () {
const redirectUrl = 'https://wombat.potato/giraffe' const redirectUrl = 'https://wombat.potato/giraffe'
beforeEach(function() { beforeEach(function () {
FileHandler.getRedirectUrl.yields(null, redirectUrl) FileHandler.getRedirectUrl.yields(null, redirectUrl)
res.redirect = sinon.stub() res.redirect = sinon.stub()
}) })
it('should redirect', function() { it('should redirect', function () {
FileController.getFile(req, res, next) FileController.getFile(req, res, next)
expect(res.redirect).to.have.been.calledWith(redirectUrl) expect(res.redirect).to.have.been.calledWith(redirectUrl)
}) })
it('should not get a file stream', function() { it('should not get a file stream', function () {
FileController.getFile(req, res, next) FileController.getFile(req, res, next)
expect(FileHandler.getFile).not.to.have.been.called expect(FileHandler.getFile).not.to.have.been.called
}) })
describe('when there is an error getting the redirect url', function() { describe('when there is an error getting the redirect url', function () {
beforeEach(function() { beforeEach(function () {
FileHandler.getRedirectUrl.yields(new Error('wombat herding error')) FileHandler.getRedirectUrl.yields(new Error('wombat herding error'))
}) })
it('should not redirect', function() { it('should not redirect', function () {
FileController.getFile(req, res, next) FileController.getFile(req, res, next)
expect(res.redirect).not.to.have.been.called expect(res.redirect).not.to.have.been.called
}) })
it('should not return an error', function() { it('should not return an error', function () {
FileController.getFile(req, res, next) FileController.getFile(req, res, next)
expect(next).not.to.have.been.called expect(next).not.to.have.been.called
}) })
it('should proxy the file', function() { it('should proxy the file', function () {
FileController.getFile(req, res, next) FileController.getFile(req, res, next)
expect(FileHandler.getFile).to.have.been.calledWith(bucket, key) expect(FileHandler.getFile).to.have.been.calledWith(bucket, key)
}) })
}) })
}) })
describe('with a range header', function() { describe('with a range header', function () {
let expectedOptions let expectedOptions
beforeEach(function() { beforeEach(function () {
expectedOptions = { expectedOptions = {
bucket, bucket,
key, key,
@ -169,7 +169,7 @@ describe('FileController', function() {
} }
}) })
it('should pass range options to FileHandler', function() { it('should pass range options to FileHandler', function () {
req.headers.range = 'bytes=0-8' req.headers.range = 'bytes=0-8'
expectedOptions.start = 0 expectedOptions.start = 0
expectedOptions.end = 8 expectedOptions.end = 8
@ -182,7 +182,7 @@ describe('FileController', function() {
) )
}) })
it('should ignore an invalid range header', function() { it('should ignore an invalid range header', function () {
req.headers.range = 'potato' req.headers.range = 'potato'
FileController.getFile(req, res, next) FileController.getFile(req, res, next)
expect(FileHandler.getFile).to.have.been.calledWith( expect(FileHandler.getFile).to.have.been.calledWith(
@ -192,7 +192,7 @@ describe('FileController', function() {
) )
}) })
it("should ignore any type other than 'bytes'", function() { it("should ignore any type other than 'bytes'", function () {
req.headers.range = 'wombats=0-8' req.headers.range = 'wombats=0-8'
FileController.getFile(req, res, next) FileController.getFile(req, res, next)
expect(FileHandler.getFile).to.have.been.calledWith( expect(FileHandler.getFile).to.have.been.calledWith(
@ -204,8 +204,8 @@ describe('FileController', function() {
}) })
}) })
describe('getFileHead', function() { describe('getFileHead', function () {
it('should return the file size in a Content-Length header', function(done) { it('should return the file size in a Content-Length header', function (done) {
res.end = () => { res.end = () => {
expect(res.status).to.have.been.calledWith(200) expect(res.status).to.have.been.calledWith(200)
expect(res.set).to.have.been.calledWith('Content-Length', fileSize) expect(res.set).to.have.been.calledWith('Content-Length', fileSize)
@ -215,12 +215,12 @@ describe('FileController', function() {
FileController.getFileHead(req, res, next) FileController.getFileHead(req, res, next)
}) })
it('should return a 404 is the file is not found', function(done) { it('should return a 404 is the file is not found', function (done) {
FileHandler.getFileSize.yields( FileHandler.getFileSize.yields(
new Errors.NotFoundError({ message: 'not found', info: {} }) new Errors.NotFoundError({ message: 'not found', info: {} })
) )
res.sendStatus = code => { res.sendStatus = (code) => {
expect(code).to.equal(404) expect(code).to.equal(404)
done() done()
} }
@ -228,7 +228,7 @@ describe('FileController', function() {
FileController.getFileHead(req, res, next) FileController.getFileHead(req, res, next)
}) })
it('should send an error on internal errors', function() { it('should send an error on internal errors', function () {
FileHandler.getFileSize.yields(error) FileHandler.getFileSize.yields(error)
FileController.getFileHead(req, res, next) FileController.getFileHead(req, res, next)
@ -236,9 +236,9 @@ describe('FileController', function() {
}) })
}) })
describe('insertFile', function() { describe('insertFile', function () {
it('should send bucket name key and res to PersistorManager', function(done) { it('should send bucket name key and res to PersistorManager', function (done) {
res.sendStatus = code => { res.sendStatus = (code) => {
expect(FileHandler.insertFile).to.have.been.calledWith(bucket, key, req) expect(FileHandler.insertFile).to.have.been.calledWith(bucket, key, req)
expect(code).to.equal(200) expect(code).to.equal(200)
done() done()
@ -247,12 +247,12 @@ describe('FileController', function() {
}) })
}) })
describe('copyFile', function() { describe('copyFile', function () {
const oldFileId = 'oldFileId' const oldFileId = 'oldFileId'
const oldProjectId = 'oldProjectid' const oldProjectId = 'oldProjectid'
const oldKey = `${oldProjectId}/${oldFileId}` const oldKey = `${oldProjectId}/${oldFileId}`
beforeEach(function() { beforeEach(function () {
req.body = { req.body = {
source: { source: {
project_id: oldProjectId, project_id: oldProjectId,
@ -261,8 +261,8 @@ describe('FileController', function() {
} }
}) })
it('should send bucket name and both keys to PersistorManager', function(done) { it('should send bucket name and both keys to PersistorManager', function (done) {
res.sendStatus = code => { res.sendStatus = (code) => {
code.should.equal(200) code.should.equal(200)
expect(PersistorManager.copyObject).to.have.been.calledWith( expect(PersistorManager.copyObject).to.have.been.calledWith(
bucket, bucket,
@ -274,29 +274,29 @@ describe('FileController', function() {
FileController.copyFile(req, res, next) FileController.copyFile(req, res, next)
}) })
it('should send a 404 if the original file was not found', function(done) { it('should send a 404 if the original file was not found', function (done) {
PersistorManager.copyObject.rejects( PersistorManager.copyObject.rejects(
new Errors.NotFoundError({ message: 'not found', info: {} }) new Errors.NotFoundError({ message: 'not found', info: {} })
) )
res.sendStatus = code => { res.sendStatus = (code) => {
code.should.equal(404) code.should.equal(404)
done() done()
} }
FileController.copyFile(req, res, next) FileController.copyFile(req, res, next)
}) })
it('should send an error if there was an error', function(done) { it('should send an error if there was an error', function (done) {
PersistorManager.copyObject.rejects(error) PersistorManager.copyObject.rejects(error)
FileController.copyFile(req, res, err => { FileController.copyFile(req, res, (err) => {
expect(err).to.equal(error) expect(err).to.equal(error)
done() done()
}) })
}) })
}) })
describe('delete file', function() { describe('delete file', function () {
it('should tell the file handler', function(done) { it('should tell the file handler', function (done) {
res.sendStatus = code => { res.sendStatus = (code) => {
code.should.equal(204) code.should.equal(204)
expect(FileHandler.deleteFile).to.have.been.calledWith(bucket, key) expect(FileHandler.deleteFile).to.have.been.calledWith(bucket, key)
done() done()
@ -304,16 +304,16 @@ describe('FileController', function() {
FileController.deleteFile(req, res, next) FileController.deleteFile(req, res, next)
}) })
it('should send a 500 if there was an error', function() { it('should send a 500 if there was an error', function () {
FileHandler.deleteFile.yields(error) FileHandler.deleteFile.yields(error)
FileController.deleteFile(req, res, next) FileController.deleteFile(req, res, next)
expect(next).to.have.been.calledWith(error) expect(next).to.have.been.calledWith(error)
}) })
}) })
describe('delete project', function() { describe('delete project', function () {
it('should tell the file handler', function(done) { it('should tell the file handler', function (done) {
res.sendStatus = code => { res.sendStatus = (code) => {
code.should.equal(204) code.should.equal(204)
expect(FileHandler.deleteProject).to.have.been.calledWith(bucket, key) expect(FileHandler.deleteProject).to.have.been.calledWith(bucket, key)
done() done()
@ -321,24 +321,24 @@ describe('FileController', function() {
FileController.deleteProject(req, res, next) FileController.deleteProject(req, res, next)
}) })
it('should send a 500 if there was an error', function() { it('should send a 500 if there was an error', function () {
FileHandler.deleteProject.yields(error) FileHandler.deleteProject.yields(error)
FileController.deleteProject(req, res, next) FileController.deleteProject(req, res, next)
expect(next).to.have.been.calledWith(error) expect(next).to.have.been.calledWith(error)
}) })
}) })
describe('directorySize', function() { describe('directorySize', function () {
it('should return total directory size bytes', function(done) { it('should return total directory size bytes', function (done) {
FileController.directorySize(req, { FileController.directorySize(req, {
json: result => { json: (result) => {
expect(result['total bytes']).to.equal(fileSize) expect(result['total bytes']).to.equal(fileSize)
done() done()
} }
}) })
}) })
it('should send a 500 if there was an error', function() { it('should send a 500 if there was an error', function () {
FileHandler.getDirectorySize.yields(error) FileHandler.getDirectorySize.yields(error)
FileController.directorySize(req, res, next) FileController.directorySize(req, res, next)
expect(next).to.have.been.calledWith(error) expect(next).to.have.been.calledWith(error)

26
services/filestore/test/unit/js/FileConverterTests.js
View file

@ -6,7 +6,7 @@ const { Errors } = require('@overleaf/object-persistor')
const modulePath = '../../../app/js/FileConverter.js' const modulePath = '../../../app/js/FileConverter.js'
describe('FileConverter', function() { describe('FileConverter', function () {
let SafeExec, FileConverter let SafeExec, FileConverter
const sourcePath = '/data/wombat.eps' const sourcePath = '/data/wombat.eps'
const destPath = '/tmp/dest.png' const destPath = '/tmp/dest.png'
@ -18,7 +18,7 @@ describe('FileConverter', function() {
} }
} }
beforeEach(function() { beforeEach(function () {
SafeExec = { SafeExec = {
promises: sinon.stub().resolves(destPath) promises: sinon.stub().resolves(destPath)
} }
@ -38,20 +38,20 @@ describe('FileConverter', function() {
}) })
}) })
describe('convert', function() { describe('convert', function () {
it('should convert the source to the requested format', async function() { it('should convert the source to the requested format', async function () {
await FileConverter.promises.convert(sourcePath, format) await FileConverter.promises.convert(sourcePath, format)
const args = SafeExec.promises.args[0][0] const args = SafeExec.promises.args[0][0]
expect(args).to.include(`${sourcePath}[0]`) expect(args).to.include(`${sourcePath}[0]`)
expect(args).to.include(`${sourcePath}.${format}`) expect(args).to.include(`${sourcePath}.${format}`)
}) })
it('should return the dest path', async function() { it('should return the dest path', async function () {
const destPath = await FileConverter.promises.convert(sourcePath, format) const destPath = await FileConverter.promises.convert(sourcePath, format)
destPath.should.equal(`${sourcePath}.${format}`) destPath.should.equal(`${sourcePath}.${format}`)
}) })
it('should wrap the error from convert', async function() { it('should wrap the error from convert', async function () {
SafeExec.promises.rejects(errorMessage) SafeExec.promises.rejects(errorMessage)
try { try {
await FileConverter.promises.convert(sourcePath, format) await FileConverter.promises.convert(sourcePath, format)
@ -62,7 +62,7 @@ describe('FileConverter', function() {
} }
}) })
it('should not accept an non approved format', async function() { it('should not accept an non approved format', async function () {
try { try {
await FileConverter.promises.convert(sourcePath, 'potato') await FileConverter.promises.convert(sourcePath, 'potato')
expect('error should have been thrown').not.to.exist expect('error should have been thrown').not.to.exist
@ -71,12 +71,12 @@ describe('FileConverter', function() {
} }
}) })
it('should prefix the command with Settings.commands.convertCommandPrefix', async function() { it('should prefix the command with Settings.commands.convertCommandPrefix', async function () {
Settings.commands.convertCommandPrefix = ['nice'] Settings.commands.convertCommandPrefix = ['nice']
await FileConverter.promises.convert(sourcePath, format) await FileConverter.promises.convert(sourcePath, format)
}) })
it('should convert the file when called as a callback', function(done) { it('should convert the file when called as a callback', function (done) {
FileConverter.convert(sourcePath, format, (err, destPath) => { FileConverter.convert(sourcePath, format, (err, destPath) => {
expect(err).not.to.exist expect(err).not.to.exist
destPath.should.equal(`${sourcePath}.${format}`) destPath.should.equal(`${sourcePath}.${format}`)
@ -89,16 +89,16 @@ describe('FileConverter', function() {
}) })
}) })
describe('thumbnail', function() { describe('thumbnail', function () {
it('should call converter resize with args', async function() { it('should call converter resize with args', async function () {
await FileConverter.promises.thumbnail(sourcePath) await FileConverter.promises.thumbnail(sourcePath)
const args = SafeExec.promises.args[0][0] const args = SafeExec.promises.args[0][0]
expect(args).to.include(`${sourcePath}[0]`) expect(args).to.include(`${sourcePath}[0]`)
}) })
}) })
describe('preview', function() { describe('preview', function () {
it('should call converter resize with args', async function() { it('should call converter resize with args', async function () {
await FileConverter.promises.preview(sourcePath) await FileConverter.promises.preview(sourcePath)
const args = SafeExec.promises.args[0][0] const args = SafeExec.promises.args[0][0]
expect(args).to.include(`${sourcePath}[0]`) expect(args).to.include(`${sourcePath}[0]`)

126
services/filestore/test/unit/js/FileHandlerTests.js
View file

@ -9,7 +9,7 @@ const { Errors } = require('@overleaf/object-persistor')
chai.use(require('sinon-chai')) chai.use(require('sinon-chai'))
chai.use(require('chai-as-promised')) chai.use(require('chai-as-promised'))
describe('FileHandler', function() { describe('FileHandler', function () {
let PersistorManager, let PersistorManager,
LocalFileWriter, LocalFileWriter,
FileConverter, FileConverter,
@ -31,7 +31,7 @@ describe('FileHandler', function() {
on: sinon.stub() on: sinon.stub()
} }
beforeEach(function() { beforeEach(function () {
PersistorManager = { PersistorManager = {
getObjectStream: sinon.stub().resolves(sourceStream), getObjectStream: sinon.stub().resolves(sourceStream),
getRedirectUrl: sinon.stub().resolves(redirectUrl), getRedirectUrl: sinon.stub().resolves(redirectUrl),
@ -89,11 +89,11 @@ describe('FileHandler', function() {
}) })
}) })
describe('insertFile', function() { describe('insertFile', function () {
const stream = 'stream' const stream = 'stream'
it('should send file to the filestore', function(done) { it('should send file to the filestore', function (done) {
FileHandler.insertFile(bucket, key, stream, err => { FileHandler.insertFile(bucket, key, stream, (err) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(PersistorManager.sendStream).to.have.been.calledWith( expect(PersistorManager.sendStream).to.have.been.calledWith(
bucket, bucket,
@ -104,39 +104,39 @@ describe('FileHandler', function() {
}) })
}) })
it('should not make a delete request for the convertedKey folder', function(done) { it('should not make a delete request for the convertedKey folder', function (done) {
FileHandler.insertFile(bucket, key, stream, err => { FileHandler.insertFile(bucket, key, stream, (err) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(PersistorManager.deleteDirectory).not.to.have.been.called expect(PersistorManager.deleteDirectory).not.to.have.been.called
done() done()
}) })
}) })
it('should accept templates-api key format', function(done) { it('should accept templates-api key format', function (done) {
KeyBuilder.getConvertedFolderKey.returns( KeyBuilder.getConvertedFolderKey.returns(
'5ecba29f1a294e007d0bccb4/v/0/pdf' '5ecba29f1a294e007d0bccb4/v/0/pdf'
) )
FileHandler.insertFile(bucket, key, stream, err => { FileHandler.insertFile(bucket, key, stream, (err) => {
expect(err).not.to.exist expect(err).not.to.exist
done() done()
}) })
}) })
it('should throw an error when the key is in the wrong format', function(done) { it('should throw an error when the key is in the wrong format', function (done) {
KeyBuilder.getConvertedFolderKey.returns('wombat') KeyBuilder.getConvertedFolderKey.returns('wombat')
FileHandler.insertFile(bucket, key, stream, err => { FileHandler.insertFile(bucket, key, stream, (err) => {
expect(err).to.exist expect(err).to.exist
done() done()
}) })
}) })
describe('when conversions are enabled', function() { describe('when conversions are enabled', function () {
beforeEach(function() { beforeEach(function () {
Settings.enableConversions = true Settings.enableConversions = true
}) })
it('should delete the convertedKey folder', function(done) { it('should delete the convertedKey folder', function (done) {
FileHandler.insertFile(bucket, key, stream, err => { FileHandler.insertFile(bucket, key, stream, (err) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(PersistorManager.deleteDirectory).to.have.been.calledWith( expect(PersistorManager.deleteDirectory).to.have.been.calledWith(
bucket, bucket,
@ -148,9 +148,9 @@ describe('FileHandler', function() {
}) })
}) })
describe('deleteFile', function() { describe('deleteFile', function () {
it('should tell the filestore manager to delete the file', function(done) { it('should tell the filestore manager to delete the file', function (done) {
FileHandler.deleteFile(bucket, key, err => { FileHandler.deleteFile(bucket, key, (err) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(PersistorManager.deleteObject).to.have.been.calledWith( expect(PersistorManager.deleteObject).to.have.been.calledWith(
bucket, bucket,
@ -160,39 +160,39 @@ describe('FileHandler', function() {
}) })
}) })
it('should not tell the filestore manager to delete the cached folder', function(done) { it('should not tell the filestore manager to delete the cached folder', function (done) {
FileHandler.deleteFile(bucket, key, err => { FileHandler.deleteFile(bucket, key, (err) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(PersistorManager.deleteDirectory).not.to.have.been.called expect(PersistorManager.deleteDirectory).not.to.have.been.called
done() done()
}) })
}) })
it('should accept templates-api key format', function(done) { it('should accept templates-api key format', function (done) {
KeyBuilder.getConvertedFolderKey.returns( KeyBuilder.getConvertedFolderKey.returns(
'5ecba29f1a294e007d0bccb4/v/0/pdf' '5ecba29f1a294e007d0bccb4/v/0/pdf'
) )
FileHandler.deleteFile(bucket, key, err => { FileHandler.deleteFile(bucket, key, (err) => {
expect(err).not.to.exist expect(err).not.to.exist
done() done()
}) })
}) })
it('should throw an error when the key is in the wrong format', function(done) { it('should throw an error when the key is in the wrong format', function (done) {
KeyBuilder.getConvertedFolderKey.returns('wombat') KeyBuilder.getConvertedFolderKey.returns('wombat')
FileHandler.deleteFile(bucket, key, err => { FileHandler.deleteFile(bucket, key, (err) => {
expect(err).to.exist expect(err).to.exist
done() done()
}) })
}) })
describe('when conversions are enabled', function() { describe('when conversions are enabled', function () {
beforeEach(function() { beforeEach(function () {
Settings.enableConversions = true Settings.enableConversions = true
}) })
it('should delete the convertedKey folder', function(done) { it('should delete the convertedKey folder', function (done) {
FileHandler.deleteFile(bucket, key, err => { FileHandler.deleteFile(bucket, key, (err) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(PersistorManager.deleteDirectory).to.have.been.calledWith( expect(PersistorManager.deleteDirectory).to.have.been.calledWith(
bucket, bucket,
@ -204,9 +204,9 @@ describe('FileHandler', function() {
}) })
}) })
describe('deleteProject', function() { describe('deleteProject', function () {
it('should tell the filestore manager to delete the folder', function(done) { it('should tell the filestore manager to delete the folder', function (done) {
FileHandler.deleteProject(bucket, projectKey, err => { FileHandler.deleteProject(bucket, projectKey, (err) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(PersistorManager.deleteDirectory).to.have.been.calledWith( expect(PersistorManager.deleteDirectory).to.have.been.calledWith(
bucket, bucket,
@ -216,16 +216,16 @@ describe('FileHandler', function() {
}) })
}) })
it('should throw an error when the key is in the wrong format', function(done) { it('should throw an error when the key is in the wrong format', function (done) {
FileHandler.deleteProject(bucket, 'wombat', err => { FileHandler.deleteProject(bucket, 'wombat', (err) => {
expect(err).to.exist expect(err).to.exist
done() done()
}) })
}) })
}) })
describe('getFile', function() { describe('getFile', function () {
it('should return the source stream no format or style are defined', function(done) { it('should return the source stream no format or style are defined', function (done) {
FileHandler.getFile(bucket, key, null, (err, stream) => { FileHandler.getFile(bucket, key, null, (err, stream) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(stream).to.equal(sourceStream) expect(stream).to.equal(sourceStream)
@ -233,9 +233,9 @@ describe('FileHandler', function() {
}) })
}) })
it('should pass options through to PersistorManager', function(done) { it('should pass options through to PersistorManager', function (done) {
const options = { start: 0, end: 8 } const options = { start: 0, end: 8 }
FileHandler.getFile(bucket, key, options, err => { FileHandler.getFile(bucket, key, options, (err) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(PersistorManager.getObjectStream).to.have.been.calledWith( expect(PersistorManager.getObjectStream).to.have.been.calledWith(
bucket, bucket,
@ -246,26 +246,26 @@ describe('FileHandler', function() {
}) })
}) })
describe('when a format is defined', function() { describe('when a format is defined', function () {
let result let result
describe('when the file is not cached', function() { describe('when the file is not cached', function () {
beforeEach(function(done) { beforeEach(function (done) {
FileHandler.getFile(bucket, key, { format: 'png' }, (err, stream) => { FileHandler.getFile(bucket, key, { format: 'png' }, (err, stream) => {
result = { err, stream } result = { err, stream }
done() done()
}) })
}) })
it('should convert the file', function() { it('should convert the file', function () {
expect(FileConverter.promises.convert).to.have.been.called expect(FileConverter.promises.convert).to.have.been.called
}) })
it('should compress the converted file', function() { it('should compress the converted file', function () {
expect(ImageOptimiser.promises.compressPng).to.have.been.called expect(ImageOptimiser.promises.compressPng).to.have.been.called
}) })
it('should return the the converted stream', function() { it('should return the the converted stream', function () {
expect(result.err).not.to.exist expect(result.err).not.to.exist
expect(result.stream).to.equal(readStream) expect(result.stream).to.equal(readStream)
expect(PersistorManager.getObjectStream).to.have.been.calledWith( expect(PersistorManager.getObjectStream).to.have.been.calledWith(
@ -275,8 +275,8 @@ describe('FileHandler', function() {
}) })
}) })
describe('when the file is cached', function() { describe('when the file is cached', function () {
beforeEach(function(done) { beforeEach(function (done) {
PersistorManager.checkIfObjectExists = sinon.stub().resolves(true) PersistorManager.checkIfObjectExists = sinon.stub().resolves(true)
FileHandler.getFile(bucket, key, { format: 'png' }, (err, stream) => { FileHandler.getFile(bucket, key, { format: 'png' }, (err, stream) => {
result = { err, stream } result = { err, stream }
@ -284,15 +284,15 @@ describe('FileHandler', function() {
}) })
}) })
it('should not convert the file', function() { it('should not convert the file', function () {
expect(FileConverter.promises.convert).not.to.have.been.called expect(FileConverter.promises.convert).not.to.have.been.called
}) })
it('should not compress the converted file again', function() { it('should not compress the converted file again', function () {
expect(ImageOptimiser.promises.compressPng).not.to.have.been.called expect(ImageOptimiser.promises.compressPng).not.to.have.been.called
}) })
it('should return the cached stream', function() { it('should return the cached stream', function () {
expect(result.err).not.to.exist expect(result.err).not.to.exist
expect(result.stream).to.equal(sourceStream) expect(result.stream).to.equal(sourceStream)
expect(PersistorManager.getObjectStream).to.have.been.calledWith( expect(PersistorManager.getObjectStream).to.have.been.calledWith(
@ -303,9 +303,9 @@ describe('FileHandler', function() {
}) })
}) })
describe('when a style is defined', function() { describe('when a style is defined', function () {
it('generates a thumbnail when requested', function(done) { it('generates a thumbnail when requested', function (done) {
FileHandler.getFile(bucket, key, { style: 'thumbnail' }, err => { FileHandler.getFile(bucket, key, { style: 'thumbnail' }, (err) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(FileConverter.promises.thumbnail).to.have.been.called expect(FileConverter.promises.thumbnail).to.have.been.called
expect(FileConverter.promises.preview).not.to.have.been.called expect(FileConverter.promises.preview).not.to.have.been.called
@ -313,8 +313,8 @@ describe('FileHandler', function() {
}) })
}) })
it('generates a preview when requested', function(done) { it('generates a preview when requested', function (done) {
FileHandler.getFile(bucket, key, { style: 'preview' }, err => { FileHandler.getFile(bucket, key, { style: 'preview' }, (err) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(FileConverter.promises.thumbnail).not.to.have.been.called expect(FileConverter.promises.thumbnail).not.to.have.been.called
expect(FileConverter.promises.preview).to.have.been.called expect(FileConverter.promises.preview).to.have.been.called
@ -324,8 +324,8 @@ describe('FileHandler', function() {
}) })
}) })
describe('getRedirectUrl', function() { describe('getRedirectUrl', function () {
beforeEach(function() { beforeEach(function () {
Settings.filestore = { Settings.filestore = {
allowRedirects: true, allowRedirects: true,
stores: { stores: {
@ -334,7 +334,7 @@ describe('FileHandler', function() {
} }
}) })
it('should return a redirect url', function(done) { it('should return a redirect url', function (done) {
FileHandler.getRedirectUrl(bucket, key, (err, url) => { FileHandler.getRedirectUrl(bucket, key, (err, url) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(url).to.equal(redirectUrl) expect(url).to.equal(redirectUrl)
@ -342,7 +342,7 @@ describe('FileHandler', function() {
}) })
}) })
it('should call the persistor to get a redirect url', function(done) { it('should call the persistor to get a redirect url', function (done) {
FileHandler.getRedirectUrl(bucket, key, () => { FileHandler.getRedirectUrl(bucket, key, () => {
expect(PersistorManager.getRedirectUrl).to.have.been.calledWith( expect(PersistorManager.getRedirectUrl).to.have.been.calledWith(
bucket, bucket,
@ -352,7 +352,7 @@ describe('FileHandler', function() {
}) })
}) })
it('should return null if options are supplied', function(done) { it('should return null if options are supplied', function (done) {
FileHandler.getRedirectUrl( FileHandler.getRedirectUrl(
bucket, bucket,
key, key,
@ -365,7 +365,7 @@ describe('FileHandler', function() {
) )
}) })
it('should return null if the bucket is not one of the defined ones', function(done) { it('should return null if the bucket is not one of the defined ones', function (done) {
FileHandler.getRedirectUrl('a_different_bucket', key, (err, url) => { FileHandler.getRedirectUrl('a_different_bucket', key, (err, url) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(url).to.be.null expect(url).to.be.null
@ -373,7 +373,7 @@ describe('FileHandler', function() {
}) })
}) })
it('should return null if redirects are not enabled', function(done) { it('should return null if redirects are not enabled', function (done) {
Settings.filestore.allowRedirects = false Settings.filestore.allowRedirects = false
FileHandler.getRedirectUrl(bucket, key, (err, url) => { FileHandler.getRedirectUrl(bucket, key, (err, url) => {
expect(err).not.to.exist expect(err).not.to.exist
@ -383,9 +383,9 @@ describe('FileHandler', function() {
}) })
}) })
describe('getDirectorySize', function() { describe('getDirectorySize', function () {
it('should call the filestore manager to get directory size', function(done) { it('should call the filestore manager to get directory size', function (done) {
FileHandler.getDirectorySize(bucket, key, err => { FileHandler.getDirectorySize(bucket, key, (err) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(PersistorManager.directorySize).to.have.been.calledWith( expect(PersistorManager.directorySize).to.have.been.calledWith(
bucket, bucket,

24
services/filestore/test/unit/js/ImageOptimiserTests.js
View file

@ -5,11 +5,11 @@ const modulePath = '../../../app/js/ImageOptimiser.js'
const { FailedCommandError } = require('../../../app/js/Errors') const { FailedCommandError } = require('../../../app/js/Errors')
const SandboxedModule = require('sandboxed-module') const SandboxedModule = require('sandboxed-module')
describe('ImageOptimiser', function() { describe('ImageOptimiser', function () {
let ImageOptimiser, SafeExec, logger let ImageOptimiser, SafeExec, logger
const sourcePath = '/wombat/potato.eps' const sourcePath = '/wombat/potato.eps'
beforeEach(function() { beforeEach(function () {
SafeExec = { SafeExec = {
promises: sinon.stub().resolves() promises: sinon.stub().resolves()
} }
@ -27,9 +27,9 @@ describe('ImageOptimiser', function() {
}) })
}) })
describe('compressPng', function() { describe('compressPng', function () {
it('should convert the file', function(done) { it('should convert the file', function (done) {
ImageOptimiser.compressPng(sourcePath, err => { ImageOptimiser.compressPng(sourcePath, (err) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(SafeExec.promises).to.have.been.calledWith([ expect(SafeExec.promises).to.have.been.calledWith([
'optipng', 'optipng',
@ -39,32 +39,32 @@ describe('ImageOptimiser', function() {
}) })
}) })
it('should return the error', function(done) { it('should return the error', function (done) {
SafeExec.promises.rejects('wombat herding failure') SafeExec.promises.rejects('wombat herding failure')
ImageOptimiser.compressPng(sourcePath, err => { ImageOptimiser.compressPng(sourcePath, (err) => {
expect(err.toString()).to.equal('wombat herding failure') expect(err.toString()).to.equal('wombat herding failure')
done() done()
}) })
}) })
}) })
describe('when optimiser is sigkilled', function() { describe('when optimiser is sigkilled', function () {
const expectedError = new FailedCommandError('', 'SIGKILL', '', '') const expectedError = new FailedCommandError('', 'SIGKILL', '', '')
let error let error
beforeEach(function(done) { beforeEach(function (done) {
SafeExec.promises.rejects(expectedError) SafeExec.promises.rejects(expectedError)
ImageOptimiser.compressPng(sourcePath, err => { ImageOptimiser.compressPng(sourcePath, (err) => {
error = err error = err
done() done()
}) })
}) })
it('should not produce an error', function() { it('should not produce an error', function () {
expect(error).not.to.exist expect(error).not.to.exist
}) })
it('should log a warning', function() { it('should log a warning', function () {
expect(logger.warn).to.have.been.calledOnce expect(logger.warn).to.have.been.calledOnce
}) })
}) })

12
services/filestore/test/unit/js/KeybuilderTests.js
View file

@ -2,30 +2,30 @@ const SandboxedModule = require('sandboxed-module')
const modulePath = '../../../app/js/KeyBuilder.js' const modulePath = '../../../app/js/KeyBuilder.js'
describe('KeybuilderTests', function() { describe('KeybuilderTests', function () {
let KeyBuilder let KeyBuilder
const key = 'wombat/potato' const key = 'wombat/potato'
beforeEach(function() { beforeEach(function () {
KeyBuilder = SandboxedModule.require(modulePath, { KeyBuilder = SandboxedModule.require(modulePath, {
requires: { 'settings-sharelatex': {} } requires: { 'settings-sharelatex': {} }
}) })
}) })
describe('cachedKey', function() { describe('cachedKey', function () {
it('should add the format to the key', function() { it('should add the format to the key', function () {
const opts = { format: 'png' } const opts = { format: 'png' }
const newKey = KeyBuilder.addCachingToKey(key, opts) const newKey = KeyBuilder.addCachingToKey(key, opts)
newKey.should.equal(`${key}-converted-cache/format-png`) newKey.should.equal(`${key}-converted-cache/format-png`)
}) })
it('should add the style to the key', function() { it('should add the style to the key', function () {
const opts = { style: 'thumbnail' } const opts = { style: 'thumbnail' }
const newKey = KeyBuilder.addCachingToKey(key, opts) const newKey = KeyBuilder.addCachingToKey(key, opts)
newKey.should.equal(`${key}-converted-cache/style-thumbnail`) newKey.should.equal(`${key}-converted-cache/style-thumbnail`)
}) })
it('should add format first, then style', function() { it('should add format first, then style', function () {
const opts = { const opts = {
style: 'thumbnail', style: 'thumbnail',
format: 'png' format: 'png'

36
services/filestore/test/unit/js/LocalFileWriterTests.js
View file

@ -6,7 +6,7 @@ const SandboxedModule = require('sandboxed-module')
const { Errors } = require('@overleaf/object-persistor') const { Errors } = require('@overleaf/object-persistor')
chai.use(require('sinon-chai')) chai.use(require('sinon-chai'))
describe('LocalFileWriter', function() { describe('LocalFileWriter', function () {
const writeStream = 'writeStream' const writeStream = 'writeStream'
const readStream = 'readStream' const readStream = 'readStream'
const settings = { path: { uploadFolder: '/uploads' } } const settings = { path: { uploadFolder: '/uploads' } }
@ -14,7 +14,7 @@ describe('LocalFileWriter', function() {
const filename = 'wombat' const filename = 'wombat'
let stream, fs, LocalFileWriter let stream, fs, LocalFileWriter
beforeEach(function() { beforeEach(function () {
fs = { fs = {
createWriteStream: sinon.stub().returns(writeStream), createWriteStream: sinon.stub().returns(writeStream),
unlink: sinon.stub().yields() unlink: sinon.stub().yields()
@ -39,8 +39,8 @@ describe('LocalFileWriter', function() {
}) })
}) })
describe('writeStream', function() { describe('writeStream', function () {
it('writes the stream to the upload folder', function(done) { it('writes the stream to the upload folder', function (done) {
LocalFileWriter.writeStream(readStream, filename, (err, path) => { LocalFileWriter.writeStream(readStream, filename, (err, path) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(fs.createWriteStream).to.have.been.calledWith(fsPath) expect(fs.createWriteStream).to.have.been.calledWith(fsPath)
@ -50,20 +50,20 @@ describe('LocalFileWriter', function() {
}) })
}) })
describe('when there is an error', function() { describe('when there is an error', function () {
const error = new Error('not enough ketchup') const error = new Error('not enough ketchup')
beforeEach(function() { beforeEach(function () {
stream.pipeline.yields(error) stream.pipeline.yields(error)
}) })
it('should wrap the error', function() { it('should wrap the error', function () {
LocalFileWriter.writeStream(readStream, filename, err => { LocalFileWriter.writeStream(readStream, filename, (err) => {
expect(err).to.exist expect(err).to.exist
expect(err.cause).to.equal(error) expect(err.cause).to.equal(error)
}) })
}) })
it('should delete the temporary file', function() { it('should delete the temporary file', function () {
LocalFileWriter.writeStream(readStream, filename, () => { LocalFileWriter.writeStream(readStream, filename, () => {
expect(fs.unlink).to.have.been.calledWith(fsPath) expect(fs.unlink).to.have.been.calledWith(fsPath)
}) })
@ -71,37 +71,37 @@ describe('LocalFileWriter', function() {
}) })
}) })
describe('deleteFile', function() { describe('deleteFile', function () {
it('should unlink the file', function(done) { it('should unlink the file', function (done) {
LocalFileWriter.deleteFile(fsPath, err => { LocalFileWriter.deleteFile(fsPath, (err) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(fs.unlink).to.have.been.calledWith(fsPath) expect(fs.unlink).to.have.been.calledWith(fsPath)
done() done()
}) })
}) })
it('should not call unlink with an empty path', function(done) { it('should not call unlink with an empty path', function (done) {
LocalFileWriter.deleteFile('', err => { LocalFileWriter.deleteFile('', (err) => {
expect(err).not.to.exist expect(err).not.to.exist
expect(fs.unlink).not.to.have.been.called expect(fs.unlink).not.to.have.been.called
done() done()
}) })
}) })
it('should not throw a error if the file does not exist', function(done) { it('should not throw a error if the file does not exist', function (done) {
const error = new Error('file not found') const error = new Error('file not found')
error.code = 'ENOENT' error.code = 'ENOENT'
fs.unlink = sinon.stub().yields(error) fs.unlink = sinon.stub().yields(error)
LocalFileWriter.deleteFile(fsPath, err => { LocalFileWriter.deleteFile(fsPath, (err) => {
expect(err).not.to.exist expect(err).not.to.exist
done() done()
}) })
}) })
it('should wrap the error', function(done) { it('should wrap the error', function (done) {
const error = new Error('failed to reticulate splines') const error = new Error('failed to reticulate splines')
fs.unlink = sinon.stub().yields(error) fs.unlink = sinon.stub().yields(error)
LocalFileWriter.deleteFile(fsPath, err => { LocalFileWriter.deleteFile(fsPath, (err) => {
expect(err).to.exist expect(err).to.exist
expect(err.cause).to.equal(error) expect(err.cause).to.equal(error)
done() done()

34
services/filestore/test/unit/js/SafeExecTests.js
View file

@ -5,10 +5,10 @@ const modulePath = '../../../app/js/SafeExec'
const { Errors } = require('@overleaf/object-persistor') const { Errors } = require('@overleaf/object-persistor')
const SandboxedModule = require('sandboxed-module') const SandboxedModule = require('sandboxed-module')
describe('SafeExec', function() { describe('SafeExec', function () {
let settings, options, safeExec let settings, options, safeExec
beforeEach(function() { beforeEach(function () {
settings = { enableConversions: true } settings = { enableConversions: true }
options = { timeout: 10 * 1000, killSignal: 'SIGTERM' } options = { timeout: 10 * 1000, killSignal: 'SIGTERM' }
@ -23,8 +23,8 @@ describe('SafeExec', function() {
}) })
}) })
describe('safeExec', function() { describe('safeExec', function () {
it('should execute a valid command', function(done) { it('should execute a valid command', function (done) {
safeExec(['/bin/echo', 'hello'], options, (err, stdout, stderr) => { safeExec(['/bin/echo', 'hello'], options, (err, stdout, stderr) => {
stdout.should.equal('hello\n') stdout.should.equal('hello\n')
stderr.should.equal('') stderr.should.equal('')
@ -33,16 +33,16 @@ describe('SafeExec', function() {
}) })
}) })
it('should error when conversions are disabled', function(done) { it('should error when conversions are disabled', function (done) {
settings.enableConversions = false settings.enableConversions = false
safeExec(['/bin/echo', 'hello'], options, err => { safeExec(['/bin/echo', 'hello'], options, (err) => {
expect(err).to.exist expect(err).to.exist
done() done()
}) })
}) })
it('should execute a command with non-zero exit status', function(done) { it('should execute a command with non-zero exit status', function (done) {
safeExec(['/usr/bin/env', 'false'], options, err => { safeExec(['/usr/bin/env', 'false'], options, (err) => {
expect(err).to.exist expect(err).to.exist
expect(err.name).to.equal('FailedCommandError') expect(err.name).to.equal('FailedCommandError')
expect(err.code).to.equal(1) expect(err.code).to.equal(1)
@ -52,18 +52,18 @@ describe('SafeExec', function() {
}) })
}) })
it('should handle an invalid command', function(done) { it('should handle an invalid command', function (done) {
safeExec(['/bin/foobar'], options, err => { safeExec(['/bin/foobar'], options, (err) => {
err.code.should.equal('ENOENT') err.code.should.equal('ENOENT')
done() done()
}) })
}) })
it('should handle a command that runs too long', function(done) { it('should handle a command that runs too long', function (done) {
safeExec( safeExec(
['/bin/sleep', '10'], ['/bin/sleep', '10'],
{ timeout: 500, killSignal: 'SIGTERM' }, { timeout: 500, killSignal: 'SIGTERM' },
err => { (err) => {
expect(err).to.exist expect(err).to.exist
expect(err.name).to.equal('FailedCommandError') expect(err.name).to.equal('FailedCommandError')
expect(err.code).to.equal('SIGTERM') expect(err.code).to.equal('SIGTERM')
@ -73,19 +73,19 @@ describe('SafeExec', function() {
}) })
}) })
describe('as a promise', function() { describe('as a promise', function () {
beforeEach(function() { beforeEach(function () {
safeExec = safeExec.promises safeExec = safeExec.promises
}) })
it('should execute a valid command', async function() { it('should execute a valid command', async function () {
const { stdout, stderr } = await safeExec(['/bin/echo', 'hello'], options) const { stdout, stderr } = await safeExec(['/bin/echo', 'hello'], options)
stdout.should.equal('hello\n') stdout.should.equal('hello\n')
stderr.should.equal('') stderr.should.equal('')
}) })
it('should throw a ConversionsDisabledError when appropriate', async function() { it('should throw a ConversionsDisabledError when appropriate', async function () {
settings.enableConversions = false settings.enableConversions = false
try { try {
await safeExec(['/bin/echo', 'hello'], options) await safeExec(['/bin/echo', 'hello'], options)
@ -96,7 +96,7 @@ describe('SafeExec', function() {
expect('method did not throw an error').not.to.exist expect('method did not throw an error').not.to.exist
}) })
it('should throw a FailedCommandError when appropriate', async function() { it('should throw a FailedCommandError when appropriate', async function () {
try { try {
await safeExec(['/usr/bin/env', 'false'], options) await safeExec(['/usr/bin/env', 'false'], options)
} catch (err) { } catch (err) {

6
services/filestore/test/unit/js/SettingsTests.js
View file

@ -2,9 +2,9 @@ const chai = require('chai')
const { expect } = chai const { expect } = chai
const SandboxedModule = require('sandboxed-module') const SandboxedModule = require('sandboxed-module')
describe('Settings', function() { describe('Settings', function () {
describe('s3', function() { describe('s3', function () {
it('should use JSONified env var if present', function() { it('should use JSONified env var if present', function () {
const s3Settings = { const s3Settings = {
bucket1: { bucket1: {
auth_key: 'bucket1_key', auth_key: 'bucket1_key',