mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-21 20:47:08 -05:00
Cleanup and promisify acceptance tests. Run tests for all backends.
This commit is contained in:
Simon Detheridge
parent
006f84abeb
commit
a8158d6c8c
5 changed files with 521 additions and 460 deletions
150
services/filestore/npm-shrinkwrap.json
generated
150
services/filestore/npm-shrinkwrap.json
generated
|
|
@ -1014,6 +1014,15 @@
|
|||
"type-detect": "^4.0.5"
|
||||
}
|
||||
},
|
||||
"chai-as-promised": {
|
||||
"version": "7.1.1",
|
||||
"resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz",
|
||||
"integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"check-error": "^1.0.2"
|
||||
}
|
||||
},
|
||||
"chalk": {
|
||||
"version": "2.4.2",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
|
||||
|
|
@ -1284,6 +1293,12 @@
|
|||
"integrity": "sha1-gAwN0eCov7yVg1wgKtIg/jF+WhI=",
|
||||
"dev": true
|
||||
},
|
||||
"disrequire": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/disrequire/-/disrequire-1.1.0.tgz",
|
||||
"integrity": "sha512-c3lya+wBcnfNipVE7XQC85J6Fty9XWsbNrUub8XT1Qk3mwO6f8tR7P6Ah3X09A3HTQ1biwjcwTLFkGlEejUzUw==",
|
||||
"dev": true
|
||||
},
|
||||
"dlv": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz",
|
||||
|
|
@ -4446,51 +4461,113 @@
|
|||
"dev": true
|
||||
},
|
||||
"request": {
|
||||
"version": "2.14.0",
|
||||
"resolved": "https://registry.npmjs.org/request/-/request-2.14.0.tgz",
|
||||
"integrity": "sha1-DYrLsLFMGrguAAt9OB+oyA0afYg=",
|
||||
"version": "2.88.0",
|
||||
"resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz",
|
||||
"integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==",
|
||||
"requires": {
|
||||
"form-data": "~0.0.3",
|
||||
"mime": "~1.2.7"
|
||||
"aws-sign2": "~0.7.0",
|
||||
"aws4": "^1.8.0",
|
||||
"caseless": "~0.12.0",
|
||||
"combined-stream": "~1.0.6",
|
||||
"extend": "~3.0.2",
|
||||
"forever-agent": "~0.6.1",
|
||||
"form-data": "~2.3.2",
|
||||
"har-validator": "~5.1.0",
|
||||
"http-signature": "~1.2.0",
|
||||
"is-typedarray": "~1.0.0",
|
||||
"isstream": "~0.1.2",
|
||||
"json-stringify-safe": "~5.0.1",
|
||||
"mime-types": "~2.1.19",
|
||||
"oauth-sign": "~0.9.0",
|
||||
"performance-now": "^2.1.0",
|
||||
"qs": "~6.5.2",
|
||||
"safe-buffer": "^5.1.2",
|
||||
"tough-cookie": "~2.4.3",
|
||||
"tunnel-agent": "^0.6.0",
|
||||
"uuid": "^3.3.2"
|
||||
},
|
||||
"dependencies": {
|
||||
"form-data": {
|
||||
"version": "0.0.7",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-0.0.7.tgz",
|
||||
"integrity": "sha1-chEYKiaiZs45cQ3IvEqBtwQIWb4=",
|
||||
"requires": {
|
||||
"async": "~0.1.9",
|
||||
"combined-stream": "~0.0.4",
|
||||
"mime": "~1.2.2"
|
||||
},
|
||||
"dependencies": {
|
||||
"async": {
|
||||
"version": "0.1.22",
|
||||
"resolved": "https://registry.npmjs.org/async/-/async-0.1.22.tgz",
|
||||
"integrity": "sha1-D8GqoIig4+8Ovi2IMbqw3PiEUGE="
|
||||
"caseless": {
|
||||
"version": "0.12.0",
|
||||
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
|
||||
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
|
||||
},
|
||||
"combined-stream": {
|
||||
"version": "0.0.4",
|
||||
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.4.tgz",
|
||||
"integrity": "sha1-LRpDNH2+lRWkonlnMuW4hHOECyI=",
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
|
||||
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
|
||||
"requires": {
|
||||
"delayed-stream": "0.0.5"
|
||||
"delayed-stream": "~1.0.0"
|
||||
}
|
||||
},
|
||||
"delayed-stream": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
|
||||
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
|
||||
},
|
||||
"forever-agent": {
|
||||
"version": "0.6.1",
|
||||
"resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
|
||||
"integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
|
||||
},
|
||||
"form-data": {
|
||||
"version": "2.3.3",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
|
||||
"integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
|
||||
"requires": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.6",
|
||||
"mime-types": "^2.1.12"
|
||||
}
|
||||
},
|
||||
"oauth-sign": {
|
||||
"version": "0.9.0",
|
||||
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
|
||||
"integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ=="
|
||||
},
|
||||
"safe-buffer": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz",
|
||||
"integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg=="
|
||||
},
|
||||
"tunnel-agent": {
|
||||
"version": "0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
|
||||
"integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
|
||||
"requires": {
|
||||
"safe-buffer": "^5.0.1"
|
||||
}
|
||||
},
|
||||
"uuid": {
|
||||
"version": "3.3.3",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz",
|
||||
"integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"request-promise-core": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.3.tgz",
|
||||
"integrity": "sha512-QIs2+ArIGQVp5ZYbWD5ZLCY29D5CfWizP8eWnm8FoGD1TX61veauETVQbrV60662V0oFBkrDOuaBI8XgtuyYAQ==",
|
||||
"requires": {
|
||||
"lodash": "^4.17.15"
|
||||
},
|
||||
"dependencies": {
|
||||
"delayed-stream": {
|
||||
"version": "0.0.5",
|
||||
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz",
|
||||
"integrity": "sha1-1LH0OpPoKW3+AmlPRoC8N6MTxz8="
|
||||
}
|
||||
}
|
||||
"lodash": {
|
||||
"version": "4.17.15",
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
|
||||
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"mime": {
|
||||
"version": "1.2.9",
|
||||
"resolved": "https://registry.npmjs.org/mime/-/mime-1.2.9.tgz",
|
||||
"integrity": "sha1-AJzUCGe9Nd5SGzuWbwTi+NTRPQk="
|
||||
}
|
||||
"request-promise-native": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.8.tgz",
|
||||
"integrity": "sha512-dapwLGqkHtwL5AEbfenuzjTYg35Jd6KPytsC2/TLkVMz8rm+tNt72MGUWT1RP/aYawMpN6HqbNGBQaRcBtjQMQ==",
|
||||
"requires": {
|
||||
"request-promise-core": "1.1.3",
|
||||
"stealthy-require": "^1.1.1",
|
||||
"tough-cookie": "^2.3.3"
|
||||
}
|
||||
},
|
||||
"require-directory": {
|
||||
|
|
@ -4878,6 +4955,11 @@
|
|||
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
|
||||
"integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow="
|
||||
},
|
||||
"stealthy-require": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
|
||||
"integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks="
|
||||
},
|
||||
"stream-browserify": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz",
|
||||
|
|
|
|||
|
|
@ -36,7 +36,8 @@
|
|||
"pngcrush": "0.0.3",
|
||||
"range-parser": "^1.0.2",
|
||||
"recluster": "^0.3.7",
|
||||
"request": "2.14.0",
|
||||
"request": "^2.88.0",
|
||||
"request-promise-native": "^1.0.8",
|
||||
"response": "0.14.0",
|
||||
"rimraf": "2.2.8",
|
||||
"settings-sharelatex": "^1.1.0",
|
||||
|
|
@ -48,6 +49,8 @@
|
|||
"babel-eslint": "^10.0.3",
|
||||
"bunyan": "^1.3.5",
|
||||
"chai": "4.2.0",
|
||||
"chai-as-promised": "^7.1.1",
|
||||
"disrequire": "^1.1.0",
|
||||
"eslint": "^6.4.0",
|
||||
"eslint-config-prettier": "^6.7.0",
|
||||
"eslint-config-standard": "^14.1.0",
|
||||
|
|
|
|||
|
|
@ -1,109 +1,112 @@
|
|||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
standard/no-callback-literal,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS205: Consider reworking code to avoid use of IIFEs
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const app = require('../../../app')
|
||||
require('logger-sharelatex').logger.level('info')
|
||||
const logger = require('logger-sharelatex')
|
||||
const Settings = require('settings-sharelatex')
|
||||
const fs = require('fs')
|
||||
const Path = require('path')
|
||||
const request = require('request')
|
||||
const { promisify } = require('util')
|
||||
const disrequire = require('disrequire')
|
||||
|
||||
const S3_TRIES = 30
|
||||
|
||||
module.exports = {
|
||||
running: false,
|
||||
initing: false,
|
||||
callbacks: [],
|
||||
ensureRunning(callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
logger.logger.level('info')
|
||||
|
||||
const fsReaddir = promisify(fs.readdir)
|
||||
|
||||
class FilestoreApp {
|
||||
constructor() {
|
||||
this.running = false
|
||||
this.initing = false
|
||||
}
|
||||
|
||||
async runServer() {
|
||||
if (this.running) {
|
||||
return callback()
|
||||
} else if (this.initing) {
|
||||
return this.callbacks.push(callback)
|
||||
} else {
|
||||
return
|
||||
}
|
||||
|
||||
if (this.initing) {
|
||||
return this.waitForInit()
|
||||
}
|
||||
this.initing = true
|
||||
this.callbacks.push(callback)
|
||||
return app.listen(
|
||||
__guard__(
|
||||
Settings.internal != null ? Settings.internal.filestore : undefined,
|
||||
x => x.port
|
||||
),
|
||||
|
||||
this.app = await FilestoreApp.requireApp()
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
this.server = this.app.listen(
|
||||
Settings.internal.filestore.port,
|
||||
'localhost',
|
||||
error => {
|
||||
if (error != null) {
|
||||
throw error
|
||||
err => {
|
||||
if (err) {
|
||||
return reject(err)
|
||||
}
|
||||
this.running = true
|
||||
logger.log('filestore running in dev mode')
|
||||
|
||||
return (() => {
|
||||
const result = []
|
||||
for (callback of Array.from(this.callbacks)) {
|
||||
result.push(callback())
|
||||
}
|
||||
return result
|
||||
})()
|
||||
resolve()
|
||||
}
|
||||
)
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
waitForS3(callback, tries) {
|
||||
if (
|
||||
!(Settings.filestore.s3 != null
|
||||
? Settings.filestore.s3.endpoint
|
||||
: undefined)
|
||||
) {
|
||||
return callback()
|
||||
if (Settings.filestore.backend === 's3') {
|
||||
try {
|
||||
await FilestoreApp.waitForS3()
|
||||
} catch (err) {
|
||||
await this.stop()
|
||||
throw err
|
||||
}
|
||||
if (!tries) {
|
||||
tries = 1
|
||||
}
|
||||
|
||||
return request.get(
|
||||
`${Settings.filestore.s3.endpoint}/`,
|
||||
(err, response) => {
|
||||
console.log(
|
||||
err,
|
||||
response != null ? response.statusCode : undefined,
|
||||
tries
|
||||
this.initing = false
|
||||
}
|
||||
|
||||
async waitForInit() {
|
||||
while (this.initing) {
|
||||
await new Promise(resolve => setTimeout(resolve, 1000))
|
||||
}
|
||||
}
|
||||
|
||||
async stop() {
|
||||
if (this.server) {
|
||||
await new Promise(resolve => {
|
||||
this.server.close(resolve)
|
||||
})
|
||||
delete this.server
|
||||
}
|
||||
}
|
||||
|
||||
static async waitForS3() {
|
||||
let tries = 0
|
||||
if (!Settings.filestore.s3.endpoint) {
|
||||
return
|
||||
}
|
||||
|
||||
let s3Available = false
|
||||
|
||||
while (tries < S3_TRIES && !s3Available) {
|
||||
try {
|
||||
const response = await promisify(request.get)(
|
||||
`${Settings.filestore.s3.endpoint}/`
|
||||
)
|
||||
if (
|
||||
!err &&
|
||||
[200, 404].includes(
|
||||
response != null ? response.statusCode : undefined
|
||||
)
|
||||
) {
|
||||
return callback()
|
||||
if ([200, 404].includes(response.statusCode)) {
|
||||
s3Available = true
|
||||
}
|
||||
|
||||
if (tries === S3_TRIES) {
|
||||
return callback('timed out waiting for S3')
|
||||
} catch (err) {
|
||||
} finally {
|
||||
tries++
|
||||
if (!s3Available) {
|
||||
await new Promise(resolve => setTimeout(resolve, 1000))
|
||||
}
|
||||
|
||||
return setTimeout(() => {
|
||||
return this.waitForS3(callback, tries + 1)
|
||||
}, 1000)
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return typeof value !== 'undefined' && value !== null
|
||||
? transform(value)
|
||||
: undefined
|
||||
static async requireApp() {
|
||||
// unload the app, as we may be doing this on multiple runs with
|
||||
// different settings, which affect startup in some cases
|
||||
const files = await fsReaddir(Path.resolve(__dirname, '../../../app/js'))
|
||||
files.forEach(file => {
|
||||
disrequire(Path.resolve(__dirname, '../../../app/js', file))
|
||||
})
|
||||
disrequire(Path.resolve(__dirname, '../../../app'))
|
||||
|
||||
return require('../../../app')
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = FilestoreApp
|
||||
|
|
|
|||
299
services/filestore/test/acceptance/js/FilestoreTests.js
Normal file
299
services/filestore/test/acceptance/js/FilestoreTests.js
Normal file
|
|
@ -0,0 +1,299 @@
|
|||
const chai = require('chai')
|
||||
const { expect } = chai
|
||||
const fs = require('fs')
|
||||
const Settings = require('settings-sharelatex')
|
||||
const Path = require('path')
|
||||
const FilestoreApp = require('./FilestoreApp')
|
||||
const rp = require('request-promise-native').defaults({
|
||||
resolveWithFullResponse: true
|
||||
})
|
||||
const Stream = require('stream')
|
||||
const request = require('request')
|
||||
const { promisify } = require('util')
|
||||
chai.use(require('chai-as-promised'))
|
||||
|
||||
const fsWriteFile = promisify(fs.writeFile)
|
||||
const fsStat = promisify(fs.stat)
|
||||
const pipeline = promisify(Stream.pipeline)
|
||||
|
||||
async function getMetric(filestoreUrl, metric) {
|
||||
const res = await rp.get(`${filestoreUrl}/metrics`)
|
||||
expect(res.statusCode).to.equal(200)
|
||||
const metricRegex = new RegExp(`^${metric}{[^}]+} ([0-9]+)$`, 'm')
|
||||
const found = metricRegex.exec(res.body)
|
||||
return parseInt(found ? found[1] : 0) || 0
|
||||
}
|
||||
|
||||
// store settings for multiple backends, so that we can test each one.
|
||||
// fs will always be available - add others if they are configured
|
||||
const BackendSettings = {
|
||||
FSPersistor: {
|
||||
backend: 'fs',
|
||||
stores: {
|
||||
user_files: Path.resolve(__dirname, '../../../user_files'),
|
||||
public_files: Path.resolve(__dirname, '../../../public_files'),
|
||||
template_files: Path.resolve(__dirname, '../../../template_files')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (process.env.AWS_ACCESS_KEY_ID) {
|
||||
BackendSettings.S3Persistor = {
|
||||
backend: 's3',
|
||||
s3: {
|
||||
key: process.env.AWS_ACCESS_KEY_ID,
|
||||
secret: process.env.AWS_SECRET_ACCESS_KEY,
|
||||
endpoint: process.env.AWS_S3_ENDPOINT
|
||||
},
|
||||
stores: {
|
||||
user_files: process.env.AWS_S3_USER_FILES_BUCKET_NAME,
|
||||
template_files: process.env.AWS_S3_TEMPLATE_FILES_BUCKET_NAME,
|
||||
public_files: process.env.AWS_S3_PUBLIC_FILES_BUCKET_NAME
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe('Filestore', function() {
|
||||
this.timeout(1000 * 10)
|
||||
const filestoreUrl = `http://localhost:${Settings.internal.filestore.port}`
|
||||
|
||||
// redefine the test suite for every available backend
|
||||
Object.keys(BackendSettings).forEach(backend => {
|
||||
describe(backend, function() {
|
||||
let app, previousEgress, previousIngress
|
||||
|
||||
before(async function() {
|
||||
// create the app with the relevant filestore settings
|
||||
Settings.filestore = BackendSettings[backend]
|
||||
app = new FilestoreApp()
|
||||
await app.runServer()
|
||||
})
|
||||
|
||||
after(async function() {
|
||||
return app.stop()
|
||||
})
|
||||
|
||||
beforeEach(async function() {
|
||||
// retrieve previous metrics from the app
|
||||
if (Settings.filestore.backend === 's3') {
|
||||
;[previousEgress, previousIngress] = await Promise.all([
|
||||
getMetric(filestoreUrl, 's3_egress'),
|
||||
getMetric(filestoreUrl, 's3_ingress')
|
||||
])
|
||||
}
|
||||
})
|
||||
|
||||
it('should send a 200 for the status endpoint', async function() {
|
||||
const response = await rp(`${filestoreUrl}/status`)
|
||||
expect(response.statusCode).to.equal(200)
|
||||
expect(response.body).to.contain('filestore')
|
||||
expect(response.body).to.contain('up')
|
||||
})
|
||||
|
||||
it('should send a 200 for the health-check endpoint', async function() {
|
||||
const response = await rp(`${filestoreUrl}/health_check`)
|
||||
expect(response.statusCode).to.equal(200)
|
||||
expect(response.body).to.equal('OK')
|
||||
})
|
||||
|
||||
describe('with a file on the server', function() {
|
||||
let fileId, fileUrl
|
||||
|
||||
const localFileReadPath =
|
||||
'/tmp/filestore_acceptance_tests_file_read.txt'
|
||||
const constantFileContent = [
|
||||
'hello world',
|
||||
`line 2 goes here ${Math.random()}`,
|
||||
'there are 3 lines in all'
|
||||
].join('\n')
|
||||
|
||||
before(async function() {
|
||||
await fsWriteFile(localFileReadPath, constantFileContent)
|
||||
})
|
||||
|
||||
beforeEach(async function() {
|
||||
fileId = Math.random()
|
||||
fileUrl = `${filestoreUrl}/project/acceptance_tests/file/${fileId}`
|
||||
|
||||
const writeStream = request.post(fileUrl)
|
||||
const readStream = fs.createReadStream(localFileReadPath)
|
||||
// consume the result to ensure the http request has been fully processed
|
||||
const resultStream = fs.createWriteStream('/dev/null')
|
||||
await pipeline(readStream, writeStream, resultStream)
|
||||
})
|
||||
|
||||
it('should return 404 for a non-existant id', async function() {
|
||||
const options = { uri: fileUrl + '___this_is_clearly_wrong___' }
|
||||
await expect(
|
||||
rp.get(options)
|
||||
).to.eventually.be.rejected.and.have.property('statusCode', 404)
|
||||
})
|
||||
|
||||
it('should return the file size on a HEAD request', async function() {
|
||||
const expectedLength = Buffer.byteLength(constantFileContent)
|
||||
const res = await rp.head(fileUrl)
|
||||
expect(res.statusCode).to.equal(200)
|
||||
expect(res.headers['content-length']).to.equal(
|
||||
expectedLength.toString()
|
||||
)
|
||||
})
|
||||
|
||||
it('should be able get the file back', async function() {
|
||||
const res = await rp.get(fileUrl)
|
||||
expect(res.body).to.equal(constantFileContent)
|
||||
})
|
||||
|
||||
it('should be able to get back the first 9 bytes of the file', async function() {
|
||||
const options = {
|
||||
uri: fileUrl,
|
||||
headers: {
|
||||
Range: 'bytes=0-8'
|
||||
}
|
||||
}
|
||||
const res = await rp.get(options)
|
||||
expect(res.body).to.equal('hello wor')
|
||||
})
|
||||
|
||||
it('should be able to get back bytes 4 through 10 of the file', async function() {
|
||||
const options = {
|
||||
uri: fileUrl,
|
||||
headers: {
|
||||
Range: 'bytes=4-10'
|
||||
}
|
||||
}
|
||||
const res = await rp.get(options)
|
||||
expect(res.body).to.equal('o world')
|
||||
})
|
||||
|
||||
it('should be able to delete the file', async function() {
|
||||
const response = await rp.del(fileUrl)
|
||||
expect(response.statusCode).to.equal(204)
|
||||
await expect(
|
||||
rp.get(fileUrl)
|
||||
).to.eventually.be.rejected.and.have.property('statusCode', 404)
|
||||
})
|
||||
|
||||
it('should be able to copy files', async function() {
|
||||
const newProjectID = 'acceptance_tests_copyied_project'
|
||||
const newFileId = Math.random()
|
||||
const newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${newFileId}`
|
||||
const opts = {
|
||||
method: 'put',
|
||||
uri: newFileUrl,
|
||||
json: {
|
||||
source: {
|
||||
project_id: 'acceptance_tests',
|
||||
file_id: fileId
|
||||
}
|
||||
}
|
||||
}
|
||||
let response = await rp(opts)
|
||||
expect(response.statusCode).to.equal(200)
|
||||
response = await rp.del(fileUrl)
|
||||
expect(response.statusCode).to.equal(204)
|
||||
response = await rp.get(newFileUrl)
|
||||
expect(response.body).to.equal(constantFileContent)
|
||||
})
|
||||
|
||||
if (backend === 'S3Persistor') {
|
||||
it('should record an egress metric for the upload', async function() {
|
||||
const metric = await getMetric(filestoreUrl, 's3_egress')
|
||||
expect(metric - previousEgress).to.equal(constantFileContent.length)
|
||||
})
|
||||
|
||||
it('should record an ingress metric when downloading the file', async function() {
|
||||
await rp.get(fileUrl)
|
||||
const metric = await getMetric(filestoreUrl, 's3_ingress')
|
||||
expect(metric - previousIngress).to.equal(
|
||||
constantFileContent.length
|
||||
)
|
||||
})
|
||||
|
||||
it('should record an ingress metric for a partial download', async function() {
|
||||
const options = {
|
||||
uri: fileUrl,
|
||||
headers: {
|
||||
Range: 'bytes=0-8'
|
||||
}
|
||||
}
|
||||
await rp.get(options)
|
||||
const metric = await getMetric(filestoreUrl, 's3_ingress')
|
||||
expect(metric - previousIngress).to.equal(9)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
describe('with a pdf file', function() {
|
||||
let fileId, fileUrl, localFileSize
|
||||
const localFileReadPath = Path.resolve(
|
||||
__dirname,
|
||||
'../../fixtures/test.pdf'
|
||||
)
|
||||
|
||||
beforeEach(async function() {
|
||||
fileId = Math.random()
|
||||
fileUrl = `${filestoreUrl}/project/acceptance_tests/file/${fileId}`
|
||||
const stat = await fsStat(localFileReadPath)
|
||||
localFileSize = stat.size
|
||||
const writeStream = request.post(fileUrl)
|
||||
const endStream = fs.createWriteStream('/dev/null')
|
||||
const readStream = fs.createReadStream(localFileReadPath)
|
||||
await pipeline(readStream, writeStream, endStream)
|
||||
})
|
||||
|
||||
it('should be able get the file back', async function() {
|
||||
const response = await rp.get(fileUrl)
|
||||
expect(response.body.substring(0, 8)).to.equal('%PDF-1.5')
|
||||
})
|
||||
|
||||
if (backend === 'S3Persistor') {
|
||||
it('should record an egress metric for the upload', async function() {
|
||||
const metric = await getMetric(filestoreUrl, 's3_egress')
|
||||
expect(metric - previousEgress).to.equal(localFileSize)
|
||||
})
|
||||
}
|
||||
|
||||
describe('getting the preview image', function() {
|
||||
this.timeout(1000 * 20)
|
||||
let previewFileUrl
|
||||
|
||||
beforeEach(function() {
|
||||
previewFileUrl = `${fileUrl}?style=preview`
|
||||
})
|
||||
|
||||
it('should not time out', async function() {
|
||||
const response = await rp.get(previewFileUrl)
|
||||
expect(response.statusCode).to.equal(200)
|
||||
})
|
||||
|
||||
it('should respond with image data', async function() {
|
||||
// note: this test relies of the imagemagick conversion working
|
||||
const response = await rp.get(previewFileUrl)
|
||||
expect(response.body.length).to.be.greaterThan(400)
|
||||
expect(response.body.substr(1, 3)).to.equal('PNG')
|
||||
})
|
||||
})
|
||||
|
||||
describe('warming the cache', function() {
|
||||
this.timeout(1000 * 20)
|
||||
let previewFileUrl
|
||||
|
||||
beforeEach(function() {
|
||||
previewFileUrl = `${fileUrl}?style=preview&cacheWarm=true`
|
||||
})
|
||||
|
||||
it('should not time out', async function() {
|
||||
const response = await rp.get(previewFileUrl)
|
||||
expect(response.statusCode).to.equal(200)
|
||||
})
|
||||
|
||||
it("should respond with only an 'OK'", async function() {
|
||||
// note: this test relies of the imagemagick conversion working
|
||||
const response = await rp.get(previewFileUrl)
|
||||
expect(response.body).to.equal('OK')
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -1,326 +0,0 @@
|
|||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-path-concat,
|
||||
no-return-assign,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const { assert } = require('chai')
|
||||
const sinon = require('sinon')
|
||||
const chai = require('chai')
|
||||
const should = chai.should()
|
||||
const { expect } = chai
|
||||
const modulePath = '../../../app/js/LocalFileWriter.js'
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
const fs = require('fs')
|
||||
const request = require('request')
|
||||
const settings = require('settings-sharelatex')
|
||||
const FilestoreApp = require('./FilestoreApp')
|
||||
const async = require('async')
|
||||
|
||||
const getMetric = (filestoreUrl, metric, cb) =>
|
||||
request.get(`${filestoreUrl}/metrics`, function(err, res) {
|
||||
expect(res.statusCode).to.equal(200)
|
||||
const metricRegex = new RegExp(`^${metric}{[^}]+} ([0-9]+)$`, 'm')
|
||||
return cb(parseInt(__guard__(metricRegex.exec(res.body), x => x[1]) || '0'))
|
||||
})
|
||||
|
||||
describe('Filestore', function() {
|
||||
before(function(done) {
|
||||
this.localFileReadPath = '/tmp/filestore_acceptence_tests_file_read.txt'
|
||||
this.localFileWritePath = '/tmp/filestore_acceptence_tests_file_write.txt'
|
||||
|
||||
this.constantFileContent = [
|
||||
'hello world',
|
||||
`line 2 goes here ${Math.random()}`,
|
||||
'there are 3 lines in all'
|
||||
].join('\n')
|
||||
|
||||
this.filestoreUrl = `http://localhost:${settings.internal.filestore.port}`
|
||||
return fs.writeFile(
|
||||
this.localFileReadPath,
|
||||
this.constantFileContent,
|
||||
function(err) {
|
||||
if (err) {
|
||||
return done(err)
|
||||
}
|
||||
return FilestoreApp.waitForS3(done)
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
beforeEach(function(done) {
|
||||
return FilestoreApp.ensureRunning(() => {
|
||||
return async.parallel(
|
||||
[
|
||||
cb => {
|
||||
return fs.unlink(this.localFileWritePath, () => cb())
|
||||
},
|
||||
cb => {
|
||||
return getMetric(this.filestoreUrl, 's3_egress', metric => {
|
||||
this.previousEgress = metric
|
||||
return cb()
|
||||
})
|
||||
},
|
||||
cb => {
|
||||
return getMetric(this.filestoreUrl, 's3_ingress', metric => {
|
||||
this.previousIngress = metric
|
||||
return cb()
|
||||
})
|
||||
}
|
||||
],
|
||||
done
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
it('should send a 200 for status endpoint', function(done) {
|
||||
return request(`${this.filestoreUrl}/status`, function(
|
||||
err,
|
||||
response,
|
||||
body
|
||||
) {
|
||||
response.statusCode.should.equal(200)
|
||||
body.indexOf('filestore').should.not.equal(-1)
|
||||
body.indexOf('up').should.not.equal(-1)
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
describe('with a file on the server', function() {
|
||||
beforeEach(function(done) {
|
||||
this.timeout(1000 * 10)
|
||||
this.file_id = Math.random()
|
||||
this.fileUrl = `${this.filestoreUrl}/project/acceptence_tests/file/${this.file_id}`
|
||||
|
||||
const writeStream = request.post(this.fileUrl)
|
||||
|
||||
writeStream.on('end', done)
|
||||
return fs.createReadStream(this.localFileReadPath).pipe(writeStream)
|
||||
})
|
||||
|
||||
it('should return 404 for a non-existant id', function(done) {
|
||||
this.timeout(1000 * 20)
|
||||
const options = { uri: this.fileUrl + '___this_is_clearly_wrong___' }
|
||||
return request.get(options, (err, response, body) => {
|
||||
response.statusCode.should.equal(404)
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
it('should record an egress metric for the upload', function(done) {
|
||||
return getMetric(this.filestoreUrl, 's3_egress', metric => {
|
||||
expect(metric - this.previousEgress).to.equal(
|
||||
this.constantFileContent.length
|
||||
)
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
it('should return the file size on a HEAD request', function(done) {
|
||||
const expectedLength = Buffer.byteLength(this.constantFileContent)
|
||||
return request.head(this.fileUrl, (err, res) => {
|
||||
expect(res.statusCode).to.equal(200)
|
||||
expect(res.headers['content-length']).to.equal(
|
||||
expectedLength.toString()
|
||||
)
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
it('should be able get the file back', function(done) {
|
||||
this.timeout(1000 * 10)
|
||||
return request.get(this.fileUrl, (err, response, body) => {
|
||||
body.should.equal(this.constantFileContent)
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
it('should record an ingress metric when downloading the file', function(done) {
|
||||
this.timeout(1000 * 10)
|
||||
return request.get(this.fileUrl, () => {
|
||||
return getMetric(this.filestoreUrl, 's3_ingress', metric => {
|
||||
expect(metric - this.previousIngress).to.equal(
|
||||
this.constantFileContent.length
|
||||
)
|
||||
return done()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
it('should be able to get back the first 9 bytes of the file', function(done) {
|
||||
this.timeout(1000 * 10)
|
||||
const options = {
|
||||
uri: this.fileUrl,
|
||||
headers: {
|
||||
Range: 'bytes=0-8'
|
||||
}
|
||||
}
|
||||
return request.get(options, (err, response, body) => {
|
||||
body.should.equal('hello wor')
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
it('should record an ingress metric for a partial download', function(done) {
|
||||
this.timeout(1000 * 10)
|
||||
const options = {
|
||||
uri: this.fileUrl,
|
||||
headers: {
|
||||
Range: 'bytes=0-8'
|
||||
}
|
||||
}
|
||||
return request.get(options, () => {
|
||||
return getMetric(this.filestoreUrl, 's3_ingress', metric => {
|
||||
expect(metric - this.previousIngress).to.equal(9)
|
||||
return done()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
it('should be able to get back bytes 4 through 10 of the file', function(done) {
|
||||
this.timeout(1000 * 10)
|
||||
const options = {
|
||||
uri: this.fileUrl,
|
||||
headers: {
|
||||
Range: 'bytes=4-10'
|
||||
}
|
||||
}
|
||||
return request.get(options, (err, response, body) => {
|
||||
body.should.equal('o world')
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
it('should be able to delete the file', function(done) {
|
||||
this.timeout(1000 * 20)
|
||||
return request.del(this.fileUrl, (err, response, body) => {
|
||||
response.statusCode.should.equal(204)
|
||||
return request.get(this.fileUrl, (err, response, body) => {
|
||||
response.statusCode.should.equal(404)
|
||||
return done()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
return it('should be able to copy files', function(done) {
|
||||
this.timeout(1000 * 20)
|
||||
|
||||
const newProjectID = 'acceptence_tests_copyied_project'
|
||||
const newFileId = Math.random()
|
||||
const newFileUrl = `${this.filestoreUrl}/project/${newProjectID}/file/${newFileId}`
|
||||
const opts = {
|
||||
method: 'put',
|
||||
uri: newFileUrl,
|
||||
json: {
|
||||
source: {
|
||||
project_id: 'acceptence_tests',
|
||||
file_id: this.file_id
|
||||
}
|
||||
}
|
||||
}
|
||||
return request(opts, (err, response, body) => {
|
||||
response.statusCode.should.equal(200)
|
||||
return request.del(this.fileUrl, (err, response, body) => {
|
||||
response.statusCode.should.equal(204)
|
||||
return request.get(newFileUrl, (err, response, body) => {
|
||||
body.should.equal(this.constantFileContent)
|
||||
return done()
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
return describe('with a pdf file', function() {
|
||||
beforeEach(function(done) {
|
||||
this.timeout(1000 * 10)
|
||||
this.file_id = Math.random()
|
||||
this.fileUrl = `${this.filestoreUrl}/project/acceptence_tests/file/${this.file_id}`
|
||||
this.localFileReadPath = __dirname + '/../../fixtures/test.pdf'
|
||||
return fs.stat(this.localFileReadPath, (err, stat) => {
|
||||
this.localFileSize = stat.size
|
||||
const writeStream = request.post(this.fileUrl)
|
||||
|
||||
writeStream.on('end', done)
|
||||
return fs.createReadStream(this.localFileReadPath).pipe(writeStream)
|
||||
})
|
||||
})
|
||||
|
||||
it('should record an egress metric for the upload', function(done) {
|
||||
return getMetric(this.filestoreUrl, 's3_egress', metric => {
|
||||
expect(metric - this.previousEgress).to.equal(this.localFileSize)
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
it('should be able get the file back', function(done) {
|
||||
this.timeout(1000 * 10)
|
||||
return request.get(this.fileUrl, (err, response, body) => {
|
||||
expect(body.substring(0, 8)).to.equal('%PDF-1.5')
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
describe('getting the preview image', function() {
|
||||
beforeEach(function() {
|
||||
return (this.previewFileUrl = `${this.fileUrl}?style=preview`)
|
||||
})
|
||||
|
||||
it('should not time out', function(done) {
|
||||
this.timeout(1000 * 20)
|
||||
return request.get(this.previewFileUrl, (err, response, body) => {
|
||||
expect(response).to.not.equal(null)
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
return it('should respond with image data', function(done) {
|
||||
// note: this test relies of the imagemagick conversion working
|
||||
this.timeout(1000 * 20)
|
||||
return request.get(this.previewFileUrl, (err, response, body) => {
|
||||
expect(response.statusCode).to.equal(200)
|
||||
expect(body.length).to.be.greaterThan(400)
|
||||
return done()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
return describe('warming the cache', function() {
|
||||
beforeEach(function() {
|
||||
return (this.fileUrl = this.fileUrl + '?style=preview&cacheWarm=true')
|
||||
})
|
||||
|
||||
it('should not time out', function(done) {
|
||||
this.timeout(1000 * 20)
|
||||
return request.get(this.fileUrl, (err, response, body) => {
|
||||
expect(response).to.not.equal(null)
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
return it("should respond with only an 'OK'", function(done) {
|
||||
// note: this test relies of the imagemagick conversion working
|
||||
this.timeout(1000 * 20)
|
||||
return request.get(this.fileUrl, (err, response, body) => {
|
||||
expect(response.statusCode).to.equal(200)
|
||||
body.should.equal('OK')
|
||||
return done()
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return typeof value !== 'undefined' && value !== null
|
||||
? transform(value)
|
||||
: undefined
|
||||
}
|
||||
Loading…
Reference in a new issue