overleaf/services/clsi/app.js

372 lines
10 KiB
JavaScript
Raw Normal View History

/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS103: Rewrite code to no longer use __guard__
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let tenMinutes
const Metrics = require('metrics-sharelatex')
Metrics.initialize('clsi')
const CompileController = require('./app/js/CompileController')
const Settings = require('settings-sharelatex')
const logger = require('logger-sharelatex')
logger.initialize('clsi')
if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) {
logger.initializeErrorReporting(Settings.sentry.dsn)
}
const smokeTest = require('smoke-test-sharelatex')
const ContentTypeMapper = require('./app/js/ContentTypeMapper')
const Errors = require('./app/js/Errors')
const Path = require('path')
const fs = require('fs')
Metrics.open_sockets.monitor(logger)
Metrics.memory.monitor(logger)
const ProjectPersistenceManager = require('./app/js/ProjectPersistenceManager')
const OutputCacheManager = require('./app/js/OutputCacheManager')
require('./app/js/db').sync()
const express = require('express')
const bodyParser = require('body-parser')
const app = express()
Metrics.injectMetricsRoute(app)
app.use(Metrics.http.monitor(logger))
// Compile requests can take longer than the default two
// minutes (including file download time), so bump up the
// timeout a bit.
const TIMEOUT = 10 * 60 * 1000
app.use(function(req, res, next) {
req.setTimeout(TIMEOUT)
res.setTimeout(TIMEOUT)
res.removeHeader('X-Powered-By')
return next()
})
app.param('project_id', function(req, res, next, project_id) {
if (project_id != null ? project_id.match(/^[a-zA-Z0-9_-]+$/) : undefined) {
return next()
} else {
return next(new Error('invalid project id'))
}
})
app.param('user_id', function(req, res, next, user_id) {
if (user_id != null ? user_id.match(/^[0-9a-f]{24}$/) : undefined) {
return next()
} else {
return next(new Error('invalid user id'))
}
})
app.param('build_id', function(req, res, next, build_id) {
if (
build_id != null
? build_id.match(OutputCacheManager.BUILD_REGEX)
: undefined
) {
return next()
} else {
return next(new Error(`invalid build id ${build_id}`))
}
})
app.post(
'/project/:project_id/compile',
bodyParser.json({ limit: Settings.compileSizeLimit }),
CompileController.compile
)
app.post('/project/:project_id/compile/stop', CompileController.stopCompile)
app.delete('/project/:project_id', CompileController.clearCache)
app.get('/project/:project_id/sync/code', CompileController.syncFromCode)
app.get('/project/:project_id/sync/pdf', CompileController.syncFromPdf)
app.get('/project/:project_id/wordcount', CompileController.wordcount)
app.get('/project/:project_id/status', CompileController.status)
// Per-user containers
app.post(
'/project/:project_id/user/:user_id/compile',
bodyParser.json({ limit: Settings.compileSizeLimit }),
CompileController.compile
)
app.post(
'/project/:project_id/user/:user_id/compile/stop',
CompileController.stopCompile
)
app.delete('/project/:project_id/user/:user_id', CompileController.clearCache)
app.get(
'/project/:project_id/user/:user_id/sync/code',
CompileController.syncFromCode
)
app.get(
'/project/:project_id/user/:user_id/sync/pdf',
CompileController.syncFromPdf
)
app.get(
'/project/:project_id/user/:user_id/wordcount',
CompileController.wordcount
)
const ForbidSymlinks = require('./app/js/StaticServerForbidSymlinks')
// create a static server which does not allow access to any symlinks
// avoids possible mismatch of root directory between middleware check
// and serving the files
const staticServer = ForbidSymlinks(express.static, Settings.path.compilesDir, {
setHeaders(res, path, stat) {
if (Path.basename(path) === 'output.pdf') {
// Calculate an etag in the same way as nginx
// https://github.com/tj/send/issues/65
const etag = (path, stat) =>
`"${Math.ceil(+stat.mtime / 1000).toString(16)}` +
'-' +
Number(stat.size).toString(16) +
'"'
res.set('Etag', etag(path, stat))
}
return res.set('Content-Type', ContentTypeMapper.map(path))
}
})
app.get('/project/:project_id/user/:user_id/build/:build_id/output/*', function(
req,
res,
next
) {
// for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
req.url =
`/${req.params.project_id}-${req.params.user_id}/` +
OutputCacheManager.path(req.params.build_id, `/${req.params[0]}`)
return staticServer(req, res, next)
})
app.get('/project/:project_id/build/:build_id/output/*', function(
req,
res,
next
) {
// for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
req.url =
`/${req.params.project_id}/` +
OutputCacheManager.path(req.params.build_id, `/${req.params[0]}`)
return staticServer(req, res, next)
})
app.get('/project/:project_id/user/:user_id/output/*', function(
req,
res,
next
) {
// for specific user get the path to the top level file
req.url = `/${req.params.project_id}-${req.params.user_id}/${req.params[0]}`
return staticServer(req, res, next)
})
app.get('/project/:project_id/output/*', function(req, res, next) {
if (
(req.query != null ? req.query.build : undefined) != null &&
req.query.build.match(OutputCacheManager.BUILD_REGEX)
) {
// for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
req.url =
`/${req.params.project_id}/` +
OutputCacheManager.path(req.query.build, `/${req.params[0]}`)
} else {
req.url = `/${req.params.project_id}/${req.params[0]}`
}
return staticServer(req, res, next)
})
app.get('/oops', function(req, res, next) {
logger.error({ err: 'hello' }, 'test error')
return res.send('error\n')
})
app.get('/status', (req, res, next) => res.send('CLSI is alive\n'))
const resCacher = {
contentType(setContentType) {
this.setContentType = setContentType
},
send(code, body) {
this.code = code
this.body = body
},
// default the server to be down
code: 500,
body: {},
setContentType: 'application/json'
}
if (Settings.smokeTest) {
let runSmokeTest
;(runSmokeTest = function() {
logger.log('running smoke tests')
smokeTest.run(require.resolve(__dirname + '/test/smoke/js/SmokeTests.js'))(
{},
resCacher
)
return setTimeout(runSmokeTest, 30 * 1000)
})()
}
app.get('/health_check', function(req, res) {
res.contentType(resCacher != null ? resCacher.setContentType : undefined)
return res
.status(resCacher != null ? resCacher.code : undefined)
.send(resCacher != null ? resCacher.body : undefined)
})
app.get('/smoke_test_force', (req, res) =>
smokeTest.run(require.resolve(__dirname + '/test/smoke/js/SmokeTests.js'))(
req,
res
)
)
const profiler = require('v8-profiler-node8')
app.get('/profile', function(req, res) {
const time = parseInt(req.query.time || '1000')
profiler.startProfiling('test')
return setTimeout(function() {
const profile = profiler.stopProfiling('test')
return res.json(profile)
}, time)
})
app.get('/heapdump', (req, res) =>
require('heapdump').writeSnapshot(
`/tmp/${Date.now()}.clsi.heapsnapshot`,
(err, filename) => res.send(filename)
)
)
app.use(function(error, req, res, next) {
if (error instanceof Errors.NotFoundError) {
logger.warn({ err: error, url: req.url }, 'not found error')
return res.sendStatus(404)
} else {
logger.error({ err: error, url: req.url }, 'server error')
return res.sendStatus((error != null ? error.statusCode : undefined) || 500)
}
})
const net = require('net')
const os = require('os')
let STATE = 'up'
const loadTcpServer = net.createServer(function(socket) {
socket.on('error', function(err) {
if (err.code === 'ECONNRESET') {
// this always comes up, we don't know why
return
}
logger.err({ err }, 'error with socket on load check')
return socket.destroy()
})
if (STATE === 'up' && Settings.internal.load_balancer_agent.report_load) {
let availableWorkingCpus
const currentLoad = os.loadavg()[0]
// staging clis's have 1 cpu core only
if (os.cpus().length === 1) {
availableWorkingCpus = 1
} else {
availableWorkingCpus = os.cpus().length - 1
}
const freeLoad = availableWorkingCpus - currentLoad
let freeLoadPercentage = Math.round((freeLoad / availableWorkingCpus) * 100)
if (freeLoadPercentage <= 0) {
freeLoadPercentage = 1 // when its 0 the server is set to drain and will move projects to different servers
}
socket.write(`up, ${freeLoadPercentage}%\n`, 'ASCII')
return socket.end()
} else {
socket.write(`${STATE}\n`, 'ASCII')
return socket.end()
}
})
const loadHttpServer = express()
loadHttpServer.post('/state/up', function(req, res, next) {
STATE = 'up'
logger.info('getting message to set server to down')
return res.sendStatus(204)
})
loadHttpServer.post('/state/down', function(req, res, next) {
STATE = 'down'
logger.info('getting message to set server to down')
return res.sendStatus(204)
})
loadHttpServer.post('/state/maint', function(req, res, next) {
STATE = 'maint'
logger.info('getting message to set server to maint')
return res.sendStatus(204)
})
const port =
__guard__(
Settings.internal != null ? Settings.internal.clsi : undefined,
x => x.port
) || 3013
const host =
__guard__(
Settings.internal != null ? Settings.internal.clsi : undefined,
x1 => x1.host
) || 'localhost'
const load_tcp_port = Settings.internal.load_balancer_agent.load_port
const load_http_port = Settings.internal.load_balancer_agent.local_port
if (!module.parent) {
// Called directly
app.listen(port, host, error =>
logger.info(`CLSI starting up, listening on ${host}:${port}`)
)
loadTcpServer.listen(load_tcp_port, host, function(error) {
if (error != null) {
throw error
}
return logger.info(`Load tcp agent listening on load port ${load_tcp_port}`)
})
loadHttpServer.listen(load_http_port, host, function(error) {
if (error != null) {
throw error
}
return logger.info(
`Load http agent listening on load port ${load_http_port}`
)
})
}
module.exports = app
2014-02-12 12:27:43 -05:00
setInterval(
() => ProjectPersistenceManager.clearExpiredProjects(),
(tenMinutes = 10 * 60 * 1000)
)
2016-04-08 08:31:23 -04:00
function __guard__(value, transform) {
return typeof value !== 'undefined' && value !== null
? transform(value)
: undefined
}