/* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns * DS103: Rewrite code to no longer use __guard__ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ let tenMinutes const Metrics = require('metrics-sharelatex') Metrics.initialize('clsi') const CompileController = require('./app/js/CompileController') const Settings = require('settings-sharelatex') const logger = require('logger-sharelatex') logger.initialize('clsi') if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) { logger.initializeErrorReporting(Settings.sentry.dsn) } const smokeTest = require('smoke-test-sharelatex') const ContentTypeMapper = require('./app/js/ContentTypeMapper') const Errors = require('./app/js/Errors') const Path = require('path') Metrics.open_sockets.monitor(logger) Metrics.memory.monitor(logger) const ProjectPersistenceManager = require('./app/js/ProjectPersistenceManager') const OutputCacheManager = require('./app/js/OutputCacheManager') require('./app/js/db').sync() const express = require('express') const bodyParser = require('body-parser') const app = express() Metrics.injectMetricsRoute(app) app.use(Metrics.http.monitor(logger)) // Compile requests can take longer than the default two // minutes (including file download time), so bump up the // timeout a bit. const TIMEOUT = 10 * 60 * 1000 app.use(function(req, res, next) { req.setTimeout(TIMEOUT) res.setTimeout(TIMEOUT) res.removeHeader('X-Powered-By') return next() }) app.param('project_id', function(req, res, next, project_id) { if (project_id != null ? project_id.match(/^[a-zA-Z0-9_-]+$/) : undefined) { return next() } else { return next(new Error('invalid project id')) } }) app.param('user_id', function(req, res, next, user_id) { if (user_id != null ? user_id.match(/^[0-9a-f]{24}$/) : undefined) { return next() } else { return next(new Error('invalid user id')) } }) app.param('build_id', function(req, res, next, build_id) { if ( build_id != null ? build_id.match(OutputCacheManager.BUILD_REGEX) : undefined ) { return next() } else { return next(new Error(`invalid build id ${build_id}`)) } }) app.post( '/project/:project_id/compile', bodyParser.json({ limit: Settings.compileSizeLimit }), CompileController.compile ) app.post('/project/:project_id/compile/stop', CompileController.stopCompile) app.delete('/project/:project_id', CompileController.clearCache) app.get('/project/:project_id/sync/code', CompileController.syncFromCode) app.get('/project/:project_id/sync/pdf', CompileController.syncFromPdf) app.get('/project/:project_id/wordcount', CompileController.wordcount) app.get('/project/:project_id/status', CompileController.status) // Per-user containers app.post( '/project/:project_id/user/:user_id/compile', bodyParser.json({ limit: Settings.compileSizeLimit }), CompileController.compile ) app.post( '/project/:project_id/user/:user_id/compile/stop', CompileController.stopCompile ) app.delete('/project/:project_id/user/:user_id', CompileController.clearCache) app.get( '/project/:project_id/user/:user_id/sync/code', CompileController.syncFromCode ) app.get( '/project/:project_id/user/:user_id/sync/pdf', CompileController.syncFromPdf ) app.get( '/project/:project_id/user/:user_id/wordcount', CompileController.wordcount ) const ForbidSymlinks = require('./app/js/StaticServerForbidSymlinks') // create a static server which does not allow access to any symlinks // avoids possible mismatch of root directory between middleware check // and serving the files const staticServer = ForbidSymlinks(express.static, Settings.path.compilesDir, { setHeaders(res, path, stat) { if (Path.basename(path) === 'output.pdf') { // Calculate an etag in the same way as nginx // https://github.com/tj/send/issues/65 const etag = (path, stat) => `"${Math.ceil(+stat.mtime / 1000).toString(16)}` + '-' + Number(stat.size).toString(16) + '"' res.set('Etag', etag(path, stat)) } return res.set('Content-Type', ContentTypeMapper.map(path)) } }) app.get('/project/:project_id/user/:user_id/build/:build_id/output/*', function( req, res, next ) { // for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId) req.url = `/${req.params.project_id}-${req.params.user_id}/` + OutputCacheManager.path(req.params.build_id, `/${req.params[0]}`) return staticServer(req, res, next) }) app.get('/project/:project_id/build/:build_id/output/*', function( req, res, next ) { // for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId) req.url = `/${req.params.project_id}/` + OutputCacheManager.path(req.params.build_id, `/${req.params[0]}`) return staticServer(req, res, next) }) app.get('/project/:project_id/user/:user_id/output/*', function( req, res, next ) { // for specific user get the path to the top level file req.url = `/${req.params.project_id}-${req.params.user_id}/${req.params[0]}` return staticServer(req, res, next) }) app.get('/project/:project_id/output/*', function(req, res, next) { if ( (req.query != null ? req.query.build : undefined) != null && req.query.build.match(OutputCacheManager.BUILD_REGEX) ) { // for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId) req.url = `/${req.params.project_id}/` + OutputCacheManager.path(req.query.build, `/${req.params[0]}`) } else { req.url = `/${req.params.project_id}/${req.params[0]}` } return staticServer(req, res, next) }) app.get('/oops', function(req, res, next) { logger.error({ err: 'hello' }, 'test error') return res.send('error\n') }) app.get('/status', (req, res, next) => res.send('CLSI is alive\n')) const resCacher = { contentType(setContentType) { this.setContentType = setContentType }, send(code, body) { this.code = code this.body = body }, // default the server to be down code: 500, body: {}, setContentType: 'application/json' } let shutdownTime if (Settings.processLifespanLimitMs) { Settings.processLifespanLimitMs += Settings.processLifespanLimitMs * (Math.random() / 10) shutdownTime = Date.now() + Settings.processLifespanLimitMs logger.info('Lifespan limited to ', shutdownTime) } const checkIfProcessIsTooOld = function(cont) { if (shutdownTime && shutdownTime < Date.now()) { logger.log('shutting down, process is too old') resCacher.send = function() {} resCacher.code = 500 resCacher.body = { processToOld: true } } else { cont() } } if (Settings.smokeTest) { const runSmokeTest = function() { checkIfProcessIsTooOld(function() { logger.log('running smoke tests') smokeTest.run( require.resolve(__dirname + '/test/smoke/js/SmokeTests.js') )({}, resCacher) return setTimeout(runSmokeTest, 30 * 1000) }) } runSmokeTest() } app.get('/health_check', function(req, res) { res.contentType(resCacher.setContentType) return res.status(resCacher.code).send(resCacher.body) }) app.get('/smoke_test_force', (req, res) => smokeTest.run(require.resolve(__dirname + '/test/smoke/js/SmokeTests.js'))( req, res ) ) app.use(function(error, req, res, next) { if (error instanceof Errors.NotFoundError) { logger.warn({ err: error, url: req.url }, 'not found error') return res.sendStatus(404) } else { logger.error({ err: error, url: req.url }, 'server error') return res.sendStatus((error != null ? error.statusCode : undefined) || 500) } }) const net = require('net') const os = require('os') let STATE = 'up' const loadTcpServer = net.createServer(function(socket) { socket.on('error', function(err) { if (err.code === 'ECONNRESET') { // this always comes up, we don't know why return } logger.err({ err }, 'error with socket on load check') return socket.destroy() }) if (STATE === 'up' && Settings.internal.load_balancer_agent.report_load) { let availableWorkingCpus const currentLoad = os.loadavg()[0] // staging clis's have 1 cpu core only if (os.cpus().length === 1) { availableWorkingCpus = 1 } else { availableWorkingCpus = os.cpus().length - 1 } const freeLoad = availableWorkingCpus - currentLoad let freeLoadPercentage = Math.round((freeLoad / availableWorkingCpus) * 100) if (freeLoadPercentage <= 0) { freeLoadPercentage = 1 // when its 0 the server is set to drain and will move projects to different servers } socket.write(`up, ${freeLoadPercentage}%\n`, 'ASCII') return socket.end() } else { socket.write(`${STATE}\n`, 'ASCII') return socket.end() } }) const loadHttpServer = express() loadHttpServer.post('/state/up', function(req, res, next) { STATE = 'up' logger.info('getting message to set server to down') return res.sendStatus(204) }) loadHttpServer.post('/state/down', function(req, res, next) { STATE = 'down' logger.info('getting message to set server to down') return res.sendStatus(204) }) loadHttpServer.post('/state/maint', function(req, res, next) { STATE = 'maint' logger.info('getting message to set server to maint') return res.sendStatus(204) }) const port = __guard__( Settings.internal != null ? Settings.internal.clsi : undefined, x => x.port ) || 3013 const host = __guard__( Settings.internal != null ? Settings.internal.clsi : undefined, x1 => x1.host ) || 'localhost' const load_tcp_port = Settings.internal.load_balancer_agent.load_port const load_http_port = Settings.internal.load_balancer_agent.local_port if (!module.parent) { // Called directly app.listen(port, host, error => logger.info(`CLSI starting up, listening on ${host}:${port}`) ) loadTcpServer.listen(load_tcp_port, host, function(error) { if (error != null) { throw error } return logger.info(`Load tcp agent listening on load port ${load_tcp_port}`) }) loadHttpServer.listen(load_http_port, host, function(error) { if (error != null) { throw error } return logger.info( `Load http agent listening on load port ${load_http_port}` ) }) } module.exports = app setInterval( () => ProjectPersistenceManager.clearExpiredProjects(), (tenMinutes = 10 * 60 * 1000) ) function __guard__(value, transform) { return typeof value !== 'undefined' && value !== null ? transform(value) : undefined }