overleaf/services/clsi/app.js

/*
 * decaffeinate suggestions:
 * DS102: Remove unnecessary code created because of implicit returns
 * DS103: Rewrite code to no longer use __guard__
 * DS207: Consider shorter variations of null checks
 * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
 */
let tenMinutes;
const Metrics = require("metrics-sharelatex");
Metrics.initialize("clsi");

const CompileController = require("./app/js/CompileController");
const Settings = require("settings-sharelatex");
const logger = require("logger-sharelatex");
logger.initialize("clsi");
if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) {
	logger.initializeErrorReporting(Settings.sentry.dsn);
}

const smokeTest = require("smoke-test-sharelatex");
const ContentTypeMapper = require("./app/js/ContentTypeMapper");
const Errors = require('./app/js/Errors');

const Path = require("path");
const fs = require("fs");


Metrics.open_sockets.monitor(logger);
Metrics.memory.monitor(logger);

const ProjectPersistenceManager = require("./app/js/ProjectPersistenceManager");
const OutputCacheManager = require("./app/js/OutputCacheManager");

require("./app/js/db").sync();

const express = require("express");
const bodyParser = require("body-parser");
const app = express();

Metrics.injectMetricsRoute(app);
app.use(Metrics.http.monitor(logger));

// Compile requests can take longer than the default two
// minutes (including file download time), so bump up the 
// timeout a bit.
const TIMEOUT = 10 * 60 * 1000;
app.use(function(req, res, next) {
	req.setTimeout(TIMEOUT);
	res.setTimeout(TIMEOUT);
	res.removeHeader("X-Powered-By");
	return next();
});

app.param('project_id', function(req, res, next, project_id) {
	if ((project_id != null ? project_id.match(/^[a-zA-Z0-9_-]+$/) : undefined)) {
		return next();
	} else {
		return next(new Error("invalid project id"));
	}
});

app.param('user_id', function(req, res, next, user_id) {
	if ((user_id != null ? user_id.match(/^[0-9a-f]{24}$/) : undefined)) {
		return next();
	} else {
		return next(new Error("invalid user id"));
	}
});

app.param('build_id', function(req, res, next, build_id) {
	if ((build_id != null ? build_id.match(OutputCacheManager.BUILD_REGEX) : undefined)) {
		return next();
	} else {
		return next(new Error(`invalid build id ${build_id}`));
	}
});


app.post("/project/:project_id/compile", bodyParser.json({limit: Settings.compileSizeLimit}), CompileController.compile);
app.post("/project/:project_id/compile/stop", CompileController.stopCompile);
app.delete("/project/:project_id", CompileController.clearCache);

app.get("/project/:project_id/sync/code", CompileController.syncFromCode);
app.get("/project/:project_id/sync/pdf", CompileController.syncFromPdf);
app.get("/project/:project_id/wordcount", CompileController.wordcount);
app.get("/project/:project_id/status", CompileController.status);

// Per-user containers
app.post("/project/:project_id/user/:user_id/compile", bodyParser.json({limit: Settings.compileSizeLimit}), CompileController.compile);
app.post("/project/:project_id/user/:user_id/compile/stop", CompileController.stopCompile);
app.delete("/project/:project_id/user/:user_id", CompileController.clearCache);

app.get("/project/:project_id/user/:user_id/sync/code", CompileController.syncFromCode);
app.get("/project/:project_id/user/:user_id/sync/pdf", CompileController.syncFromPdf);
app.get("/project/:project_id/user/:user_id/wordcount", CompileController.wordcount);

const ForbidSymlinks = require("./app/js/StaticServerForbidSymlinks");

// create a static server which does not allow access to any symlinks
// avoids possible mismatch of root directory between middleware check
// and serving the files
const staticServer = ForbidSymlinks(express.static, Settings.path.compilesDir, { setHeaders(res, path, stat) {
	if (Path.basename(path) === "output.pdf") {
		// Calculate an etag in the same way as nginx
		// https://github.com/tj/send/issues/65
		const etag = (path, stat) =>
			`"${Math.ceil(+stat.mtime / 1000).toString(16)}` +
			'-' + Number(stat.size).toString(16) + '"'
		;
		res.set("Etag", etag(path, stat));
	}
	return res.set("Content-Type", ContentTypeMapper.map(path));
}
}
);

app.get("/project/:project_id/user/:user_id/build/:build_id/output/*", function(req, res, next) {
	// for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
	req.url = `/${req.params.project_id}-${req.params.user_id}/` + OutputCacheManager.path(req.params.build_id, `/${req.params[0]}`);
	return staticServer(req, res, next);
});

app.get("/project/:project_id/build/:build_id/output/*", function(req, res, next) {
	// for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
	req.url = `/${req.params.project_id}/` + OutputCacheManager.path(req.params.build_id, `/${req.params[0]}`);
	return staticServer(req, res, next);
});

app.get("/project/:project_id/user/:user_id/output/*", function(req, res, next) {
	// for specific user get the path to the top level file
	req.url = `/${req.params.project_id}-${req.params.user_id}/${req.params[0]}`;
	return staticServer(req, res, next);
});

app.get("/project/:project_id/output/*", function(req, res, next) {
	if (((req.query != null ? req.query.build : undefined) != null) && req.query.build.match(OutputCacheManager.BUILD_REGEX)) {
		// for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
		req.url = `/${req.params.project_id}/` + OutputCacheManager.path(req.query.build, `/${req.params[0]}`);
	} else {
		req.url = `/${req.params.project_id}/${req.params[0]}`;
	}
	return staticServer(req, res, next);
});

app.get("/oops", function(req, res, next) {
	logger.error({err: "hello"}, "test error");
	return res.send("error\n");
});


app.get("/status", (req, res, next) => res.send("CLSI is alive\n"));

const resCacher = {
	contentType(setContentType){
		this.setContentType = setContentType;
	},
	send(code, body){
		this.code = code;
		this.body = body;
	},

	//default the server to be down
	code:500,
	body:{},
	setContentType:"application/json"
};

if (Settings.smokeTest) {
	let runSmokeTest;
	(runSmokeTest = function() {
		logger.log("running smoke tests");
		smokeTest.run(require.resolve(__dirname + "/test/smoke/js/SmokeTests.js"))({}, resCacher);
		return setTimeout(runSmokeTest, 30 * 1000);
	})();
}

app.get("/health_check", function(req, res){
	res.contentType(resCacher != null ? resCacher.setContentType : undefined);
	return res.status(resCacher != null ? resCacher.code : undefined).send(resCacher != null ? resCacher.body : undefined);
});

app.get("/smoke_test_force", (req, res)=> smokeTest.run(require.resolve(__dirname + "/test/smoke/js/SmokeTests.js"))(req, res));

const profiler = require("v8-profiler-node8");
app.get("/profile", function(req, res) {
	const time = parseInt(req.query.time || "1000");
	profiler.startProfiling("test");
	return setTimeout(function() {
		const profile = profiler.stopProfiling("test");
		return res.json(profile);
	}
	, time);
});

app.get("/heapdump", (req, res)=>
	require('heapdump').writeSnapshot(`/tmp/${Date.now()}.clsi.heapsnapshot`, (err, filename)=> res.send(filename))
);

app.use(function(error, req, res, next) {
	if (error instanceof Errors.NotFoundError) {
		logger.warn({err: error, url: req.url}, "not found error");
		return res.sendStatus(404);
	} else {
		logger.error({err: error, url: req.url}, "server error");
		return res.sendStatus((error != null ? error.statusCode : undefined) || 500);
	}
});

const net = require("net");
const os = require("os");

let STATE = "up";


const loadTcpServer = net.createServer(function(socket) {
	socket.on("error", function(err){
		if (err.code === "ECONNRESET") {
			// this always comes up, we don't know why
			return;
		}
		logger.err({err}, "error with socket on load check");
		return socket.destroy();
	});
	
	if ((STATE === "up") && Settings.internal.load_balancer_agent.report_load) {
		let availableWorkingCpus;
		const currentLoad = os.loadavg()[0];

		// staging clis's have 1 cpu core only
		if (os.cpus().length === 1) {
			availableWorkingCpus = 1;
		} else {
			availableWorkingCpus = os.cpus().length - 1;
		}

		const freeLoad = availableWorkingCpus - currentLoad;
		let freeLoadPercentage = Math.round((freeLoad / availableWorkingCpus) * 100);
		if (freeLoadPercentage <= 0) {
			freeLoadPercentage = 1; // when its 0 the server is set to drain and will move projects to different servers
		}
		socket.write(`up, ${freeLoadPercentage}%\n`, "ASCII");
		return socket.end();
	} else {
		socket.write(`${STATE}\n`, "ASCII");
		return socket.end();
	}
});

const loadHttpServer = express();

loadHttpServer.post("/state/up", function(req, res, next) {
	STATE = "up";
	logger.info("getting message to set server to down");
	return res.sendStatus(204);
});

loadHttpServer.post("/state/down", function(req, res, next) {
	STATE = "down";
	logger.info("getting message to set server to down");
	return res.sendStatus(204);
});

loadHttpServer.post("/state/maint", function(req, res, next) {
	STATE = "maint";
	logger.info("getting message to set server to maint");
	return res.sendStatus(204);
});
	

const port = (__guard__(Settings.internal != null ? Settings.internal.clsi : undefined, x => x.port) || 3013);
const host = (__guard__(Settings.internal != null ? Settings.internal.clsi : undefined, x1 => x1.host) || "localhost");

const load_tcp_port = Settings.internal.load_balancer_agent.load_port;
const load_http_port = Settings.internal.load_balancer_agent.local_port;

if (!module.parent) { // Called directly
	app.listen(port, host, error => logger.info(`CLSI starting up, listening on ${host}:${port}`));

	loadTcpServer.listen(load_tcp_port, host, function(error) {
		if (error != null) { throw error; }
		return logger.info(`Load tcp agent listening on load port ${load_tcp_port}`);
	});

	loadHttpServer.listen(load_http_port, host, function(error) {
		if (error != null) { throw error; }
		return logger.info(`Load http agent listening on load port ${load_http_port}`);
	});
}

module.exports = app;

setInterval(() => ProjectPersistenceManager.clearExpiredProjects()
, (tenMinutes = 10 * 60 * 1000));


function __guard__(value, transform) {
  return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
}