Merge pull request #86 from sharelatex/ho-dockerise

Dockerised clsi

services/clsi/.dockerignore

@@ -0,0 +1,9 @@
+node_modules/*
+gitrev
+.git
+.gitignore
+.npm
+.nvmrc
+nodemon.json
+app.js
+**/js/*

services/clsi/.gitignore

@@ -12,5 +12,7 @@ app.js
 cache
 .vagrant
 db.sqlite
+db.sqlite-wal
+db.sqlite-shm
 config/*
-bin/synctex
+npm-debug.log

services/clsi/.nvmrc

@@ -1 +1 @@
-6.11.2
+6.14.1

services/clsi/.viminfo

@@ -0,0 +1,35 @@
+# This viminfo file was generated by Vim 7.4.
+# You may edit it if you're careful!
+
+# Value of 'encoding' when this file was written
+*encoding=latin1
+
+
+# hlsearch on (H) or off (h):
+~h
+# Command Line History (newest to oldest):
+:x
+
+# Search String History (newest to oldest):
+
+# Expression History (newest to oldest):
+
+# Input Line History (newest to oldest):
+
+# Input Line History (newest to oldest):
+
+# Registers:
+
+# File marks:
+'0  1  0  ~/hello
+
+# Jumplist (newest first):
+-'  1  0  ~/hello
+
+# History of marks within files (newest to oldest):
+
+> ~/hello
+	"	1	0
+	^	1	1
+	.	1	0
+	+	1	0

services/clsi/Dockerfile

@@ -0,0 +1,23 @@
+FROM node:6.14.1 as app
+
+WORKDIR /app
+
+#wildcard as some files may not be in all repos
+COPY package*.json npm-shrink*.json /app/
+
+RUN npm install --quiet
+
+COPY . /app
+
+
+RUN npm run compile:all
+
+FROM node:6.14.1
+
+COPY --from=app /app /app
+
+WORKDIR /app
+RUN chmod 0755 ./install_deps.sh && ./install_deps.sh
+ENTRYPOINT ["/bin/sh", "entrypoint.sh"]
+
+CMD ["node","app.js"]

services/clsi/Jenkinsfile

@@ -1,79 +1,69 @@
-pipeline {
+String cron_string = BRANCH_NAME == "master" ? "@daily" : ""
 
+pipeline {
   agent any
 
+  environment {
+    GIT_PROJECT = "clsi-sharelatex"
+    JENKINS_WORKFLOW = "clsi-sharelatex"
+    TARGET_URL = "${env.JENKINS_URL}blue/organizations/jenkins/${JENKINS_WORKFLOW}/detail/$BRANCH_NAME/$BUILD_NUMBER/pipeline"
+    GIT_API_URL = "https://api.github.com/repos/sharelatex/${GIT_PROJECT}/statuses/$GIT_COMMIT"
+  }
+
   triggers {
     pollSCM('* * * * *')
-    cron('@daily')
+    cron(cron_string)
   }
 
   stages {
-    stage('Clean') {
-      steps {
-        // This is a terrible hack to set the file ownership to jenkins:jenkins so we can cleanup the directory
-        sh 'docker run -v $(pwd):/app --rm busybox /bin/chown -R 111:119 /app'
-        sh 'rm -fr node_modules'
-      }
-    }
     stage('Install') {
-      agent {
-        docker {
-          image 'node:6.11.2'
-          args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp"
-          reuseNode true
-        }
-      }
       steps {
-        sh 'git config --global core.logallrefupdates false'
+        withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
-        sh 'rm -fr node_modules'
+          sh "curl $GIT_API_URL \
-        checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: '_docker-runner'], [$class: 'CloneOption', shallow: true]], userRemoteConfigs: [[credentialsId: 'GIT_DEPLOY_KEY', url: 'git@github.com:sharelatex/docker-runner-sharelatex']]])
+            --data '{ \
-        sh 'npm install ./_docker-runner'
+            \"state\" : \"pending\", \
-        sh 'rm -fr ./_docker-runner ./_docker-runner@tmp'
+            \"target_url\": \"$TARGET_URL\", \
-        sh 'npm install'
+            \"description\": \"Your build is underway\", \
-        sh 'npm rebuild'
+            \"context\": \"ci/jenkins\" }' \
-        sh 'npm install --quiet grunt-cli'
+            -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
         }
       }
-    stage('Compile and Test') {
-      agent {
-        docker {
-          image 'node:6.11.2'
-          reuseNode true
-        }
     }
+
+    stage('Build') {
       steps {
-        sh 'node_modules/.bin/grunt compile:app'
+        sh 'make build'
-        sh 'node_modules/.bin/grunt compile:acceptance_tests'
-        sh 'NODE_ENV=development node_modules/.bin/grunt test:unit'
       }
     }
+
+    stage('Unit Tests') {
+      steps {
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'
+      }
+    }
+
     stage('Acceptance Tests') {
-      environment {
-        TEXLIVE_IMAGE="quay.io/sharelatex/texlive-full:2017.1"
-      }
       steps {
-        sh 'mkdir -p compiles cache'
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance'
-        // Not yet running, due to volumes/sibling containers
-        sh 'docker container prune -f || true'
-        sh 'docker pull $TEXLIVE_IMAGE'
-        sh 'docker pull sharelatex/acceptance-test-runner:clsi-6.11.2'
-        sh 'docker run --rm -e SIBLING_CONTAINER_USER=root -e SANDBOXED_COMPILES_HOST_DIR=$(pwd)/compiles -e SANDBOXED_COMPILES_SIBLING_CONTAINERS=true -e TEXLIVE_IMAGE=$TEXLIVE_IMAGE -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd):/app sharelatex/acceptance-test-runner:clsi-6.11.2'
-        // This is a terrible hack to set the file ownership to jenkins:jenkins so we can cleanup the directory
-        sh 'docker run -v $(pwd):/app --rm busybox /bin/chown -R 111:119 /app'
-        sh 'rm -r compiles cache server.log db.sqlite config/settings.defaults.coffee'
       }
     }
-    stage('Package') {
+
+    stage('Package and publish build') {
       steps {
-        sh 'echo ${BUILD_NUMBER} > build_number.txt'
+        
-        sh 'touch build.tar.gz' // Avoid tar warning about files changing during read
+        withCredentials([file(credentialsId: 'gcr.io_overleaf-ops', variable: 'DOCKER_REPO_KEY_PATH')]) {
-        sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .'
+          sh 'docker login -u _json_key --password-stdin https://gcr.io/overleaf-ops < ${DOCKER_REPO_KEY_PATH}'
+        }
+        sh 'DOCKER_REPO=gcr.io/overleaf-ops make publish'
+        sh 'docker logout https://gcr.io/overleaf-ops'
+        
       }
     }
-    stage('Publish') {
+
+    stage('Publish build number') {
       steps {
+        sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt'
         withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
-            s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
             // The deployment process uses this file to figure out the latest build
             s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest")
         }
@@ -82,11 +72,37 @@ pipeline {
   }
 
   post {
+    always {
+      sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean'
+      sh 'make clean'
+    }
+
+    success {
+      withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
+        sh "curl $GIT_API_URL \
+          --data '{ \
+          \"state\" : \"success\", \
+          \"target_url\": \"$TARGET_URL\", \
+          \"description\": \"Your build succeeded!\", \
+          \"context\": \"ci/jenkins\" }' \
+          -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
+      }
+    }
+
     failure {
       mail(from: "${EMAIL_ALERT_FROM}",
            to: "${EMAIL_ALERT_TO}",
            subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}",
            body: "Build: ${BUILD_URL}")
+      withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
+        sh "curl $GIT_API_URL \
+          --data '{ \
+          \"state\" : \"failure\", \
+          \"target_url\": \"$TARGET_URL\", \
+          \"description\": \"Your build failed\", \
+          \"context\": \"ci/jenkins\" }' \
+          -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
+      }
     }
   }
 

services/clsi/Makefile

@@ -0,0 +1,47 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.1.9
+
+BUILD_NUMBER ?= local
+BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
+PROJECT_NAME = clsi
+DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
+DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
+	BRANCH_NAME=$(BRANCH_NAME) \
+	PROJECT_NAME=$(PROJECT_NAME) \
+	MOCHA_GREP=${MOCHA_GREP} \
+	docker-compose ${DOCKER_COMPOSE_FLAGS}
+
+
+clean:
+	docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
+	docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
+	rm -f app.js
+	rm -rf app/js
+	rm -rf test/unit/js
+	rm -rf test/acceptance/js
+
+test: test_unit test_acceptance
+
+test_unit:
+	@[ ! -d test/unit ] && echo "clsi has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit
+
+test_acceptance: test_clean test_acceptance_pre_run # clear the database before each acceptance test run
+	@[ ! -d test/acceptance ] && echo "clsi has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance
+
+test_clean:
+	$(DOCKER_COMPOSE) down -v -t 0
+
+test_acceptance_pre_run:
+	@[ ! -f test/acceptance/scripts/pre-run ] && echo "clsi has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run
+build:
+	docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
+		--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
+		.
+
+publish:
+
+	docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
+
+.PHONY: clean test test_unit test_acceptance test_clean build publish

services/clsi/app.coffee

@@ -35,6 +35,7 @@ TIMEOUT = 6 * 60 * 1000
 app.use (req, res, next) ->
 	req.setTimeout TIMEOUT
 	res.setTimeout TIMEOUT
+	res.removeHeader("X-Powered-By")
 	next()
 
 app.param 'project_id', (req, res, next, project_id) ->
@@ -139,6 +140,14 @@ app.get "/health_check", (req, res)->
 	res.contentType(resCacher?.setContentType)
 	res.status(resCacher?.code).send(resCacher?.body)
 
+app.get "/smoke_test_force", (req, res)->
+	smokeTest.run(require.resolve(__dirname + "/test/smoke/js/SmokeTests.js"))(req, res)
+
+
+#TODO delete this
+app.get "/settings", (req, res)->
+	res.json(Settings)
+
 profiler = require "v8-profiler"
 app.get "/profile", (req, res) ->
 	time = parseInt(req.query.time || "1000")
@@ -160,9 +169,77 @@ app.use (error, req, res, next) ->
 		logger.error {err: error, url: req.url}, "server error"
 		res.sendStatus(error?.statusCode || 500)
 
-app.listen port = (Settings.internal?.clsi?.port or 3013), host = (Settings.internal?.clsi?.host or "localhost"), (error) ->
+net = require "net"
+os = require "os"
+
+STATE = "up"
+
+
+loadTcpServer = net.createServer (socket) ->
+	socket.on "error", (err)->
+		if err.code == "ECONNRESET"
+			# this always comes up, we don't know why
+			return
+		logger.err err:err, "error with socket on load check"
+		socket.destroy()
+	
+	if STATE == "up" and Settings.internal.load_balancer_agent.report_load
+		currentLoad = os.loadavg()[0]
+
+		# staging clis's have 1 cpu core only
+		if os.cpus().length == 1
+			availableWorkingCpus = 1
+		else
+			availableWorkingCpus = os.cpus().length - 1
+
+		freeLoad = availableWorkingCpus - currentLoad
+		freeLoadPercentage = Math.round((freeLoad / availableWorkingCpus) * 100)
+		if freeLoadPercentage <= 0
+			freeLoadPercentage = 1 # when its 0 the server is set to drain and will move projects to different servers
+		socket.write("up, #{freeLoadPercentage}%\n", "ASCII")
+		socket.end()
+	else
+		socket.write("#{STATE}\n", "ASCII")
+		socket.end()
+
+loadHttpServer = express()
+
+loadHttpServer.post "/state/up", (req, res, next) ->
+	STATE = "up"
+	logger.info "getting message to set server to down"
+	res.sendStatus 204
+
+loadHttpServer.post "/state/down", (req, res, next) ->
+	STATE = "down"
+	logger.info "getting message to set server to down"
+	res.sendStatus 204
+
+loadHttpServer.post "/state/maint", (req, res, next) ->
+	STATE = "maint"
+	logger.info "getting message to set server to maint"
+	res.sendStatus 204
+	
+
+port = (Settings.internal?.clsi?.port or 3013)
+host = (Settings.internal?.clsi?.host or "localhost")
+
+load_tcp_port = Settings.internal.load_balancer_agent.load_port
+load_http_port = Settings.internal.load_balancer_agent.local_port
+
+if !module.parent # Called directly
+	app.listen port, host, (error) ->
 		logger.info "CLSI starting up, listening on #{host}:#{port}"
 
+	loadTcpServer.listen load_tcp_port, host, (error) ->
+		throw error if error?
+		logger.info "Load tcp agent listening on load port #{load_tcp_port}"
+
+	loadHttpServer.listen load_http_port, host, (error) ->
+		throw error if error?
+		logger.info "Load http agent listening on load port #{load_http_port}"
+
+module.exports = app
+
 setInterval () ->
 	ProjectPersistenceManager.clearExpiredProjects()
 , tenMinutes = 10 * 60 * 1000

services/clsi/app/coffee/CommandRunner.coffee

@@ -1,44 +1,11 @@
-spawn = require("child_process").spawn
+Settings = require "settings-sharelatex"
 logger = require "logger-sharelatex"
 
-logger.info "using standard command runner"
+if Settings.clsi?.dockerRunner == true
+	commandRunnerPath = "./DockerRunner"
+else 
+	commandRunnerPath = "./LocalCommandRunner"
+logger.info commandRunnerPath:commandRunnerPath, "selecting command runner for clsi"
+CommandRunner = require(commandRunnerPath)
 
-module.exports = CommandRunner =
+module.exports = CommandRunner
-	run: (project_id, command, directory, image, timeout, environment, callback = (error) ->) ->
-		command = (arg.replace('$COMPILE_DIR', directory) for arg in command)
-		logger.log project_id: project_id, command: command, directory: directory, "running command"
-		logger.warn "timeouts and sandboxing are not enabled with CommandRunner"
-
-		# merge environment settings
-		env = {}
-		env[key] = value for key, value of process.env
-		env[key] = value for key, value of environment
-
-		# run command as detached process so it has its own process group (which can be killed if needed)
-		proc = spawn command[0], command.slice(1), stdio: "inherit", cwd: directory, detached: true, env: env
-
-		proc.on "error", (err)->
-			logger.err err:err, project_id:project_id, command: command, directory: directory, "error running command"
-			callback(err)
-
-		proc.on "close", (code, signal) ->
-			logger.info code:code, signal:signal, project_id:project_id, "command exited"
-			if signal is 'SIGTERM' # signal from kill method below
-				err = new Error("terminated")
-				err.terminated = true
-				return callback(err)
-			else if code is 1 # exit status from chktex
-				err = new Error("exited")
-				err.code = code
-				return callback(err)
-			else
-				callback()
-
-		return proc.pid # return process id to allow job to be killed if necessary
-
-	kill: (pid, callback = (error) ->) ->
-		try
-			process.kill -pid # kill all processes in group
-		catch err
-			return callback(err)
-		callback()

services/clsi/app/coffee/CompileController.coffee

@@ -34,11 +34,16 @@ module.exports = CompileController =
 							status = "error"
 							code = 500
 							logger.error err: error, project_id: request.project_id, "error running compile"
+
 					else
 						status = "failure"
 						for file in outputFiles
 							if file.path?.match(/output\.pdf$/)
 								status = "success"
+								
+						if status == "failure"
+							logger.err project_id: request.project_id, outputFiles:outputFiles, "project failed to compile successfully, no output.pdf generated"
+
 						# log an error if any core files are found
 						for file in outputFiles
 							if file.path is "core"
@@ -77,7 +82,6 @@ module.exports = CompileController =
 		column = parseInt(req.query.column, 10)
 		project_id = req.params.project_id
 		user_id = req.params.user_id
-
 		CompileManager.syncFromCode project_id, user_id, file, line, column, (error, pdfPositions) ->
 			return next(error) if error?
 			res.send JSON.stringify {
@@ -90,7 +94,6 @@ module.exports = CompileController =
 		v      = parseFloat(req.query.v)
 		project_id = req.params.project_id
 		user_id = req.params.user_id
-
 		CompileManager.syncFromPdf project_id, user_id, page, h, v, (error, codePositions) ->
 			return next(error) if error?
 			res.send JSON.stringify {

services/clsi/app/coffee/CompileManager.coffee

@@ -15,10 +15,7 @@ fse = require "fs-extra"
 os = require("os")
 async = require "async"
 Errors = require './Errors'
-
+CommandRunner = require "./CommandRunner"
-commandRunner = Settings.clsi?.commandRunner or "./CommandRunner"
-logger.info commandRunner:commandRunner, "selecting command runner for clsi"
-CommandRunner = require(commandRunner)
 
 getCompileName = (project_id, user_id) ->
 	if user_id? then "#{project_id}-#{user_id}" else project_id
@@ -41,7 +38,6 @@ module.exports = CompileManager =
 
 	doCompile: (request, callback = (error, outputFiles) ->) ->
 		compileDir = getCompileDir(request.project_id, request.user_id)
-
 		timer = new Metrics.Timer("write-to-disk")
 		logger.log project_id: request.project_id, user_id: request.user_id, "syncing resources to disk"
 		ResourceWriter.syncResourcesToDisk request, compileDir, (error, resourceList) ->
@@ -205,18 +201,28 @@ module.exports = CompileManager =
 		base_dir = Settings.path.synctexBaseDir(compileName)
 		file_path = base_dir + "/" + file_name
 		compileDir = getCompileDir(project_id, user_id)
-		synctex_path = Path.join(compileDir, "output.pdf")
+		synctex_path =  "#{base_dir}/output.pdf"
-		CompileManager._runSynctex ["code", synctex_path, file_path, line, column], (error, stdout) ->
+		command = ["code", synctex_path, file_path, line, column]
+		fse.ensureDir compileDir, (error) ->
+			if error?
+				logger.err {error, project_id, user_id, file_name}, "error ensuring dir for sync from code"
+				return callback(error)
+			CompileManager._runSynctex project_id, user_id, command, (error, stdout) ->
 				return callback(error) if error?
-			logger.log project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, stdout: stdout, "synctex code output"
+				logger.log project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, command:command, stdout: stdout, "synctex code output"
 				callback null, CompileManager._parseSynctexFromCodeOutput(stdout)
 
 	syncFromPdf: (project_id, user_id, page, h, v, callback = (error, filePositions) ->) ->
 		compileName = getCompileName(project_id, user_id)
-		base_dir = Settings.path.synctexBaseDir(compileName)
 		compileDir = getCompileDir(project_id, user_id)
-		synctex_path = Path.join(compileDir, "output.pdf")
+		base_dir = Settings.path.synctexBaseDir(compileName)
-		CompileManager._runSynctex ["pdf", synctex_path, page, h, v], (error, stdout) ->
+		synctex_path =  "#{base_dir}/output.pdf"
+		command = ["pdf", synctex_path, page, h, v]
+		fse.ensureDir compileDir, (error) ->
+			if error?
+				logger.err {error, project_id, user_id, file_name}, "error ensuring dir for sync to code"
+				return callback(error)
+			CompileManager._runSynctex  project_id, user_id, command, (error, stdout) ->
 				return callback(error) if error?
 				logger.log project_id: project_id, user_id:user_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output"
 				callback null, CompileManager._parseSynctexFromPdfOutput(stdout, base_dir)
@@ -235,19 +241,19 @@ module.exports = CompileManager =
 				return callback(new Error("not a file")) if not stats?.isFile()
 				callback()
 
-	_runSynctex: (args, callback = (error, stdout) ->) ->
+	_runSynctex: (project_id, user_id, command, callback = (error, stdout) ->) ->
-		bin_path = Path.resolve(__dirname + "/../../bin/synctex")
 		seconds = 1000
-		outputFilePath = args[1]
+
-		CompileManager._checkFileExists outputFilePath, (error) ->
+		command.unshift("/opt/synctex")
-			return callback(error) if error?
+
-			if Settings.clsi?.synctexCommandWrapper?
+		directory = getCompileDir(project_id, user_id)
-				[bin_path, args] = Settings.clsi?.synctexCommandWrapper bin_path, args
+		timeout = 60 * 1000 # increased to allow for large projects
-			child_process.execFile bin_path, args, timeout: 10 * seconds, (error, stdout, stderr) ->
+		compileName = getCompileName(project_id, user_id)
+		CommandRunner.run compileName, command, directory, Settings.clsi.docker.image, timeout, {}, (error, output) ->
 			if error?
-					logger.err err:error, args:args, "error running synctex"
+				logger.err err:error, command:command, project_id:project_id, user_id:user_id, "error running synctex"
 				return callback(error)
-				callback(null, stdout)
+			callback(null, output.stdout)
 
 	_parseSynctexFromCodeOutput: (output) ->
 		results = []
@@ -276,19 +282,24 @@ module.exports = CompileManager =
 				}
 		return results
 
+
 	wordcount: (project_id, user_id, file_name, image, callback = (error, pdfPositions) ->) ->
 		logger.log project_id:project_id, user_id:user_id, file_name:file_name, image:image, "running wordcount"
 		file_path = "$COMPILE_DIR/" + file_name
 		command = [ "texcount", '-nocol', '-inc', file_path, "-out=" + file_path + ".wc"]
-		directory = getCompileDir(project_id, user_id)
+		compileDir = getCompileDir(project_id, user_id)
-		timeout = 60 * 1000 # increased to allow for large projects
+		timeout = 60 * 1000
 		compileName = getCompileName(project_id, user_id)
-
+		fse.ensureDir compileDir, (error) ->
-		CommandRunner.run compileName, command, directory, image, timeout, {}, (error) ->
+			if error?
+				logger.err {error, project_id, user_id, file_name}, "error ensuring dir for sync from code"
+				return callback(error)
+			CommandRunner.run compileName, command, compileDir, image, timeout, {}, (error) ->
 				return callback(error) if error?
-			fs.readFile directory + "/" + file_name + ".wc", "utf-8", (err, stdout) ->
+				fs.readFile compileDir + "/" + file_name + ".wc", "utf-8", (err, stdout) ->
 					if err?
-					logger.err err:err, command:command, directory:directory, project_id:project_id, user_id:user_id, "error reading word count output"
+						#call it node_err so sentry doesn't use random path error as unique id so it can't be ignored
+						logger.err node_err:err, command:command, compileDir:compileDir, project_id:project_id, user_id:user_id, "error reading word count output"
 						return callback(err)
 					results = CompileManager._parseWordcountFromOutput(stdout)
 					logger.log project_id:project_id, user_id:user_id, wordcount: results, "word count results"

services/clsi/app/coffee/DbQueue.coffee

@@ -0,0 +1,13 @@
+async = require "async"
+Settings = require "settings-sharelatex"
+
+queue = async.queue((task, cb)->
+		task(cb)
+	, Settings.parallelSqlQueryLimit)
+
+queue.drain = ()->
+    console.log('HI all items have been processed')
+
+module.exports = 
+	queue: queue
+

services/clsi/app/coffee/DockerLockManager.coffee

@@ -0,0 +1,56 @@
+logger = require "logger-sharelatex"
+
+LockState = {} # locks for docker container operations, by container name
+
+module.exports = LockManager =
+
+	MAX_LOCK_HOLD_TIME: 15000  # how long we can keep a lock
+	MAX_LOCK_WAIT_TIME: 10000  # how long we wait for a lock
+	LOCK_TEST_INTERVAL: 1000   # retry time
+
+	tryLock: (key, callback = (err, gotLock) ->) ->
+		existingLock = LockState[key]
+		if existingLock? # the lock is already taken, check how old it is
+			lockAge = Date.now() - existingLock.created
+			if lockAge < LockManager.MAX_LOCK_HOLD_TIME
+				return callback(null, false) # we didn't get the lock, bail out
+			else
+				logger.error {key: key, lock: existingLock, age:lockAge}, "taking old lock by force"
+		# take the lock
+		LockState[key] = lockValue = {created: Date.now()}
+		callback(null, true, lockValue)
+
+	getLock: (key, callback = (error, lockValue) ->) ->
+		startTime = Date.now()
+		do attempt = () ->
+			LockManager.tryLock key, (error, gotLock, lockValue) ->
+				return callback(error) if error?
+				if gotLock
+					callback(null, lockValue)
+				else if Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME
+					e = new Error("Lock timeout")
+					e.key = key
+					return callback(e)
+				else
+					setTimeout attempt, LockManager.LOCK_TEST_INTERVAL
+
+	releaseLock: (key, lockValue, callback = (error) ->) ->
+		existingLock = LockState[key]
+		if existingLock is lockValue # lockValue is an object, so we can test by reference
+			delete LockState[key] # our lock, so we can free it
+			callback()
+		else if existingLock? # lock exists but doesn't match ours
+			logger.error {key:key, lock: existingLock}, "tried to release lock taken by force"
+			callback()
+		else
+			logger.error {key:key, lock: existingLock}, "tried to release lock that has gone"
+			callback()
+
+	runWithLock: (key, runner = ( (releaseLock = (error) ->) -> ), callback = ( (error) -> )) ->
+		LockManager.getLock key, (error, lockValue) ->
+			return callback(error) if error?
+			runner (error1, args...) ->
+				LockManager.releaseLock key, lockValue, (error2) ->
+					error = error1 or error2
+					return callback(error) if error?
+					callback(null, args...)

services/clsi/app/coffee/DockerRunner.coffee

@@ -0,0 +1,358 @@
+Settings = require "settings-sharelatex"
+logger = require "logger-sharelatex"
+Docker = require("dockerode")
+dockerode = new Docker()
+crypto = require "crypto"
+async = require "async"
+LockManager = require "./DockerLockManager"
+fs = require "fs"
+Path = require 'path'
+_ = require "underscore"
+
+logger.info "using docker runner"
+
+usingSiblingContainers = () ->
+	Settings?.path?.sandboxedCompilesHostDir?
+
+module.exports = DockerRunner =
+	ERR_NOT_DIRECTORY: new Error("not a directory")
+	ERR_TERMINATED: new Error("terminated")
+	ERR_EXITED: new Error("exited")
+	ERR_TIMED_OUT: new Error("container timed out")
+
+	run: (project_id, command, directory, image, timeout, environment, callback = (error, output) ->) ->
+
+		if usingSiblingContainers()
+			_newPath = Settings.path.sandboxedCompilesHostDir
+			logger.log {path: _newPath}, "altering bind path for sibling containers"
+			# Server Pro, example:
+			#   '/var/lib/sharelatex/data/compiles/<project-id>'
+			#   ... becomes ...
+			#   '/opt/sharelatex_data/data/compiles/<project-id>'
+			directory = Path.join(Settings.path.sandboxedCompilesHostDir, Path.basename(directory))
+
+		volumes = {}
+		volumes[directory] = "/compile"
+
+		command = (arg.toString().replace?('$COMPILE_DIR', "/compile") for arg in command)
+		if !image?
+			image = Settings.clsi.docker.image
+
+		if Settings.texliveImageNameOveride?
+			img = image.split("/")
+			image = "#{Settings.texliveImageNameOveride}/#{img[2]}"
+
+		options      = DockerRunner._getContainerOptions(command, image, volumes, timeout, environment)
+		fingerprint  = DockerRunner._fingerprintContainer(options)
+		options.name = name = "project-#{project_id}-#{fingerprint}"
+
+		# logOptions = _.clone(options)
+		# logOptions?.HostConfig?.SecurityOpt = "secomp used, removed in logging"
+		logger.log project_id: project_id, "running docker container"
+		DockerRunner._runAndWaitForContainer options, volumes, timeout, (error, output) ->
+			if error?.message?.match("HTTP code is 500")
+				logger.log err: error, project_id: project_id, "error running container so destroying and retrying"
+				DockerRunner.destroyContainer name, null, true, (error) ->
+					return callback(error) if error?
+					DockerRunner._runAndWaitForContainer options, volumes, timeout, callback
+			else
+				callback(error, output)
+
+		return name # pass back the container name to allow it to be killed
+
+	kill: (container_id, callback = (error) ->) ->
+		logger.log container_id: container_id, "sending kill signal to container"
+		container = dockerode.getContainer(container_id)
+		container.kill (error) ->
+			if error? and error?.message?.match?(/Cannot kill container .* is not running/)
+				logger.warn err: error, container_id: container_id, "container not running, continuing"
+				error = null
+			if error?
+				logger.error err: error, container_id: container_id, "error killing container"
+				return callback(error)
+			else
+				callback()
+
+	_runAndWaitForContainer: (options, volumes, timeout, _callback = (error, output) ->) ->
+		callback = (args...) ->
+			_callback(args...)
+			# Only call the callback once
+			_callback = () ->
+		
+		name = options.name
+
+		streamEnded = false
+		containerReturned = false
+		output = {}
+
+		callbackIfFinished = () ->
+			if streamEnded and containerReturned
+				callback(null, output)
+
+		attachStreamHandler = (error, _output) ->
+			return callback(error) if error?
+			output = _output
+			streamEnded = true
+			callbackIfFinished()
+
+		DockerRunner.startContainer options, volumes, attachStreamHandler, (error, containerId) ->
+			return callback(error) if error?
+			
+			DockerRunner.waitForContainer name, timeout, (error, exitCode) ->
+				return callback(error) if error?
+				if exitCode is 137  # exit status from kill -9
+					err = DockerRunner.ERR_TERMINATED
+					err.terminated = true
+					return callback(err)
+				if exitCode is 1 # exit status from chktex
+					err = DockerRunner.ERR_EXITED
+					err.code = exitCode
+					return callback(err)
+				containerReturned = true
+				options?.HostConfig?.SecurityOpt = null #small log line
+				logger.log err:err, exitCode:exitCode, options:options, "docker container has exited"
+				callbackIfFinished()
+
+	_getContainerOptions: (command, image, volumes, timeout, environment) ->
+		timeoutInSeconds = timeout / 1000
+		
+		dockerVolumes = {}
+		for hostVol, dockerVol of volumes
+			dockerVolumes[dockerVol] = {}
+
+			if volumes[hostVol].slice(-3).indexOf(":r") == -1
+				volumes[hostVol] = "#{dockerVol}:rw"
+
+		# merge settings and environment parameter
+		env = {}
+		for src in [Settings.clsi.docker.env, environment or {}]
+			env[key] = value for key, value of src
+		# set the path based on the image year
+		if m = image.match /:([0-9]+)\.[0-9]+/
+			year = m[1]
+		else
+			year = "2014"
+		env['PATH'] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/#{year}/bin/x86_64-linux/"
+		options =
+			"Cmd"             : command,
+			"Image"           : image
+			"Volumes"         : dockerVolumes
+			"WorkingDir"      : "/compile"
+			"NetworkDisabled" : true
+			"Memory"          : 1024 * 1024 * 1024 * 1024 # 1 Gb
+			"User"            : Settings.clsi.docker.user
+			"Env"             : ("#{key}=#{value}" for key, value of env)	# convert the environment hash to an array
+			"HostConfig"      :
+				"Binds": ("#{hostVol}:#{dockerVol}" for hostVol, dockerVol of volumes)
+				"LogConfig": {"Type": "none", "Config": {}}
+				"Ulimits": [{'Name': 'cpu', 'Soft': timeoutInSeconds+5, 'Hard': timeoutInSeconds+10}]
+				"CapDrop": "ALL"
+				"SecurityOpt": ["no-new-privileges"]
+		
+
+		if Settings.path?.synctexBinHostPath?
+			options["HostConfig"]["Binds"].push("#{Settings.path.synctexBinHostPath}:/opt/synctex:ro")
+
+		if Settings.clsi.docker.seccomp_profile?
+			options.HostConfig.SecurityOpt.push "seccomp=#{Settings.clsi.docker.seccomp_profile}"
+
+		return options
+
+	_fingerprintContainer: (containerOptions) ->
+		# Yay, Hashing!
+		json = JSON.stringify(containerOptions)
+		return crypto.createHash("md5").update(json).digest("hex")
+
+	startContainer: (options, volumes, attachStreamHandler, callback) ->
+		LockManager.runWithLock options.name, (releaseLock) ->
+			# Check that volumes exist before starting the container.
+			# When a container is started with volume pointing to a
+			# non-existent directory then docker creates the directory but
+			# with root ownership.
+			DockerRunner._checkVolumes options, volumes, (err) ->
+				return releaseLock(err) if err?
+				DockerRunner._startContainer options, volumes, attachStreamHandler, releaseLock
+		, callback
+
+	# Check that volumes exist and are directories
+	_checkVolumes: (options, volumes, callback = (error, containerName) ->) ->
+		if usingSiblingContainers()
+			# Server Pro, with sibling-containers active, skip checks
+			return callback(null)
+
+		checkVolume = (path, cb) ->
+			fs.stat path, (err, stats) ->
+				return cb(err) if err?
+				return cb(DockerRunner.ERR_NOT_DIRECTORY) if not stats?.isDirectory()
+				cb()
+		jobs = []
+		for vol of volumes
+			do (vol) ->
+				jobs.push (cb) -> checkVolume(vol, cb)
+		async.series jobs, callback
+
+	_startContainer: (options, volumes, attachStreamHandler, callback = ((error, output) ->)) ->
+		callback = _.once(callback)
+		name = options.name
+
+		logger.log {container_name: name}, "starting container"
+		container = dockerode.getContainer(name)
+
+		createAndStartContainer = ->
+			dockerode.createContainer options, (error, container) ->
+				return callback(error) if error?
+				startExistingContainer()
+
+		startExistingContainer = ->
+			DockerRunner.attachToContainer options.name, attachStreamHandler, (error)->
+				return callback(error) if error?
+				container.start (error) ->
+					if error? and error?.statusCode != 304 #already running
+						return callback(error)
+					else
+						callback()
+
+		container.inspect (error, stats)->
+			if error?.statusCode == 404
+				createAndStartContainer()
+			else if error?
+				logger.err {container_name: name, error:error}, "unable to inspect container to start"
+				return callback(error)
+			else
+				startExistingContainer()
+
+
+	attachToContainer: (containerId, attachStreamHandler, attachStartCallback) ->
+		container = dockerode.getContainer(containerId)
+		container.attach {stdout: 1, stderr: 1, stream: 1}, (error, stream) ->
+			if error?
+				logger.error err: error, container_id: containerId, "error attaching to container"
+				return attachStartCallback(error)
+			else
+				attachStartCallback()
+
+
+			logger.log container_id: containerId, "attached to container"
+
+			MAX_OUTPUT = 1024 * 1024 # limit output to 1MB
+			createStringOutputStream = (name) ->
+				return {
+					data: ""
+					overflowed: false
+					write: (data) ->
+						return if @overflowed
+						if @data.length < MAX_OUTPUT
+							@data += data
+						else
+							logger.error container_id: containerId, length: @data.length, maxLen: MAX_OUTPUT, "#{name} exceeds max size"
+							@data += "(...truncated at #{MAX_OUTPUT} chars...)"
+							@overflowed = true
+							# kill container if too much output
+							# docker.containers.kill(containerId, () ->)
+				}
+
+			stdout = createStringOutputStream "stdout"
+			stderr = createStringOutputStream "stderr"
+
+			container.modem.demuxStream(stream, stdout, stderr)
+
+			stream.on "error", (err) ->
+				logger.error err: err, container_id: containerId, "error reading from container stream"
+
+			stream.on "end", () ->
+				attachStreamHandler null, {stdout: stdout.data, stderr: stderr.data}
+
+	waitForContainer: (containerId, timeout, _callback = (error, exitCode) ->) ->
+		callback = (args...) ->
+			_callback(args...)
+			# Only call the callback once
+			_callback = () ->
+
+		container = dockerode.getContainer(containerId)
+
+		timedOut = false
+		timeoutId = setTimeout () ->
+			timedOut = true
+			logger.log container_id: containerId, "timeout reached, killing container"
+			container.kill(() ->)
+		, timeout
+			
+		logger.log container_id: containerId, "waiting for docker container"
+		container.wait (error, res) ->
+			if error?
+				clearTimeout timeoutId
+				logger.error err: error, container_id: containerId, "error waiting for container"
+				return callback(error)
+			if timedOut
+				logger.log containerId: containerId, "docker container timed out"
+				error = DockerRunner.ERR_TIMED_OUT
+				error.timedout = true
+				callback error
+			else
+				clearTimeout timeoutId
+				logger.log container_id: containerId, exitCode: res.StatusCode, "docker container returned"
+				callback null, res.StatusCode
+
+	destroyContainer: (containerName, containerId, shouldForce, callback = (error) ->) ->
+		# We want the containerName for the lock and, ideally, the
+		# containerId to delete.  There is a bug in the docker.io module
+		# where if you delete by name and there is an error, it throws an
+		# async exception, but if you delete by id it just does a normal
+		# error callback. We fall back to deleting by name if no id is
+		# supplied.
+		LockManager.runWithLock containerName, (releaseLock) ->
+			DockerRunner._destroyContainer containerId or containerName, shouldForce, releaseLock
+		, callback
+
+	_destroyContainer: (containerId, shouldForce, callback = (error) ->) ->
+		logger.log container_id: containerId, "destroying docker container"
+		container = dockerode.getContainer(containerId)
+		container.remove {force: shouldForce == true}, (error) ->
+			if error? and error?.statusCode == 404
+				logger.warn err: error, container_id: containerId, "container not found, continuing"
+				error = null
+			if error?
+				logger.error err: error, container_id: containerId, "error destroying container"
+			else
+				logger.log container_id: containerId, "destroyed container"
+			callback(error)
+
+	# handle expiry of docker containers
+
+	MAX_CONTAINER_AGE: Settings.clsi.docker.maxContainerAge or oneHour = 60 * 60 * 1000
+
+	examineOldContainer: (container, callback = (error, name, id, ttl)->) ->
+		name = container.Name or container.Names?[0]
+		created = container.Created * 1000 # creation time is returned in seconds
+		now = Date.now()
+		age = now - created
+		maxAge = DockerRunner.MAX_CONTAINER_AGE
+		ttl = maxAge - age
+		logger.log {containerName: name, created: created, now: now, age: age, maxAge: maxAge, ttl: ttl}, "checking whether to destroy container"
+		callback(null, name, container.Id, ttl)
+
+	destroyOldContainers: (callback = (error) ->) ->
+		dockerode.listContainers all: true, (error, containers) ->
+			return callback(error) if error?
+			jobs = []
+			for container in containers or []
+				do (container) ->
+					DockerRunner.examineOldContainer container, (err, name, id, ttl) ->
+						if name.slice(0, 9) == '/project-' && ttl <= 0
+							jobs.push	(cb) ->
+								DockerRunner.destroyContainer name, id, false, () -> cb()
+								# Ignore errors because some containers get stuck but
+								# will be destroyed next time
+			async.series jobs, callback
+
+	startContainerMonitor: () ->
+		logger.log {maxAge: DockerRunner.MAX_CONTAINER_AGE}, "starting container expiry"
+		# randomise the start time
+		randomDelay = Math.floor(Math.random() * 5 * 60 * 1000)
+		setTimeout () ->
+			setInterval () ->
+				DockerRunner.destroyOldContainers()
+			, oneHour = 60 * 60 * 1000
+		, randomDelay
+
+DockerRunner.startContainerMonitor()

services/clsi/app/coffee/LatexRunner.coffee

@@ -2,7 +2,7 @@ Path = require "path"
 Settings = require "settings-sharelatex"
 logger = require "logger-sharelatex"
 Metrics = require "./Metrics"
-CommandRunner = require(Settings.clsi?.commandRunner or "./CommandRunner")
+CommandRunner = require "./CommandRunner"
 
 ProcessTable = {}  # table of currently running jobs (pids or docker container names)
 

services/clsi/app/coffee/LocalCommandRunner.coffee

@@ -0,0 +1,44 @@
+spawn = require("child_process").spawn
+logger = require "logger-sharelatex"
+
+logger.info "using standard command runner"
+
+module.exports = CommandRunner =
+	run: (project_id, command, directory, image, timeout, environment, callback = (error) ->) ->
+		command = (arg.replace('$COMPILE_DIR', directory) for arg in command)
+		logger.log project_id: project_id, command: command, directory: directory, "running command"
+		logger.warn "timeouts and sandboxing are not enabled with CommandRunner"
+
+		# merge environment settings
+		env = {}
+		env[key] = value for key, value of process.env
+		env[key] = value for key, value of environment
+
+		# run command as detached process so it has its own process group (which can be killed if needed)
+		proc = spawn command[0], command.slice(1), stdio: "inherit", cwd: directory, detached: true, env: env
+
+		proc.on "error", (err)->
+			logger.err err:err, project_id:project_id, command: command, directory: directory, "error running command"
+			callback(err)
+
+		proc.on "close", (code, signal) ->
+			logger.info code:code, signal:signal, project_id:project_id, "command exited"
+			if signal is 'SIGTERM' # signal from kill method below
+				err = new Error("terminated")
+				err.terminated = true
+				return callback(err)
+			else if code is 1 # exit status from chktex
+				err = new Error("exited")
+				err.code = code
+				return callback(err)
+			else
+				callback()
+
+		return proc.pid # return process id to allow job to be killed if necessary
+
+	kill: (pid, callback = (error) ->) ->
+		try
+			process.kill -pid # kill all processes in group
+		catch err
+			return callback(err)
+		callback()

services/clsi/app/coffee/LockManager.coffee

@@ -2,7 +2,8 @@ Settings = require('settings-sharelatex')
 logger = require "logger-sharelatex"
 Lockfile = require('lockfile') # from https://github.com/npm/lockfile
 Errors = require "./Errors"
-
+fs = require("fs")
+Path = require("path")
 module.exports = LockManager =
 	LOCK_TEST_INTERVAL: 1000 # 50ms between each test of the lock
 	MAX_LOCK_WAIT_TIME: 15000 # 10s maximum time to spend trying to get the lock
@@ -14,8 +15,15 @@ module.exports = LockManager =
 			pollPeriod: @LOCK_TEST_INTERVAL
 			stale: @LOCK_STALE
 		Lockfile.lock path, lockOpts, (error) ->
-			return callback new Errors.AlreadyCompilingError("compile in progress")	if error?.code is 'EEXIST'
+			if error?.code is 'EEXIST'
-			return callback(error) if error?
+				return callback new Errors.AlreadyCompilingError("compile in progress")
+			else if error?
+				fs.lstat path, (statLockErr, statLock)->
+					fs.lstat Path.dirname(path), (statDirErr, statDir)->
+						fs.readdir Path.dirname(path), (readdirErr, readdirDir)->
+							logger.err error:error, path:path, statLock:statLock, statLockErr:statLockErr, statDir:statDir, statDirErr: statDirErr, readdirErr:readdirErr, readdirDir:readdirDir, "unable to get lock"
+							return callback(error)
+			else
 				runner (error1, args...) ->
 					Lockfile.unlock path, (error2) ->
 						error = error1 or error2

services/clsi/app/coffee/OutputFileFinder.coffee

@@ -10,8 +10,6 @@ module.exports = OutputFileFinder =
 		for resource in resources
 			incomingResources[resource.path] = true
 			
-		logger.log directory: directory, "getting output files"
-
 		OutputFileFinder._getAllFiles directory, (error, allFiles = []) ->
 			if error?
 				logger.err err:error, "error finding all output files"

services/clsi/app/coffee/ProjectPersistenceManager.coffee

@@ -1,6 +1,7 @@
 UrlCache = require "./UrlCache"
 CompileManager = require "./CompileManager"
 db = require "./db"
+dbQueue = require "./DbQueue"
 async = require "async"
 logger = require "logger-sharelatex"
 oneDay = 24 * 60 * 60 * 1000
@@ -11,14 +12,17 @@ module.exports = ProjectPersistenceManager =
 	EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5
 
 	markProjectAsJustAccessed: (project_id, callback = (error) ->) ->
+		job = (cb)->
 			db.Project.findOrCreate(where: {project_id: project_id})
 				.spread(
 					(project, created) ->
 						project.updateAttributes(lastAccessed: new Date())
-						.then(() -> callback())
+							.then(() -> cb())
-						.error callback
+							.error cb
 				)
-			.error callback
+				.error cb
+		dbQueue.queue.push(job, callback)
+
 
 	clearExpiredProjects: (callback = (error) ->) ->
 		ProjectPersistenceManager._findExpiredProjectIds (error, project_ids) ->
@@ -47,20 +51,34 @@ module.exports = ProjectPersistenceManager =
 	clearProjectFromCache: (project_id, callback = (error) ->) ->
 		logger.log project_id: project_id, "clearing project from cache"
 		UrlCache.clearProject project_id, (error) ->
-			return callback(error) if error?
+			if error?
+				logger.err error:error, project_id: project_id, "error clearing project from cache"
+				return callback(error) 
 			ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) ->
-				return callback(error) if error?
+				if error?
-				callback()
+					logger.err error:error, project_id:project_id, "error clearing project from database"
+				callback(error)
 
 	_clearProjectFromDatabase: (project_id, callback = (error) ->) ->
+		logger.log project_id:project_id, "clearing project from database"
+		job = (cb)->
 			db.Project.destroy(where: {project_id: project_id})
-			.then(() -> callback())
+				.then(() -> cb())
-			.error callback
+				.error cb
+		dbQueue.queue.push(job, callback)
+
 
 	_findExpiredProjectIds: (callback = (error, project_ids) ->) ->
-		db.Project.findAll(where: ["lastAccessed < ?", new Date(Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT)])
+		job = (cb)->
+			keepProjectsFrom = new Date(Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT)
+			q = {}
+			q[db.op.lt] = keepProjectsFrom
+			db.Project.findAll(where:{lastAccessed:q})
 				.then((projects) ->
-				callback null, projects.map((project) -> project.project_id)
+					cb null, projects.map((project) -> project.project_id)
-			).error callback
+				).error cb
+
+		dbQueue.queue.push(job, callback)
+
 
 logger.log {EXPIRY_TIMEOUT: ProjectPersistenceManager.EXPIRY_TIMEOUT}, "project assets kept timeout"

services/clsi/app/coffee/RequestParser.coffee

@@ -1,3 +1,5 @@
+settings = require("settings-sharelatex")
+
 module.exports = RequestParser =
 	VALID_COMPILERS: ["pdflatex", "latex", "xelatex", "lualatex"]
 	MAX_TIMEOUT: 300

services/clsi/app/coffee/ResourceWriter.coffee

@@ -120,7 +120,11 @@ module.exports = ResourceWriter =
 							logger.err err:err, project_id:project_id, path:path, resource_url:resource.url, modified:resource.modified, "error downloading file for resources"
 						callback() #try and continue compiling even if http resource can not be downloaded at this time
 				else
+					process = require("process")
 					fs.writeFile path, resource.content, callback
+					try 
+						result = fs.lstatSync(path)
+					catch e
 
 	checkPath: (basePath, resourcePath, callback) ->
 		path = Path.normalize(Path.join(basePath, resourcePath))

services/clsi/app/coffee/UrlCache.coffee

@@ -1,4 +1,5 @@
 db = require("./db")
+dbQueue = require "./DbQueue"
 UrlFetcher = require("./UrlFetcher")
 Settings = require("settings-sharelatex")
 crypto = require("crypto")
@@ -51,7 +52,6 @@ module.exports = UrlCache =
 	_doesUrlNeedDownloading: (project_id, url, lastModified, callback = (error, needsDownloading) ->) ->
 		if !lastModified?
 			return callback null, true
-
 		UrlCache._findUrlDetails project_id, url, (error, urlDetails) ->
 			return callback(error) if error?
 			if !urlDetails? or !urlDetails.lastModified? or urlDetails.lastModified.getTime() < lastModified.getTime()
@@ -94,32 +94,41 @@ module.exports = UrlCache =
 				return callback()
 
 	_findUrlDetails: (project_id, url, callback = (error, urlDetails) ->) ->
+		job = (cb)->
 			db.UrlCache.find(where: { url: url, project_id: project_id })
-			.then((urlDetails) -> callback null, urlDetails)
+				.then((urlDetails) -> cb null, urlDetails)
-			.error callback
+				.error cb
+		dbQueue.queue.push job, callback
 
 	_updateOrCreateUrlDetails: (project_id, url, lastModified, callback = (error) ->) ->
+		job = (cb)->
 			db.UrlCache.findOrCreate(where: {url: url, project_id: project_id})
 				.spread(
 					(urlDetails, created) ->
 						urlDetails.updateAttributes(lastModified: lastModified)
-						.then(() -> callback())
+							.then(() -> cb())
-						.error(callback)
+							.error(cb)
 				)
-			.error callback
+				.error cb
+		dbQueue.queue.push(job, callback)
 
 	_clearUrlDetails: (project_id, url, callback = (error) ->) ->
+		job = (cb)->
 			db.UrlCache.destroy(where: {url: url, project_id: project_id})
-			.then(() -> callback null)
+				.then(() -> cb null)
-			.error callback
+				.error cb
+		dbQueue.queue.push(job, callback)
+
 
 	_findAllUrlsInProject: (project_id, callback = (error, urls) ->) ->
+		job = (cb)->
 			db.UrlCache.findAll(where: { project_id: project_id })
 				.then(
 					(urlEntries) ->
-					callback null, urlEntries.map((entry) -> entry.url)
+						cb null, urlEntries.map((entry) -> entry.url)
 				)
-			.error callback
+				.error cb
+		dbQueue.queue.push(job, callback)
 
 		
 		

services/clsi/app/coffee/UrlFetcher.coffee

@@ -1,6 +1,8 @@
 request = require("request").defaults(jar: false)
 fs = require("fs")
 logger = require "logger-sharelatex"
+settings = require("settings-sharelatex")
+URL = require('url');
 
 oneMinute = 60 * 1000
 
@@ -11,6 +13,9 @@ module.exports = UrlFetcher =
 			_callback(error)
 			_callback = () ->
 
+		if settings.filestoreDomainOveride?
+			p = URL.parse(url).path
+			url = "#{settings.filestoreDomainOveride}#{p}"
 		timeoutHandler = setTimeout () ->
 			timeoutHandler = null
 			logger.error url:url, filePath: filePath, "Timed out downloading file to cache"

services/clsi/app/coffee/db.coffee

@@ -1,9 +1,12 @@
 Sequelize = require("sequelize")
 Settings = require("settings-sharelatex")
 _ = require("underscore")
+logger = require "logger-sharelatex"
 
 options = _.extend {logging:false}, Settings.mysql.clsi
 
+logger.log dbPath:Settings.mysql.clsi.storage, "connecting to db"
+
 sequelize = new Sequelize(
 	Settings.mysql.clsi.database,
 	Settings.mysql.clsi.username,
@@ -11,6 +14,12 @@ sequelize = new Sequelize(
 	options
 )
 
+if Settings.mysql.clsi.dialect == "sqlite"
+	logger.log "running PRAGMA journal_mode=WAL;"
+	sequelize.query("PRAGMA journal_mode=WAL;")
+	sequelize.query("PRAGMA synchronous=OFF;")
+	sequelize.query("PRAGMA read_uncommitted = true;")
+
 module.exports =
 	UrlCache: sequelize.define("UrlCache", {
 		url: Sequelize.STRING
@@ -32,5 +41,15 @@ module.exports =
 		]
 	})
 
-	sync: () -> sequelize.sync()
+	op: Sequelize.Op
+	
+	sync: () -> 
+		logger.log dbPath:Settings.mysql.clsi.storage, "syncing db schema"
+		sequelize.sync()
+			.then(-> 
+				logger.log "db sync complete"
+			).catch((err)->
+				console.log err, "error syncing"
+			)
+
 	

services/clsi/bin/acceptance_test

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e;
+MOCHA="node_modules/.bin/mocha --recursive --reporter spec --timeout 15000"
+$MOCHA "$@"

services/clsi/bin/install_texlive_gce.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+METADATA=http://metadata.google.internal./computeMetadata/v1
+SVC_ACCT=$METADATA/instance/service-accounts/default
+PROJECT_URL=$METADATA/project/project-id
+ACCESS_TOKEN=$(curl -s -H 'Metadata-Flavor: Google' $SVC_ACCT/token | cut -d'"' -f 4)
+if [ -z "$ACCESS_TOKEN" ]; then
+	echo "No acccess token to download texlive-full images from google container, continuing without downloading. This is likely not a google cloud enviroment."
+	exit 0
+fi
+PROJECT=$(curl -s -H 'Metadata-Flavor: Google' $PROJECT_URL)
+if [ -z "$PROJECT" ]; then
+	echo "No project name to download texlive-full images from google container, continuing without downloading. This is likely not a google cloud enviroment."
+	exit 0
+fi
+docker login -u '_token' -p $ACCESS_TOKEN https://gcr.io
+docker pull --all-tags gcr.io/$PROJECT/texlive-full
+cp /app/bin/synctex /app/bin/synctex-mount/synctex
+
+echo "Finished downloading texlive-full images"
+
+

services/clsi/config/settings.defaults.coffee

@@ -7,9 +7,13 @@ module.exports =
 		clsi:
 			database: "clsi"
 			username: "clsi"
-			password: null
 			dialect: "sqlite"
-			storage: Path.resolve(__dirname + "/../db.sqlite")
+			storage: process.env["SQLITE_PATH"] or Path.resolve(__dirname + "/../db.sqlite")
+			pool:
+				max: 1
+				min: 1
+			retry:
+				max: 10
 
 	path:
 		compilesDir:  Path.resolve(__dirname + "/../compiles")
@@ -21,18 +25,27 @@ module.exports =
 			port: 3013
 			host: process.env["LISTEN_ADDRESS"] or "localhost"
 			
-	
+		load_balancer_agent:
+			report_load:true
+			load_port: 3048
+			local_port: 3049
 	apis:
 		clsi:
-			url: "http://localhost:3013"
+			url: "http://#{process.env['CLSI_HOST'] or 'localhost'}:3013"
 			
-	smokeTest: false
+	smokeTest: process.env["SMOKE_TEST"] or false
 	project_cache_length_ms: 1000 * 60 * 60 * 24
-	parallelFileDownloads:1
+	parallelFileDownloads: process.env["FILESTORE_PARALLEL_FILE_DOWNLOADS"] or 1
+	parallelSqlQueryLimit: process.env["FILESTORE_PARALLEL_SQL_QUERY_LIMIT"] or 1
+	filestoreDomainOveride: process.env["FILESTORE_DOMAIN_OVERRIDE"]
+	texliveImageNameOveride: process.env["TEX_LIVE_IMAGE_NAME_OVERRIDE"]
+	sentry:
+		dsn: process.env['SENTRY_DSN']
 
-if process.env["COMMAND_RUNNER"]
+
+if process.env["DOCKER_RUNNER"]
 	module.exports.clsi =
-		commandRunner: process.env["COMMAND_RUNNER"]
+		dockerRunner: process.env["DOCKER_RUNNER"] == "true"
 		docker:
 			image: process.env["TEXLIVE_IMAGE"] or "quay.io/sharelatex/texlive-full:2017.1"
 			env:
@@ -41,4 +54,15 @@ if process.env["COMMAND_RUNNER"]
 			user: process.env["TEXLIVE_IMAGE_USER"] or "tex"
 		expireProjectAfterIdleMs: 24 * 60 * 60 * 1000
 		checkProjectsIntervalMs: 10 * 60 * 1000
+
+	try
+		seccomp_profile_path = Path.resolve(__dirname + "/../seccomp/clsi-profile.json")
+		module.exports.clsi.docker.seccomp_profile = JSON.stringify(JSON.parse(require("fs").readFileSync(seccomp_profile_path)))
+	catch error
+		console.log error, "could not load seccom profile from #{seccomp_profile_path}"
+
+	module.exports.path.synctexBaseDir = -> "/compile"	
+	
 	module.exports.path.sandboxedCompilesHostDir = process.env["COMPILES_HOST_DIR"]
+
+	module.exports.path.synctexBinHostPath = process.env["SYNCTEX_BIN_HOST_PATH"]

services/clsi/debug

@@ -0,0 +1,5 @@
+#!/bin/bash
+echo "hello world"
+sleep 3
+echo "awake"
+/opt/synctex pdf /compile/output.pdf 1 100 200

services/clsi/docker-compose-config.yml

@@ -0,0 +1,32 @@
+version: "2"
+
+services:
+  dev:
+    environment:
+      TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
+      TEXLIVE_IMAGE_USER: "tex"
+      SHARELATEX_CONFIG: /app/config/settings.defaults.coffee
+      DOCKER_RUNNER: "true"
+      COMPILES_HOST_DIR: $PWD/compiles
+      SYNCTEX_BIN_HOST_PATH: $PWD/bin/synctex
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./compiles:/app/compiles
+      - ./cache:/app/cache
+      - ./bin/synctex:/app/bin/synctex
+
+      
+  ci:
+    environment:
+      TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
+      TEXLIVE_IMAGE_USER: "tex"
+      SHARELATEX_CONFIG: /app/config/settings.defaults.coffee
+      DOCKER_RUNNER: "true"
+      COMPILES_HOST_DIR: $PWD/compiles
+      SYNCTEX_BIN_HOST_PATH: $PWD/bin/synctex
+      SQLITE_PATH: /app/compiles/db.sqlite
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:rw
+      - ./compiles:/app/compiles
+      - ./cache:/app/cache
+      - ./bin/synctex:/app/bin/synctex

services/clsi/docker-compose.ci.yml

@@ -0,0 +1,34 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.1.9
+
+version: "2"
+
+services:
+  test_unit:
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    command: npm run test:unit:_run
+
+  test_acceptance:
+    build: .
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    extends:
+      file: docker-compose-config.yml
+      service: ci
+    environment:
+      ELASTIC_SEARCH_DSN: es:9200
+      REDIS_HOST: redis
+      MONGO_HOST: mongo
+      POSTGRES_HOST: postgres
+      MOCHA_GREP: ${MOCHA_GREP}
+    depends_on:
+      - mongo
+      - redis
+    command: npm run test:acceptance:_run
+
+  redis:
+    image: redis
+
+  mongo:
+    image: mongo:3.4

services/clsi/docker-compose.yml

@@ -0,0 +1,42 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.1.9
+
+version: "2"
+
+services:
+  test_unit:
+    build: .
+    volumes:
+      - .:/app
+    working_dir: /app
+    environment:
+      MOCHA_GREP: ${MOCHA_GREP}
+    command: npm run test:unit
+
+  test_acceptance:
+    build: .
+    volumes:
+      - .:/app
+    working_dir: /app
+    extends:
+      file: docker-compose-config.yml
+      service: dev
+    environment:
+      ELASTIC_SEARCH_DSN: es:9200
+      REDIS_HOST: redis
+      MONGO_HOST: mongo
+      POSTGRES_HOST: postgres
+      MOCHA_GREP: ${MOCHA_GREP}
+    depends_on:
+      - mongo
+      - redis
+    command: npm run test:acceptance
+
+  redis:
+    image: redis
+
+  mongo:
+    image: mongo:3.4
+

services/clsi/entrypoint.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+echo "Changing permissions of /var/run/docker.sock for sibling containers"
+ls -al /var/run/docker.sock
+docker --version
+cat /etc/passwd
+usermod -aG docker node
+chown root:docker /var/run/docker.sock
+
+mkdir -p /app/cache
+chown -R node:node /app/cache
+
+mkdir -p /app/compiles
+chown -R node:node /app/compiles
+
+chown -R node:node /app/bin/synctex
+mkdir -p /app/test/acceptance/fixtures/tmp/
+chown -R node:node /app
+
+chown -R node:node /app/bin
+
+./bin/install_texlive_gce.sh
+exec runuser -u node "$@"

services/clsi/install_deps.sh

@@ -0,0 +1,4 @@
+/bin/sh
+wget -qO- https://get.docker.com/ | sh
+apt-get install poppler-utils vim ghostscript --yes
+npm rebuild

services/clsi/kube.yaml

@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: clsi
+  namespace: default
+spec:
+  type: LoadBalancer
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    run: clsi
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: clsi
+  namespace: default
+spec:
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        run: clsi
+    spec:
+      containers:
+      - name: clsi
+        image: gcr.io/henry-terraform-admin/clsi
+        imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: status
+            port: 80
+          periodSeconds: 5
+          initialDelaySeconds: 0
+          failureThreshold: 3
+          successThreshold: 1
+
+
+

services/clsi/nodemon.json

@@ -0,0 +1,19 @@
+{
+  "ignore": [
+    ".git",
+    "node_modules/"
+  ],
+  "verbose": true,
+  "legacyWatch": true,
+  "execMap": {
+    "js": "npm run start"
+  },
+
+  "watch": [
+    "app/coffee/",
+    "app.coffee",
+    "config/"
+  ],
+  "ext": "coffee"
+
+}

services/clsi/package.json

@@ -7,13 +7,23 @@
     "url": "https://github.com/sharelatex/clsi-sharelatex.git"
   },
   "scripts": {
-    "compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee",
+    "compile:app": "([ -e app/coffee ] && coffee $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
-    "start": "npm run compile:app && node app.js"
+    "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
+    "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js",
+    "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
+    "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
+    "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP",
+    "compile:unit_tests": "[ ! -e test/unit/coffee ] &&  echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee",
+    "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee",
+    "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests",
+    "nodemon": "nodemon --config nodemon.json",
+    "compile:smoke_tests": "[ ! -e test/smoke/coffee ] &&  echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee"
   },
   "author": "James Allen <james@sharelatex.com>",
   "dependencies": {
     "async": "0.2.9",
     "body-parser": "^1.2.0",
+    "dockerode": "^2.5.3",
     "express": "^4.2.0",
     "fs-extra": "^0.16.3",
     "grunt-mkdir": "^1.0.0",
@@ -21,20 +31,20 @@
     "lockfile": "^1.0.3",
     "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.4",
     "lynx": "0.0.11",
-    "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.5.0",
+    "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.8.1",
     "mkdirp": "0.3.5",
     "mysql": "2.6.2",
     "request": "^2.21.0",
-    "sequelize": "^2.1.3",
+    "sequelize": "^4.38.0",
     "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",
     "smoke-test-sharelatex": "git+https://github.com/sharelatex/smoke-test-sharelatex.git#v0.2.0",
-    "sqlite3": "~3.1.8",
+    "sqlite3": "^4.0.2",
     "underscore": "^1.8.2",
     "v8-profiler": "^5.2.4",
     "wrench": "~1.5.4"
   },
   "devDependencies": {
-    "mocha": "1.10.0",
+    "mocha": "^4.0.1",
     "coffee-script": "1.6.0",
     "chai": "~1.8.1",
     "sinon": "~1.7.3",

services/clsi/patch-texlive-dockerfile

@@ -0,0 +1,3 @@
+FROM quay.io/sharelatex/texlive-full:2017.1
+
+# RUN usermod -u 1001 tex

services/clsi/seccomp/clsi-profile.json

@@ -0,0 +1,832 @@
+{
+    "defaultAction": "SCMP_ACT_ERRNO",
+    "architectures": [
+        "SCMP_ARCH_X86_64",
+        "SCMP_ARCH_X86",
+        "SCMP_ARCH_X32"
+    ],
+    "syscalls": [
+        {
+            "name": "access",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "arch_prctl",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "brk",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "chdir",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "chmod",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "clock_getres",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "clock_gettime",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "clock_nanosleep",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "clone",
+            "action": "SCMP_ACT_ALLOW",
+            "args": [
+                {
+                    "index": 0,
+                    "value": 2080505856,
+                    "valueTwo": 0,
+                    "op": "SCMP_CMP_MASKED_EQ"
+                }
+            ]
+        },
+        {
+            "name": "close",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "copy_file_range",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "creat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "dup",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "dup2",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "dup3",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "execve",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "execveat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "exit",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "exit_group",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "faccessat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fadvise64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fadvise64_64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fallocate",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fchdir",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fchmod",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fchmodat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fcntl",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fcntl64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fdatasync",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fork",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fstat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fstat64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fstatat64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fstatfs",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fstatfs64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fsync",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "ftruncate",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "ftruncate64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "futex",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "futimesat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getcpu",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getcwd",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getdents",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getdents64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getegid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getegid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "geteuid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "geteuid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getgid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getgid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getgroups",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getgroups32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getpgid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getpgrp",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getpid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getppid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getpriority",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getresgid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getresgid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getresuid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getresuid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getrlimit",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "get_robust_list",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getrusage",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getsid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "gettid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getuid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getuid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "ioctl",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "kill",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "_llseek",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "lseek",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "lstat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "lstat64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "madvise",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mkdir",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mkdirat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mmap",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mmap2",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mprotect",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mremap",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "munmap",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "newfstatat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "open",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "openat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pause",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pipe",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pipe2",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "prctl",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pread64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "preadv",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "prlimit64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pwrite64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pwritev",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "read",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "readlink",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "readlinkat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "readv",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rename",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "renameat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "renameat2",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "restart_syscall",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rmdir",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigaction",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigpending",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigprocmask",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigqueueinfo",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigreturn",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigsuspend",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigtimedwait",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_tgsigqueueinfo",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_getaffinity",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_getparam",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_get_priority_max",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_get_priority_min",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_getscheduler",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_rr_get_interval",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_yield",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sendfile",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sendfile64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "setgroups",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "setgroups32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "set_robust_list",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "set_tid_address",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sigaltstack",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "stat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "stat64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "statfs",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "statfs64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sync",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sync_file_range",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "syncfs",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sysinfo",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "tgkill",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timer_create",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timer_delete",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timer_getoverrun",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timer_gettime",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timer_settime",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "times",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "tkill",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "truncate",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "truncate64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "umask",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "uname",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "unlink",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "unlinkat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "utime",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "utimensat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "utimes",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "vfork",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "vhangup",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "wait4",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "waitid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "write",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "writev",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pread",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "setgid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "setuid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "capget",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "capset",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fchown",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+          "name": "gettimeofday",
+          "action": "SCMP_ACT_ALLOW",
+          "args": []
+        }
+    ]
+}

services/clsi/synctex.profile

@@ -0,0 +1,34 @@
+include /etc/firejail/disable-common.inc
+include /etc/firejail/disable-devel.inc
+# include /etc/firejail/disable-mgmt.inc  ## removed in 0.9.40
+# include /etc/firejail/disable-secret.inc ## removed in 0.9.40
+
+read-only /bin
+blacklist /boot
+blacklist /dev
+read-only /etc
+blacklist /home # blacklisted for synctex
+read-only /lib
+read-only /lib64
+blacklist /media
+blacklist /mnt
+blacklist /opt
+blacklist /root
+read-only /run
+blacklist /sbin
+blacklist /selinux
+blacklist /src
+blacklist /sys
+read-only /usr
+
+caps.drop all
+noroot
+nogroups
+net none
+private-tmp
+private-dev
+shell none
+seccomp
+nonewprivs
+
+

services/clsi/test/acceptance/coffee/BrokenLatexFileTests.coffee

@@ -1,9 +1,10 @@
 Client = require "./helpers/Client"
 request = require "request"
 require("chai").should()
+ClsiApp = require "./helpers/ClsiApp"
 
 describe "Broken LaTeX file", ->
-	before ->
+	before (done)->
 		@broken_request =
 			resources: [
 				path: "main.tex"
@@ -24,6 +25,7 @@ describe "Broken LaTeX file", ->
 					\\end{document}
 				'''
 			]
+		ClsiApp.ensureRunning done
 		
 	describe "on first run", ->
 		before (done) ->

services/clsi/test/acceptance/coffee/DeleteOldFilesTest.coffee

@@ -1,9 +1,10 @@
 Client = require "./helpers/Client"
 request = require "request"
 require("chai").should()
+ClsiApp = require "./helpers/ClsiApp"
 
 describe "Deleting Old Files", ->
-	before ->
+	before (done)->
 		@request =
 			resources: [
 				path: "main.tex"
@@ -14,6 +15,7 @@ describe "Deleting Old Files", ->
 					\\end{document}
 				'''
 			]
+		ClsiApp.ensureRunning done
 
 	describe "on first run", ->
 		before (done) ->

services/clsi/test/acceptance/coffee/ExampleDocumentTests.coffee

@@ -3,15 +3,23 @@ request = require "request"
 require("chai").should()
 fs = require "fs"
 ChildProcess = require "child_process"
-
+ClsiApp = require "./helpers/ClsiApp"
-fixturePath = (path) -> __dirname + "/../fixtures/" + path
+logger = require("logger-sharelatex")
-
+Path = require("path")
+fixturePath = (path) -> Path.normalize(__dirname + "/../fixtures/" + path)
+process = require "process"
+console.log process.pid, process.ppid, process.getuid(),process.getgroups(), "PID"
 try
+	console.log "creating tmp directory", fixturePath("tmp")
 	fs.mkdirSync(fixturePath("tmp"))
-catch e
+catch err
+	console.log err, fixturePath("tmp"), "unable to create fixture tmp path"
 
 convertToPng = (pdfPath, pngPath, callback = (error) ->) ->
-	convert = ChildProcess.exec "convert #{fixturePath(pdfPath)} #{fixturePath(pngPath)}"
+	command = "convert #{fixturePath(pdfPath)} #{fixturePath(pngPath)}"
+	console.log "COMMAND"
+	console.log command
+	convert = ChildProcess.exec command
 	stdout = ""
 	convert.stdout.on "data", (chunk) -> console.log "STDOUT", chunk.toString()
 	convert.stderr.on "data", (chunk) -> console.log "STDERR", chunk.toString()
@@ -40,7 +48,6 @@ checkPdfInfo = (pdfPath, callback = (error, output) ->) ->
 		if stdout.match(/Optimized:\s+yes/)
 			callback null, true
 		else
-			console.log "pdfinfo result", stdout
 			callback null, false
 
 compareMultiplePages = (project_id, callback = (error) ->) ->
@@ -57,6 +64,8 @@ compareMultiplePages = (project_id, callback = (error) ->) ->
 	compareNext 0, callback
 
 comparePdf = (project_id, example_dir, callback = (error) ->) ->
+	console.log "CONVERT"
+	console.log "tmp/#{project_id}.pdf", "tmp/#{project_id}-generated.png"
 	convertToPng "tmp/#{project_id}.pdf", "tmp/#{project_id}-generated.png", (error) =>
 		throw error if error?
 		convertToPng "examples/#{example_dir}/output.pdf", "tmp/#{project_id}-source.png", (error) =>
@@ -75,6 +84,7 @@ comparePdf = (project_id, example_dir, callback = (error) ->) ->
 downloadAndComparePdf = (project_id, example_dir, url, callback = (error) ->) ->
 	writeStream = fs.createWriteStream(fixturePath("tmp/#{project_id}.pdf"))
 	request.get(url).pipe(writeStream)
+	console.log("writing file out", fixturePath("tmp/#{project_id}.pdf"))
 	writeStream.on "close", () =>
 		checkPdfInfo "tmp/#{project_id}.pdf", (error, optimised) =>
 			throw error if error?
@@ -85,7 +95,9 @@ Client.runServer(4242, fixturePath("examples"))
 
 describe "Example Documents", ->
 	before (done) ->
-		ChildProcess.exec("rm test/acceptance/fixtures/tmp/*").on "exit", () -> done()
+		ChildProcess.exec("rm test/acceptance/fixtures/tmp/*").on "exit", () -> 
+			ClsiApp.ensureRunning done
+
 
 	for example_dir in fs.readdirSync fixturePath("examples")
 		do (example_dir) ->

services/clsi/test/acceptance/coffee/SimpleLatexFileTests.coffee

@@ -1,6 +1,7 @@
 Client = require "./helpers/Client"
 request = require "request"
 require("chai").should()
+ClsiApp = require "./helpers/ClsiApp"
 
 describe "Simple LaTeX file", ->
 	before (done) ->
@@ -15,6 +16,7 @@ describe "Simple LaTeX file", ->
 					\\end{document}
 				'''
 			]
+		ClsiApp.ensureRunning =>
 			Client.compile @project_id, @request, (@error, @res, @body) => done()
 
 	it "should return the PDF", ->

services/clsi/test/acceptance/coffee/SynctexTests.coffee

@@ -2,20 +2,24 @@ Client = require "./helpers/Client"
 request = require "request"
 require("chai").should()
 expect = require("chai").expect
+ClsiApp = require "./helpers/ClsiApp"
+crypto = require("crypto")
 
 describe "Syncing", ->
 	before (done) ->
-		@request =
+		content = '''
-			resources: [
-				path: "main.tex"
-				content: '''
 					\\documentclass{article}
 					\\begin{document}
 					Hello world
 					\\end{document}
 				'''
+		@request =
+			resources: [
+				path: "main.tex"
+				content: content
 			]
 		@project_id = Client.randomId()
+		ClsiApp.ensureRunning =>
 			Client.compile @project_id, @request, (@error, @res, @body) => done()
 
 	describe "from code to pdf", ->
@@ -29,7 +33,7 @@ describe "Syncing", ->
 
 	describe "from pdf to code", ->
 		it "should return the correct location", (done) ->
-			Client.syncFromPdf @project_id, 1, 100, 200, (error, codePositions) ->
+			Client.syncFromPdf @project_id, 1, 100, 200, (error, codePositions) =>
 				throw error if error?
 				expect(codePositions).to.deep.equal(
 					code: [ { file: 'main.tex', line: 3, column: -1 } ]

services/clsi/test/acceptance/coffee/TimeoutTests.coffee

@@ -1,23 +1,26 @@
 Client = require "./helpers/Client"
 request = require "request"
 require("chai").should()
+ClsiApp = require "./helpers/ClsiApp"
+
 
 describe "Timed out compile", ->
 	before (done) ->
 		@request =
 			options:
-				timeout: 1 #seconds
+				timeout: 10 #seconds
 			resources: [
 				path: "main.tex"
 				content: '''
 					\\documentclass{article}
 					\\begin{document}
-					Hello world
+					\\def\\x{Hello!\\par\\x}
-					\\input{|"sleep 10"}
+					\\x					
 					\\end{document}
 				'''
 			]
 		@project_id = Client.randomId()
+		ClsiApp.ensureRunning =>
 			Client.compile @project_id, @request, (@error, @res, @body) => done()
 
 	it "should return a timeout error", ->

services/clsi/test/acceptance/coffee/UrlCachingTests.coffee

@@ -2,6 +2,7 @@ Client = require "./helpers/Client"
 request = require "request"
 require("chai").should()
 sinon = require "sinon"
+ClsiApp = require "./helpers/ClsiApp"
 
 host = "localhost"
 
@@ -46,6 +47,7 @@ describe "Url Caching", ->
 				}]
 
 			sinon.spy Server, "getFile"
+			ClsiApp.ensureRunning =>
 				Client.compile @project_id, @request, (@error, @res, @body) => done()
 
 		afterEach ->

services/clsi/test/acceptance/coffee/WordcountTests.coffee

@@ -4,6 +4,7 @@ require("chai").should()
 expect = require("chai").expect
 path = require("path")
 fs = require("fs")
+ClsiApp = require "./helpers/ClsiApp"
 
 describe "Syncing", ->
 	before (done) ->
@@ -13,6 +14,7 @@ describe "Syncing", ->
 				content: fs.readFileSync(path.join(__dirname,"../fixtures/naugty_strings.txt"),"utf-8")
 			]
 		@project_id = Client.randomId()
+		ClsiApp.ensureRunning =>
 			Client.compile @project_id, @request, (@error, @res, @body) => done()
 
 	describe "wordcount file", ->

services/clsi/test/acceptance/coffee/helpers/Client.coffee

@@ -30,6 +30,7 @@ module.exports = Client =
 		express = require("express")
 		app = express()
 		app.use express.static(directory)
+		console.log("starting test server on", port, host)
 		app.listen(port, host).on "error", (error) ->
 			console.error "error starting server:", error.message
 			process.exit(1)

services/clsi/test/acceptance/coffee/helpers/ClsiApp.coffee

@@ -0,0 +1,24 @@
+app = require('../../../../app')
+require("logger-sharelatex").logger.level("info")
+logger = require("logger-sharelatex")
+Settings = require("settings-sharelatex")
+
+module.exports =
+	running: false
+	initing: false
+	callbacks: []
+	ensureRunning: (callback = (error) ->) ->
+		if @running
+			return callback()
+		else if @initing
+			@callbacks.push callback
+		else
+			@initing = true
+			@callbacks.push callback
+			app.listen Settings.internal?.clsi?.port, "localhost", (error) => 
+				throw error if error?
+				@running = true
+				logger.log("clsi running in dev mode")
+
+				for callback in @callbacks
+					callback()

services/clsi/test/unit/coffee/CompileControllerTests.coffee

@@ -14,7 +14,7 @@ describe "CompileController", ->
 					clsi:
 						url: "http://clsi.example.com"
 			"./ProjectPersistenceManager": @ProjectPersistenceManager = {}
-			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
+			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), err:sinon.stub() }
 		@Settings.externalUrl = "http://www.example.com"
 		@req = {}
 		@res = {}

services/clsi/test/unit/coffee/CompileManagerTests.coffee

@@ -13,7 +13,14 @@ describe "CompileManager", ->
 			"./ResourceWriter": @ResourceWriter = {}
 			"./OutputFileFinder": @OutputFileFinder = {}
 			"./OutputCacheManager": @OutputCacheManager = {}
-			"settings-sharelatex": @Settings = { path: compilesDir: "/compiles/dir" }
+			"settings-sharelatex": @Settings = 
+				path:
+					compilesDir: "/compiles/dir"
+				synctexBaseDir: -> "/compile"
+				clsi:
+					docker:
+						image: "SOMEIMAGE"
+
 			"logger-sharelatex": @logger = { log: sinon.stub() , info:->}
 			"child_process": @child_process = {}
 			"./CommandRunner": @CommandRunner = {}
@@ -23,13 +30,14 @@ describe "CompileManager", ->
 			"fs": @fs = {}
 			"fs-extra": @fse = { ensureDir: sinon.stub().callsArg(1) }
 		@callback = sinon.stub()
-
+		@project_id = "project-id-123"
+		@user_id = "1234"
 	describe "doCompileWithLock", ->
 		beforeEach ->
 			@request =
 				resources: @resources = "mock-resources"
-				project_id: @project_id = "project-id-123"
+				project_id: @project_id
-				user_id: @user_id = "1234"
+				user_id: @user_id
 			@output_files = ["foo", "bar"]
 			@Settings.compileDir = "compiles"
 			@compileDir = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"
@@ -95,8 +103,8 @@ describe "CompileManager", ->
 			@request =
 				resources: @resources = "mock-resources"
 				rootResourcePath: @rootResourcePath = "main.tex"
-				project_id: @project_id = "project-id-123"
+				project_id: @project_id
-				user_id: @user_id = "1234"
+				user_id: @user_id
 				compiler: @compiler = "pdflatex"
 				timeout: @timeout = 42000
 				imageName: @image = "example.com/image"
@@ -247,16 +255,23 @@ describe "CompileManager", ->
 		describe "syncFromCode", ->
 			beforeEach ->
 				@fs.stat = sinon.stub().callsArgWith(1, null,{isFile: ()->true})
-				@child_process.execFile.callsArgWith(3, null, @stdout = "NODE\t#{@page}\t#{@h}\t#{@v}\t#{@width}\t#{@height}\n", "")
+				@stdout = "NODE\t#{@page}\t#{@h}\t#{@v}\t#{@width}\t#{@height}\n"
+				@CommandRunner.run = sinon.stub().callsArgWith(6, null, {stdout:@stdout})
 				@CompileManager.syncFromCode @project_id, @user_id, @file_name, @line, @column, @callback
 
 			it "should execute the synctex binary", ->
 				bin_path = Path.resolve(__dirname + "/../../../bin/synctex")
 				synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/output.pdf"
 				file_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/#{@file_name}"
-				@child_process.execFile
+				@CommandRunner.run
-					.calledWith(bin_path, ["code", synctex_path, file_path, @line, @column], timeout: 10000)
+					.calledWith(
-					.should.equal true
+						"#{@project_id}-#{@user_id}",
+						['/opt/synctex', 'code', synctex_path, file_path, @line, @column],
+						"#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}",
+						@Settings.clsi.docker.image,
+						60000,
+						{}
+						).should.equal true
 
 			it "should call the callback with the parsed output", ->
 				@callback
@@ -272,15 +287,21 @@ describe "CompileManager", ->
 		describe "syncFromPdf", ->
 			beforeEach ->
 				@fs.stat = sinon.stub().callsArgWith(1, null,{isFile: ()->true})
-				@child_process.execFile.callsArgWith(3, null, @stdout = "NODE\t#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/#{@file_name}\t#{@line}\t#{@column}\n", "")
+				@stdout = "NODE\t#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/#{@file_name}\t#{@line}\t#{@column}\n"
+				@CommandRunner.run = sinon.stub().callsArgWith(6, null, {stdout:@stdout})
 				@CompileManager.syncFromPdf @project_id, @user_id, @page, @h, @v, @callback
 
 			it "should execute the synctex binary", ->
 				bin_path = Path.resolve(__dirname + "/../../../bin/synctex")
 				synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/output.pdf"
-				@child_process.execFile
+				@CommandRunner.run
-					.calledWith(bin_path, ["pdf", synctex_path, @page, @h, @v], timeout: 10000)
+					.calledWith(
-					.should.equal true
+						"#{@project_id}-#{@user_id}",
+						['/opt/synctex', "pdf", synctex_path, @page, @h, @v],				
+						"#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}",
+						@Settings.clsi.docker.image,
+						60000,
+						{}).should.equal true
 
 			it "should call the callback with the parsed output", ->
 				@callback
@@ -297,7 +318,7 @@ describe "CompileManager", ->
 			@fs.readFile = sinon.stub().callsArgWith(2, null, @stdout = "Encoding: ascii\nWords in text: 2")
 			@callback  = sinon.stub()
 
-			@project_id = "project-id-123"
+			@project_id
 			@timeout = 60 * 1000
 			@file_name = "main.tex"
 			@Settings.path.compilesDir = "/local/compile/directory"

services/clsi/test/unit/coffee/DockerLockManagerTests.coffee

@@ -0,0 +1,145 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+require "coffee-script"
+modulePath = require('path').join __dirname, '../../../app/coffee/DockerLockManager'
+
+describe "LockManager", ->
+	beforeEach ->
+		@LockManager = SandboxedModule.require modulePath, requires:
+			"settings-sharelatex": @Settings =
+				clsi: docker: {}
+			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
+
+	describe "runWithLock", ->
+		describe "with a single lock", ->
+			beforeEach (done) ->
+				@callback  = sinon.stub()
+				@LockManager.runWithLock "lock-one", (releaseLock) ->
+					setTimeout () ->
+						releaseLock(null, "hello", "world")
+					, 100
+				, (err, args...) =>
+					@callback(err,args...)
+					done()
+
+			it "should call the callback", ->
+				@callback.calledWith(null,"hello","world").should.equal true
+
+		describe "with two locks", ->
+			beforeEach (done) ->
+				@callback1 = sinon.stub()
+				@callback2 = sinon.stub()
+				@LockManager.runWithLock "lock-one", (releaseLock) ->
+					setTimeout () ->
+						releaseLock(null, "hello", "world","one")
+					, 100
+				, (err, args...) =>
+					@callback1(err,args...)
+				@LockManager.runWithLock "lock-two", (releaseLock) ->
+					setTimeout () ->
+						releaseLock(null, "hello", "world","two")
+					, 200
+				, (err, args...) =>
+					@callback2(err,args...)
+					done()
+
+			it "should call the first callback", ->
+				@callback1.calledWith(null,"hello","world","one").should.equal true
+
+			it "should call the second callback", ->
+				@callback2.calledWith(null,"hello","world","two").should.equal true
+
+		describe "with lock contention", ->
+			describe "where the first lock is released quickly", ->
+				beforeEach (done) ->
+					@LockManager.MAX_LOCK_WAIT_TIME = 1000
+					@LockManager.LOCK_TEST_INTERVAL = 100
+					@callback1 = sinon.stub()
+					@callback2 = sinon.stub()
+					@LockManager.runWithLock "lock", (releaseLock) ->
+						setTimeout () ->
+							releaseLock(null, "hello", "world","one")
+						, 100
+					, (err, args...) =>
+						@callback1(err,args...)
+					@LockManager.runWithLock "lock", (releaseLock) ->
+						setTimeout () ->
+							releaseLock(null, "hello", "world","two")
+						, 200
+					, (err, args...) =>
+						@callback2(err,args...)
+						done()
+
+				it "should call the first callback", ->
+					@callback1.calledWith(null,"hello","world","one").should.equal true
+
+				it "should call the second callback", ->
+					@callback2.calledWith(null,"hello","world","two").should.equal true
+
+			describe "where the first lock is held longer than the waiting time", ->
+				beforeEach (done) ->
+					@LockManager.MAX_LOCK_HOLD_TIME = 10000
+					@LockManager.MAX_LOCK_WAIT_TIME = 1000
+					@LockManager.LOCK_TEST_INTERVAL = 100
+					@callback1 = sinon.stub()
+					@callback2 = sinon.stub()
+					doneOne = doneTwo = false
+					finish = (key) ->
+						doneOne = true if key is 1
+						doneTwo = true if key is 2
+						done() if doneOne and doneTwo
+					@LockManager.runWithLock "lock", (releaseLock) ->
+						setTimeout () ->
+							releaseLock(null, "hello", "world","one")
+						, 1100
+					, (err, args...) =>
+						@callback1(err,args...)
+						finish(1)
+					@LockManager.runWithLock "lock", (releaseLock) ->
+						setTimeout () ->
+							releaseLock(null, "hello", "world","two")
+						, 100
+					, (err, args...) =>
+						@callback2(err,args...)
+						finish(2)
+
+				it "should call the first callback", ->
+					@callback1.calledWith(null,"hello","world","one").should.equal true
+
+				it "should call the second callback with an error", ->
+					error = sinon.match.instanceOf Error
+					@callback2.calledWith(error).should.equal true
+
+			describe "where the first lock is held longer than the max holding time", ->
+				beforeEach (done) ->
+					@LockManager.MAX_LOCK_HOLD_TIME = 1000
+					@LockManager.MAX_LOCK_WAIT_TIME = 2000
+					@LockManager.LOCK_TEST_INTERVAL = 100
+					@callback1 = sinon.stub()
+					@callback2 = sinon.stub()
+					doneOne = doneTwo = false
+					finish = (key) ->
+						doneOne = true if key is 1
+						doneTwo = true if key is 2
+						done() if doneOne and doneTwo
+					@LockManager.runWithLock "lock", (releaseLock) ->
+						setTimeout () ->
+							releaseLock(null, "hello", "world","one")
+						, 1500
+					, (err, args...) =>
+						@callback1(err,args...)
+						finish(1)
+					@LockManager.runWithLock "lock", (releaseLock) ->
+						setTimeout () ->
+							releaseLock(null, "hello", "world","two")
+						, 100
+					, (err, args...) =>
+						@callback2(err,args...)
+						finish(2)
+
+				it "should call the first callback", ->
+					@callback1.calledWith(null,"hello","world","one").should.equal true
+
+				it "should call the second callback", ->
+					@callback2.calledWith(null,"hello","world","two").should.equal true

services/clsi/test/unit/coffee/DockerRunnerTests.coffee

@@ -0,0 +1,509 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+expect = require('chai').expect
+require "coffee-script"
+modulePath = require('path').join __dirname, '../../../app/coffee/DockerRunner'
+Path = require "path"
+
+describe "DockerRunner", ->
+	beforeEach ->
+		@container = container = {}
+		@DockerRunner = SandboxedModule.require modulePath, requires:
+			"settings-sharelatex": @Settings =
+				clsi: docker: {}
+				path: {}
+			"logger-sharelatex": @logger = {
+				log: sinon.stub(),
+				error: sinon.stub(),
+				info: sinon.stub(),
+				warn: sinon.stub()
+			}
+			"dockerode": class Docker
+				getContainer: sinon.stub().returns(container)
+				createContainer: sinon.stub().yields(null, container)
+				listContainers: sinon.stub()
+			"fs": @fs = { stat: sinon.stub().yields(null,{isDirectory:()->true}) }
+			"./Metrics":
+				Timer: class Timer
+					done: () ->
+			"./LockManager":
+				runWithLock: (key, runner, callback) -> runner(callback)
+		@Docker = Docker
+		@getContainer = Docker::getContainer
+		@createContainer = Docker::createContainer
+		@listContainers = Docker::listContainers
+
+		@directory = "/local/compile/directory"
+		@mainFile  = "main-file.tex"
+		@compiler  = "pdflatex"
+		@image     = "example.com/sharelatex/image:2016.2"
+		@env       = {}
+		@callback  = sinon.stub()
+		@project_id = "project-id-123"
+		@volumes =
+			"/local/compile/directory": "/compile"
+		@Settings.clsi.docker.image = @defaultImage = "default-image"
+		@Settings.clsi.docker.env   = PATH: "mock-path"
+
+	describe "run", ->
+		beforeEach (done)->
+			@DockerRunner._getContainerOptions = sinon.stub().returns(@options = {mockoptions: "foo"})
+			@DockerRunner._fingerprintContainer = sinon.stub().returns(@fingerprint = "fingerprint")
+
+			@name = "project-#{@project_id}-#{@fingerprint}"
+
+			@command = ["mock", "command", "--outdir=$COMPILE_DIR"]
+			@command_with_dir = ["mock", "command", "--outdir=/compile"]
+			@timeout = 42000
+			done()
+
+		describe "successfully", ->
+			beforeEach (done)->
+				@DockerRunner._runAndWaitForContainer = sinon.stub().callsArgWith(3, null, @output = "mock-output")
+				@DockerRunner.run @project_id, @command, @directory, @image, @timeout, @env, (err, output)=>
+					@callback(err, output)
+					done()
+
+			it "should generate the options for the container", ->
+				@DockerRunner._getContainerOptions
+					.calledWith(@command_with_dir, @image, @volumes, @timeout)
+					.should.equal true
+
+			it "should generate the fingerprint from the returned options", ->
+				@DockerRunner._fingerprintContainer
+					.calledWith(@options)
+					.should.equal true
+
+			it "should do the run", ->
+				@DockerRunner._runAndWaitForContainer	
+					.calledWith(@options, @volumes, @timeout)		
+					.should.equal true
+
+			it "should call the callback", ->
+				@callback.calledWith(null, @output).should.equal true
+
+		describe 'when path.sandboxedCompilesHostDir is set', ->
+
+			beforeEach ->
+				@Settings.path.sandboxedCompilesHostDir = '/some/host/dir/compiles'
+				@directory = '/var/lib/sharelatex/data/compiles/xyz'
+				@DockerRunner._runAndWaitForContainer = sinon.stub().callsArgWith(3, null, @output = "mock-output")
+				@DockerRunner.run @project_id, @command, @directory, @image, @timeout, @env, @callback
+
+			it 'should re-write the bind directory', ->
+				volumes = @DockerRunner._runAndWaitForContainer.lastCall.args[1]
+				expect(volumes).to.deep.equal {
+					'/some/host/dir/compiles/xyz': '/compile'
+				}
+
+			it "should call the callback", ->
+				@callback.calledWith(null, @output).should.equal true
+
+		describe "when the run throws an error", ->
+			beforeEach ->
+				firstTime = true
+				@output = "mock-output"
+				@DockerRunner._runAndWaitForContainer = (options, volumes, timeout, callback = (error, output)->) =>
+					if firstTime
+						firstTime = false
+						callback new Error("HTTP code is 500 which indicates error: server error")
+					else
+						callback(null, @output)
+				sinon.spy @DockerRunner, "_runAndWaitForContainer"
+				@DockerRunner.destroyContainer = sinon.stub().callsArg(3)
+				@DockerRunner.run @project_id, @command, @directory, @image, @timeout, @env, @callback
+
+			it "should do the run twice", ->
+				@DockerRunner._runAndWaitForContainer	
+					.calledTwice.should.equal true
+
+			it "should destroy the container in between", ->
+				@DockerRunner.destroyContainer
+					.calledWith(@name, null)
+					.should.equal true
+
+			it "should call the callback", ->
+				@callback.calledWith(null, @output).should.equal true
+		
+		describe "with no image", ->
+			beforeEach ->
+				@DockerRunner._runAndWaitForContainer = sinon.stub().callsArgWith(3, null, @output = "mock-output")
+				@DockerRunner.run @project_id, @command, @directory, null, @timeout, @env, @callback
+
+			it "should use the default image", ->
+				@DockerRunner._getContainerOptions
+					.calledWith(@command_with_dir, @defaultImage, @volumes, @timeout)
+					.should.equal true
+		
+		describe "with image override", ->
+			beforeEach ->
+				@Settings.texliveImageNameOveride = "overrideimage.com/something"
+				@DockerRunner._runAndWaitForContainer = sinon.stub().callsArgWith(3, null, @output = "mock-output")
+				@DockerRunner.run @project_id, @command, @directory, @image, @timeout, @env, @callback
+
+			it "should use the override and keep the tag", ->
+				image = @DockerRunner._getContainerOptions.args[0][1]
+				image.should.equal "overrideimage.com/something/image:2016.2"
+
+	describe "_runAndWaitForContainer", ->
+		beforeEach ->
+			@options = {mockoptions: "foo", name: @name = "mock-name"}
+			@DockerRunner.startContainer = (options, volumes, attachStreamHandler, callback) =>
+				attachStreamHandler(null, @output = "mock-output")
+				callback(null, @containerId = "container-id")
+			sinon.spy @DockerRunner, "startContainer"
+			@DockerRunner.waitForContainer = sinon.stub().callsArgWith(2, null, @exitCode = 42)
+			@DockerRunner._runAndWaitForContainer @options, @volumes, @timeout, @callback
+
+		it "should create/start the container", ->
+			@DockerRunner.startContainer
+				.calledWith(@options, @volumes)
+				.should.equal true
+
+		it "should wait for the container to finish", ->
+			@DockerRunner.waitForContainer
+				.calledWith(@name, @timeout)
+				.should.equal true
+
+		it "should call the callback with the output", ->
+			@callback.calledWith(null, @output).should.equal true
+
+	describe "startContainer", ->
+		beforeEach ->
+			@attachStreamHandler = sinon.stub()
+			@attachStreamHandler.cock = true
+			@options = {mockoptions: "foo", name: "mock-name"}
+			@container.inspect = sinon.stub().callsArgWith(0)
+			@DockerRunner.attachToContainer = (containerId, attachStreamHandler, cb)=> 
+				attachStreamHandler()
+				cb()
+			sinon.spy @DockerRunner, "attachToContainer"
+
+
+
+		describe "when the container exists", ->
+			beforeEach ->
+				@container.inspect = sinon.stub().callsArgWith(0)
+				@container.start = sinon.stub().yields()
+
+				@DockerRunner.startContainer @options, @volumes, @callback, ->
+
+			it "should start the container with the given name", ->
+				@getContainer
+					.calledWith(@options.name)
+					.should.equal true
+				@container.start
+					.called
+					.should.equal true
+
+			it "should not try to create the container", ->
+				@createContainer.called.should.equal false
+			
+			it "should attach to the container", ->
+				@DockerRunner.attachToContainer.called.should.equal true
+
+			it "should call the callback", ->
+				@callback.called.should.equal true
+	
+			it "should attach before the container starts", ->
+				sinon.assert.callOrder(@DockerRunner.attachToContainer, @container.start)
+						
+		describe "when the container does not exist", ->
+			beforeEach ()->
+				exists = false
+				@container.start = sinon.stub().yields()
+				@container.inspect = sinon.stub().callsArgWith(0, {statusCode:404})
+				@DockerRunner.startContainer @options, @volumes, @attachStreamHandler, @callback
+
+			it "should create the container", ->
+				@createContainer
+					.calledWith(@options)
+					.should.equal true
+
+			it "should call the callback and stream handler", ->
+				@attachStreamHandler.called.should.equal true
+				@callback.called.should.equal true
+			
+			it "should attach to the container", ->
+				@DockerRunner.attachToContainer.called.should.equal true
+
+			it "should attach before the container starts", ->
+				sinon.assert.callOrder(@DockerRunner.attachToContainer, @container.start)
+
+
+		describe "when the container is already running", ->
+			beforeEach ->
+				error = new Error("HTTP code is 304 which indicates error: server error - start: Cannot start container #{@name}: The container MOCKID is already running.")
+				error.statusCode = 304
+				@container.start = sinon.stub().yields(error)
+				@container.inspect = sinon.stub().callsArgWith(0)
+				@DockerRunner.startContainer @options, @volumes, @attachStreamHandler, @callback
+
+			it "should not try to create the container", ->
+				@createContainer.called.should.equal false
+
+			it "should call the callback  and stream handler without an error", ->
+				@attachStreamHandler.called.should.equal true
+				@callback.called.should.equal true
+
+		describe "when a volume does not exist", ->
+			beforeEach ()->
+				@fs.stat = sinon.stub().yields(new Error("no such path"))
+				@DockerRunner.startContainer @options, @volumes, @attachStreamHandler, @callback
+
+			it "should not try to create the container", ->
+				@createContainer.called.should.equal false
+
+			it "should call the callback with an error", ->
+				@callback.calledWith(new Error()).should.equal true
+
+		describe "when a volume exists but is not a directory", ->
+			beforeEach ->
+				@fs.stat = sinon.stub().yields(null, {isDirectory: () -> return false})
+				@DockerRunner.startContainer @options, @volumes, @attachStreamHandler, @callback
+
+			it "should not try to create the container", ->
+				@createContainer.called.should.equal false
+
+			it "should call the callback with an error", ->
+				@callback.calledWith(new Error()).should.equal true
+
+		describe "when a volume does not exist, but sibling-containers are used", ->
+			beforeEach ->
+				@fs.stat = sinon.stub().yields(new Error("no such path"))
+				@Settings.path.sandboxedCompilesHostDir = '/some/path'
+				@container.start = sinon.stub().yields()
+				@DockerRunner.startContainer @options, @volumes, @callback
+
+			afterEach ->
+				delete @Settings.path.sandboxedCompilesHostDir
+
+			it "should start the container with the given name", ->
+				@getContainer
+					.calledWith(@options.name)
+					.should.equal true
+				@container.start
+					.called
+					.should.equal true
+
+			it "should not try to create the container", ->
+				@createContainer.called.should.equal false
+
+			it "should call the callback", ->
+				@callback.called.should.equal true
+				@callback.calledWith(new Error()).should.equal false
+
+		describe "when the container tries to be created, but already has been (race condition)", ->
+
+	describe "waitForContainer", ->
+		beforeEach ->
+			@containerId = "container-id"
+			@timeout = 5000
+			@container.wait = sinon.stub().yields(null, StatusCode: @statusCode = 42)
+			@container.kill = sinon.stub().yields()
+			
+		describe "when the container returns in time", ->
+			beforeEach ->
+				@DockerRunner.waitForContainer @containerId, @timeout, @callback
+
+			it "should wait for the container", ->
+				@getContainer
+					.calledWith(@containerId)
+					.should.equal true
+				@container.wait
+					.called
+					.should.equal true
+
+			it "should call the callback with the exit", ->
+				@callback
+					.calledWith(null, @statusCode)
+					.should.equal true
+
+		describe "when the container does not return before the timeout", ->
+			beforeEach (done) ->
+				@container.wait = (callback = (error, exitCode) ->) ->
+					setTimeout () ->
+						callback(null, StatusCode: 42)
+					, 100
+				@timeout = 5
+				@DockerRunner.waitForContainer @containerId, @timeout, (args...) =>
+					@callback(args...)
+					done()
+
+			it "should call kill on the container", ->
+				@getContainer
+					.calledWith(@containerId)
+					.should.equal true
+				@container.kill
+					.called
+					.should.equal true
+
+			it "should call the callback with an error", ->
+				error = new Error("container timed out")
+				error.timedout = true
+				@callback
+					.calledWith(error)
+					.should.equal true
+
+	describe "destroyOldContainers", ->
+		beforeEach (done) ->
+			oneHourInSeconds = 60 * 60
+			oneHourInMilliseconds = oneHourInSeconds * 1000
+			nowInSeconds = Date.now()/1000
+			@containers = [{
+				Name: "/project-old-container-name"
+				Id: "old-container-id"
+				Created: nowInSeconds - oneHourInSeconds - 100
+			}, {
+				Name: "/project-new-container-name"
+				Id: "new-container-id"
+				Created: nowInSeconds - oneHourInSeconds + 100
+			}, {
+				Name: "/totally-not-a-project-container"
+				Id: "some-random-id"
+				Created: nowInSeconds - (2 * oneHourInSeconds )
+			}]
+			@DockerRunner.MAX_CONTAINER_AGE = oneHourInMilliseconds
+			@listContainers.callsArgWith(1, null, @containers)
+			@DockerRunner.destroyContainer = sinon.stub().callsArg(3)
+			@DockerRunner.destroyOldContainers (error) =>
+				@callback(error)
+				done()
+
+		it "should list all containers", ->
+			@listContainers
+				.calledWith(all: true)
+				.should.equal true
+
+		it "should destroy old containers", ->
+			@DockerRunner.destroyContainer
+				.callCount
+				.should.equal 1
+			@DockerRunner.destroyContainer
+				.calledWith("/project-old-container-name", "old-container-id")
+				.should.equal true
+
+		it "should not destroy new containers", ->
+			@DockerRunner.destroyContainer
+				.calledWith("/project-new-container-name", "new-container-id")
+				.should.equal false
+
+		it "should not destroy non-project containers", ->
+			@DockerRunner.destroyContainer
+				.calledWith("/totally-not-a-project-container", "some-random-id")
+				.should.equal false
+
+		it "should callback the callback", ->
+			@callback.called.should.equal true
+
+
+	describe '_destroyContainer', ->
+		beforeEach ->
+			@containerId = 'some_id'
+			@fakeContainer =
+				remove: sinon.stub().callsArgWith(1, null)
+			@Docker::getContainer = sinon.stub().returns(@fakeContainer)
+
+		it 'should get the container', (done) ->
+			@DockerRunner._destroyContainer @containerId, false, (err) =>
+				@Docker::getContainer.callCount.should.equal 1
+				@Docker::getContainer.calledWith(@containerId).should.equal true
+				done()
+
+		it 'should try to force-destroy the container when shouldForce=true', (done) ->
+			@DockerRunner._destroyContainer @containerId, true, (err) =>
+				@fakeContainer.remove.callCount.should.equal 1
+				@fakeContainer.remove.calledWith({force: true}).should.equal true
+				done()
+
+		it 'should not try to force-destroy the container when shouldForce=false', (done) ->
+			@DockerRunner._destroyContainer @containerId, false, (err) =>
+				@fakeContainer.remove.callCount.should.equal 1
+				@fakeContainer.remove.calledWith({force: false}).should.equal true
+				done()
+
+		it 'should not produce an error', (done) ->
+			@DockerRunner._destroyContainer @containerId, false, (err) =>
+				expect(err).to.equal null
+				done()
+
+		describe 'when the container is already gone', ->
+			beforeEach ->
+				@fakeError = new Error('woops')
+				@fakeError.statusCode = 404
+				@fakeContainer =
+					remove: sinon.stub().callsArgWith(1, @fakeError)
+				@Docker::getContainer = sinon.stub().returns(@fakeContainer)
+
+			it 'should not produce an error', (done) ->
+				@DockerRunner._destroyContainer @containerId, false, (err) =>
+					expect(err).to.equal null
+					done()
+
+		describe 'when container.destroy produces an error', (done) ->
+			beforeEach ->
+				@fakeError = new Error('woops')
+				@fakeError.statusCode = 500
+				@fakeContainer =
+					remove: sinon.stub().callsArgWith(1, @fakeError)
+				@Docker::getContainer = sinon.stub().returns(@fakeContainer)
+
+			it 'should produce an error', (done) ->
+				@DockerRunner._destroyContainer @containerId, false, (err) =>
+					expect(err).to.not.equal null
+					expect(err).to.equal @fakeError
+					done()
+
+
+	describe 'kill', ->
+		beforeEach ->
+			@containerId = 'some_id'
+			@fakeContainer =
+				kill: sinon.stub().callsArgWith(0, null)
+			@Docker::getContainer = sinon.stub().returns(@fakeContainer)
+
+		it 'should get the container', (done) ->
+			@DockerRunner.kill @containerId, (err) =>
+				@Docker::getContainer.callCount.should.equal 1
+				@Docker::getContainer.calledWith(@containerId).should.equal true
+				done()
+
+		it 'should try to force-destroy the container', (done) ->
+			@DockerRunner.kill @containerId, (err) =>
+				@fakeContainer.kill.callCount.should.equal 1
+				done()
+
+		it 'should not produce an error', (done) ->
+			@DockerRunner.kill @containerId, (err) =>
+				expect(err).to.equal undefined
+				done()
+
+		describe 'when the container is not actually running', ->
+			beforeEach ->
+				@fakeError = new Error('woops')
+				@fakeError.statusCode = 500
+				@fakeError.message = 'Cannot kill container <whatever> is not running'
+				@fakeContainer =
+					kill: sinon.stub().callsArgWith(0, @fakeError)
+				@Docker::getContainer = sinon.stub().returns(@fakeContainer)
+
+			it 'should not produce an error', (done) ->
+				@DockerRunner.kill @containerId, (err) =>
+					expect(err).to.equal undefined
+					done()
+
+		describe 'when container.kill produces a legitimate error', (done) ->
+			beforeEach ->
+				@fakeError = new Error('woops')
+				@fakeError.statusCode = 500
+				@fakeError.message = 'Totally legitimate reason to throw an error'
+				@fakeContainer =
+					kill: sinon.stub().callsArgWith(0, @fakeError)
+				@Docker::getContainer = sinon.stub().returns(@fakeContainer)
+
+			it 'should produce an error', (done) ->
+				@DockerRunner.kill @containerId, (err) =>
+					expect(err).to.not.equal undefined
+					expect(err).to.equal @fakeError
+					done()

services/clsi/test/unit/coffee/LockManagerTests.coffee

@@ -5,11 +5,14 @@ modulePath = require('path').join __dirname, '../../../app/js/LockManager'
 Path = require "path"
 Errors = require "../../../app/js/Errors"
 
-describe "LockManager", ->
+describe "DockerLockManager", ->
 	beforeEach ->
 		@LockManager = SandboxedModule.require modulePath, requires:
 			"settings-sharelatex": {}
-			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
+			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), err:-> }
+			"fs":
+				lstat:sinon.stub().callsArgWith(1)
+				readdir: sinon.stub().callsArgWith(1)
 			"lockfile": @Lockfile = {}
 		@lockFile = "/local/compile/directory/.project-lock"
 

services/clsi/test/unit/coffee/RequestParserTests.coffee

@@ -16,10 +16,12 @@ describe "RequestParser", ->
 			compile:
 				token: "token-123"
 				options:
+					imageName: "basicImageName/here:2017-1"
 					compiler: "pdflatex"
 					timeout:  42
 				resources: []
-		@RequestParser = SandboxedModule.require modulePath
+		@RequestParser = SandboxedModule.require modulePath, requires:
+			"settings-sharelatex": @settings = {}
 	
 	afterEach ->
 		tk.reset()
@@ -57,6 +59,13 @@ describe "RequestParser", ->
 		it "should set the compiler to pdflatex by default", ->
 			@data.compiler.should.equal "pdflatex"
 
+	describe "with imageName set", ->
+		beforeEach ->
+			@RequestParser.parse @validRequest, (error, @data) =>
+
+		it "should set the imageName", ->
+			@data.imageName.should.equal "basicImageName/here:2017-1"
+
 	describe "without a timeout specified", ->
 		beforeEach ->
 			delete @validRequest.compile.options.timeout

services/clsi/test/unit/coffee/UrlFetcherTests.coffee

@@ -7,17 +7,18 @@ EventEmitter = require("events").EventEmitter
 describe "UrlFetcher", ->
 	beforeEach ->
 		@callback = sinon.stub()
-		@url = "www.example.com/file"
+		@url = "https://www.example.com/file/here?query=string"
 		@UrlFetcher = SandboxedModule.require modulePath, requires:
 			request: defaults: @defaults = sinon.stub().returns(@request = {})
 			fs: @fs = {}
 			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
+			"settings-sharelatex": @settings = {}
 
 	it "should turn off the cookie jar in request", ->
 		@defaults.calledWith(jar: false)
 			.should.equal true
 
-	describe "_pipeUrlToFile", ->
+	describe "rewrite url domain if filestoreDomainOveride is set", ->
 		beforeEach ->
 			@path = "/path/to/file/on/disk"
 			@request.get = sinon.stub().returns(@urlStream = new EventEmitter)
@@ -26,21 +27,54 @@ describe "UrlFetcher", ->
 			@urlStream.resume = sinon.stub()
 			@fs.createWriteStream = sinon.stub().returns(@fileStream = new EventEmitter)
 			@fs.unlink = (file, callback) -> callback()
-			@UrlFetcher.pipeUrlToFile(@url, @path, @callback)
+
+		it "should use the normal domain when override not set", (done)->
+			@UrlFetcher.pipeUrlToFile @url, @path, =>
+				@request.get.args[0][0].url.should.equal @url
+				done()
+			@res = statusCode: 200
+			@urlStream.emit "response", @res
+			@urlStream.emit "end"
+			@fileStream.emit "finish"
+
+
+		it "should use override domain when filestoreDomainOveride is set", (done)->
+			@settings.filestoreDomainOveride = "192.11.11.11"
+			@UrlFetcher.pipeUrlToFile @url, @path, =>
+				@request.get.args[0][0].url.should.equal "192.11.11.11/file/here?query=string"
+				done()
+			@res = statusCode: 200
+			@urlStream.emit "response", @res
+			@urlStream.emit "end"
+			@fileStream.emit "finish"
+
+	describe "pipeUrlToFile", ->
+		beforeEach (done)->
+			@path = "/path/to/file/on/disk"
+			@request.get = sinon.stub().returns(@urlStream = new EventEmitter)
+			@urlStream.pipe = sinon.stub()
+			@urlStream.pause = sinon.stub()
+			@urlStream.resume = sinon.stub()
+			@fs.createWriteStream = sinon.stub().returns(@fileStream = new EventEmitter)
+			@fs.unlink = (file, callback) -> callback()
+			done()
+
+		describe "successfully", ->
+			beforeEach (done)->
+				@UrlFetcher.pipeUrlToFile @url, @path, =>
+					@callback()
+					done()
+				@res = statusCode: 200
+				@urlStream.emit "response", @res
+				@urlStream.emit "end"
+				@fileStream.emit "finish"
+
 
 			it "should request the URL", ->
 				@request.get
 					.calledWith(sinon.match {"url": @url})
 					.should.equal true
 
-
-		describe "successfully", ->
-			beforeEach ->
-				@res = statusCode: 200
-				@urlStream.emit "response", @res
-				@urlStream.emit "end"
-				@fileStream.emit "finish"
-
 			it "should open the file for writing", ->
 				@fs.createWriteStream
 					.calledWith(@path)
@@ -55,7 +89,10 @@ describe "UrlFetcher", ->
 				@callback.called.should.equal true
 
 		describe "with non success status code", ->
-			beforeEach ->
+			beforeEach (done)->
+				@UrlFetcher.pipeUrlToFile @url, @path, (err)=>
+					@callback(err)
+					done()	
 				@res = statusCode: 404
 				@urlStream.emit "response", @res
 				@urlStream.emit "end"
@@ -66,7 +103,10 @@ describe "UrlFetcher", ->
 					.should.equal true
 
 		describe "with error", ->
-			beforeEach ->
+			beforeEach (done)->
+				@UrlFetcher.pipeUrlToFile @url, @path, (err)=>
+					@callback(err)
+					done()
 				@urlStream.emit "error", @error = new Error("something went wrong")
 
 			it "should call the callback with the error", ->