mvp

needs hacked pacth in docker runner

wip

most tests pass

services/clsi/.gitignore

@@ -14,3 +14,4 @@ cache
 db.sqlite
 config/*
 bin/synctex
+npm-debug.log

services/clsi/Dockerfile

@@ -2,17 +2,21 @@ FROM node:6.9.5
 
 RUN wget -qO- https://get.docker.com/ | sh
 
-# ---- Copy Files/Build ----
+run apt-get install poppler-utils vim ghostscript --yes
-WORKDIR /app
+
+# run git build-essential --yes
+# RUN git clone https://github.com/netblue30/firejail.git
+# RUN cd firejail && ./configure && make && make install-strip
+# run mkdir /data
+
 COPY ./ /app
-# Build react/vue/angular bundle static files
+
-# RUN npm run build
+WORKDIR /app
+
 RUN npm install
 
 RUN npm run compile
 
-EXPOSE 3013
-
 ENV SHARELATEX_CONFIG /app/config/settings.production.coffee
 ENV NODE_ENV production
 

services/clsi/Jenkinsfile

@@ -9,34 +9,9 @@ pipeline {
   }
 
   stages {
-    stage('Install') {
+    stage('Build') {
-      agent {
-        docker {
-          image 'node:6.9.5'
-          args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp"
-          reuseNode true
-        }
-      }
       steps {
-        // we need to disable logallrefupdates, else git clones 
+        sh 'make build'
-        // during the npm install will require git to lookup the 
-        // user id which does not exist in the container's 
-        // /etc/passwd file, causing the clone to fail.
-        sh 'git config --global core.logallrefupdates false'
-        sh 'rm -rf node_modules'
-        sh 'npm install && npm rebuild'
-      }
-    }
-
-    stage('Compile') {
-      agent {
-        docker {
-          image 'node:6.9.5'
-          reuseNode true
-        }
-      }
-      steps {
-        sh 'npm run compile:all'
       }
     }
 
@@ -54,12 +29,7 @@ pipeline {
 
     stage('Package and publish build') {
       steps {
-        sh 'echo ${BUILD_NUMBER} > build_number.txt'
+        sh 'make publish'
-        sh 'touch build.tar.gz' // Avoid tar warning about files changing during read
-        sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .'
-        withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
-            s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
-        }
       }
     }
 

services/clsi/Makefile

@@ -25,5 +25,10 @@ test_acceptance: test_clean # clear the database before each acceptance test run
 
 test_clean:
 	$(DOCKER_COMPOSE) down
+build:
+	docker build --pull --tag quay.io/sharelatex/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) .
+
+publish:
+	docker push quay.io/sharelatex/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 
 .PHONY: clean test test_unit test_acceptance test_clean build publish

services/clsi/app/coffee/CommandRunner.coffee

@@ -5,7 +5,9 @@ logger.info "using standard command runner"
 
 module.exports = CommandRunner =
 	run: (project_id, command, directory, image, timeout, environment, callback = (error) ->) ->
+		console.log("Command runner", directory)
 		command = (arg.replace('$COMPILE_DIR', directory) for arg in command)
+		console.log("Command runner 2", command)
 		logger.log project_id: project_id, command: command, directory: directory, "running command"
 		logger.warn "timeouts and sandboxing are not enabled with CommandRunner"
 

services/clsi/app/coffee/CompileController.coffee

@@ -34,11 +34,14 @@ module.exports = CompileController =
 							status = "error"
 							code = 500
 							logger.error err: error, project_id: request.project_id, "error running compile"
+
 					else
 						status = "failure"
 						for file in outputFiles
 							if file.path?.match(/output\.pdf$/)
 								status = "success"
+						if status == "failure"
+							logger.err project_id: request.project_id, outputFiles:outputFiles, "project failed to compile successfully, no output.pdf generated"
 
 					timer.done()
 					res.status(code or 200).send {

services/clsi/app/coffee/CompileManager.coffee

@@ -41,6 +41,7 @@ module.exports = CompileManager =
 
 	doCompile: (request, callback = (error, outputFiles) ->) ->
 		compileDir = getCompileDir(request.project_id, request.user_id)
+		console.log("doCompile",compileDir )
 
 		timer = new Metrics.Timer("write-to-disk")
 		logger.log project_id: request.project_id, user_id: request.user_id, "syncing resources to disk"
@@ -206,9 +207,14 @@ module.exports = CompileManager =
 		file_path = base_dir + "/" + file_name
 		compileDir = getCompileDir(project_id, user_id)
 		synctex_path = Path.join(compileDir, "output.pdf")
-		CompileManager._runSynctex ["code", synctex_path, file_path, line, column], (error, stdout) ->
+		command = ["code", synctex_path, file_path, line, column]
+		CompileManager._runSynctex command, (error, stdout) ->
 			return callback(error) if error?
-			logger.log project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, stdout: stdout, "synctex code output"
+			if stdout.toLowerCase().indexOf("warning") == -1
+				logType = "log"
+			else
+				logType = "err"
+			logger[logType] project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, command:command, stdout: stdout, "synctex code output"
 			callback null, CompileManager._parseSynctexFromCodeOutput(stdout)
 
 	syncFromPdf: (project_id, user_id, page, h, v, callback = (error, filePositions) ->) ->
@@ -216,6 +222,7 @@ module.exports = CompileManager =
 		base_dir = Settings.path.synctexBaseDir(compileName)
 		compileDir = getCompileDir(project_id, user_id)
 		synctex_path = Path.join(compileDir, "output.pdf")
+		logger.log({base_dir, project_id, synctex_path}, "base diiir")
 		CompileManager._runSynctex ["pdf", synctex_path, page, h, v], (error, stdout) ->
 			return callback(error) if error?
 			logger.log project_id: project_id, user_id:user_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output"
@@ -243,6 +250,7 @@ module.exports = CompileManager =
 			return callback(error) if error?
 			if Settings.clsi?.synctexCommandWrapper?
 				[bin_path, args] = Settings.clsi?.synctexCommandWrapper bin_path, args
+			logger.log({bin_path, args}, "synctex being run")
 			child_process.execFile bin_path, args, timeout: 10 * seconds, (error, stdout, stderr) ->
 				if error?
 					logger.err err:error, args:args, "error running synctex"

services/clsi/app/coffee/LatexRunner.coffee

@@ -8,6 +8,7 @@ ProcessTable = {}  # table of currently running jobs (pids or docker container n
 
 module.exports = LatexRunner =
 	runLatex: (project_id, options, callback = (error) ->) ->
+		console.log("LatexRunner", options.directory)
 		{directory, mainFile, compiler, timeout, image, environment} = options
 		compiler ||= "pdflatex"
 		timeout  ||= 60000 # milliseconds

services/clsi/app/coffee/OutputFileFinder.coffee

@@ -10,8 +10,6 @@ module.exports = OutputFileFinder =
 		for resource in resources
 			incomingResources[resource.path] = true
 			
-		logger.log directory: directory, "getting output files"
-
 		OutputFileFinder._getAllFiles directory, (error, allFiles = []) ->
 			if error?
 				logger.err err:error, "error finding all output files"

services/clsi/app/coffee/ResourceWriter.coffee

@@ -109,6 +109,7 @@ module.exports = ResourceWriter =
 				callback()
 
 	_writeResourceToDisk: (project_id, resource, basePath, callback = (error) ->) ->
+		console.log("_writeResourceToDisk", basePath, resource.path)
 		ResourceWriter.checkPath basePath, resource.path, (error, path) ->
 			return callback(error) if error?
 			mkdirp Path.dirname(path), (error) ->

services/clsi/config/settings.defaults.coffee

@@ -40,4 +40,7 @@ if process.env["COMMAND_RUNNER"]
 			user: process.env["TEXLIVE_IMAGE_USER"] or "tex"
 		expireProjectAfterIdleMs: 24 * 60 * 60 * 1000
 		checkProjectsIntervalMs: 10 * 60 * 1000
+
 	module.exports.path.sandboxedCompilesHostDir = process.env["COMPILES_HOST_DIR"]
+	
+console.log module.exports

services/clsi/docker-compose.ci.yml

@@ -7,17 +7,11 @@ version: "2"
 
 services:
   test_unit:
-    image: node:6.9.5
+    image: quay.io/sharelatex/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
-    volumes:
-      - .:/app
-    working_dir: /app
     entrypoint: npm run test:unit:_run
 
   test_acceptance:
-    image: node:6.9.5
+    image: quay.io/sharelatex/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
-    volumes:
-      - .:/app
-    working_dir: /app
     environment:
       REDIS_HOST: redis
       MONGO_HOST: mongo

services/clsi/docker-compose.yml

@@ -20,9 +20,7 @@ services:
     environment:
       REDIS_HOST: redis
       MONGO_HOST: mongo
-      CLSI_HOST: clsi
     depends_on:
-      - clsi
       - redis
       - mongo
     working_dir: /app
@@ -34,18 +32,18 @@ services:
   mongo:
     image: mongo:3.4
 
-  clsi:
+  app:
     image: gcr.io/henry-terraform-admin/clsi
     build: .
     environment:
-      TEXLIVE_IMAGE: quay.io/sharelatex/texlive-small:latest
+      TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
       TEXLIVE_IMAGE_USER: root # Not ideal, but makes running in dev very simple
       COMMAND_RUNNER: docker-runner-sharelatex
       SHARELATEX_CONFIG: /app/config/settings.defaults.coffee
       COMPILES_HOST_DIR: $PWD/compiles
     volumes:
-      - .:/app:cached
       - /var/run/docker.sock:/var/run/docker.sock
       - ./docker-runner:/app/node_modules/docker-runner-sharelatex
+      - /Users/henry/Projects/sharelatex-dev-environment/clsi/compiles:/app/compiles
     ports:
       - 3013:3013

services/clsi/docker-runner

@@ -1 +1 @@
-Subproject commit f861a1c810ad844a6e00e82f2ebedac26dcad8b7
+Subproject commit 9a732b2594f014a722f0c16150cf848b9512430f

services/clsi/package.json

@@ -1,67 +1,67 @@
 {
-  "name": "node-clsi", 
+  "name": "node-clsi",
-  "description": "A Node.js implementation of the CLSI LaTeX web-API", 
+  "description": "A Node.js implementation of the CLSI LaTeX web-API",
-  "version": "0.1.4", 
+  "version": "0.1.4",
   "repository": {
-    "type": "git", 
+    "type": "git",
     "url": "https://github.com/sharelatex/clsi-sharelatex.git"
-  }, 
+  },
   "scripts": {
-    "test:acceptance:wait_for_app": "echo 'Waiting for app to be accessible' && while (! curl -s -o /dev/null localhost:3013/status) do sleep 1; done", 
+    "test:acceptance:wait_for_app": "echo 'Waiting for app to be accessible' && while (! curl -s -o /dev/null localhost:3013/status) do sleep 1; done",
-    "test:acceptance:run": "mocha --recursive --reporter spec --timeout 15000", 
+    "test:acceptance:run": "mocha --recursive --reporter spec --timeout 15000",
-    "test:acceptance:dir": "npm -q run test:acceptance:wait_for_app && npm -q run test:acceptance:run -- $@", 
+    "test:acceptance:dir": "npm -q run test:acceptance:wait_for_app && npm -q run test:acceptance:run -- $@",
-    "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- $@", 
+    "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- $@",
-    "compile:test:acceptance": "coffee -o test/acceptance/js -c test/acceptance/coffee", 
+    "compile:test:acceptance": "coffee -o test/acceptance/js -c test/acceptance/coffee",
-    "compile:test:smoke": "coffee -o test/smoke/js -c test/smoke/coffee", 
+    "compile:test:smoke": "coffee -o test/smoke/js -c test/smoke/coffee",
-    "compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee", 
+    "compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee",
-    "compile": "npm run compile:app && npm run compile:test:acceptance && npm run compile:test:smoke", 
+    "compile": "npm run compile:app && npm run compile:test:acceptance && npm run compile:test:smoke",
-    "start": "npm run compile:app && node app.js", 
+    "start": "npm run compile:app && node app.js",
-    "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 $@ test/acceptance/js", 
+    "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
-    "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", 
+    "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
-    "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- $@", 
+    "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- $@",
-    "compile:unit_tests": "[ -e test/unit ] && coffee -o test/unit/js -c test/unit/coffee || echo 'No unit tests to compile'", 
+    "compile:unit_tests": "[ -e test/unit ] && coffee -o test/unit/js -c test/unit/coffee || echo 'No unit tests to compile'",
-    "compile:acceptance_tests": "[ -e test/acceptance ] && coffee -o test/acceptance/js -c test/acceptance/coffee || echo 'No acceptance tests to compile'", 
+    "compile:acceptance_tests": "[ -e test/acceptance ] && coffee -o test/acceptance/js -c test/acceptance/coffee || echo 'No acceptance tests to compile'",
-    "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && compile:test:smoke", 
+    "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests",
     "nodemon": "nodemon --config nodemon.json"
-  }, 
+  },
-  "author": "James Allen <james@sharelatex.com>", 
+  "author": "James Allen <james@sharelatex.com>",
   "dependencies": {
-    "async": "0.2.9", 
+    "async": "0.2.9",
-    "body-parser": "^1.2.0", 
+    "body-parser": "^1.2.0",
-    "dockerode": "^2.5.3", 
+    "dockerode": "^2.5.3",
-    "express": "^4.2.0", 
+    "express": "^4.2.0",
-    "fs-extra": "^0.16.3", 
+    "fs-extra": "^0.16.3",
-    "grunt-mkdir": "^1.0.0", 
+    "grunt-mkdir": "^1.0.0",
-    "heapdump": "^0.3.5", 
+    "heapdump": "^0.3.5",
-    "lockfile": "^1.0.3", 
+    "lockfile": "^1.0.3",
-    "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.4", 
+    "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.4",
-    "lynx": "0.0.11", 
+    "lynx": "0.0.11",
-    "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.5.0", 
+    "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.5.0",
-    "mkdirp": "0.3.5", 
+    "mkdirp": "0.3.5",
-    "mysql": "2.6.2", 
+    "mysql": "2.6.2",
-    "request": "^2.21.0", 
+    "request": "^2.21.0",
-    "sequelize": "^2.1.3", 
+    "sequelize": "^2.1.3",
-    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0", 
+    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",
-    "smoke-test-sharelatex": "git+https://github.com/sharelatex/smoke-test-sharelatex.git#v0.2.0", 
+    "smoke-test-sharelatex": "git+https://github.com/sharelatex/smoke-test-sharelatex.git#v0.2.0",
-    "sqlite3": "~3.1.8", 
+    "sqlite3": "~3.1.8",
-    "underscore": "^1.8.2", 
+    "underscore": "^1.8.2",
-    "v8-profiler": "^5.2.4", 
+    "v8-profiler": "^5.2.4",
     "wrench": "~1.5.4"
-  }, 
+  },
   "devDependencies": {
-    "mocha": "1.10.0", 
+    "mocha": "^4.0.1",
-    "coffee-script": "1.6.0", 
+    "coffee-script": "1.6.0",
-    "chai": "~1.8.1", 
+    "chai": "~1.8.1",
-    "sinon": "~1.7.3", 
+    "sinon": "~1.7.3",
-    "grunt": "~0.4.2", 
+    "grunt": "~0.4.2",
-    "grunt-contrib-coffee": "~0.7.0", 
+    "grunt-contrib-coffee": "~0.7.0",
-    "grunt-contrib-clean": "~0.5.0", 
+    "grunt-contrib-clean": "~0.5.0",
-    "grunt-shell": "~0.6.1", 
+    "grunt-shell": "~0.6.1",
-    "grunt-mocha-test": "~0.8.1", 
+    "grunt-mocha-test": "~0.8.1",
-    "sandboxed-module": "~0.3.0", 
+    "sandboxed-module": "~0.3.0",
-    "timekeeper": "0.0.4", 
+    "timekeeper": "0.0.4",
-    "grunt-execute": "^0.1.5", 
+    "grunt-execute": "^0.1.5",
-    "bunyan": "^0.22.1", 
+    "bunyan": "^0.22.1",
     "grunt-bunyan": "^0.5.0"
   }
 }

services/clsi/test/acceptance/coffee/ExampleDocumentTests.coffee

@@ -11,7 +11,8 @@ try
 catch e
 
 convertToPng = (pdfPath, pngPath, callback = (error) ->) ->
-	convert = ChildProcess.exec "convert #{fixturePath(pdfPath)} #{fixturePath(pngPath)}"
+	command = "convert #{fixturePath(pdfPath)} #{fixturePath(pngPath)}"
+	convert = ChildProcess.exec command
 	stdout = ""
 	convert.stdout.on "data", (chunk) -> console.log "STDOUT", chunk.toString()
 	convert.stderr.on "data", (chunk) -> console.log "STDERR", chunk.toString()

services/clsi/test/acceptance/coffee/helpers/Client.coffee

@@ -31,6 +31,7 @@ module.exports = Client =
 		express = require("express")
 		app = express()
 		app.use express.static(directory)
+		console.log("starting test server on", port, host)
 		app.listen(port, host).on "error", (error) ->
 			console.error "error starting server:", error.message
 			process.exit(1)