needs hacked pacth in docker runner

wip

most tests pass
This commit is contained in:
Henry Oswald 2018-02-13 12:05:10 +00:00
parent df21b9de53
commit 5cf3c904cb
19 changed files with 3008 additions and 1912 deletions

View file

@ -14,3 +14,4 @@ cache
db.sqlite
config/*
bin/synctex
npm-debug.log

View file

@ -2,17 +2,21 @@ FROM node:6.9.5
RUN wget -qO- https://get.docker.com/ | sh
# ---- Copy Files/Build ----
WORKDIR /app
run apt-get install poppler-utils vim ghostscript --yes
# run git build-essential --yes
# RUN git clone https://github.com/netblue30/firejail.git
# RUN cd firejail && ./configure && make && make install-strip
# run mkdir /data
COPY ./ /app
# Build react/vue/angular bundle static files
# RUN npm run build
WORKDIR /app
RUN npm install
RUN npm run compile
EXPOSE 3013
ENV SHARELATEX_CONFIG /app/config/settings.production.coffee
ENV NODE_ENV production

View file

@ -9,34 +9,9 @@ pipeline {
}
stages {
stage('Install') {
agent {
docker {
image 'node:6.9.5'
args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp"
reuseNode true
}
}
stage('Build') {
steps {
// we need to disable logallrefupdates, else git clones
// during the npm install will require git to lookup the
// user id which does not exist in the container's
// /etc/passwd file, causing the clone to fail.
sh 'git config --global core.logallrefupdates false'
sh 'rm -rf node_modules'
sh 'npm install && npm rebuild'
}
}
stage('Compile') {
agent {
docker {
image 'node:6.9.5'
reuseNode true
}
}
steps {
sh 'npm run compile:all'
sh 'make build'
}
}
@ -54,12 +29,7 @@ pipeline {
stage('Package and publish build') {
steps {
sh 'echo ${BUILD_NUMBER} > build_number.txt'
sh 'touch build.tar.gz' // Avoid tar warning about files changing during read
sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .'
withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
}
sh 'make publish'
}
}

View file

@ -25,5 +25,10 @@ test_acceptance: test_clean # clear the database before each acceptance test run
test_clean:
$(DOCKER_COMPOSE) down
build:
docker build --pull --tag quay.io/sharelatex/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) .
publish:
docker push quay.io/sharelatex/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
.PHONY: clean test test_unit test_acceptance test_clean build publish

View file

@ -5,7 +5,9 @@ logger.info "using standard command runner"
module.exports = CommandRunner =
run: (project_id, command, directory, image, timeout, environment, callback = (error) ->) ->
console.log("Command runner", directory)
command = (arg.replace('$COMPILE_DIR', directory) for arg in command)
console.log("Command runner 2", command)
logger.log project_id: project_id, command: command, directory: directory, "running command"
logger.warn "timeouts and sandboxing are not enabled with CommandRunner"

View file

@ -34,11 +34,14 @@ module.exports = CompileController =
status = "error"
code = 500
logger.error err: error, project_id: request.project_id, "error running compile"
else
status = "failure"
for file in outputFiles
if file.path?.match(/output\.pdf$/)
status = "success"
if status == "failure"
logger.err project_id: request.project_id, outputFiles:outputFiles, "project failed to compile successfully, no output.pdf generated"
timer.done()
res.status(code or 200).send {

View file

@ -41,6 +41,7 @@ module.exports = CompileManager =
doCompile: (request, callback = (error, outputFiles) ->) ->
compileDir = getCompileDir(request.project_id, request.user_id)
console.log("doCompile",compileDir )
timer = new Metrics.Timer("write-to-disk")
logger.log project_id: request.project_id, user_id: request.user_id, "syncing resources to disk"
@ -206,9 +207,14 @@ module.exports = CompileManager =
file_path = base_dir + "/" + file_name
compileDir = getCompileDir(project_id, user_id)
synctex_path = Path.join(compileDir, "output.pdf")
CompileManager._runSynctex ["code", synctex_path, file_path, line, column], (error, stdout) ->
command = ["code", synctex_path, file_path, line, column]
CompileManager._runSynctex command, (error, stdout) ->
return callback(error) if error?
logger.log project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, stdout: stdout, "synctex code output"
if stdout.toLowerCase().indexOf("warning") == -1
logType = "log"
else
logType = "err"
logger[logType] project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, command:command, stdout: stdout, "synctex code output"
callback null, CompileManager._parseSynctexFromCodeOutput(stdout)
syncFromPdf: (project_id, user_id, page, h, v, callback = (error, filePositions) ->) ->
@ -216,6 +222,7 @@ module.exports = CompileManager =
base_dir = Settings.path.synctexBaseDir(compileName)
compileDir = getCompileDir(project_id, user_id)
synctex_path = Path.join(compileDir, "output.pdf")
logger.log({base_dir, project_id, synctex_path}, "base diiir")
CompileManager._runSynctex ["pdf", synctex_path, page, h, v], (error, stdout) ->
return callback(error) if error?
logger.log project_id: project_id, user_id:user_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output"
@ -243,6 +250,7 @@ module.exports = CompileManager =
return callback(error) if error?
if Settings.clsi?.synctexCommandWrapper?
[bin_path, args] = Settings.clsi?.synctexCommandWrapper bin_path, args
logger.log({bin_path, args}, "synctex being run")
child_process.execFile bin_path, args, timeout: 10 * seconds, (error, stdout, stderr) ->
if error?
logger.err err:error, args:args, "error running synctex"

View file

@ -8,6 +8,7 @@ ProcessTable = {} # table of currently running jobs (pids or docker container n
module.exports = LatexRunner =
runLatex: (project_id, options, callback = (error) ->) ->
console.log("LatexRunner", options.directory)
{directory, mainFile, compiler, timeout, image, environment} = options
compiler ||= "pdflatex"
timeout ||= 60000 # milliseconds

View file

@ -10,8 +10,6 @@ module.exports = OutputFileFinder =
for resource in resources
incomingResources[resource.path] = true
logger.log directory: directory, "getting output files"
OutputFileFinder._getAllFiles directory, (error, allFiles = []) ->
if error?
logger.err err:error, "error finding all output files"

View file

@ -109,6 +109,7 @@ module.exports = ResourceWriter =
callback()
_writeResourceToDisk: (project_id, resource, basePath, callback = (error) ->) ->
console.log("_writeResourceToDisk", basePath, resource.path)
ResourceWriter.checkPath basePath, resource.path, (error, path) ->
return callback(error) if error?
mkdirp Path.dirname(path), (error) ->

BIN
services/clsi/bin/synctex Executable file

Binary file not shown.

View file

@ -40,4 +40,7 @@ if process.env["COMMAND_RUNNER"]
user: process.env["TEXLIVE_IMAGE_USER"] or "tex"
expireProjectAfterIdleMs: 24 * 60 * 60 * 1000
checkProjectsIntervalMs: 10 * 60 * 1000
module.exports.path.sandboxedCompilesHostDir = process.env["COMPILES_HOST_DIR"]
console.log module.exports

View file

@ -7,17 +7,11 @@ version: "2"
services:
test_unit:
image: node:6.9.5
volumes:
- .:/app
working_dir: /app
image: quay.io/sharelatex/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
entrypoint: npm run test:unit:_run
test_acceptance:
image: node:6.9.5
volumes:
- .:/app
working_dir: /app
image: quay.io/sharelatex/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
REDIS_HOST: redis
MONGO_HOST: mongo

View file

@ -20,9 +20,7 @@ services:
environment:
REDIS_HOST: redis
MONGO_HOST: mongo
CLSI_HOST: clsi
depends_on:
- clsi
- redis
- mongo
working_dir: /app
@ -34,18 +32,18 @@ services:
mongo:
image: mongo:3.4
clsi:
app:
image: gcr.io/henry-terraform-admin/clsi
build: .
environment:
TEXLIVE_IMAGE: quay.io/sharelatex/texlive-small:latest
TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
TEXLIVE_IMAGE_USER: root # Not ideal, but makes running in dev very simple
COMMAND_RUNNER: docker-runner-sharelatex
SHARELATEX_CONFIG: /app/config/settings.defaults.coffee
COMPILES_HOST_DIR: $PWD/compiles
volumes:
- .:/app:cached
- /var/run/docker.sock:/var/run/docker.sock
- ./docker-runner:/app/node_modules/docker-runner-sharelatex
- /Users/henry/Projects/sharelatex-dev-environment/clsi/compiles:/app/compiles
ports:
- 3013:3013

@ -1 +1 @@
Subproject commit f861a1c810ad844a6e00e82f2ebedac26dcad8b7
Subproject commit 9a732b2594f014a722f0c16150cf848b9512430f

File diff suppressed because it is too large Load diff

View file

@ -1,67 +1,67 @@
{
"name": "node-clsi",
"description": "A Node.js implementation of the CLSI LaTeX web-API",
"version": "0.1.4",
"name": "node-clsi",
"description": "A Node.js implementation of the CLSI LaTeX web-API",
"version": "0.1.4",
"repository": {
"type": "git",
"type": "git",
"url": "https://github.com/sharelatex/clsi-sharelatex.git"
},
},
"scripts": {
"test:acceptance:wait_for_app": "echo 'Waiting for app to be accessible' && while (! curl -s -o /dev/null localhost:3013/status) do sleep 1; done",
"test:acceptance:run": "mocha --recursive --reporter spec --timeout 15000",
"test:acceptance:dir": "npm -q run test:acceptance:wait_for_app && npm -q run test:acceptance:run -- $@",
"test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- $@",
"compile:test:acceptance": "coffee -o test/acceptance/js -c test/acceptance/coffee",
"compile:test:smoke": "coffee -o test/smoke/js -c test/smoke/coffee",
"compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee",
"compile": "npm run compile:app && npm run compile:test:acceptance && npm run compile:test:smoke",
"start": "npm run compile:app && node app.js",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 $@ test/acceptance/js",
"test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
"test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- $@",
"compile:unit_tests": "[ -e test/unit ] && coffee -o test/unit/js -c test/unit/coffee || echo 'No unit tests to compile'",
"compile:acceptance_tests": "[ -e test/acceptance ] && coffee -o test/acceptance/js -c test/acceptance/coffee || echo 'No acceptance tests to compile'",
"compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && compile:test:smoke",
"test:acceptance:wait_for_app": "echo 'Waiting for app to be accessible' && while (! curl -s -o /dev/null localhost:3013/status) do sleep 1; done",
"test:acceptance:run": "mocha --recursive --reporter spec --timeout 15000",
"test:acceptance:dir": "npm -q run test:acceptance:wait_for_app && npm -q run test:acceptance:run -- $@",
"test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- $@",
"compile:test:acceptance": "coffee -o test/acceptance/js -c test/acceptance/coffee",
"compile:test:smoke": "coffee -o test/smoke/js -c test/smoke/coffee",
"compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee",
"compile": "npm run compile:app && npm run compile:test:acceptance && npm run compile:test:smoke",
"start": "npm run compile:app && node app.js",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
"test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- $@",
"compile:unit_tests": "[ -e test/unit ] && coffee -o test/unit/js -c test/unit/coffee || echo 'No unit tests to compile'",
"compile:acceptance_tests": "[ -e test/acceptance ] && coffee -o test/acceptance/js -c test/acceptance/coffee || echo 'No acceptance tests to compile'",
"compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests",
"nodemon": "nodemon --config nodemon.json"
},
"author": "James Allen <james@sharelatex.com>",
},
"author": "James Allen <james@sharelatex.com>",
"dependencies": {
"async": "0.2.9",
"body-parser": "^1.2.0",
"dockerode": "^2.5.3",
"express": "^4.2.0",
"fs-extra": "^0.16.3",
"grunt-mkdir": "^1.0.0",
"heapdump": "^0.3.5",
"lockfile": "^1.0.3",
"logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.4",
"lynx": "0.0.11",
"metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.5.0",
"mkdirp": "0.3.5",
"mysql": "2.6.2",
"request": "^2.21.0",
"sequelize": "^2.1.3",
"settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",
"smoke-test-sharelatex": "git+https://github.com/sharelatex/smoke-test-sharelatex.git#v0.2.0",
"sqlite3": "~3.1.8",
"underscore": "^1.8.2",
"v8-profiler": "^5.2.4",
"async": "0.2.9",
"body-parser": "^1.2.0",
"dockerode": "^2.5.3",
"express": "^4.2.0",
"fs-extra": "^0.16.3",
"grunt-mkdir": "^1.0.0",
"heapdump": "^0.3.5",
"lockfile": "^1.0.3",
"logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.4",
"lynx": "0.0.11",
"metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.5.0",
"mkdirp": "0.3.5",
"mysql": "2.6.2",
"request": "^2.21.0",
"sequelize": "^2.1.3",
"settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",
"smoke-test-sharelatex": "git+https://github.com/sharelatex/smoke-test-sharelatex.git#v0.2.0",
"sqlite3": "~3.1.8",
"underscore": "^1.8.2",
"v8-profiler": "^5.2.4",
"wrench": "~1.5.4"
},
},
"devDependencies": {
"mocha": "1.10.0",
"coffee-script": "1.6.0",
"chai": "~1.8.1",
"sinon": "~1.7.3",
"grunt": "~0.4.2",
"grunt-contrib-coffee": "~0.7.0",
"grunt-contrib-clean": "~0.5.0",
"grunt-shell": "~0.6.1",
"grunt-mocha-test": "~0.8.1",
"sandboxed-module": "~0.3.0",
"timekeeper": "0.0.4",
"grunt-execute": "^0.1.5",
"bunyan": "^0.22.1",
"mocha": "^4.0.1",
"coffee-script": "1.6.0",
"chai": "~1.8.1",
"sinon": "~1.7.3",
"grunt": "~0.4.2",
"grunt-contrib-coffee": "~0.7.0",
"grunt-contrib-clean": "~0.5.0",
"grunt-shell": "~0.6.1",
"grunt-mocha-test": "~0.8.1",
"sandboxed-module": "~0.3.0",
"timekeeper": "0.0.4",
"grunt-execute": "^0.1.5",
"bunyan": "^0.22.1",
"grunt-bunyan": "^0.5.0"
}
}

View file

@ -11,7 +11,8 @@ try
catch e
convertToPng = (pdfPath, pngPath, callback = (error) ->) ->
convert = ChildProcess.exec "convert #{fixturePath(pdfPath)} #{fixturePath(pngPath)}"
command = "convert #{fixturePath(pdfPath)} #{fixturePath(pngPath)}"
convert = ChildProcess.exec command
stdout = ""
convert.stdout.on "data", (chunk) -> console.log "STDOUT", chunk.toString()
convert.stderr.on "data", (chunk) -> console.log "STDERR", chunk.toString()

View file

@ -31,6 +31,7 @@ module.exports = Client =
express = require("express")
app = express()
app.use express.static(directory)
console.log("starting test server on", port, host)
app.listen(port, host).on "error", (error) ->
console.error "error starting server:", error.message
process.exit(1)