mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-07 20:31:06 -05:00
mvp
needs hacked pacth in docker runner wip most tests pass
This commit is contained in:
parent
df21b9de53
commit
5cf3c904cb
19 changed files with 3008 additions and 1912 deletions
1
services/clsi/.gitignore
vendored
1
services/clsi/.gitignore
vendored
|
@ -14,3 +14,4 @@ cache
|
|||
db.sqlite
|
||||
config/*
|
||||
bin/synctex
|
||||
npm-debug.log
|
||||
|
|
|
@ -2,17 +2,21 @@ FROM node:6.9.5
|
|||
|
||||
RUN wget -qO- https://get.docker.com/ | sh
|
||||
|
||||
# ---- Copy Files/Build ----
|
||||
WORKDIR /app
|
||||
run apt-get install poppler-utils vim ghostscript --yes
|
||||
|
||||
# run git build-essential --yes
|
||||
# RUN git clone https://github.com/netblue30/firejail.git
|
||||
# RUN cd firejail && ./configure && make && make install-strip
|
||||
# run mkdir /data
|
||||
|
||||
COPY ./ /app
|
||||
# Build react/vue/angular bundle static files
|
||||
# RUN npm run build
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN npm install
|
||||
|
||||
RUN npm run compile
|
||||
|
||||
EXPOSE 3013
|
||||
|
||||
ENV SHARELATEX_CONFIG /app/config/settings.production.coffee
|
||||
ENV NODE_ENV production
|
||||
|
||||
|
|
36
services/clsi/Jenkinsfile
vendored
36
services/clsi/Jenkinsfile
vendored
|
@ -9,34 +9,9 @@ pipeline {
|
|||
}
|
||||
|
||||
stages {
|
||||
stage('Install') {
|
||||
agent {
|
||||
docker {
|
||||
image 'node:6.9.5'
|
||||
args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp"
|
||||
reuseNode true
|
||||
}
|
||||
}
|
||||
stage('Build') {
|
||||
steps {
|
||||
// we need to disable logallrefupdates, else git clones
|
||||
// during the npm install will require git to lookup the
|
||||
// user id which does not exist in the container's
|
||||
// /etc/passwd file, causing the clone to fail.
|
||||
sh 'git config --global core.logallrefupdates false'
|
||||
sh 'rm -rf node_modules'
|
||||
sh 'npm install && npm rebuild'
|
||||
}
|
||||
}
|
||||
|
||||
stage('Compile') {
|
||||
agent {
|
||||
docker {
|
||||
image 'node:6.9.5'
|
||||
reuseNode true
|
||||
}
|
||||
}
|
||||
steps {
|
||||
sh 'npm run compile:all'
|
||||
sh 'make build'
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -54,12 +29,7 @@ pipeline {
|
|||
|
||||
stage('Package and publish build') {
|
||||
steps {
|
||||
sh 'echo ${BUILD_NUMBER} > build_number.txt'
|
||||
sh 'touch build.tar.gz' // Avoid tar warning about files changing during read
|
||||
sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .'
|
||||
withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
|
||||
s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
|
||||
}
|
||||
sh 'make publish'
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -25,5 +25,10 @@ test_acceptance: test_clean # clear the database before each acceptance test run
|
|||
|
||||
test_clean:
|
||||
$(DOCKER_COMPOSE) down
|
||||
build:
|
||||
docker build --pull --tag quay.io/sharelatex/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) .
|
||||
|
||||
publish:
|
||||
docker push quay.io/sharelatex/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
|
||||
|
||||
.PHONY: clean test test_unit test_acceptance test_clean build publish
|
||||
|
|
|
@ -5,7 +5,9 @@ logger.info "using standard command runner"
|
|||
|
||||
module.exports = CommandRunner =
|
||||
run: (project_id, command, directory, image, timeout, environment, callback = (error) ->) ->
|
||||
console.log("Command runner", directory)
|
||||
command = (arg.replace('$COMPILE_DIR', directory) for arg in command)
|
||||
console.log("Command runner 2", command)
|
||||
logger.log project_id: project_id, command: command, directory: directory, "running command"
|
||||
logger.warn "timeouts and sandboxing are not enabled with CommandRunner"
|
||||
|
||||
|
|
|
@ -34,11 +34,14 @@ module.exports = CompileController =
|
|||
status = "error"
|
||||
code = 500
|
||||
logger.error err: error, project_id: request.project_id, "error running compile"
|
||||
|
||||
else
|
||||
status = "failure"
|
||||
for file in outputFiles
|
||||
if file.path?.match(/output\.pdf$/)
|
||||
status = "success"
|
||||
if status == "failure"
|
||||
logger.err project_id: request.project_id, outputFiles:outputFiles, "project failed to compile successfully, no output.pdf generated"
|
||||
|
||||
timer.done()
|
||||
res.status(code or 200).send {
|
||||
|
|
|
@ -41,6 +41,7 @@ module.exports = CompileManager =
|
|||
|
||||
doCompile: (request, callback = (error, outputFiles) ->) ->
|
||||
compileDir = getCompileDir(request.project_id, request.user_id)
|
||||
console.log("doCompile",compileDir )
|
||||
|
||||
timer = new Metrics.Timer("write-to-disk")
|
||||
logger.log project_id: request.project_id, user_id: request.user_id, "syncing resources to disk"
|
||||
|
@ -206,9 +207,14 @@ module.exports = CompileManager =
|
|||
file_path = base_dir + "/" + file_name
|
||||
compileDir = getCompileDir(project_id, user_id)
|
||||
synctex_path = Path.join(compileDir, "output.pdf")
|
||||
CompileManager._runSynctex ["code", synctex_path, file_path, line, column], (error, stdout) ->
|
||||
command = ["code", synctex_path, file_path, line, column]
|
||||
CompileManager._runSynctex command, (error, stdout) ->
|
||||
return callback(error) if error?
|
||||
logger.log project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, stdout: stdout, "synctex code output"
|
||||
if stdout.toLowerCase().indexOf("warning") == -1
|
||||
logType = "log"
|
||||
else
|
||||
logType = "err"
|
||||
logger[logType] project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, command:command, stdout: stdout, "synctex code output"
|
||||
callback null, CompileManager._parseSynctexFromCodeOutput(stdout)
|
||||
|
||||
syncFromPdf: (project_id, user_id, page, h, v, callback = (error, filePositions) ->) ->
|
||||
|
@ -216,6 +222,7 @@ module.exports = CompileManager =
|
|||
base_dir = Settings.path.synctexBaseDir(compileName)
|
||||
compileDir = getCompileDir(project_id, user_id)
|
||||
synctex_path = Path.join(compileDir, "output.pdf")
|
||||
logger.log({base_dir, project_id, synctex_path}, "base diiir")
|
||||
CompileManager._runSynctex ["pdf", synctex_path, page, h, v], (error, stdout) ->
|
||||
return callback(error) if error?
|
||||
logger.log project_id: project_id, user_id:user_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output"
|
||||
|
@ -243,6 +250,7 @@ module.exports = CompileManager =
|
|||
return callback(error) if error?
|
||||
if Settings.clsi?.synctexCommandWrapper?
|
||||
[bin_path, args] = Settings.clsi?.synctexCommandWrapper bin_path, args
|
||||
logger.log({bin_path, args}, "synctex being run")
|
||||
child_process.execFile bin_path, args, timeout: 10 * seconds, (error, stdout, stderr) ->
|
||||
if error?
|
||||
logger.err err:error, args:args, "error running synctex"
|
||||
|
|
|
@ -8,6 +8,7 @@ ProcessTable = {} # table of currently running jobs (pids or docker container n
|
|||
|
||||
module.exports = LatexRunner =
|
||||
runLatex: (project_id, options, callback = (error) ->) ->
|
||||
console.log("LatexRunner", options.directory)
|
||||
{directory, mainFile, compiler, timeout, image, environment} = options
|
||||
compiler ||= "pdflatex"
|
||||
timeout ||= 60000 # milliseconds
|
||||
|
|
|
@ -10,8 +10,6 @@ module.exports = OutputFileFinder =
|
|||
for resource in resources
|
||||
incomingResources[resource.path] = true
|
||||
|
||||
logger.log directory: directory, "getting output files"
|
||||
|
||||
OutputFileFinder._getAllFiles directory, (error, allFiles = []) ->
|
||||
if error?
|
||||
logger.err err:error, "error finding all output files"
|
||||
|
|
|
@ -109,6 +109,7 @@ module.exports = ResourceWriter =
|
|||
callback()
|
||||
|
||||
_writeResourceToDisk: (project_id, resource, basePath, callback = (error) ->) ->
|
||||
console.log("_writeResourceToDisk", basePath, resource.path)
|
||||
ResourceWriter.checkPath basePath, resource.path, (error, path) ->
|
||||
return callback(error) if error?
|
||||
mkdirp Path.dirname(path), (error) ->
|
||||
|
|
BIN
services/clsi/bin/synctex
Executable file
BIN
services/clsi/bin/synctex
Executable file
Binary file not shown.
|
@ -40,4 +40,7 @@ if process.env["COMMAND_RUNNER"]
|
|||
user: process.env["TEXLIVE_IMAGE_USER"] or "tex"
|
||||
expireProjectAfterIdleMs: 24 * 60 * 60 * 1000
|
||||
checkProjectsIntervalMs: 10 * 60 * 1000
|
||||
|
||||
module.exports.path.sandboxedCompilesHostDir = process.env["COMPILES_HOST_DIR"]
|
||||
|
||||
console.log module.exports
|
|
@ -7,17 +7,11 @@ version: "2"
|
|||
|
||||
services:
|
||||
test_unit:
|
||||
image: node:6.9.5
|
||||
volumes:
|
||||
- .:/app
|
||||
working_dir: /app
|
||||
image: quay.io/sharelatex/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
|
||||
entrypoint: npm run test:unit:_run
|
||||
|
||||
test_acceptance:
|
||||
image: node:6.9.5
|
||||
volumes:
|
||||
- .:/app
|
||||
working_dir: /app
|
||||
image: quay.io/sharelatex/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
|
||||
environment:
|
||||
REDIS_HOST: redis
|
||||
MONGO_HOST: mongo
|
||||
|
|
|
@ -20,9 +20,7 @@ services:
|
|||
environment:
|
||||
REDIS_HOST: redis
|
||||
MONGO_HOST: mongo
|
||||
CLSI_HOST: clsi
|
||||
depends_on:
|
||||
- clsi
|
||||
- redis
|
||||
- mongo
|
||||
working_dir: /app
|
||||
|
@ -34,18 +32,18 @@ services:
|
|||
mongo:
|
||||
image: mongo:3.4
|
||||
|
||||
clsi:
|
||||
app:
|
||||
image: gcr.io/henry-terraform-admin/clsi
|
||||
build: .
|
||||
environment:
|
||||
TEXLIVE_IMAGE: quay.io/sharelatex/texlive-small:latest
|
||||
TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
|
||||
TEXLIVE_IMAGE_USER: root # Not ideal, but makes running in dev very simple
|
||||
COMMAND_RUNNER: docker-runner-sharelatex
|
||||
SHARELATEX_CONFIG: /app/config/settings.defaults.coffee
|
||||
COMPILES_HOST_DIR: $PWD/compiles
|
||||
volumes:
|
||||
- .:/app:cached
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- ./docker-runner:/app/node_modules/docker-runner-sharelatex
|
||||
- /Users/henry/Projects/sharelatex-dev-environment/clsi/compiles:/app/compiles
|
||||
ports:
|
||||
- 3013:3013
|
|
@ -1 +1 @@
|
|||
Subproject commit f861a1c810ad844a6e00e82f2ebedac26dcad8b7
|
||||
Subproject commit 9a732b2594f014a722f0c16150cf848b9512430f
|
4702
services/clsi/package-lock.json
generated
4702
services/clsi/package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
@ -1,67 +1,67 @@
|
|||
{
|
||||
"name": "node-clsi",
|
||||
"description": "A Node.js implementation of the CLSI LaTeX web-API",
|
||||
"version": "0.1.4",
|
||||
"name": "node-clsi",
|
||||
"description": "A Node.js implementation of the CLSI LaTeX web-API",
|
||||
"version": "0.1.4",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"type": "git",
|
||||
"url": "https://github.com/sharelatex/clsi-sharelatex.git"
|
||||
},
|
||||
},
|
||||
"scripts": {
|
||||
"test:acceptance:wait_for_app": "echo 'Waiting for app to be accessible' && while (! curl -s -o /dev/null localhost:3013/status) do sleep 1; done",
|
||||
"test:acceptance:run": "mocha --recursive --reporter spec --timeout 15000",
|
||||
"test:acceptance:dir": "npm -q run test:acceptance:wait_for_app && npm -q run test:acceptance:run -- $@",
|
||||
"test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- $@",
|
||||
"compile:test:acceptance": "coffee -o test/acceptance/js -c test/acceptance/coffee",
|
||||
"compile:test:smoke": "coffee -o test/smoke/js -c test/smoke/coffee",
|
||||
"compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee",
|
||||
"compile": "npm run compile:app && npm run compile:test:acceptance && npm run compile:test:smoke",
|
||||
"start": "npm run compile:app && node app.js",
|
||||
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 $@ test/acceptance/js",
|
||||
"test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
|
||||
"test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- $@",
|
||||
"compile:unit_tests": "[ -e test/unit ] && coffee -o test/unit/js -c test/unit/coffee || echo 'No unit tests to compile'",
|
||||
"compile:acceptance_tests": "[ -e test/acceptance ] && coffee -o test/acceptance/js -c test/acceptance/coffee || echo 'No acceptance tests to compile'",
|
||||
"compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && compile:test:smoke",
|
||||
"test:acceptance:wait_for_app": "echo 'Waiting for app to be accessible' && while (! curl -s -o /dev/null localhost:3013/status) do sleep 1; done",
|
||||
"test:acceptance:run": "mocha --recursive --reporter spec --timeout 15000",
|
||||
"test:acceptance:dir": "npm -q run test:acceptance:wait_for_app && npm -q run test:acceptance:run -- $@",
|
||||
"test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- $@",
|
||||
"compile:test:acceptance": "coffee -o test/acceptance/js -c test/acceptance/coffee",
|
||||
"compile:test:smoke": "coffee -o test/smoke/js -c test/smoke/coffee",
|
||||
"compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee",
|
||||
"compile": "npm run compile:app && npm run compile:test:acceptance && npm run compile:test:smoke",
|
||||
"start": "npm run compile:app && node app.js",
|
||||
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
|
||||
"test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
|
||||
"test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- $@",
|
||||
"compile:unit_tests": "[ -e test/unit ] && coffee -o test/unit/js -c test/unit/coffee || echo 'No unit tests to compile'",
|
||||
"compile:acceptance_tests": "[ -e test/acceptance ] && coffee -o test/acceptance/js -c test/acceptance/coffee || echo 'No acceptance tests to compile'",
|
||||
"compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests",
|
||||
"nodemon": "nodemon --config nodemon.json"
|
||||
},
|
||||
"author": "James Allen <james@sharelatex.com>",
|
||||
},
|
||||
"author": "James Allen <james@sharelatex.com>",
|
||||
"dependencies": {
|
||||
"async": "0.2.9",
|
||||
"body-parser": "^1.2.0",
|
||||
"dockerode": "^2.5.3",
|
||||
"express": "^4.2.0",
|
||||
"fs-extra": "^0.16.3",
|
||||
"grunt-mkdir": "^1.0.0",
|
||||
"heapdump": "^0.3.5",
|
||||
"lockfile": "^1.0.3",
|
||||
"logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.4",
|
||||
"lynx": "0.0.11",
|
||||
"metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.5.0",
|
||||
"mkdirp": "0.3.5",
|
||||
"mysql": "2.6.2",
|
||||
"request": "^2.21.0",
|
||||
"sequelize": "^2.1.3",
|
||||
"settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",
|
||||
"smoke-test-sharelatex": "git+https://github.com/sharelatex/smoke-test-sharelatex.git#v0.2.0",
|
||||
"sqlite3": "~3.1.8",
|
||||
"underscore": "^1.8.2",
|
||||
"v8-profiler": "^5.2.4",
|
||||
"async": "0.2.9",
|
||||
"body-parser": "^1.2.0",
|
||||
"dockerode": "^2.5.3",
|
||||
"express": "^4.2.0",
|
||||
"fs-extra": "^0.16.3",
|
||||
"grunt-mkdir": "^1.0.0",
|
||||
"heapdump": "^0.3.5",
|
||||
"lockfile": "^1.0.3",
|
||||
"logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.4",
|
||||
"lynx": "0.0.11",
|
||||
"metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.5.0",
|
||||
"mkdirp": "0.3.5",
|
||||
"mysql": "2.6.2",
|
||||
"request": "^2.21.0",
|
||||
"sequelize": "^2.1.3",
|
||||
"settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",
|
||||
"smoke-test-sharelatex": "git+https://github.com/sharelatex/smoke-test-sharelatex.git#v0.2.0",
|
||||
"sqlite3": "~3.1.8",
|
||||
"underscore": "^1.8.2",
|
||||
"v8-profiler": "^5.2.4",
|
||||
"wrench": "~1.5.4"
|
||||
},
|
||||
},
|
||||
"devDependencies": {
|
||||
"mocha": "1.10.0",
|
||||
"coffee-script": "1.6.0",
|
||||
"chai": "~1.8.1",
|
||||
"sinon": "~1.7.3",
|
||||
"grunt": "~0.4.2",
|
||||
"grunt-contrib-coffee": "~0.7.0",
|
||||
"grunt-contrib-clean": "~0.5.0",
|
||||
"grunt-shell": "~0.6.1",
|
||||
"grunt-mocha-test": "~0.8.1",
|
||||
"sandboxed-module": "~0.3.0",
|
||||
"timekeeper": "0.0.4",
|
||||
"grunt-execute": "^0.1.5",
|
||||
"bunyan": "^0.22.1",
|
||||
"mocha": "^4.0.1",
|
||||
"coffee-script": "1.6.0",
|
||||
"chai": "~1.8.1",
|
||||
"sinon": "~1.7.3",
|
||||
"grunt": "~0.4.2",
|
||||
"grunt-contrib-coffee": "~0.7.0",
|
||||
"grunt-contrib-clean": "~0.5.0",
|
||||
"grunt-shell": "~0.6.1",
|
||||
"grunt-mocha-test": "~0.8.1",
|
||||
"sandboxed-module": "~0.3.0",
|
||||
"timekeeper": "0.0.4",
|
||||
"grunt-execute": "^0.1.5",
|
||||
"bunyan": "^0.22.1",
|
||||
"grunt-bunyan": "^0.5.0"
|
||||
}
|
||||
}
|
||||
|
|
|
@ -11,7 +11,8 @@ try
|
|||
catch e
|
||||
|
||||
convertToPng = (pdfPath, pngPath, callback = (error) ->) ->
|
||||
convert = ChildProcess.exec "convert #{fixturePath(pdfPath)} #{fixturePath(pngPath)}"
|
||||
command = "convert #{fixturePath(pdfPath)} #{fixturePath(pngPath)}"
|
||||
convert = ChildProcess.exec command
|
||||
stdout = ""
|
||||
convert.stdout.on "data", (chunk) -> console.log "STDOUT", chunk.toString()
|
||||
convert.stderr.on "data", (chunk) -> console.log "STDERR", chunk.toString()
|
||||
|
|
|
@ -31,6 +31,7 @@ module.exports = Client =
|
|||
express = require("express")
|
||||
app = express()
|
||||
app.use express.static(directory)
|
||||
console.log("starting test server on", port, host)
|
||||
app.listen(port, host).on "error", (error) ->
|
||||
console.error "error starting server:", error.message
|
||||
process.exit(1)
|
||||
|
|
Loading…
Reference in a new issue