From 4c96abd6c5fa14b6d658b258cbe091c401aac4ad Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 29 Mar 2018 13:07:55 +0100 Subject: [PATCH] grep works with command updated build scripts acceptence tests break, files are written as root when user is node --- services/clsi/Makefile | 7 ++++--- services/clsi/app.coffee | 4 ++-- services/clsi/app/coffee/ResourceWriter.coffee | 6 ++++++ services/clsi/docker-compose-config.yml | 4 +++- services/clsi/docker-compose.ci.yml | 6 ++---- services/clsi/docker-compose.yml | 10 ++++++---- services/clsi/entrypoint.sh | 12 +++++++++--- services/clsi/install_deps.sh | 9 --------- services/clsi/package.json | 4 ++-- .../coffee/ExampleDocumentTests.coffee | 17 +++++++++++++---- 10 files changed, 47 insertions(+), 32 deletions(-) diff --git a/services/clsi/Makefile b/services/clsi/Makefile index 2a1196c2ca..2db377abee 100644 --- a/services/clsi/Makefile +++ b/services/clsi/Makefile @@ -10,6 +10,7 @@ DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ BRANCH_NAME=$(BRANCH_NAME) \ PROJECT_NAME=$(PROJECT_NAME) \ + MOCHA_GREP=${MOCHA_GREP} \ docker-compose ${DOCKER_COMPOSE_FLAGS} clean: @@ -21,13 +22,13 @@ clean: test: test_unit test_acceptance test_unit: - @[ ! -d test/unit ] && echo "clsi has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit -- ${MOCHA_ARGS} + @[ ! -d test/unit ] && echo "clsi has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit test_acceptance: test_clean # clear the database before each acceptance test run - @[ ! -d test/acceptance ] && echo "clsi has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance -- ${MOCHA_ARGS} + @[ ! -d test/acceptance ] && echo "clsi has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance test_clean: - $(DOCKER_COMPOSE) down -t -v 0 + $(DOCKER_COMPOSE) down -v -t 0 build: docker build --pull --tag quay.io/sharelatex/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) . diff --git a/services/clsi/app.coffee b/services/clsi/app.coffee index a3f4ef17e1..11981a72aa 100644 --- a/services/clsi/app.coffee +++ b/services/clsi/app.coffee @@ -182,7 +182,7 @@ server = net.createServer (socket) -> logger.err err:err, "error with socket on load check" socket.destroy() - if STATE == "up" and settings.load_balancer_agent.report_load + if STATE == "up" and Settings.load_balancer_agent.report_load currentLoad = os.loadavg()[0] # staging clis's have 1 cpu core only @@ -206,7 +206,7 @@ server = net.createServer (socket) -> port = (Settings.internal?.clsi?.port or 3013) host = (Settings.internal?.clsi?.host or "localhost") -load_port = settings.internal.clsi.load_port or 3048 +load_port = Settings.internal.clsi.load_port or 3048 diff --git a/services/clsi/app/coffee/ResourceWriter.coffee b/services/clsi/app/coffee/ResourceWriter.coffee index 0b6aef5b72..90ada043b8 100644 --- a/services/clsi/app/coffee/ResourceWriter.coffee +++ b/services/clsi/app/coffee/ResourceWriter.coffee @@ -120,7 +120,13 @@ module.exports = ResourceWriter = logger.err err:err, project_id:project_id, path:path, resource_url:resource.url, modified:resource.modified, "error downloading file for resources" callback() #try and continue compiling even if http resource can not be downloaded at this time else + process = require("process") + console.log "writing file out", path, process.getuid() fs.writeFile path, resource.content, callback + try + result = fs.lstatSync(path) + console.log "path stats", result + catch e checkPath: (basePath, resourcePath, callback) -> path = Path.normalize(Path.join(basePath, resourcePath)) diff --git a/services/clsi/docker-compose-config.yml b/services/clsi/docker-compose-config.yml index f2bb8631c3..b6deacffcd 100644 --- a/services/clsi/docker-compose-config.yml +++ b/services/clsi/docker-compose-config.yml @@ -12,6 +12,8 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock - ./compiles:/app/compiles + - ./cache:/app/cache + ci: environment: @@ -24,4 +26,4 @@ services: SQLITE_PATH: /app/compiles/db.sqlite volumes: - /var/run/docker.sock:/var/run/docker.sock:rw - - ./compiles:/app/compiles \ No newline at end of file + - ./cache:/app/cache diff --git a/services/clsi/docker-compose.ci.yml b/services/clsi/docker-compose.ci.yml index 370e2a80a2..85512cf7f1 100644 --- a/services/clsi/docker-compose.ci.yml +++ b/services/clsi/docker-compose.ci.yml @@ -8,8 +8,7 @@ version: "2" services: test_unit: image: quay.io/sharelatex/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER - user: node - entrypoint: npm run test:unit:_run + command: npm run test:unit:_run test_acceptance: build: . @@ -23,8 +22,7 @@ services: depends_on: - redis - mongo - user: node - entrypoint: npm run test:acceptance:_run + command: npm run test:acceptance:_run redis: image: redis diff --git a/services/clsi/docker-compose.yml b/services/clsi/docker-compose.yml index 32cbf61e55..5f53d69191 100644 --- a/services/clsi/docker-compose.yml +++ b/services/clsi/docker-compose.yml @@ -11,8 +11,9 @@ services: volumes: - .:/app working_dir: /app - user: node - entrypoint: npm run test:unit + environment: + MOCHA_GREP: ${MOCHA_GREP} + command: npm run test:unit test_acceptance: build: . @@ -25,11 +26,12 @@ services: environment: REDIS_HOST: redis MONGO_HOST: mongo - user: node + environment: + MOCHA_GREP: ${MOCHA_GREP} depends_on: - redis - mongo - entrypoint: npm run test:acceptance + command: npm run test:acceptance redis: image: redis diff --git a/services/clsi/entrypoint.sh b/services/clsi/entrypoint.sh index ee5ea47f90..32776fa553 100755 --- a/services/clsi/entrypoint.sh +++ b/services/clsi/entrypoint.sh @@ -1,7 +1,10 @@ #!/bin/sh echo "Changing permissions of /var/run/docker.sock for sibling containers" - +ls -al /var/run/docker.sock +docker --version +cat /etc/passwd +usermod -aG docker node chown root:docker /var/run/docker.sock mkdir -p /app/cache @@ -10,7 +13,10 @@ chown -R node:node /app/cache mkdir -p /app/compiles chown -R node:node /app/compiles +chown -R node:node /app/bin/synctex +mkdir -p /app/test/acceptance/fixtures/tmp/ +chown -R node:node /app + + ./bin/install_texlive_gce.sh -echo "HELOOOo" -echo "$@" exec runuser -u node "$@" \ No newline at end of file diff --git a/services/clsi/install_deps.sh b/services/clsi/install_deps.sh index 4b06f22d88..49bdc5c963 100755 --- a/services/clsi/install_deps.sh +++ b/services/clsi/install_deps.sh @@ -2,12 +2,3 @@ wget -qO- https://get.docker.com/ | sh apt-get install poppler-utils vim ghostscript --yes npm rebuild -usermod -aG docker node - -mkdir -p /app/cache -chown -R node:node /app/cache - -mkdir -p /app/compiles -chown -R node:node /app/compiles - -chown -R node:node /app/bin/synctex diff --git a/services/clsi/package.json b/services/clsi/package.json index e0e4fdafc1..fe2bd26023 100644 --- a/services/clsi/package.json +++ b/services/clsi/package.json @@ -11,9 +11,9 @@ "compile": "npm run compile:app && npm run compile:test:acceptance && npm run compile:test:smoke", "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", - "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- $@", + "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP", "test:unit:_run": "mocha --recursive --exit --reporter spec $@ test/unit/js", - "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- $@", + "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP", "compile:unit_tests": "[ -e test/unit ] && coffee -o test/unit/js -c test/unit/coffee || echo 'No unit tests to compile'", "compile:acceptance_tests": "[ -e test/acceptance ] && coffee -o test/acceptance/js -c test/acceptance/coffee || echo 'No acceptance tests to compile'", "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests", diff --git a/services/clsi/test/acceptance/coffee/ExampleDocumentTests.coffee b/services/clsi/test/acceptance/coffee/ExampleDocumentTests.coffee index ec569796b8..d4bd19f55c 100644 --- a/services/clsi/test/acceptance/coffee/ExampleDocumentTests.coffee +++ b/services/clsi/test/acceptance/coffee/ExampleDocumentTests.coffee @@ -4,15 +4,21 @@ require("chai").should() fs = require "fs" ChildProcess = require "child_process" ClsiApp = require "./helpers/ClsiApp" - -fixturePath = (path) -> __dirname + "/../fixtures/" + path - +logger = require("logger-sharelatex") +Path = require("path") +fixturePath = (path) -> Path.normalize(__dirname + "/../fixtures/" + path) +process = require "process" +console.log process.pid, process.ppid, process.getuid(),process.getgroups(), "PID" try + console.log "creating tmp directory", fixturePath("tmp") fs.mkdirSync(fixturePath("tmp")) -catch e +catch err + console.log err, fixturePath("tmp"), "unable to create fixture tmp path" convertToPng = (pdfPath, pngPath, callback = (error) ->) -> command = "convert #{fixturePath(pdfPath)} #{fixturePath(pngPath)}" + console.log "COMMAND" + console.log command convert = ChildProcess.exec command stdout = "" convert.stdout.on "data", (chunk) -> console.log "STDOUT", chunk.toString() @@ -58,6 +64,8 @@ compareMultiplePages = (project_id, callback = (error) ->) -> compareNext 0, callback comparePdf = (project_id, example_dir, callback = (error) ->) -> + console.log "CONVERT" + console.log "tmp/#{project_id}.pdf", "tmp/#{project_id}-generated.png" convertToPng "tmp/#{project_id}.pdf", "tmp/#{project_id}-generated.png", (error) => throw error if error? convertToPng "examples/#{example_dir}/output.pdf", "tmp/#{project_id}-source.png", (error) => @@ -76,6 +84,7 @@ comparePdf = (project_id, example_dir, callback = (error) ->) -> downloadAndComparePdf = (project_id, example_dir, url, callback = (error) ->) -> writeStream = fs.createWriteStream(fixturePath("tmp/#{project_id}.pdf")) request.get(url).pipe(writeStream) + console.log("writing file out", fixturePath("tmp/#{project_id}.pdf")) writeStream.on "close", () => checkPdfInfo "tmp/#{project_id}.pdf", (error, optimised) => throw error if error?