From 5b1656b88482c48893a4819046d67b53b3cfee1f Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 19 Feb 2020 10:32:50 +0000 Subject: [PATCH 1/4] [misc] drop debug output and log docker version on stderr --- services/clsi/entrypoint.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/services/clsi/entrypoint.sh b/services/clsi/entrypoint.sh index 07d902acc2..e28bbe6624 100755 --- a/services/clsi/entrypoint.sh +++ b/services/clsi/entrypoint.sh @@ -1,9 +1,6 @@ #!/bin/sh -echo "Changing permissions of /var/run/docker.sock for sibling containers" -ls -al /var/run/docker.sock -docker --version -cat /etc/passwd +docker --version >&2 DOCKER_GROUP=$(stat -c '%g' /var/run/docker.sock) groupadd --non-unique --gid ${DOCKER_GROUP} dockeronhost From 3309adf2ad1415ec4ba33381f56bf996beb9c3ff Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 19 Feb 2020 12:23:20 +0100 Subject: [PATCH 2/4] [misc] move the sqlite database into a db/ directory --- services/clsi/.dockerignore | 1 + services/clsi/Dockerfile | 2 ++ services/clsi/buildscript.txt | 1 + services/clsi/config/settings.defaults.js | 8 +++----- services/clsi/db/.gitignore | 2 ++ 5 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 services/clsi/db/.gitignore diff --git a/services/clsi/.dockerignore b/services/clsi/.dockerignore index ba1c3442de..35f8905ec5 100644 --- a/services/clsi/.dockerignore +++ b/services/clsi/.dockerignore @@ -5,3 +5,4 @@ gitrev .npm .nvmrc nodemon.json +db/ diff --git a/services/clsi/Dockerfile b/services/clsi/Dockerfile index 27158b5d6a..3fbae08b32 100644 --- a/services/clsi/Dockerfile +++ b/services/clsi/Dockerfile @@ -24,5 +24,7 @@ COPY . /app FROM base COPY --from=app /app /app +RUN mkdir -p db \ +&& chown node:node db CMD ["node", "--expose-gc", "app.js"] diff --git a/services/clsi/buildscript.txt b/services/clsi/buildscript.txt index 1f0893492f..72b0f6a3d6 100644 --- a/services/clsi/buildscript.txt +++ b/services/clsi/buildscript.txt @@ -1,5 +1,6 @@ clsi --acceptance-creds=None +--data-dirs=db --dependencies= --docker-repos=gcr.io/overleaf-ops --env-add= diff --git a/services/clsi/config/settings.defaults.js b/services/clsi/config/settings.defaults.js index b0fd0cbd35..021c9cd74f 100644 --- a/services/clsi/config/settings.defaults.js +++ b/services/clsi/config/settings.defaults.js @@ -9,7 +9,7 @@ module.exports = { username: 'clsi', dialect: 'sqlite', storage: - process.env.SQLITE_PATH || Path.resolve(__dirname + '/../db.sqlite'), + process.env.SQLITE_PATH || Path.resolve(__dirname + '/../db/db.sqlite'), pool: { max: 1, min: 1 @@ -65,8 +65,7 @@ if (process.env.DOCKER_RUNNER) { dockerRunner: process.env.DOCKER_RUNNER === 'true', docker: { image: - process.env.TEXLIVE_IMAGE || - 'quay.io/sharelatex/texlive-full:2017.1', + process.env.TEXLIVE_IMAGE || 'quay.io/sharelatex/texlive-full:2017.1', env: { HOME: '/tmp' }, @@ -93,8 +92,7 @@ if (process.env.DOCKER_RUNNER) { module.exports.path.synctexBaseDir = () => '/compile' - module.exports.path.sandboxedCompilesHostDir = - process.env.COMPILES_HOST_DIR + module.exports.path.sandboxedCompilesHostDir = process.env.COMPILES_HOST_DIR module.exports.path.synctexBinHostPath = process.env.SYNCTEX_BIN_HOST_PATH } diff --git a/services/clsi/db/.gitignore b/services/clsi/db/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/services/clsi/db/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore From b9deec10951aef07caca13ef4ebf1b9a90e801de Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 19 Feb 2020 12:06:28 +0100 Subject: [PATCH 3/4] [misc] narrow down write access/ownership for the run-time user --- services/clsi/.dockerignore | 2 ++ services/clsi/Dockerfile | 4 ++-- services/clsi/buildscript.txt | 2 +- services/clsi/entrypoint.sh | 17 ++++++++--------- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/services/clsi/.dockerignore b/services/clsi/.dockerignore index 35f8905ec5..74fdc35e80 100644 --- a/services/clsi/.dockerignore +++ b/services/clsi/.dockerignore @@ -5,4 +5,6 @@ gitrev .npm .nvmrc nodemon.json +cache/ +compiles/ db/ diff --git a/services/clsi/Dockerfile b/services/clsi/Dockerfile index 3fbae08b32..40615ad8c3 100644 --- a/services/clsi/Dockerfile +++ b/services/clsi/Dockerfile @@ -24,7 +24,7 @@ COPY . /app FROM base COPY --from=app /app /app -RUN mkdir -p db \ -&& chown node:node db +RUN mkdir -p cache compiles db \ +&& chown node:node cache compiles db CMD ["node", "--expose-gc", "app.js"] diff --git a/services/clsi/buildscript.txt b/services/clsi/buildscript.txt index 72b0f6a3d6..81d65464f9 100644 --- a/services/clsi/buildscript.txt +++ b/services/clsi/buildscript.txt @@ -1,6 +1,6 @@ clsi --acceptance-creds=None ---data-dirs=db +--data-dirs=cache,compiles,db --dependencies= --docker-repos=gcr.io/overleaf-ops --env-add= diff --git a/services/clsi/entrypoint.sh b/services/clsi/entrypoint.sh index e28bbe6624..3e3f838258 100755 --- a/services/clsi/entrypoint.sh +++ b/services/clsi/entrypoint.sh @@ -2,22 +2,21 @@ docker --version >&2 +# add the node user to the docker group on the host DOCKER_GROUP=$(stat -c '%g' /var/run/docker.sock) groupadd --non-unique --gid ${DOCKER_GROUP} dockeronhost usermod -aG dockeronhost node -mkdir -p /app/cache -chown -R node:node /app/cache +# compatibility: initial volume setup +chown node:node /app/cache +chown node:node /app/compiles +chown node:node /app/db -mkdir -p /app/compiles -chown -R node:node /app/compiles - -chown -R node:node /app/bin/synctex +# acceptance tests mkdir -p /app/test/acceptance/fixtures/tmp/ -chown -R node:node /app - -chown -R node:node /app/bin +chown -R node:node /app/test/acceptance/fixtures +# make synctex available for remount in compiles cp /app/bin/synctex /app/bin/synctex-mount/synctex exec runuser -u node -- "$@" From 101c4ea4dc487637bd87c7508ffbd47ae19336a5 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 27 Mar 2020 11:10:27 +0100 Subject: [PATCH 4/4] [misc] use a directory in /tmp for temporary data --- services/clsi/entrypoint.sh | 4 --- .../acceptance/js/ExampleDocumentTests.js | 29 +++++++++++-------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/services/clsi/entrypoint.sh b/services/clsi/entrypoint.sh index 3e3f838258..2696574873 100755 --- a/services/clsi/entrypoint.sh +++ b/services/clsi/entrypoint.sh @@ -12,10 +12,6 @@ chown node:node /app/cache chown node:node /app/compiles chown node:node /app/db -# acceptance tests -mkdir -p /app/test/acceptance/fixtures/tmp/ -chown -R node:node /app/test/acceptance/fixtures - # make synctex available for remount in compiles cp /app/bin/synctex /app/bin/synctex-mount/synctex diff --git a/services/clsi/test/acceptance/js/ExampleDocumentTests.js b/services/clsi/test/acceptance/js/ExampleDocumentTests.js index 110b5d6f9a..0134c0e106 100644 --- a/services/clsi/test/acceptance/js/ExampleDocumentTests.js +++ b/services/clsi/test/acceptance/js/ExampleDocumentTests.js @@ -19,11 +19,17 @@ const Client = require('./helpers/Client') const request = require('request') require('chai').should() const fs = require('fs') +const fsExtra = require('fs-extra') const ChildProcess = require('child_process') const ClsiApp = require('./helpers/ClsiApp') const logger = require('logger-sharelatex') const Path = require('path') -const fixturePath = path => Path.normalize(__dirname + '/../fixtures/' + path) +const fixturePath = path => { + if (path.slice(0, 3) === 'tmp') { + return '/tmp/clsi_acceptance_tests' + path.slice(3) + } + return Path.normalize(__dirname + '/../fixtures/' + path) +} const process = require('process') console.log( process.pid, @@ -32,13 +38,6 @@ console.log( process.getgroups(), 'PID' ) -try { - console.log('creating tmp directory', fixturePath('tmp')) - fs.mkdirSync(fixturePath('tmp')) -} catch (error) { - const err = error - console.log(err, fixturePath('tmp'), 'unable to create fixture tmp path') -} const MOCHA_LATEX_TIMEOUT = 60 * 1000 @@ -201,10 +200,16 @@ Client.runServer(4242, fixturePath('examples')) describe('Example Documents', function() { before(function(done) { - return ChildProcess.exec('rm test/acceptance/fixtures/tmp/*').on( - 'exit', - () => ClsiApp.ensureRunning(done) - ) + ClsiApp.ensureRunning(done) + }) + before(function(done) { + fsExtra.remove(fixturePath('tmp'), done) + }) + before(function(done) { + fs.mkdir(fixturePath('tmp'), done) + }) + after(function(done) { + fsExtra.remove(fixturePath('tmp'), done) }) return Array.from(fs.readdirSync(fixturePath('examples'))).map(example_dir =>