From 41e442d403dbcca4b8c3d4d518b7cb8a52834783 Mon Sep 17 00:00:00 2001 From: Joe Green Date: Fri, 8 Sep 2017 14:06:04 +0100 Subject: [PATCH] Add jenkinsfile (#72) * create Jenkinsfile * allow textlive image to be set with env vars * log error message in test * use sandboxed compiles variables * Add SANDBOXED_COMPILES_HOST_DIR var to test config * add SIBLING_CONTAINER_USER env var --- services/clsi/Jenkinsfile | 97 +++++++++++++++++++ .../coffee/ExampleDocumentTests.coffee | 4 +- .../acceptance/scripts/settings.test.coffee | 5 +- 3 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 services/clsi/Jenkinsfile diff --git a/services/clsi/Jenkinsfile b/services/clsi/Jenkinsfile new file mode 100644 index 0000000000..97efd51f20 --- /dev/null +++ b/services/clsi/Jenkinsfile @@ -0,0 +1,97 @@ +pipeline { + + agent any + + triggers { + pollSCM('* * * * *') + cron('@daily') + } + + stages { + stage('Clean') { + steps { + // This is a terrible hack to set the file ownership to jenkins:jenkins so we can cleanup the directory + sh 'docker run -v $(pwd):/app --rm busybox /bin/chown -R 111:119 /app' + } + } + stage('Install') { + agent { + docker { + image 'node:4.2.1' + args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp" + reuseNode true + } + } + steps { + sh 'git config --global core.logallrefupdates false' + sh 'rm -fr node_modules' + checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: '_docker-runner'], [$class: 'CloneOption', shallow: true]], userRemoteConfigs: [[credentialsId: 'GIT_DEPLOY_KEY', url: 'git@github.com:sharelatex/docker-runner-sharelatex']]]) + sh 'npm install ./_docker-runner' + sh 'rm -fr ./_docker-runner' + sh 'npm install' + sh 'npm rebuild' + sh 'npm install --quiet grunt-cli sqlite3' + } + } + stage('Compile and Test') { + agent { + docker { + image 'node:4.2.1' + reuseNode true + } + } + steps { + sh 'node_modules/.bin/grunt compile:app' + sh 'node_modules/.bin/grunt compile:acceptance_tests' + sh 'NODE_ENV=development node_modules/.bin/grunt test:unit' + } + } + stage('Acceptance Tests') { + environment { + TEXLIVE_IMAGE="quay.io/sharelatex/texlive-full:2017.1" + } + steps { + sh 'mkdir -p compiles' + // Not yet running, due to volumes/sibling containers + sh 'docker container prune -f' + sh 'docker pull $TEXLIVE_IMAGE' + sh 'docker pull sharelatex/acceptance-test-runner:clsi-4.2.1' + sh 'docker run --rm -e SIBLING_CONTAINER_USER=root -e SANDBOXED_COMPILES_HOST_DIR=$(pwd)/compiles -e SANDBOXED_COMPILES_SIBLING_CONTAINERS=true -e TEXLIVE_IMAGE=$TEXLIVE_IMAGE -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd):/app sharelatex/acceptance-test-runner:clsi-4.2.1' + } + } + stage('Package') { + steps { + sh 'echo ${BUILD_NUMBER} > build_number.txt' + sh 'touch build.tar.gz' // Avoid tar warning about files changing during read + sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .' + } + } + stage('Publish') { + steps { + withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { + s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz") + // The deployment process uses this file to figure out the latest build + s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest") + } + } + } + } + + post { + failure { + mail(from: "${EMAIL_ALERT_FROM}", + to: "${EMAIL_ALERT_TO}", + subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}", + body: "Build: ${BUILD_URL}") + } + } + + // The options directive is for configuration that applies to the whole job. + options { + // we'd like to make sure remove old builds, so we don't fill up our storage! + buildDiscarder(logRotator(numToKeepStr:'50')) + + // And we'd really like to be sure that this build doesn't hang forever, so let's time it out after: + timeout(time: 30, unit: 'MINUTES') + } +} diff --git a/services/clsi/test/acceptance/coffee/ExampleDocumentTests.coffee b/services/clsi/test/acceptance/coffee/ExampleDocumentTests.coffee index a49f5d62cb..4e36c6ac05 100644 --- a/services/clsi/test/acceptance/coffee/ExampleDocumentTests.coffee +++ b/services/clsi/test/acceptance/coffee/ExampleDocumentTests.coffee @@ -92,14 +92,14 @@ describe "Example Documents", -> it "should generate the correct pdf", (done) -> Client.compileDirectory @project_id, fixturePath("examples"), example_dir, 4242, (error, res, body) => - if body?.compile?.status is "failure" + if error || body?.compile?.status is "failure" console.log "DEBUG: error", error, "body", JSON.stringify(body) pdf = Client.getOutputFile body, "pdf" downloadAndComparePdf(@project_id, example_dir, pdf.url, done) it "should generate the correct pdf on the second run as well", (done) -> Client.compileDirectory @project_id, fixturePath("examples"), example_dir, 4242, (error, res, body) => - if body?.compile?.status is "failure" + if error || body?.compile?.status is "failure" console.log "DEBUG: error", error, "body", JSON.stringify(body) pdf = Client.getOutputFile body, "pdf" downloadAndComparePdf(@project_id, example_dir, pdf.url, done) diff --git a/services/clsi/test/acceptance/scripts/settings.test.coffee b/services/clsi/test/acceptance/scripts/settings.test.coffee index cf84c6baba..e3d3b70bb3 100644 --- a/services/clsi/test/acceptance/scripts/settings.test.coffee +++ b/services/clsi/test/acceptance/scripts/settings.test.coffee @@ -16,6 +16,7 @@ module.exports = clsiCacheDir: Path.resolve(__dirname + "/../../../cache") #synctexBaseDir: (project_id) -> Path.join(@compilesDir, project_id) synctexBaseDir: () -> "/compile" + sandboxedCompilesHostDir: process.env['SANDBOXED_COMPILES_HOST_DIR'] clsi: #strace: true @@ -23,13 +24,13 @@ module.exports = commandRunner: "docker-runner-sharelatex" latexmkCommandPrefix: ["/usr/bin/time", "-v"] # on Linux docker: - image: "texlive-full:2017.1-opt" + image: process.env.TEXLIVE_IMAGE || "texlive-full:2017.1-opt" env: PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/2017/bin/x86_64-linux/" HOME: "/tmp" modem: socketPath: false - user: "111" + user: process.env.SIBLING_CONTAINER_USER ||"111" internal: clsi: