From b0b96ee014b90e665e89cda372458fe3a3ee6b08 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 24 Jan 2020 18:31:56 +0100 Subject: [PATCH] [misc] add support for offline s3 tests update the build scripts to 1.3.4 --- services/track-changes/Dockerfile | 2 +- services/track-changes/Jenkinsfile | 4 +- services/track-changes/Makefile | 5 +- .../track-changes/app/coffee/MongoAWS.coffee | 2 + services/track-changes/buildscript.txt | 8 +-- .../config/settings.defaults.coffee | 2 + services/track-changes/docker-compose.ci.yml | 18 +++++-- services/track-changes/docker-compose.yml | 18 +++++-- services/track-changes/package.json | 2 +- .../coffee/ArchivingUpdatesTests.coffee | 2 +- .../coffee/helpers/TrackChangesClient.coffee | 49 +++++++++++-------- 11 files changed, 69 insertions(+), 43 deletions(-) diff --git a/services/track-changes/Dockerfile b/services/track-changes/Dockerfile index 5f69a4721f..3066d5d478 100644 --- a/services/track-changes/Dockerfile +++ b/services/track-changes/Dockerfile @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.3.3 +# Version: 1.3.4 FROM node:6.11.2 as app diff --git a/services/track-changes/Jenkinsfile b/services/track-changes/Jenkinsfile index b1ea80110c..23be6d5d33 100644 --- a/services/track-changes/Jenkinsfile +++ b/services/track-changes/Jenkinsfile @@ -45,9 +45,7 @@ pipeline { stage('Acceptance Tests') { steps { - withCredentials([usernamePassword(credentialsId: 'S3_DOCSTORE_TEST_AWS_KEYS', passwordVariable: 'AWS_SECRET_ACCESS_KEY', usernameVariable: 'AWS_ACCESS_KEY_ID')]) { - sh 'AWS_BUCKET="sl-acceptance-tests" AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance' - } + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance' } } diff --git a/services/track-changes/Makefile b/services/track-changes/Makefile index becf4c28f9..6be6b7acb2 100644 --- a/services/track-changes/Makefile +++ b/services/track-changes/Makefile @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.3.3 +# Version: 1.3.4 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) @@ -11,9 +11,6 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ BRANCH_NAME=$(BRANCH_NAME) \ PROJECT_NAME=$(PROJECT_NAME) \ MOCHA_GREP=${MOCHA_GREP} \ - AWS_BUCKET=${AWS_BUCKET} \ - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \ - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \ docker-compose ${DOCKER_COMPOSE_FLAGS} clean: diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index 4308763a7a..0223a89981 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -14,6 +14,8 @@ createStream = (streamConstructor, project_id, doc_id, pack_id) -> AWS_CONFIG = accessKeyId: settings.trackchanges.s3.key secretAccessKey: settings.trackchanges.s3.secret + endpoint: settings.trackchanges.s3.endpoint + s3ForcePathStyle: settings.trackchanges.s3.pathStyle return streamConstructor new AWS.S3(AWS_CONFIG), { "Bucket": settings.trackchanges.stores.doc_history, diff --git a/services/track-changes/buildscript.txt b/services/track-changes/buildscript.txt index 98b6e9a4a4..83d66e52e5 100644 --- a/services/track-changes/buildscript.txt +++ b/services/track-changes/buildscript.txt @@ -1,10 +1,10 @@ track-changes --public-repo=True --language=coffeescript ---env-add= +--env-add=AWS_BUCKET=bucket --node-version=6.11.2 ---acceptance-creds=aws ---dependencies=mongo,redis +--acceptance-creds=None +--dependencies=mongo,redis,s3 --docker-repos=gcr.io/overleaf-ops --env-pass-through= ---script-version=1.3.3 +--script-version=1.3.4 diff --git a/services/track-changes/config/settings.defaults.coffee b/services/track-changes/config/settings.defaults.coffee index fa5d730f33..897c5a0536 100755 --- a/services/track-changes/config/settings.defaults.coffee +++ b/services/track-changes/config/settings.defaults.coffee @@ -38,6 +38,8 @@ module.exports = s3: key: process.env['AWS_ACCESS_KEY_ID'] secret: process.env['AWS_SECRET_ACCESS_KEY'] + endpoint: process.env['AWS_S3_ENDPOINT'] + pathStyle: process.env['AWS_S3_PATH_STYLE'] == 'true' stores: doc_history: process.env['AWS_BUCKET'] continueOnError: process.env['TRACK_CHANGES_CONTINUE_ON_ERROR'] or false diff --git a/services/track-changes/docker-compose.ci.yml b/services/track-changes/docker-compose.ci.yml index d3fd49dfae..58033f8156 100644 --- a/services/track-changes/docker-compose.ci.yml +++ b/services/track-changes/docker-compose.ci.yml @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.3.3 +# Version: 1.3.4 version: "2.1" @@ -22,16 +22,20 @@ services: REDIS_HOST: redis MONGO_HOST: mongo POSTGRES_HOST: postgres - AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} - AWS_BUCKET: ${AWS_BUCKET} + AWS_S3_ENDPOINT: http://s3:9090 + AWS_S3_PATH_STYLE: 'true' + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake MOCHA_GREP: ${MOCHA_GREP} NODE_ENV: test + AWS_BUCKET: bucket depends_on: mongo: condition: service_healthy redis: condition: service_healthy + s3: + condition: service_healthy user: node command: npm run test:acceptance:_run @@ -48,3 +52,9 @@ services: mongo: image: mongo:3.6 + s3: + image: adobe/s3mock + environment: + - initialBuckets=fake_user_files,fake_template_files,fake_public_files,bucket + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9090"] diff --git a/services/track-changes/docker-compose.yml b/services/track-changes/docker-compose.yml index 450979f592..d9bee3a71b 100644 --- a/services/track-changes/docker-compose.yml +++ b/services/track-changes/docker-compose.yml @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.3.3 +# Version: 1.3.4 version: "2.1" @@ -27,18 +27,22 @@ services: REDIS_HOST: redis MONGO_HOST: mongo POSTGRES_HOST: postgres - AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} - AWS_BUCKET: ${AWS_BUCKET} + AWS_S3_ENDPOINT: http://s3:9090 + AWS_S3_PATH_STYLE: 'true' + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ERROR NODE_ENV: test + AWS_BUCKET: bucket user: node depends_on: mongo: condition: service_healthy redis: condition: service_healthy + s3: + condition: service_healthy command: npm run test:acceptance redis: @@ -47,3 +51,9 @@ services: mongo: image: mongo:3.6 + s3: + image: adobe/s3mock + environment: + - initialBuckets=fake_user_files,fake_template_files,fake_public_files,bucket + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9090"] diff --git a/services/track-changes/package.json b/services/track-changes/package.json index b5320f4f5d..72e3e7dbed 100644 --- a/services/track-changes/package.json +++ b/services/track-changes/package.json @@ -9,7 +9,7 @@ "scripts": { "compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')", "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js", - "test:acceptance:_run": "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY AWS_BUCKET=$AWS_BUCKET AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js", + "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js", "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP", "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP", diff --git a/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee b/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee index 0ff62d70d9..25397cfa61 100644 --- a/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee +++ b/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee @@ -17,7 +17,7 @@ MockWebApi = require "./helpers/MockWebApi" describe "Archiving updates", -> before (done) -> if Settings?.trackchanges?.s3?.key.length < 1 - message = "s3 keys not setup, this test setup will fail" + message = new Error("s3 keys not setup, this test setup will fail") return done(message) @now = Date.now() diff --git a/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee b/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee index e927333004..9f17939656 100644 --- a/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee +++ b/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee @@ -6,6 +6,15 @@ rclient = require("redis-sharelatex").createClient(Settings.redis.history) # Onl Keys = Settings.redis.history.key_schema {db, ObjectId} = require "../../../../app/js/mongojs" +aws = require "aws-sdk" +s3 = new aws.S3( + accessKeyId: Settings.trackchanges.s3.key + secretAccessKey: Settings.trackchanges.s3.secret + endpoint: Settings.trackchanges.s3.endpoint + s3ForcePathStyle: Settings.trackchanges.s3.pathStyle +) +S3_BUCKET = Settings.trackchanges.stores.doc_history + module.exports = TrackChangesClient = flushAndGetCompressedUpdates: (project_id, doc_id, callback = (error, updates) ->) -> TrackChangesClient.flushDoc project_id, doc_id, (error) -> @@ -91,32 +100,30 @@ module.exports = TrackChangesClient = response.statusCode.should.equal 204 callback(error) - buildS3Options: (content, key)-> - return { - aws: - key: Settings.trackchanges.s3.key - secret: Settings.trackchanges.s3.secret - bucket: Settings.trackchanges.stores.doc_history - timeout: 30 * 1000 - json: content - uri:"https://#{Settings.trackchanges.stores.doc_history}.s3.amazonaws.com/#{key}" - } - getS3Doc: (project_id, doc_id, pack_id, callback = (error, body) ->) -> - options = TrackChangesClient.buildS3Options(true, project_id+"/changes-"+doc_id+"/pack-"+pack_id) - options.encoding = null - request.get options, (err, res, body) -> + params = + Bucket: S3_BUCKET + Key: "#{project_id}/changes-#{doc_id}/pack-#{pack_id}" + + s3.getObject params, (error, data) -> return callback(error) if error? + body = data.Body return callback(new Error("empty response from s3")) if not body? zlib.gunzip body, (err, result) -> return callback(err) if err? callback(null, JSON.parse(result.toString())) removeS3Doc: (project_id, doc_id, callback = (error, res, body) ->) -> - options = TrackChangesClient.buildS3Options(true, "?prefix=" + project_id + "/changes-" +doc_id) - request.get options, (error, res, body) -> - keys = body.match /[0-9a-f]{24}\/changes-[0-9a-f]{24}\/pack-[0-9a-f]{24}/g - async.eachSeries keys, (key, cb) -> - options = TrackChangesClient.buildS3Options(true, key) - request.del options, cb - , callback + params = + Bucket: S3_BUCKET + Prefix: "#{project_id}/changes-#{doc_id}" + + s3.listObjects params, (error, data) -> + return callback(error) if error? + + params = + Bucket: S3_BUCKET + Delete: + Objects: data.Contents.map((s3object) -> {Key: s3object.Key}) + + s3.deleteObjects params, callback