diff --git a/server-ce/Dockerfile b/server-ce/Dockerfile index ff8aabd9a6..ac09bb8215 100644 --- a/server-ce/Dockerfile +++ b/server-ce/Dockerfile @@ -53,7 +53,11 @@ COPY server-ce/init_scripts/ /etc/my_init.d/ # Copy app settings files # ----------------------- -COPY server-ce/settings.js /etc/sharelatex/settings.js +COPY server-ce/config/settings.js /etc/sharelatex/settings.js + +# Copy history-v1 files +# ----------------------- +COPY server-ce/config/production.json /overleaf/services/history-v1/config/production.json # Copy grunt thin wrapper # ----------------------- diff --git a/server-ce/config/production.json b/server-ce/config/production.json new file mode 100644 index 0000000000..167f6e1442 --- /dev/null +++ b/server-ce/config/production.json @@ -0,0 +1,28 @@ +{ + "persistor": { + "backend": "fs", + "useSubdirectories": true + }, + "blobStore": { + "globalBucket": "/var/lib/sharelatex/history/overleaf-blobs", + "projectBucket": "/var/lib/sharelatex/history/overleaf-project-blobs" + }, + "chunkStore": { + "bucket": "/var/lib/sharelatex/history/overleaf-chunks" + }, + "zipStore": { + "bucket": "/var/lib/sharelatex/history/overleaf-zips" + }, + "analytics": { + "bucket": "/var/lib/sharelatex/history/overleaf-analytics" + }, + "basicHttpAuth": { + "password": "password" + }, + "useDeleteObjects": "false", + "jwtAuth": { + "key": "secureKey", + "algorithm": "HS256" + }, + "mongo": {} +} diff --git a/server-ce/settings.js b/server-ce/config/settings.js similarity index 97% rename from server-ce/settings.js rename to server-ce/config/settings.js index e33b2be07b..6b78333b18 100644 --- a/server-ce/settings.js +++ b/server-ce/config/settings.js @@ -151,6 +151,20 @@ const settings = { api: redisConfig, pubsub: redisConfig, project_history: redisConfig, + + project_history_migration: { + host: redisConfig.host, + port: redisConfig.port, + password: redisConfig.password, + maxRetriesPerRequest: parseInt( + process.env.REDIS_MAX_RETRIES_PER_REQUEST || '20' + ), + key_schema: { + projectHistoryOps({ projectId }) { + return `ProjectHistory:Ops:{${projectId}}` // NOTE: the extra braces are intentional + }, + }, + }, }, // File storage @@ -273,6 +287,17 @@ const settings = { user: httpAuthUser, pass: httpAuthPass, }, + project_history: { + sendProjectStructureOps: true, + initializeHistoryForNewProjects: true, + displayHistoryForNewProjects: true, + url: 'http://localhost:3054', + }, + v1_history: { + url: 'http://localhost:3100/api', + user: 'staging', + pass: process.env.STAGING_PASSWORD, + } }, references: {}, notifications: undefined, diff --git a/server-ce/init_scripts/00_regen_sharelatex_secrets.sh b/server-ce/init_scripts/00_regen_sharelatex_secrets.sh index 365be0869f..3bc7d31f29 100755 --- a/server-ce/init_scripts/00_regen_sharelatex_secrets.sh +++ b/server-ce/init_scripts/00_regen_sharelatex_secrets.sh @@ -5,15 +5,21 @@ set -e -o pipefail # https://github.com/phusion/baseimage-docker#centrally-defining-your-own-environment-variables WEB_API_PASSWORD_FILE=/etc/container_environment/WEB_API_PASSWORD +STAGING_PASSWORD_FILE=/etc/container_environment/STAGING_PASSWORD # HTTP auth for history-v1 +V1_HISTORY_PASSWORD_FILE=/etc/container_environment/V1_HISTORY_PASSWORD CRYPTO_RANDOM_FILE=/etc/container_environment/CRYPTO_RANDOM -if [ ! -f "$WEB_API_PASSWORD_FILE" ] || [ ! -f "$CRYPTO_RANDOM_FILE" ]; then +if [ ! -f "$WEB_API_PASSWORD_FILE" ] || [ ! -f "$STAGING_PASSWORD_FILE" ] || [ ! -f "$CRYPTO_RANDOM_FILE" ]; then echo "generating random secrets" SECRET=$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 -w 0 | rev | cut -b 2- | rev | tr -d '\n+/') echo ${SECRET} > ${WEB_API_PASSWORD_FILE} + SECRET=$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 -w 0 | rev | cut -b 2- | rev | tr -d '\n+/') + echo ${SECRET} > ${STAGING_PASSWORD_FILE} + echo ${SECRET} > ${V1_HISTORY_PASSWORD_FILE} + SECRET=$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 -w 0 | rev | cut -b 2- | rev | tr -d '\n+/') echo ${SECRET} > ${CRYPTO_RANDOM_FILE} fi diff --git a/server-ce/runit/history-v1-sharelatex/run b/server-ce/runit/history-v1-sharelatex/run index d818808def..37173585bd 100755 --- a/server-ce/runit/history-v1-sharelatex/run +++ b/server-ce/runit/history-v1-sharelatex/run @@ -6,4 +6,4 @@ if [ "$DEBUG_NODE" == "true" ]; then NODE_PARAMS="--inspect=0.0.0.0:30640" fi -MONGO_CONNECTION_STRING=$SHARELATEX_MONGO_URL NODE_CONFIG_DIR=/overleaf/services/history-v1/config NODE_CONFIG_ENV=overleaf-ce exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/history-v1/app.js >> /var/log/sharelatex/history-v1.log 2>&1 +MONGO_CONNECTION_STRING=$SHARELATEX_MONGO_URL NODE_CONFIG_DIR=/overleaf/services/history-v1/config exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/history-v1/app.js >> /var/log/sharelatex/history-v1.log 2>&1 diff --git a/services/history-v1/config/overleaf-ce.json b/services/history-v1/config/overleaf-ce.json deleted file mode 100644 index 21f17031c3..0000000000 --- a/services/history-v1/config/overleaf-ce.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "persistor": { - "s3": { - "endpoint": "http://s3:8080", - "pathStyle": "true" - } - }, - "blobStore": { - "globalBucket": "overleaf-blobs", - "projectBucket": "overleaf-project-blobs" - }, - "chunkStore": { - "bucket": "overleaf-chunks" - }, - "zipStore": { - "bucket": "overleaf-zips" - }, - "analytics": { - "bucket": "overleaf-analytics" - }, - "useDeleteObjects": "false", - "basicHttpAuth": { - "password": "password" - }, - "jwtAuth": { - "key": "secureKey", - "algorithm": "HS256" - }, - "mongo": {} -} diff --git a/services/web/app/src/Features/Project/ProjectCreationHandler.js b/services/web/app/src/Features/Project/ProjectCreationHandler.js index 47935ca1f8..08cf1abc88 100644 --- a/services/web/app/src/Features/Project/ProjectCreationHandler.js +++ b/services/web/app/src/Features/Project/ProjectCreationHandler.js @@ -171,7 +171,6 @@ async function _createBlankProject( // (to allow scripted creation of projects without full project history) const historyId = project.overleaf.history.id if ( - Features.hasFeature('history-v1') && Settings.apis.project_history.displayHistoryForNewProjects && historyId != null ) { diff --git a/services/web/app/src/Features/Project/ProjectDeleter.js b/services/web/app/src/Features/Project/ProjectDeleter.js index 7abdbca1df..a71fe69e28 100644 --- a/services/web/app/src/Features/Project/ProjectDeleter.js +++ b/services/web/app/src/Features/Project/ProjectDeleter.js @@ -1,4 +1,3 @@ -const Features = require('../../infrastructure/Features') const _ = require('lodash') const { db, ObjectId } = require('../../infrastructure/mongodb') const { callbackify } = require('util') @@ -376,12 +375,10 @@ async function expireDeletedProject(projectId) { await Promise.all([ DocstoreManager.promises.destroyProject(deletedProject.project._id), - Features.hasFeature('history-v1') - ? HistoryManager.promises.deleteProject( - deletedProject.project._id, - historyId - ) - : Promise.resolve(), + HistoryManager.promises.deleteProject( + deletedProject.project._id, + historyId + ), FilestoreHandler.promises.deleteProject(deletedProject.project._id), TpdsUpdateSender.promises.deleteProject({ projectId: deletedProject.project._id, diff --git a/services/web/app/src/infrastructure/Features.js b/services/web/app/src/infrastructure/Features.js index badfd40655..5c420a2486 100644 --- a/services/web/app/src/infrastructure/Features.js +++ b/services/web/app/src/infrastructure/Features.js @@ -6,9 +6,6 @@ const publicRegistrationModuleAvailable = const supportModuleAvailable = Settings.moduleImportSequence.includes('support') -const historyV1ModuleAvailable = - Settings.moduleImportSequence.includes('history-v1') - const trackChangesModuleAvailable = Settings.moduleImportSequence.includes('track-changes') @@ -71,8 +68,6 @@ const Features = { return Boolean(Settings.oauth) case 'templates-server-pro': return !Settings.overleaf - case 'history-v1': - return historyV1ModuleAvailable case 'affiliations': case 'analytics': return Boolean(_.get(Settings, ['apis', 'v1', 'url'])) diff --git a/services/web/test/acceptance/config/settings.test.defaults.js b/services/web/test/acceptance/config/settings.test.defaults.js index f292b33cf0..52587290bc 100644 --- a/services/web/test/acceptance/config/settings.test.defaults.js +++ b/services/web/test/acceptance/config/settings.test.defaults.js @@ -76,6 +76,17 @@ module.exports = { notifications: { url: 'http://localhost:23042', }, + project_history: { + sendProjectStructureOps: true, + initializeHistoryForNewProjects: true, + displayHistoryForNewProjects: true, + url: `http://localhost:23054`, + }, + v1_history: { + url: `http://localhost:23100/api`, + user: 'overleaf', + pass: 'password', + }, webpack: { url: 'http://localhost:23808', }, diff --git a/services/web/test/acceptance/config/settings.test.saas.js b/services/web/test/acceptance/config/settings.test.saas.js index 27cf80d3ab..a1201883a0 100644 --- a/services/web/test/acceptance/config/settings.test.saas.js +++ b/services/web/test/acceptance/config/settings.test.saas.js @@ -17,13 +17,6 @@ const overrides = { analytics: { url: `http://localhost:23050`, }, - project_history: { - sendProjectStructureOps: true, - initializeHistoryForNewProjects: true, - displayHistoryForNewProjects: true, - url: `http://localhost:23054`, - }, - recurly: { url: 'http://localhost:26034', subdomain: 'test', @@ -42,12 +35,6 @@ const overrides = { user: 'overleaf', pass: 'password', }, - - v1_history: { - url: `http://localhost:23100/api`, - user: 'overleaf', - pass: 'password', - }, }, oauthProviders: { diff --git a/services/web/test/acceptance/src/Init.js b/services/web/test/acceptance/src/Init.js index 6bd267306d..7d6a70b93a 100644 --- a/services/web/test/acceptance/src/Init.js +++ b/services/web/test/acceptance/src/Init.js @@ -27,11 +27,11 @@ MockFilestoreApi.initialize(23009, mockOpts) MockNotificationsApi.initialize(23042, mockOpts) MockSpellingApi.initialize(23005, mockOpts) MockHaveIBeenPwnedApi.initialize(1337, mockOpts) +MockProjectHistoryApi.initialize(23054, mockOpts) +MockV1HistoryApi.initialize(23100, mockOpts) if (Features.hasFeature('saas')) { MockAnalyticsApi.initialize(23050, mockOpts) - MockProjectHistoryApi.initialize(23054, mockOpts) MockV1Api.initialize(25000, mockOpts) - MockV1HistoryApi.initialize(23100, mockOpts) MockThirdPartyDataStoreApi.initialize(23002, mockOpts) } diff --git a/services/web/test/unit/src/Project/ProjectDeleterTests.js b/services/web/test/unit/src/Project/ProjectDeleterTests.js index 12161ceec8..51e8f04f6b 100644 --- a/services/web/test/unit/src/Project/ProjectDeleterTests.js +++ b/services/web/test/unit/src/Project/ProjectDeleterTests.js @@ -513,53 +513,6 @@ describe('ProjectDeleter', function () { }) }) - describe('when history-v1 is not available', function () { - beforeEach(async function () { - this.Features.hasFeature.returns(false) - - this.ProjectMock.expects('findById') - .withArgs(this.deletedProjects[0].deleterData.deletedProjectId) - .chain('exec') - .resolves(null) - this.DeletedProjectMock.expects('updateOne') - .withArgs( - { - _id: this.deletedProjects[0]._id, - }, - { - $set: { - 'deleterData.deleterIpAddress': null, - project: null, - }, - } - ) - .chain('exec') - .resolves() - - this.DeletedProjectMock.expects('findOne') - .withArgs({ - 'deleterData.deletedProjectId': this.deletedProjects[0].project._id, - }) - .chain('exec') - .resolves(this.deletedProjects[0]) - - await this.ProjectDeleter.promises.expireDeletedProject( - this.deletedProjects[0].project._id - ) - }) - - it('should destroy the docs in docstore', function () { - expect( - this.DocstoreManager.promises.destroyProject - ).to.have.been.calledWith(this.deletedProjects[0].project._id) - }) - - it('should not call project history', function () { - expect(this.HistoryManager.promises.deleteProject).to.not.have.been - .called - }) - }) - describe('on an active project (from an incomplete delete)', function () { beforeEach(async function () { this.ProjectMock.expects('findById')