Merge pull request #11346 from overleaf/msm-config-history-v1-sp

Configuration changes for FPH in CE/SP

GitOrigin-RevId: 990eb0fa6158d4815740413da085759d2cc5e231

server-ce/Dockerfile

@@ -53,7 +53,11 @@ COPY server-ce/init_scripts/ /etc/my_init.d/
 
 # Copy app settings files
 # -----------------------
-COPY server-ce/settings.js /etc/sharelatex/settings.js
+COPY server-ce/config/settings.js /etc/sharelatex/settings.js
+
+# Copy history-v1 files
+# -----------------------
+COPY server-ce/config/production.json /overleaf/services/history-v1/config/production.json
 
 # Copy grunt thin wrapper
 # -----------------------

server-ce/config/production.json

@@ -0,0 +1,28 @@
+{
+  "persistor": {
+    "backend": "fs",
+    "useSubdirectories": true
+  },
+  "blobStore": {
+    "globalBucket": "/var/lib/sharelatex/history/overleaf-blobs",
+    "projectBucket": "/var/lib/sharelatex/history/overleaf-project-blobs"
+  },
+  "chunkStore": {
+    "bucket": "/var/lib/sharelatex/history/overleaf-chunks"
+  },
+  "zipStore": {
+    "bucket": "/var/lib/sharelatex/history/overleaf-zips"
+  },
+  "analytics": {
+    "bucket": "/var/lib/sharelatex/history/overleaf-analytics"
+  },
+  "basicHttpAuth": {
+    "password": "password"
+  },
+  "useDeleteObjects": "false",
+  "jwtAuth": {
+    "key": "secureKey",
+    "algorithm": "HS256"
+  },
+  "mongo": {}
+}

server-ce/config/settings.js

@@ -151,6 +151,20 @@ const settings = {
     api: redisConfig,
     pubsub: redisConfig,
     project_history: redisConfig,
+
+    project_history_migration: {
+      host: redisConfig.host,
+      port: redisConfig.port,
+      password: redisConfig.password,
+      maxRetriesPerRequest: parseInt(
+        process.env.REDIS_MAX_RETRIES_PER_REQUEST || '20'
+      ),
+      key_schema: {
+        projectHistoryOps({ projectId }) {
+          return `ProjectHistory:Ops:{${projectId}}` // NOTE: the extra braces are intentional
+        },
+      },
+    },
   },
 
   // File storage
@@ -273,6 +287,17 @@ const settings = {
       user: httpAuthUser,
       pass: httpAuthPass,
     },
+    project_history: {
+      sendProjectStructureOps: true,
+      initializeHistoryForNewProjects: true,
+      displayHistoryForNewProjects: true,
+      url: 'http://localhost:3054',
+    },
+    v1_history: {
+      url: 'http://localhost:3100/api',
+      user: 'staging',
+      pass: process.env.STAGING_PASSWORD,
+    }
   },
   references: {},
   notifications: undefined,

server-ce/init_scripts/00_regen_sharelatex_secrets.sh

@@ -5,15 +5,21 @@ set -e -o pipefail
 # https://github.com/phusion/baseimage-docker#centrally-defining-your-own-environment-variables
 
 WEB_API_PASSWORD_FILE=/etc/container_environment/WEB_API_PASSWORD
+STAGING_PASSWORD_FILE=/etc/container_environment/STAGING_PASSWORD # HTTP auth for history-v1
+V1_HISTORY_PASSWORD_FILE=/etc/container_environment/V1_HISTORY_PASSWORD
 CRYPTO_RANDOM_FILE=/etc/container_environment/CRYPTO_RANDOM
 
-if [ ! -f "$WEB_API_PASSWORD_FILE" ] || [ ! -f "$CRYPTO_RANDOM_FILE" ]; then
+if [ ! -f "$WEB_API_PASSWORD_FILE" ] || [ ! -f "$STAGING_PASSWORD_FILE" ] || [ ! -f "$CRYPTO_RANDOM_FILE" ]; then
 
     echo "generating random secrets"
 
     SECRET=$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 -w 0 | rev | cut -b 2- | rev | tr -d '\n+/')
     echo ${SECRET} > ${WEB_API_PASSWORD_FILE}
 
+    SECRET=$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 -w 0 | rev | cut -b 2- | rev | tr -d '\n+/')
+    echo ${SECRET} > ${STAGING_PASSWORD_FILE}
+    echo ${SECRET} > ${V1_HISTORY_PASSWORD_FILE}
+
     SECRET=$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 -w 0 | rev | cut -b 2- | rev | tr -d '\n+/')
     echo ${SECRET} > ${CRYPTO_RANDOM_FILE}
 fi

server-ce/runit/history-v1-sharelatex/run

@@ -6,4 +6,4 @@ if [ "$DEBUG_NODE" == "true" ]; then
     NODE_PARAMS="--inspect=0.0.0.0:30640"
 fi
 
-MONGO_CONNECTION_STRING=$SHARELATEX_MONGO_URL NODE_CONFIG_DIR=/overleaf/services/history-v1/config NODE_CONFIG_ENV=overleaf-ce exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS  /overleaf/services/history-v1/app.js >> /var/log/sharelatex/history-v1.log 2>&1
+MONGO_CONNECTION_STRING=$SHARELATEX_MONGO_URL NODE_CONFIG_DIR=/overleaf/services/history-v1/config exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS  /overleaf/services/history-v1/app.js >> /var/log/sharelatex/history-v1.log 2>&1

services/history-v1/config/overleaf-ce.json

@@ -1,30 +0,0 @@
-{
-  "persistor": {
-    "s3": {
-      "endpoint": "http://s3:8080",
-      "pathStyle": "true"
-    }
-  },
-  "blobStore": {
-    "globalBucket": "overleaf-blobs",
-    "projectBucket": "overleaf-project-blobs"
-  },
-  "chunkStore": {
-    "bucket": "overleaf-chunks"
-  },
-  "zipStore": {
-    "bucket": "overleaf-zips"
-  },
-  "analytics": {
-    "bucket": "overleaf-analytics"
-  },
-  "useDeleteObjects": "false",
-  "basicHttpAuth": {
-    "password": "password"
-  },
-  "jwtAuth": {
-    "key": "secureKey",
-    "algorithm": "HS256"
-  },
-  "mongo": {}
-}

services/web/app/src/Features/Project/ProjectCreationHandler.js

@@ -171,7 +171,6 @@ async function _createBlankProject(
   // (to allow scripted creation of projects without full project history)
   const historyId = project.overleaf.history.id
   if (
-    Features.hasFeature('history-v1') &&
     Settings.apis.project_history.displayHistoryForNewProjects &&
     historyId != null
   ) {

services/web/app/src/Features/Project/ProjectDeleter.js

@@ -1,4 +1,3 @@
-const Features = require('../../infrastructure/Features')
 const _ = require('lodash')
 const { db, ObjectId } = require('../../infrastructure/mongodb')
 const { callbackify } = require('util')
@@ -376,12 +375,10 @@ async function expireDeletedProject(projectId) {
 
     await Promise.all([
       DocstoreManager.promises.destroyProject(deletedProject.project._id),
-      Features.hasFeature('history-v1')
+      HistoryManager.promises.deleteProject(
-        ? HistoryManager.promises.deleteProject(
         deletedProject.project._id,
         historyId
-          )
+      ),
-        : Promise.resolve(),
       FilestoreHandler.promises.deleteProject(deletedProject.project._id),
       TpdsUpdateSender.promises.deleteProject({
         projectId: deletedProject.project._id,

services/web/app/src/infrastructure/Features.js

@@ -6,9 +6,6 @@ const publicRegistrationModuleAvailable =
 
 const supportModuleAvailable = Settings.moduleImportSequence.includes('support')
 
-const historyV1ModuleAvailable =
-  Settings.moduleImportSequence.includes('history-v1')
-
 const trackChangesModuleAvailable =
   Settings.moduleImportSequence.includes('track-changes')
 
@@ -71,8 +68,6 @@ const Features = {
         return Boolean(Settings.oauth)
       case 'templates-server-pro':
         return !Settings.overleaf
-      case 'history-v1':
-        return historyV1ModuleAvailable
       case 'affiliations':
       case 'analytics':
         return Boolean(_.get(Settings, ['apis', 'v1', 'url']))

services/web/test/acceptance/config/settings.test.defaults.js

@@ -76,6 +76,17 @@ module.exports = {
     notifications: {
       url: 'http://localhost:23042',
     },
+    project_history: {
+      sendProjectStructureOps: true,
+      initializeHistoryForNewProjects: true,
+      displayHistoryForNewProjects: true,
+      url: `http://localhost:23054`,
+    },
+    v1_history: {
+      url: `http://localhost:23100/api`,
+      user: 'overleaf',
+      pass: 'password',
+    },
     webpack: {
       url: 'http://localhost:23808',
     },

services/web/test/acceptance/config/settings.test.saas.js

@@ -17,13 +17,6 @@ const overrides = {
     analytics: {
       url: `http://localhost:23050`,
     },
-    project_history: {
-      sendProjectStructureOps: true,
-      initializeHistoryForNewProjects: true,
-      displayHistoryForNewProjects: true,
-      url: `http://localhost:23054`,
-    },
-
     recurly: {
       url: 'http://localhost:26034',
       subdomain: 'test',
@@ -42,12 +35,6 @@ const overrides = {
       user: 'overleaf',
       pass: 'password',
     },
-
-    v1_history: {
-      url: `http://localhost:23100/api`,
-      user: 'overleaf',
-      pass: 'password',
-    },
   },
 
   oauthProviders: {

services/web/test/acceptance/src/Init.js

@@ -27,11 +27,11 @@ MockFilestoreApi.initialize(23009, mockOpts)
 MockNotificationsApi.initialize(23042, mockOpts)
 MockSpellingApi.initialize(23005, mockOpts)
 MockHaveIBeenPwnedApi.initialize(1337, mockOpts)
+MockProjectHistoryApi.initialize(23054, mockOpts)
+MockV1HistoryApi.initialize(23100, mockOpts)
 
 if (Features.hasFeature('saas')) {
   MockAnalyticsApi.initialize(23050, mockOpts)
-  MockProjectHistoryApi.initialize(23054, mockOpts)
   MockV1Api.initialize(25000, mockOpts)
-  MockV1HistoryApi.initialize(23100, mockOpts)
   MockThirdPartyDataStoreApi.initialize(23002, mockOpts)
 }

services/web/test/unit/src/Project/ProjectDeleterTests.js

@@ -513,53 +513,6 @@ describe('ProjectDeleter', function () {
       })
     })
 
-    describe('when history-v1 is not available', function () {
-      beforeEach(async function () {
-        this.Features.hasFeature.returns(false)
-
-        this.ProjectMock.expects('findById')
-          .withArgs(this.deletedProjects[0].deleterData.deletedProjectId)
-          .chain('exec')
-          .resolves(null)
-        this.DeletedProjectMock.expects('updateOne')
-          .withArgs(
-            {
-              _id: this.deletedProjects[0]._id,
-            },
-            {
-              $set: {
-                'deleterData.deleterIpAddress': null,
-                project: null,
-              },
-            }
-          )
-          .chain('exec')
-          .resolves()
-
-        this.DeletedProjectMock.expects('findOne')
-          .withArgs({
-            'deleterData.deletedProjectId': this.deletedProjects[0].project._id,
-          })
-          .chain('exec')
-          .resolves(this.deletedProjects[0])
-
-        await this.ProjectDeleter.promises.expireDeletedProject(
-          this.deletedProjects[0].project._id
-        )
-      })
-
-      it('should destroy the docs in docstore', function () {
-        expect(
-          this.DocstoreManager.promises.destroyProject
-        ).to.have.been.calledWith(this.deletedProjects[0].project._id)
-      })
-
-      it('should not call project history', function () {
-        expect(this.HistoryManager.promises.deleteProject).to.not.have.been
-          .called
-      })
-    })
-
     describe('on an active project (from an incomplete delete)', function () {
       beforeEach(async function () {
         this.ProjectMock.expects('findById')