Merge pull request #11346 from overleaf/msm-config-history-v1-sp

Configuration changes for FPH in CE/SP GitOrigin-RevId: 990eb0fa6158d4815740413da085759d2cc5e231
2024-11-21 20:47:08 -05:00 · 2023-01-20 12:06:26 +01:00 · 2023-01-20 12:06:26 +01:00 · 4a84922a2f
commit 4a84922a2f
parent 0b986d5625
13 changed files with 83 additions and 108 deletions
--- a/server-ce/Dockerfile
+++ b/server-ce/Dockerfile
@ -53,7 +53,11 @@ COPY server-ce/init_scripts/ /etc/my_init.d/

 # Copy app settings files
 # -----------------------
-COPY server-ce/settings.js /etc/sharelatex/settings.js
+COPY server-ce/config/settings.js /etc/sharelatex/settings.js
+
+# Copy history-v1 files
+# -----------------------
+COPY server-ce/config/production.json /overleaf/services/history-v1/config/production.json

 # Copy grunt thin wrapper
 # -----------------------
--- a/server-ce/config/production.json
+++ b/server-ce/config/production.json
@ -0,0 +1,28 @@
+{
+  "persistor": {
+    "backend": "fs",
+    "useSubdirectories": true
+  },
+  "blobStore": {
+    "globalBucket": "/var/lib/sharelatex/history/overleaf-blobs",
+    "projectBucket": "/var/lib/sharelatex/history/overleaf-project-blobs"
+  },
+  "chunkStore": {
+    "bucket": "/var/lib/sharelatex/history/overleaf-chunks"
+  },
+  "zipStore": {
+    "bucket": "/var/lib/sharelatex/history/overleaf-zips"
+  },
+  "analytics": {
+    "bucket": "/var/lib/sharelatex/history/overleaf-analytics"
+  },
+  "basicHttpAuth": {
+    "password": "password"
+  },
+  "useDeleteObjects": "false",
+  "jwtAuth": {
+    "key": "secureKey",
+    "algorithm": "HS256"
+  },
+  "mongo": {}
+}
--- a/server-ce/config/settings.js
+++ b/server-ce/config/settings.js
@ -151,6 +151,20 @@ const settings = {
    api: redisConfig,
    pubsub: redisConfig,
    project_history: redisConfig,
+
+    project_history_migration: {
+      host: redisConfig.host,
+      port: redisConfig.port,
+      password: redisConfig.password,
+      maxRetriesPerRequest: parseInt(
+        process.env.REDIS_MAX_RETRIES_PER_REQUEST || '20'
+      ),
+      key_schema: {
+        projectHistoryOps({ projectId }) {
+          return `ProjectHistory:Ops:{${projectId}}` // NOTE: the extra braces are intentional
+        },
+      },
+    },
  },

  // File storage
@ -273,6 +287,17 @@ const settings = {
      user: httpAuthUser,
      pass: httpAuthPass,
    },
+    project_history: {
+      sendProjectStructureOps: true,
+      initializeHistoryForNewProjects: true,
+      displayHistoryForNewProjects: true,
+      url: 'http://localhost:3054',
+    },
+    v1_history: {
+      url: 'http://localhost:3100/api',
+      user: 'staging',
+      pass: process.env.STAGING_PASSWORD,
+    }
  },
  references: {},
  notifications: undefined,
--- a/server-ce/init_scripts/00_regen_sharelatex_secrets.sh
+++ b/server-ce/init_scripts/00_regen_sharelatex_secrets.sh
@ -5,15 +5,21 @@ set -e -o pipefail
 # https://github.com/phusion/baseimage-docker#centrally-defining-your-own-environment-variables

 WEB_API_PASSWORD_FILE=/etc/container_environment/WEB_API_PASSWORD
+STAGING_PASSWORD_FILE=/etc/container_environment/STAGING_PASSWORD # HTTP auth for history-v1
+V1_HISTORY_PASSWORD_FILE=/etc/container_environment/V1_HISTORY_PASSWORD
 CRYPTO_RANDOM_FILE=/etc/container_environment/CRYPTO_RANDOM

-if [ ! -f "$WEB_API_PASSWORD_FILE" ] || [ ! -f "$CRYPTO_RANDOM_FILE" ]; then
+if [ ! -f "$WEB_API_PASSWORD_FILE" ] || [ ! -f "$STAGING_PASSWORD_FILE" ] || [ ! -f "$CRYPTO_RANDOM_FILE" ]; then

    echo "generating random secrets"

    SECRET=$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 -w 0 | rev | cut -b 2- | rev | tr -d '\n+/')
    echo ${SECRET} > ${WEB_API_PASSWORD_FILE}

+    SECRET=$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 -w 0 | rev | cut -b 2- | rev | tr -d '\n+/')
+    echo ${SECRET} > ${STAGING_PASSWORD_FILE}
+    echo ${SECRET} > ${V1_HISTORY_PASSWORD_FILE}
+
    SECRET=$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 -w 0 | rev | cut -b 2- | rev | tr -d '\n+/')
    echo ${SECRET} > ${CRYPTO_RANDOM_FILE}
 fi
--- a/server-ce/runit/history-v1-sharelatex/run
+++ b/server-ce/runit/history-v1-sharelatex/run
@ -6,4 +6,4 @@ if [ "$DEBUG_NODE" == "true" ]; then
    NODE_PARAMS="--inspect=0.0.0.0:30640"
 fi

-MONGO_CONNECTION_STRING=$SHARELATEX_MONGO_URL NODE_CONFIG_DIR=/overleaf/services/history-v1/config NODE_CONFIG_ENV=overleaf-ce exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS  /overleaf/services/history-v1/app.js >> /var/log/sharelatex/history-v1.log 2>&1
+MONGO_CONNECTION_STRING=$SHARELATEX_MONGO_URL NODE_CONFIG_DIR=/overleaf/services/history-v1/config exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS  /overleaf/services/history-v1/app.js >> /var/log/sharelatex/history-v1.log 2>&1
--- a/services/history-v1/config/overleaf-ce.json
+++ b/services/history-v1/config/overleaf-ce.json
@ -1,30 +0,0 @@
-{
-  "persistor": {
-    "s3": {
-      "endpoint": "http://s3:8080",
-      "pathStyle": "true"
-    }
-  },
-  "blobStore": {
-    "globalBucket": "overleaf-blobs",
-    "projectBucket": "overleaf-project-blobs"
-  },
-  "chunkStore": {
-    "bucket": "overleaf-chunks"
-  },
-  "zipStore": {
-    "bucket": "overleaf-zips"
-  },
-  "analytics": {
-    "bucket": "overleaf-analytics"
-  },
-  "useDeleteObjects": "false",
-  "basicHttpAuth": {
-    "password": "password"
-  },
-  "jwtAuth": {
-    "key": "secureKey",
-    "algorithm": "HS256"
-  },
-  "mongo": {}
-}
--- a/services/web/app/src/Features/Project/ProjectCreationHandler.js
+++ b/services/web/app/src/Features/Project/ProjectCreationHandler.js
@ -171,7 +171,6 @@ async function _createBlankProject(
  // (to allow scripted creation of projects without full project history)
  const historyId = project.overleaf.history.id
  if (
-    Features.hasFeature('history-v1') &&
    Settings.apis.project_history.displayHistoryForNewProjects &&
    historyId != null
  ) {
--- a/services/web/app/src/Features/Project/ProjectDeleter.js
+++ b/services/web/app/src/Features/Project/ProjectDeleter.js
@ -1,4 +1,3 @@
-const Features = require('../../infrastructure/Features')
 const _ = require('lodash')
 const { db, ObjectId } = require('../../infrastructure/mongodb')
 const { callbackify } = require('util')
@ -376,12 +375,10 @@ async function expireDeletedProject(projectId) {

    await Promise.all([
      DocstoreManager.promises.destroyProject(deletedProject.project._id),
-      Features.hasFeature('history-v1')
-        ? HistoryManager.promises.deleteProject(
-            deletedProject.project._id,
-            historyId
-          )
-        : Promise.resolve(),
+      HistoryManager.promises.deleteProject(
+        deletedProject.project._id,
+        historyId
+      ),
      FilestoreHandler.promises.deleteProject(deletedProject.project._id),
      TpdsUpdateSender.promises.deleteProject({
        projectId: deletedProject.project._id,
--- a/services/web/app/src/infrastructure/Features.js
+++ b/services/web/app/src/infrastructure/Features.js
@ -6,9 +6,6 @@ const publicRegistrationModuleAvailable =

 const supportModuleAvailable = Settings.moduleImportSequence.includes('support')

-const historyV1ModuleAvailable =
-  Settings.moduleImportSequence.includes('history-v1')
-
 const trackChangesModuleAvailable =
  Settings.moduleImportSequence.includes('track-changes')

@ -71,8 +68,6 @@ const Features = {
        return Boolean(Settings.oauth)
      case 'templates-server-pro':
        return !Settings.overleaf
-      case 'history-v1':
-        return historyV1ModuleAvailable
      case 'affiliations':
      case 'analytics':
        return Boolean(_.get(Settings, ['apis', 'v1', 'url']))
--- a/services/web/test/acceptance/config/settings.test.defaults.js
+++ b/services/web/test/acceptance/config/settings.test.defaults.js
@ -76,6 +76,17 @@ module.exports = {
    notifications: {
      url: 'http://localhost:23042',
    },
+    project_history: {
+      sendProjectStructureOps: true,
+      initializeHistoryForNewProjects: true,
+      displayHistoryForNewProjects: true,
+      url: `http://localhost:23054`,
+    },
+    v1_history: {
+      url: `http://localhost:23100/api`,
+      user: 'overleaf',
+      pass: 'password',
+    },
    webpack: {
      url: 'http://localhost:23808',
    },
--- a/services/web/test/acceptance/config/settings.test.saas.js
+++ b/services/web/test/acceptance/config/settings.test.saas.js
@ -17,13 +17,6 @@ const overrides = {
    analytics: {
      url: `http://localhost:23050`,
    },
-    project_history: {
-      sendProjectStructureOps: true,
-      initializeHistoryForNewProjects: true,
-      displayHistoryForNewProjects: true,
-      url: `http://localhost:23054`,
-    },
-
    recurly: {
      url: 'http://localhost:26034',
      subdomain: 'test',
@ -42,12 +35,6 @@ const overrides = {
      user: 'overleaf',
      pass: 'password',
    },
-
-    v1_history: {
-      url: `http://localhost:23100/api`,
-      user: 'overleaf',
-      pass: 'password',
-    },
  },

  oauthProviders: {
--- a/services/web/test/acceptance/src/Init.js
+++ b/services/web/test/acceptance/src/Init.js
@ -27,11 +27,11 @@ MockFilestoreApi.initialize(23009, mockOpts)
 MockNotificationsApi.initialize(23042, mockOpts)
 MockSpellingApi.initialize(23005, mockOpts)
 MockHaveIBeenPwnedApi.initialize(1337, mockOpts)
+MockProjectHistoryApi.initialize(23054, mockOpts)
+MockV1HistoryApi.initialize(23100, mockOpts)

 if (Features.hasFeature('saas')) {
  MockAnalyticsApi.initialize(23050, mockOpts)
-  MockProjectHistoryApi.initialize(23054, mockOpts)
  MockV1Api.initialize(25000, mockOpts)
-  MockV1HistoryApi.initialize(23100, mockOpts)
  MockThirdPartyDataStoreApi.initialize(23002, mockOpts)
 }
--- a/services/web/test/unit/src/Project/ProjectDeleterTests.js
+++ b/services/web/test/unit/src/Project/ProjectDeleterTests.js
@ -513,53 +513,6 @@ describe('ProjectDeleter', function () {
      })
    })

-    describe('when history-v1 is not available', function () {
-      beforeEach(async function () {
-        this.Features.hasFeature.returns(false)
-
-        this.ProjectMock.expects('findById')
-          .withArgs(this.deletedProjects[0].deleterData.deletedProjectId)
-          .chain('exec')
-          .resolves(null)
-        this.DeletedProjectMock.expects('updateOne')
-          .withArgs(
-            {
-              _id: this.deletedProjects[0]._id,
-            },
-            {
-              $set: {
-                'deleterData.deleterIpAddress': null,
-                project: null,
-              },
-            }
-          )
-          .chain('exec')
-          .resolves()
-
-        this.DeletedProjectMock.expects('findOne')
-          .withArgs({
-            'deleterData.deletedProjectId': this.deletedProjects[0].project._id,
-          })
-          .chain('exec')
-          .resolves(this.deletedProjects[0])
-
-        await this.ProjectDeleter.promises.expireDeletedProject(
-          this.deletedProjects[0].project._id
-        )
-      })
-
-      it('should destroy the docs in docstore', function () {
-        expect(
-          this.DocstoreManager.promises.destroyProject
-        ).to.have.been.calledWith(this.deletedProjects[0].project._id)
-      })
-
-      it('should not call project history', function () {
-        expect(this.HistoryManager.promises.deleteProject).to.not.have.been
-          .called
-      })
-    })
-
    describe('on an active project (from an incomplete delete)', function () {
      beforeEach(async function () {
        this.ProjectMock.expects('findById')