Merge pull request #13274 from overleaf/csh-issue-13004-cleanup-custom-code

Remove apiScheme from GCS GitOrigin-RevId: c948287cb00c7bc432c797bfdfa92a79b84d1bed
2024-11-21 20:47:08 -05:00 · 2023-06-01 09:43:56 +01:00 · 2023-06-01 09:43:56 +01:00 · 5fbd5e21a2
commit 5fbd5e21a2
parent 9ddc27d919
15 changed files with 11 additions and 41 deletions
--- a/libraries/object-persistor/README.md
+++ b/libraries/object-persistor/README.md
@ -295,7 +295,6 @@ GCS authentication is configured automatically via the local service account, or
 - `gcs.deleteConcurrency`: when recursively deleting a directory, the maximum number of delete requests that will be used at once (default 50)
 - `gcs.unsignedUrls`: For testing - do not sign GCS download URLs
 - `gcs.endpoint.apiEndpoint`: For testing - specify a different GCS endpoint to use
- `gcs.endpoint.apiScheme`: For testing - specify a scheme to use for the GCS endpoint (`http` or `https`)
 - `gcs.endpoint.projectId`: For testing - the GCS project ID to supply to the overridden backend

 #### Notes
--- a/libraries/object-persistor/src/GcsPersistor.js
+++ b/libraries/object-persistor/src/GcsPersistor.js
@ -16,21 +16,6 @@ module.exports = class GcsPersistor extends AbstractPersistor {
    // endpoint settings will be null by default except for tests
    // that's OK - GCS uses the locally-configured service account by default
    this.storage = new Storage(this.settings.endpoint)
-    // workaround for broken uploads with custom endpoints:
-    // https://github.com/googleapis/nodejs-storage/issues/898
-    if (this.settings.endpoint && this.settings.endpoint.apiEndpoint) {
-      this.storage.interceptors.push({
-        request: reqOpts => {
-          const url = new URL(reqOpts.uri)
-          url.host = this.settings.endpoint.apiEndpoint
-          if (this.settings.endpoint.apiScheme) {
-            url.protocol = this.settings.endpoint.apiScheme
-          }
-          reqOpts.uri = url.toString()
-          return reqOpts
-        },
-      })
-    }
  }

  async sendFile(bucketName, key, fsPath) {
@ -142,10 +127,9 @@ module.exports = class GcsPersistor extends AbstractPersistor {
    if (this.settings.unsignedUrls) {
      // Construct a direct URL to the object download endpoint
      // (see https://cloud.google.com/storage/docs/request-endpoints#json-api)
-      const apiScheme = this.settings.endpoint.apiScheme || 'https://'
      const apiEndpoint =
-        this.settings.endpoint.apiEndpoint || 'storage.googleapis.com'
-      return `${apiScheme}://${apiEndpoint}/download/storage/v1/b/${bucketName}/o/${key}?alt=media`
+        this.settings.endpoint.apiEndpoint || 'https://storage.googleapis.com'
+      return `${apiEndpoint}/download/storage/v1/b/${bucketName}/o/${key}?alt=media`
    }
    try {
      const [url] = await this.storage
--- a/libraries/object-persistor/test/unit/GcsPersistorTests.js
+++ b/libraries/object-persistor/test/unit/GcsPersistorTests.js
@ -283,8 +283,7 @@ describe('GcsPersistorTests', function () {
      beforeEach(async function () {
        GcsPersistor.settings.unsignedUrls = true
        GcsPersistor.settings.endpoint = {
-          apiScheme: 'http',
-          apiEndpoint: 'custom.endpoint',
+          apiEndpoint: 'http://custom.endpoint',
        }
        signedUrl = await GcsPersistor.getRedirectUrl(bucket, key)
      })
--- a/services/docstore/config/settings.defaults.js
+++ b/services/docstore/config/settings.defaults.js
@ -67,7 +67,6 @@ if (
 if (process.env.GCS_API_ENDPOINT) {
  Settings.docstore.gcs.endpoint = {
    apiEndpoint: process.env.GCS_API_ENDPOINT,
-    apiScheme: process.env.GCS_API_SCHEME,
    projectId: process.env.GCS_PROJECT_ID,
  }
 }
--- a/services/docstore/docker-compose.ci.yml
+++ b/services/docstore/docker-compose.ci.yml
@ -27,8 +27,7 @@ services:
      ANALYTICS_QUEUES_REDIS_HOST: redis
      MONGO_HOST: mongo
      POSTGRES_HOST: postgres
-      GCS_API_ENDPOINT: gcs:9090
-      GCS_API_SCHEME: http
+      GCS_API_ENDPOINT: http://gcs:9090
      GCS_PROJECT_ID: fake
      STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
      MOCHA_GREP: ${MOCHA_GREP}
--- a/services/docstore/docker-compose.yml
+++ b/services/docstore/docker-compose.yml
@ -36,8 +36,7 @@ services:
      ANALYTICS_QUEUES_REDIS_HOST: redis
      MONGO_HOST: mongo
      POSTGRES_HOST: postgres
-      GCS_API_ENDPOINT: gcs:9090
-      GCS_API_SCHEME: http
+      GCS_API_ENDPOINT: http://gcs:9090
      GCS_PROJECT_ID: fake
      STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
      MOCHA_GREP: ${MOCHA_GREP}
--- a/services/filestore/config/settings.defaults.js
+++ b/services/filestore/config/settings.defaults.js
@ -53,7 +53,6 @@ const settings = {
      endpoint: process.env.GCS_API_ENDPOINT
        ? {
            apiEndpoint: process.env.GCS_API_ENDPOINT,
-            apiScheme: process.env.GCS_API_SCHEME,
            projectId: process.env.GCS_PROJECT_ID,
          }
        : undefined,
--- a/services/filestore/docker-compose.ci.yml
+++ b/services/filestore/docker-compose.ci.yml
@ -28,8 +28,7 @@ services:
      AWS_S3_PATH_STYLE: 'true'
      AWS_ACCESS_KEY_ID: fake
      AWS_SECRET_ACCESS_KEY: fake
-      GCS_API_ENDPOINT: gcs:9090
-      GCS_API_SCHEME: http
+      GCS_API_ENDPOINT: http://gcs:9090
      GCS_PROJECT_ID: fake
      STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
      MOCHA_GREP: ${MOCHA_GREP}
--- a/services/filestore/docker-compose.yml
+++ b/services/filestore/docker-compose.yml
@ -43,8 +43,7 @@ services:
      AWS_S3_PATH_STYLE: 'true'
      AWS_ACCESS_KEY_ID: fake
      AWS_SECRET_ACCESS_KEY: fake
-      GCS_API_ENDPOINT: gcs:9090
-      GCS_API_SCHEME: http
+      GCS_API_ENDPOINT: http://gcs:9090
      GCS_PROJECT_ID: fake
      STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
      MOCHA_GREP: ${MOCHA_GREP}
--- a/services/filestore/test/acceptance/js/TestConfig.js
+++ b/services/filestore/test/acceptance/js/TestConfig.js
@ -23,7 +23,6 @@ function gcsConfig() {
  return {
    endpoint: {
      apiEndpoint: process.env.GCS_API_ENDPOINT,
-      apiScheme: process.env.GCS_API_SCHEME,
      projectId: 'fake',
    },
    directoryKeyRegex: /^[0-9a-fA-F]{24}\/[0-9a-fA-F]{24}/,
--- a/services/history-v1/config/custom-environment-variables.json
+++ b/services/history-v1/config/custom-environment-variables.json
@ -18,7 +18,6 @@
      "unlockBeforeDelete": "GCS_UNLOCK_BEFORE_DELETE",
      "endpoint": {
        "apiEndpoint": "GCS_API_ENDPOINT",
-        "apiScheme": "GCS_API_SCHEME",
        "projectId": "GCS_PROJECT_ID"
      }
    },
--- a/services/history-v1/config/development.json
+++ b/services/history-v1/config/development.json
@ -8,8 +8,7 @@
    "gcs": {
      "unsignedUrls": "true",
      "endpoint": {
-        "apiEndpoint": "fake-gcs:9090",
-        "apiScheme": "http",
+        "apiEndpoint": "http://fake-gcs:9090",
        "projectId": "fake"
      }
    }
--- a/services/history-v1/config/test.json
+++ b/services/history-v1/config/test.json
@ -5,8 +5,7 @@
    "gcs": {
      "unsignedUrls": "true",
      "endpoint": {
-        "apiEndpoint": "gcs:9090",
-        "apiScheme": "http",
+        "apiEndpoint": "http://gcs:9090",
        "projectId": "fake"
      }
    }
--- a/services/history-v1/docker-compose.ci.yml
+++ b/services/history-v1/docker-compose.ci.yml
@ -24,8 +24,7 @@ services:
      ANALYTICS_QUEUES_REDIS_HOST: redis
      MONGO_HOST: mongo
      POSTGRES_HOST: postgres
-      GCS_API_ENDPOINT: gcs:9090
-      GCS_API_SCHEME: http
+      GCS_API_ENDPOINT: http://gcs:9090
      GCS_PROJECT_ID: fake
      STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
      MOCHA_GREP: ${MOCHA_GREP}
--- a/services/history-v1/docker-compose.yml
+++ b/services/history-v1/docker-compose.yml
@ -33,8 +33,7 @@ services:
      ANALYTICS_QUEUES_REDIS_HOST: redis
      MONGO_HOST: mongo
      POSTGRES_HOST: postgres
-      GCS_API_ENDPOINT: gcs:9090
-      GCS_API_SCHEME: http
+      GCS_API_ENDPOINT: http://gcs:9090
      GCS_PROJECT_ID: fake
      STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
      MOCHA_GREP: ${MOCHA_GREP}