Merge pull request #19282 from overleaf/jpa-filestore-sharding

[filestore] add sharding to test_acceptance in CI GitOrigin-RevId: dd16a0370878d76a934d277838083e08ae0ad8c2
2024-11-14 20:40:17 -05:00 · 2024-07-12 11:46:33 +02:00 · 2024-07-12 11:46:33 +02:00 · cbe7a2f5a9
commit cbe7a2f5a9
parent 788009330c
24 changed files with 82 additions and 41 deletions
--- a/services/chat/Makefile
+++ b/services/chat/Makefile
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 	MOCHA_GREP=${MOCHA_GREP} \
 	docker compose ${DOCKER_COMPOSE_FLAGS}

+COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_ACCEPTANCE = \
-	COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)

+COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
-	COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)

 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
--- a/services/chat/docker-compose.ci.yml
+++ b/services/chat/docker-compose.ci.yml
@ -28,7 +28,7 @@ services:
      mongo:
        condition: service_healthy
    user: node
-    command: npm run test:acceptance:_run
+    command: npm run test:acceptance


  tar:
--- a/services/clsi/Makefile
+++ b/services/clsi/Makefile
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 	MOCHA_GREP=${MOCHA_GREP} \
 	docker compose ${DOCKER_COMPOSE_FLAGS}

+COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_ACCEPTANCE = \
-	COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)

+COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
-	COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)

 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
--- a/services/clsi/docker-compose.ci.yml
+++ b/services/clsi/docker-compose.ci.yml
@ -29,7 +29,7 @@ services:
      TEXLIVE_IMAGE:
      ENABLE_PDF_CACHING: "true"
      PDF_CACHING_ENABLE_WORKER_POOL: "true"
-    command: npm run test:acceptance:_run
+    command: npm run test:acceptance


  tar:
--- a/services/contacts/Makefile
+++ b/services/contacts/Makefile
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 	MOCHA_GREP=${MOCHA_GREP} \
 	docker compose ${DOCKER_COMPOSE_FLAGS}

+COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_ACCEPTANCE = \
-	COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)

+COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
-	COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)

 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
--- a/services/contacts/docker-compose.ci.yml
+++ b/services/contacts/docker-compose.ci.yml
@ -28,7 +28,7 @@ services:
      mongo:
        condition: service_healthy
    user: node
-    command: npm run test:acceptance:_run
+    command: npm run test:acceptance


  tar:
--- a/services/docstore/Makefile
+++ b/services/docstore/Makefile
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 	MOCHA_GREP=${MOCHA_GREP} \
 	docker compose ${DOCKER_COMPOSE_FLAGS}

+COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_ACCEPTANCE = \
-	COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)

+COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
-	COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)

 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
--- a/services/docstore/docker-compose.ci.yml
+++ b/services/docstore/docker-compose.ci.yml
@ -36,7 +36,7 @@ services:
      gcs:
        condition: service_healthy
    user: node
-    command: npm run test:acceptance:_run
+    command: npm run test:acceptance


  tar:
--- a/services/document-updater/Makefile
+++ b/services/document-updater/Makefile
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 	MOCHA_GREP=${MOCHA_GREP} \
 	docker compose ${DOCKER_COMPOSE_FLAGS}

+COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_ACCEPTANCE = \
-	COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)

+COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
-	COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)

 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
--- a/services/document-updater/docker-compose.ci.yml
+++ b/services/document-updater/docker-compose.ci.yml
@ -33,7 +33,7 @@ services:
      redis:
        condition: service_healthy
    user: node
-    command: npm run test:acceptance:_run
+    command: npm run test:acceptance


  tar:
--- a/services/filestore/Makefile
+++ b/services/filestore/Makefile
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 	MOCHA_GREP=${MOCHA_GREP} \
 	docker compose ${DOCKER_COMPOSE_FLAGS}

+COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_ACCEPTANCE = \
-	COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)

+COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
-	COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)

 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
--- a/services/filestore/buildscript.txt
+++ b/services/filestore/buildscript.txt
@ -8,3 +8,4 @@ filestore
 --node-version=18.20.2
 --public-repo=True
 --script-version=4.5.0
+--test-acceptance-shards=SHARD_01_,SHARD_02_,SHARD_03_
--- a/services/filestore/docker-compose.ci.yml
+++ b/services/filestore/docker-compose.ci.yml
@ -45,7 +45,7 @@ services:
      gcs:
        condition: service_healthy
    user: node
-    command: npm run test:acceptance:_run
+    command: npm run test:acceptance


  tar:
--- a/services/filestore/test/acceptance/js/TestConfig.js
+++ b/services/filestore/test/acceptance/js/TestConfig.js
@ -1,3 +1,4 @@
+const fs = require('fs')
 const Path = require('path')

 // use functions to get a fresh copy, not a reference, each time
@ -64,26 +65,26 @@ function fallbackStores(primaryConfig, fallbackConfig) {
 }

 module.exports = {
-  FSPersistor: {
+  SHARD_01_FSPersistor: {
    backend: 'fs',
    stores: fsStores(),
  },
-  S3Persistor: {
+  SHARD_01_S3Persistor: {
    backend: 's3',
    s3: s3Config(),
    stores: s3Stores(),
  },
-  S3PersistorDefaultProviderCredentials: {
+  SHARD_01_S3PersistorDefaultProviderCredentials: {
    backend: 's3',
    s3: s3ConfigDefaultProviderCredentials(),
    stores: s3Stores(),
  },
-  GcsPersistor: {
+  SHARD_01_GcsPersistor: {
    backend: 'gcs',
    gcs: gcsConfig(),
    stores: gcsStores(),
  },
-  FallbackS3ToFSPersistor: {
+  SHARD_02_FallbackS3ToFSPersistor: {
    backend: 's3',
    s3: s3Config(),
    stores: s3Stores(),
@ -92,7 +93,7 @@ module.exports = {
      buckets: fallbackStores(s3Stores(), fsStores()),
    },
  },
-  FallbackFSToS3Persistor: {
+  SHARD_02_FallbackFSToS3Persistor: {
    backend: 'fs',
    s3: s3Config(),
    stores: fsStores(),
@ -101,7 +102,7 @@ module.exports = {
      buckets: fallbackStores(fsStores(), s3Stores()),
    },
  },
-  FallbackGcsToS3Persistor: {
+  SHARD_03_FallbackGcsToS3Persistor: {
    backend: 'gcs',
    gcs: gcsConfig(),
    stores: gcsStores(),
@ -111,7 +112,7 @@ module.exports = {
      buckets: fallbackStores(gcsStores(), s3Stores()),
    },
  },
-  FallbackS3ToGcsPersistor: {
+  SHARD_03_FallbackS3ToGcsPersistor: {
    backend: 's3',
    // can use the same bucket names for gcs and s3 (in tests)
    stores: s3Stores(),
@ -123,3 +124,20 @@ module.exports = {
    },
  },
 }
+
+function checkForUnexpectedTestFile() {
+  const awareOfSharding = [
+    'FilestoreApp.js',
+    'FilestoreTests.js',
+    'TestConfig.js',
+    'TestHelper.js',
+  ]
+  for (const file of fs.readdirSync(__dirname).sort()) {
+    if (!awareOfSharding.includes(file)) {
+      throw new Error(
+        `Found new test file ${file}: All tests must be aware of the SHARD_ prefix.`
+      )
+    }
+  }
+}
+checkForUnexpectedTestFile()
--- a/services/history-v1/Makefile
+++ b/services/history-v1/Makefile
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 	MOCHA_GREP=${MOCHA_GREP} \
 	docker compose ${DOCKER_COMPOSE_FLAGS}

+COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_ACCEPTANCE = \
-	COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)

+COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
-	COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)

 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
--- a/services/history-v1/docker-compose.ci.yml
+++ b/services/history-v1/docker-compose.ci.yml
@ -35,7 +35,7 @@ services:
      gcs:
        condition: service_healthy
    user: node
-    command: npm run test:acceptance:_run
+    command: npm run test:acceptance


  tar:
--- a/services/notifications/Makefile
+++ b/services/notifications/Makefile
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 	MOCHA_GREP=${MOCHA_GREP} \
 	docker compose ${DOCKER_COMPOSE_FLAGS}

+COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_ACCEPTANCE = \
-	COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)

+COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
-	COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)

 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
--- a/services/notifications/docker-compose.ci.yml
+++ b/services/notifications/docker-compose.ci.yml
@ -28,7 +28,7 @@ services:
      mongo:
        condition: service_healthy
    user: node
-    command: npm run test:acceptance:_run
+    command: npm run test:acceptance


  tar:
--- a/services/project-history/Makefile
+++ b/services/project-history/Makefile
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 	MOCHA_GREP=${MOCHA_GREP} \
 	docker compose ${DOCKER_COMPOSE_FLAGS}

+COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_ACCEPTANCE = \
-	COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)

+COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
-	COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)

 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
--- a/services/project-history/docker-compose.ci.yml
+++ b/services/project-history/docker-compose.ci.yml
@ -33,7 +33,7 @@ services:
      redis:
        condition: service_healthy
    user: node
-    command: npm run test:acceptance:_run
+    command: npm run test:acceptance


  tar:
--- a/services/real-time/Makefile
+++ b/services/real-time/Makefile
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 	MOCHA_GREP=${MOCHA_GREP} \
 	docker compose ${DOCKER_COMPOSE_FLAGS}

+COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_ACCEPTANCE = \
-	COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)

+COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
-	COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)

 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
--- a/services/real-time/docker-compose.ci.yml
+++ b/services/real-time/docker-compose.ci.yml
@ -31,7 +31,7 @@ services:
      redis:
        condition: service_healthy
    user: node
-    command: npm run test:acceptance:_run
+    command: npm run test:acceptance


  tar:
--- a/services/spelling/Makefile
+++ b/services/spelling/Makefile
@ -14,11 +14,13 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
 	MOCHA_GREP=${MOCHA_GREP} \
 	docker compose ${DOCKER_COMPOSE_FLAGS}

+COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_ACCEPTANCE = \
-	COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)

+COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
 DOCKER_COMPOSE_TEST_UNIT = \
-	COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE)
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)

 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
--- a/services/spelling/docker-compose.ci.yml
+++ b/services/spelling/docker-compose.ci.yml
@ -25,7 +25,7 @@ services:
      NODE_ENV: test
      NODE_OPTIONS: "--unhandled-rejections=strict"
    user: node
-    command: npm run test:acceptance:_run
+    command: npm run test:acceptance


  tar: