Merge pull request #12575 from overleaf/em-fix-chunk-replace-transaction

Fix history chunk replacement transaction

GitOrigin-RevId: fc0fa8f4a55ae5f402ba87db5e4ac2856fe6627b

libraries/access-token-encryptor/scripts/helpers/re-encrypt-tokens.js

@@ -56,7 +56,7 @@ async function reEncryptTokensInCollection({
   const cursor = collection.find(
     {},
     {
-      readPreference: ReadPreference.SECONDARY,
+      readPreference: ReadPreference.secondaryPreferred,
       projection,
     }
   )

services/chat/Makefile

@@ -79,6 +79,13 @@ test_acceptance_clean:
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
+		while ! mongo --eval "db.version()" > /dev/null; do \
+			echo "Waiting for Mongo..."; \
+			sleep 1; \
+		done; \
+		mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
 endif

services/chat/docker-compose.ci.yml

@@ -43,6 +43,7 @@ services:
     user: root
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
     healthcheck:
       test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
       interval: 1s

services/chat/docker-compose.yml

@@ -45,6 +45,7 @@ services:
 
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
     healthcheck:
       test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
       interval: 1s

services/contacts/Makefile

@@ -79,6 +79,13 @@ test_acceptance_clean:
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
+		while ! mongo --eval "db.version()" > /dev/null; do \
+			echo "Waiting for Mongo..."; \
+			sleep 1; \
+		done; \
+		mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
 endif

services/contacts/docker-compose.ci.yml

@@ -43,6 +43,7 @@ services:
     user: root
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
     healthcheck:
       test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
       interval: 1s

services/contacts/docker-compose.yml

@@ -45,6 +45,7 @@ services:
 
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
     healthcheck:
       test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
       interval: 1s

services/docstore/Makefile

@@ -79,6 +79,13 @@ test_acceptance_clean:
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
+		while ! mongo --eval "db.version()" > /dev/null; do \
+			echo "Waiting for Mongo..."; \
+			sleep 1; \
+		done; \
+		mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
 endif

services/docstore/docker-compose.ci.yml

@@ -52,6 +52,7 @@ services:
     user: root
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
     healthcheck:
       test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
       interval: 1s

services/docstore/docker-compose.yml

@@ -54,6 +54,7 @@ services:
 
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
     healthcheck:
       test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
       interval: 1s

services/document-updater/Makefile

@@ -79,6 +79,13 @@ test_acceptance_clean:
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
+		while ! mongo --eval "db.version()" > /dev/null; do \
+			echo "Waiting for Mongo..."; \
+			sleep 1; \
+		done; \
+		mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
 endif

services/document-updater/docker-compose.ci.yml

@@ -52,6 +52,7 @@ services:
 
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
     healthcheck:
       test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
       interval: 1s

services/document-updater/docker-compose.yml

@@ -54,6 +54,7 @@ services:
 
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
     healthcheck:
       test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
       interval: 1s

services/history-v1/Makefile

@@ -79,6 +79,13 @@ test_acceptance_clean:
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
+		while ! mongo --eval "db.version()" > /dev/null; do \
+			echo "Waiting for Mongo..."; \
+			sleep 1; \
+		done; \
+		mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
 endif

services/history-v1/docker-compose.ci.yml

@@ -51,6 +51,7 @@ services:
     user: root
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
     healthcheck:
       test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
       interval: 1s

services/history-v1/docker-compose.yml

@@ -53,6 +53,7 @@ services:
 
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
     healthcheck:
       test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
       interval: 1s

services/history-v1/storage/lib/chunk_store/mongo.js

@@ -113,7 +113,7 @@ async function insertPendingChunk(projectId, chunk) {
 /**
  * Record that a new chunk was created.
  */
-async function confirmCreate(projectId, chunk, chunkId) {
+async function confirmCreate(projectId, chunk, chunkId, mongoOpts = {}) {
   assert.mongoId(projectId, 'bad projectId')
   assert.instance(chunk, Chunk, 'bad chunk')
   assert.mongoId(chunkId, 'bad chunkId')
@@ -126,7 +126,8 @@ async function confirmCreate(projectId, chunk, chunkId) {
         projectId: ObjectId(projectId),
         state: 'pending',
       },
-      { $set: { state: 'active', updatedAt: new Date() } }
+      { $set: { state: 'active', updatedAt: new Date() } },
+      mongoOpts
     )
   } catch (err) {
     if (err.code === DUPLICATE_KEY_ERROR_CODE) {
@@ -155,8 +156,8 @@ async function confirmUpdate(projectId, oldChunkId, newChunk, newChunkId) {
   const session = mongodb.client.startSession()
   try {
     await session.withTransaction(async () => {
-      await deleteChunk(projectId, oldChunkId)
+      await deleteChunk(projectId, oldChunkId, { session })
-      await confirmCreate(projectId, newChunk, newChunkId)
+      await confirmCreate(projectId, newChunk, newChunkId, { session })
     })
   } finally {
     await session.endSession()
@@ -170,13 +171,14 @@ async function confirmUpdate(projectId, oldChunkId, newChunk, newChunkId) {
  * @param {number} chunkId
  * @return {Promise}
  */
-async function deleteChunk(projectId, chunkId) {
+async function deleteChunk(projectId, chunkId, mongoOpts = {}) {
   assert.mongoId(projectId, 'bad projectId')
   assert.mongoId(chunkId, 'bad chunkId')
 
   await mongodb.chunks.updateOne(
     { _id: ObjectId(chunkId), projectId: ObjectId(projectId) },
-    { $set: { state: 'deleted', updatedAt: new Date() } }
+    { $set: { state: 'deleted', updatedAt: new Date() } },
+    mongoOpts
   )
 }
 

services/notifications/Makefile

@@ -79,6 +79,13 @@ test_acceptance_clean:
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
+		while ! mongo --eval "db.version()" > /dev/null; do \
+			echo "Waiting for Mongo..."; \
+			sleep 1; \
+		done; \
+		mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
 endif

services/notifications/docker-compose.ci.yml

@@ -43,6 +43,7 @@ services:
     user: root
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
     healthcheck:
       test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
       interval: 1s

services/notifications/docker-compose.yml

@@ -45,6 +45,7 @@ services:
 
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
     healthcheck:
       test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
       interval: 1s

services/project-history/Makefile

@@ -79,6 +79,13 @@ test_acceptance_clean:
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
+		while ! mongo --eval "db.version()" > /dev/null; do \
+			echo "Waiting for Mongo..."; \
+			sleep 1; \
+		done; \
+		mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
 endif

services/project-history/docker-compose.ci.yml

@@ -52,6 +52,7 @@ services:
 
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
     healthcheck:
       test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
       interval: 1s

services/project-history/docker-compose.yml

@@ -54,6 +54,7 @@ services:
 
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
     healthcheck:
       test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
       interval: 1s

services/track-changes/Makefile

@@ -79,6 +79,13 @@ test_acceptance_clean:
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
 
 test_acceptance_pre_run:
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
+	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
+		while ! mongo --eval "db.version()" > /dev/null; do \
+			echo "Waiting for Mongo..."; \
+			sleep 1; \
+		done; \
+		mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
 ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
 	$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
 endif

services/track-changes/docker-compose.ci.yml

@@ -59,6 +59,7 @@ services:
 
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
     healthcheck:
       test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
       interval: 1s

services/track-changes/docker-compose.yml

@@ -61,6 +61,7 @@ services:
 
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
     healthcheck:
       test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
       interval: 1s

services/web/Makefile

@@ -170,6 +170,15 @@ test_frontend_ct_modules:
 # Acceptance tests
 #
 
+TEST_ACCEPTANCE_MONGO_INIT := \
+	$(DOCKER_COMPOSE) up -d mongo; \
+	$(DOCKER_COMPOSE) exec -T mongo sh -c ' \
+		while ! mongo --eval "db.version()" > /dev/null; do \
+			echo "Waiting for Mongo..."; \
+			sleep 1; \
+		done; \
+		mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
+
 test_acceptance: test_acceptance_app test_acceptance_modules
 test_acceptance_saas: test_acceptance_app_saas test_acceptance_modules_merged_saas
 test_acceptance_server_ce: test_acceptance_app_server_ce test_acceptance_modules_merged_server_ce
@@ -190,6 +199,7 @@ test_acceptance_app_server_pro: export SHARELATEX_CONFIG=$(CFG_SERVER_PRO)
 
 $(TEST_ACCEPTANCE_APP):
 	$(DOCKER_COMPOSE) down -v -t 0
+	$(TEST_ACCEPTANCE_MONGO_INIT)
 	$(DOCKER_COMPOSE) run --rm test_acceptance
 	$(DOCKER_COMPOSE) down -v -t 0
 
@@ -338,6 +348,7 @@ TEST_ACCEPTANCE_MODULES_MERGED_VARIANTS = \
 
 $(TEST_ACCEPTANCE_MODULES_MERGED_VARIANTS):
 	$(DOCKER_COMPOSE) down -v -t 0
+	$(TEST_ACCEPTANCE_MONGO_INIT)
 	$(DOCKER_COMPOSE) run --rm test_acceptance make test_acceptance_modules_merged_inner
 	$(DOCKER_COMPOSE) down -v -t 0
 
@@ -360,6 +371,7 @@ $(TEST_ACCEPTANCE_MODULES_MERGED_SPLIT_SAAS): export BASE_CONFIG = $(CFG_SAAS)
 
 $(TEST_ACCEPTANCE_MODULES_MERGED_SPLIT_SAAS): test_acceptance_modules_merged_saas_%:
 	$(DOCKER_COMPOSE) down -v -t 0
+	$(TEST_ACCEPTANCE_MONGO_INIT)
 	$(DOCKER_COMPOSE) run --rm test_acceptance make test_acceptance_modules_merged_inner_$*
 	$(DOCKER_COMPOSE) down -v -t 0
 

services/web/app/src/infrastructure/mongodb.js

@@ -1,5 +1,6 @@
-const { ObjectId } = require('mongodb')
+const { ObjectId, ReadPreference } = require('mongodb')
 const OError = require('@overleaf/o-error')
+const Settings = require('@overleaf/settings')
 const { getNativeDb } = require('./Mongoose')
 
 if (
@@ -11,6 +12,11 @@ if (
   )
 }
 
+const READ_PREFERENCE_PRIMARY = ReadPreference.primary
+const READ_PREFERENCE_SECONDARY = Settings.mongo.hasSecondaries
+  ? ReadPreference.secondary
+  : ReadPreference.secondaryPreferred
+
 let setupDbPromise
 async function waitForDb() {
   if (!setupDbPromise) {
@@ -112,4 +118,6 @@ module.exports = {
   getCollectionInternal,
   dropTestDatabase,
   waitForDb,
+  READ_PREFERENCE_PRIMARY,
+  READ_PREFERENCE_SECONDARY,
 }

services/web/config/settings.defaults.js

@@ -97,6 +97,7 @@ module.exports = {
       process.env.MONGO_CONNECTION_STRING ||
       process.env.MONGO_URL ||
       `mongodb://${process.env.MONGO_HOST || '127.0.0.1'}/sharelatex`,
+    hasSecondaries: process.env.MONGO_HAS_SECONDARIES === 'true',
   },
 
   redis: {

services/web/docker-compose.ci.yml

@@ -94,6 +94,7 @@ services:
 
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
 
   ldap:
     restart: always

services/web/docker-compose.yml

@@ -90,6 +90,7 @@ services:
 
   mongo:
     image: mongo:4.4.16
+    command: --replSet overleaf
 
   ldap:
     restart: always

services/web/modules/history-migration/app/src/HistoryUpgradeHelper.js

@@ -1,5 +1,8 @@
-const { ReadPreference, ObjectId } = require('mongodb')
+const { ObjectId } = require('mongodb')
-const { db } = require('../../../../app/src/infrastructure/mongodb')
+const {
+  db,
+  READ_PREFERENCE_SECONDARY,
+} = require('../../../../app/src/infrastructure/mongodb')
 const Settings = require('@overleaf/settings')
 
 const ProjectHistoryHandler = require('../../../../app/src/Features/Project/ProjectHistoryHandler')
@@ -311,7 +314,7 @@ async function shouldPreserveHistory(project) {
         { preserveHistory: { $eq: true } },
       ],
     },
-    { readPreference: ReadPreference.SECONDARY }
+    { readPreference: READ_PREFERENCE_SECONDARY }
   )
 }
 
@@ -320,7 +323,7 @@ async function anyDocHistoryExists(project) {
     { project_id: { $eq: project._id } },
     {
       projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
+      readPreference: READ_PREFERENCE_SECONDARY,
     }
   )
 }
@@ -330,7 +333,7 @@ async function anyDocHistoryIndexExists(project) {
     { project_id: { $eq: project._id } },
     {
       projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
+      readPreference: READ_PREFERENCE_SECONDARY,
     }
   )
 }

services/web/scripts/back_fill_dummy_doc_meta.js

@@ -1,6 +1,10 @@
 const { promisify } = require('util')
-const { ObjectId, ReadPreference } = require('mongodb')
+const { ObjectId } = require('mongodb')
-const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
+const {
+  db,
+  waitForDb,
+  READ_PREFERENCE_SECONDARY,
+} = require('../app/src/infrastructure/mongodb')
 const sleep = promisify(setTimeout)
 const _ = require('lodash')
 
@@ -64,7 +68,7 @@ async function main(options) {
       },
     }
     const docs = await db.docs
-      .find(query, { readPreference: ReadPreference.SECONDARY })
+      .find(query, { readPreference: READ_PREFERENCE_SECONDARY })
       .project({ _id: 1, project_id: 1 })
       .limit(options.batchSize)
       .toArray()

services/web/scripts/back_fill_staff_access.js

@@ -1,5 +1,8 @@
-const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
+const {
-const { ReadPreference } = require('mongodb')
+  db,
+  waitForDb,
+  READ_PREFERENCE_SECONDARY,
+} = require('../app/src/infrastructure/mongodb')
 const UserSessionsManager = require('../app/src/Features/User/UserSessionsManager')
 
 const COMMIT = process.argv.includes('--commit')
@@ -46,7 +49,7 @@ async function main() {
           email: 1,
           staffAccess: 1,
         },
-        readPreference: ReadPreference.SECONDARY,
+        readPreference: READ_PREFERENCE_SECONDARY,
       }
     )
     .toArray()

services/web/scripts/count_encrypted_access_tokens.js

@@ -2,8 +2,11 @@ const TEN_MINUTES = 1000 * 60 * 10
 process.env.MONGO_SOCKET_TIMEOUT =
   process.env.MONGO_SOCKET_TIMEOUT || TEN_MINUTES.toString()
 
-const { ReadPreference } = require('mongodb')
+const {
-const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
+  db,
+  waitForDb,
+  READ_PREFERENCE_SECONDARY,
+} = require('../app/src/infrastructure/mongodb')
 const _ = require('lodash')
 const {
   formatTokenUsageStats,
@@ -32,7 +35,7 @@ async function count(collectionName, paths) {
   const cursor = collection.find(
     {},
     {
-      readPreference: ReadPreference.SECONDARY,
+      readPreference: READ_PREFERENCE_SECONDARY,
       projection,
     }
   )

services/web/scripts/count_image_files.js

@@ -1,5 +1,8 @@
-const { ReadPreference } = require('mongodb')
+const {
-const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
+  db,
+  waitForDb,
+  READ_PREFERENCE_SECONDARY,
+} = require('../app/src/infrastructure/mongodb')
 const { extname } = require('node:path')
 
 const FILE_TYPES = [
@@ -22,7 +25,10 @@ async function main() {
   await waitForDb()
   const projects = db.projects.find(
     {},
-    { projection: { rootFolder: 1 }, readPreference: ReadPreference.SECONDARY }
+    {
+      projection: { rootFolder: 1 },
+      readPreference: READ_PREFERENCE_SECONDARY,
+    }
   )
   let projectsProcessed = 0
   const result = new Map(FILE_TYPES.map(fileType => [fileType, 0]))

services/web/scripts/delete_orphaned_data_helper.js

@@ -1,5 +1,8 @@
-const { ReadPreference } = require('mongodb')
+const {
-const { db } = require('../app/src/infrastructure/mongodb')
+  db,
+  READ_PREFERENCE_PRIMARY,
+  READ_PREFERENCE_SECONDARY,
+} = require('../app/src/infrastructure/mongodb')
 const { promiseMapWithLimit } = require('../app/src/util/promises')
 
 async function getDeletedProject(projectId, readPreference) {
@@ -57,14 +60,14 @@ async function checkProjectExistsWithReadPreference(projectId, readPreference) {
 async function checkProjectExistsOnPrimary(projectId) {
   return await checkProjectExistsWithReadPreference(
     projectId,
-    ReadPreference.PRIMARY
+    READ_PREFERENCE_PRIMARY
   )
 }
 
 async function checkProjectExistsOnSecondary(projectId) {
   return await checkProjectExistsWithReadPreference(
     projectId,
-    ReadPreference.SECONDARY
+    READ_PREFERENCE_SECONDARY
   )
 }
 

services/web/scripts/delete_orphaned_docs_online_check.js

@@ -1,7 +1,12 @@
 const DocstoreManager = require('../app/src/Features/Docstore/DocstoreManager')
 const { promisify } = require('util')
-const { ObjectId, ReadPreference } = require('mongodb')
+const { ObjectId } = require('mongodb')
-const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
+const {
+  db,
+  waitForDb,
+  READ_PREFERENCE_PRIMARY,
+  READ_PREFERENCE_SECONDARY,
+} = require('../app/src/infrastructure/mongodb')
 const { promiseMapWithLimit } = require('../app/src/util/promises')
 const { getHardDeletedProjectIds } = require('./delete_orphaned_data_helper')
 const sleep = promisify(setTimeout)
@@ -52,7 +57,7 @@ async function main() {
       },
     }
     const docs = await db.docs
-      .find(query, { readPreference: ReadPreference.SECONDARY })
+      .find(query, { readPreference: READ_PREFERENCE_SECONDARY })
       .project({ project_id: 1 })
       .sort({ project_id: 1 })
       .limit(BATCH_SIZE)
@@ -97,7 +102,7 @@ async function getProjectDocs(projectId) {
       { project_id: projectId },
       {
         projection: { _id: 1 },
-        readPreference: ReadPreference.PRIMARY,
+        readPreference: READ_PREFERENCE_PRIMARY,
       }
     )
     .toArray()

services/web/scripts/find_malformed_filetrees.js

@@ -1,11 +1,17 @@
-const { ReadPreference } = require('mongodb')
+const {
-const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
+  db,
+  waitForDb,
+  READ_PREFERENCE_SECONDARY,
+} = require('../app/src/infrastructure/mongodb')
 
 async function main() {
   await waitForDb()
   const projects = db.projects.find(
     {},
-    { projection: { rootFolder: 1 }, readPreference: ReadPreference.SECONDARY }
+    {
+      projection: { rootFolder: 1 },
+      readPreference: READ_PREFERENCE_SECONDARY,
+    }
   )
   let projectsProcessed = 0
   for await (const project of projects) {

services/web/scripts/helpers/batchedUpdate.js

@@ -1,5 +1,9 @@
-const { ReadPreference, ObjectId } = require('mongodb')
+const { ObjectId } = require('mongodb')
-const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
+const {
+  db,
+  waitForDb,
+  READ_PREFERENCE_SECONDARY,
+} = require('../../app/src/infrastructure/mongodb')
 
 const ONE_MONTH_IN_MS = 1000 * 60 * 60 * 24 * 31
 let ID_EDGE_PAST
@@ -129,7 +133,7 @@ async function batchedUpdate(
   refreshGlobalOptionsForBatchedUpdate(batchedUpdateOptions)
 
   findOptions = findOptions || {}
-  findOptions.readPreference = ReadPreference.SECONDARY
+  findOptions.readPreference = READ_PREFERENCE_SECONDARY
 
   projection = projection || { _id: 1 }
   let nextBatch

services/web/scripts/history/downgrade_project.js

@@ -5,8 +5,12 @@ process.env.MONGO_SOCKET_TIMEOUT =
 
 const PROJECT_ID = process.env.PROJECT_ID
 
-const { ReadPreference, ObjectId } = require('mongodb')
+const { ObjectId } = require('mongodb')
-const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
+const {
+  db,
+  waitForDb,
+  READ_PREFERENCE_SECONDARY,
+} = require('../../app/src/infrastructure/mongodb')
 const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler')
 
 console.log({
@@ -43,7 +47,7 @@ async function shouldPreserveHistory(project) {
         { preserveHistory: { $eq: true } },
       ],
     },
-    { readPreference: ReadPreference.SECONDARY }
+    { readPreference: READ_PREFERENCE_SECONDARY }
   )
 }
 

services/web/scripts/history/unset_allow_downgrade.js

@@ -1,6 +1,10 @@
 const { promisify } = require('util')
-const { ObjectId, ReadPreference } = require('mongodb')
+const { ObjectId } = require('mongodb')
-const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
+const {
+  db,
+  waitForDb,
+  READ_PREFERENCE_SECONDARY,
+} = require('../../app/src/infrastructure/mongodb')
 const sleep = promisify(setTimeout)
 const _ = require('lodash')
 
@@ -63,7 +67,7 @@ async function main(options) {
       'overleaf.history.allowDowngrade': true,
     }
     const projects = await db.projects
-      .find(query, { readPreference: ReadPreference.SECONDARY })
+      .find(query, { readPreference: READ_PREFERENCE_SECONDARY })
       .project({ _id: 1 })
       .limit(options.batchSize)
       .toArray()

services/web/scripts/history/upgrade_none_without_conversion_if_no_sl_history.js

@@ -14,8 +14,10 @@ process.env.BATCH_SIZE = BATCH_SIZE
 process.env.MONGO_SOCKET_TIMEOUT =
   parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
 
-const { ReadPreference } = require('mongodb')
+const {
-const { db } = require('../../app/src/infrastructure/mongodb')
+  db,
+  READ_PREFERENCE_SECONDARY,
+} = require('../../app/src/infrastructure/mongodb')
 const { promiseMapWithLimit } = require('../../app/src/util/promises')
 const { batchedUpdate } = require('../helpers/batchedUpdate')
 const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler')
@@ -172,7 +174,7 @@ async function anyDocHistoryExists(project) {
     { project_id: { $eq: project._id } },
     {
       projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
+      readPreference: READ_PREFERENCE_SECONDARY,
     }
   )
 }
@@ -182,7 +184,7 @@ async function anyDocHistoryIndexExists(project) {
     { project_id: { $eq: project._id } },
     {
       projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
+      readPreference: READ_PREFERENCE_SECONDARY,
     }
   )
 }

services/web/scripts/history/upgrade_project.js

@@ -1,5 +1,9 @@
-const { ReadPreference, ObjectId } = require('mongodb')
+const { ObjectId } = require('mongodb')
-const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
+const {
+  db,
+  waitForDb,
+  READ_PREFERENCE_SECONDARY,
+} = require('../../app/src/infrastructure/mongodb')
 const {
   upgradeProject,
 } = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
@@ -20,7 +24,7 @@ async function main() {
   }
   const options = {
     projection,
-    readPreference: ReadPreference.SECONDARY,
+    readPreference: READ_PREFERENCE_SECONDARY,
   }
   const project = await db.projects.findOne(query, options)
   if (project) {

services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js

@@ -16,8 +16,12 @@ process.env.MONGO_SOCKET_TIMEOUT =
 
 const PROJECT_ID = process.env.PROJECT_ID
 
-const { ReadPreference, ObjectId } = require('mongodb')
+const { ObjectId } = require('mongodb')
-const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
+const {
+  db,
+  waitForDb,
+  READ_PREFERENCE_SECONDARY,
+} = require('../../app/src/infrastructure/mongodb')
 const { promiseMapWithLimit } = require('../../app/src/util/promises')
 const { batchedUpdate } = require('../helpers/batchedUpdate')
 const ProjectHistoryController = require('../../modules/history-migration/app/src/ProjectHistoryController')
@@ -165,7 +169,7 @@ async function shouldPreserveHistory(project) {
         { preserveHistory: { $eq: true } },
       ],
     },
-    { readPreference: ReadPreference.SECONDARY }
+    { readPreference: READ_PREFERENCE_SECONDARY }
   )
 }
 
@@ -174,7 +178,7 @@ async function anyDocHistoryExists(project) {
     { project_id: { $eq: project._id } },
     {
       projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
+      readPreference: READ_PREFERENCE_SECONDARY,
     }
   )
 }
@@ -184,7 +188,7 @@ async function anyDocHistoryIndexExists(project) {
     { project_id: { $eq: project._id } },
     {
       projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
+      readPreference: READ_PREFERENCE_SECONDARY,
     }
   )
 }

services/web/scripts/history/upgrade_v1_without_conversion_if_created_after_fph_enabled.js

@@ -10,8 +10,11 @@ process.env.BATCH_SIZE = BATCH_SIZE
 process.env.MONGO_SOCKET_TIMEOUT =
   parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
 
-const { ReadPreference, ObjectId } = require('mongodb')
+const { ObjectId } = require('mongodb')
-const { db } = require('../../app/src/infrastructure/mongodb')
+const {
+  db,
+  READ_PREFERENCE_SECONDARY,
+} = require('../../app/src/infrastructure/mongodb')
 const { promiseMapWithLimit } = require('../../app/src/util/promises')
 const { batchedUpdate } = require('../helpers/batchedUpdate')
 
@@ -111,7 +114,7 @@ async function anyDocHistoryExists(project) {
     { project_id: { $eq: project._id } },
     {
       projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
+      readPreference: READ_PREFERENCE_SECONDARY,
     }
   )
 }
@@ -121,7 +124,7 @@ async function anyDocHistoryIndexExists(project) {
     { project_id: { $eq: project._id } },
     {
       projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
+      readPreference: READ_PREFERENCE_SECONDARY,
     }
   )
 }

services/web/scripts/history/upgrade_v1_without_conversion_if_no_sl_history.js

@@ -12,8 +12,10 @@ process.env.BATCH_SIZE = BATCH_SIZE
 process.env.MONGO_SOCKET_TIMEOUT =
   parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
 
-const { ReadPreference } = require('mongodb')
+const {
-const { db } = require('../../app/src/infrastructure/mongodb')
+  db,
+  READ_PREFERENCE_SECONDARY,
+} = require('../../app/src/infrastructure/mongodb')
 const { promiseMapWithLimit } = require('../../app/src/util/promises')
 const { batchedUpdate } = require('../helpers/batchedUpdate')
 
@@ -99,7 +101,7 @@ async function shouldPreserveHistory(project) {
         { preserveHistory: { $eq: true } },
       ],
     },
-    { readPreference: ReadPreference.SECONDARY }
+    { readPreference: READ_PREFERENCE_SECONDARY }
   )
 }
 
@@ -108,7 +110,7 @@ async function anyDocHistoryExists(project) {
     { project_id: { $eq: project._id } },
     {
       projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
+      readPreference: READ_PREFERENCE_SECONDARY,
     }
   )
 }
@@ -118,7 +120,7 @@ async function anyDocHistoryIndexExists(project) {
     { project_id: { $eq: project._id } },
     {
       projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
+      readPreference: READ_PREFERENCE_SECONDARY,
     }
   )
 }

services/web/scripts/oauth/remove_client.js

@@ -1,6 +1,9 @@
 const minimist = require('minimist')
-const { ReadPreference } = require('mongodb')
+const {
-const { waitForDb, db } = require('../../app/src/infrastructure/mongodb')
+  waitForDb,
+  db,
+  READ_PREFERENCE_SECONDARY,
+} = require('../../app/src/infrastructure/mongodb')
 
 async function main() {
   const opts = parseArgs()
@@ -49,7 +52,7 @@ async function countAccessTokens(applicationId) {
     {
       oauthApplication_id: applicationId,
     },
-    { readPreference: ReadPreference.secondary }
+    { readPreference: READ_PREFERENCE_SECONDARY }
   )
 }
 
@@ -58,7 +61,7 @@ async function countAuthorizationCodes(applicationId) {
     {
       oauthApplication_id: applicationId,
     },
-    { readPreference: ReadPreference.secondary }
+    { readPreference: READ_PREFERENCE_SECONDARY }
   )
 }
 

services/web/scripts/regenerate_duplicate_referral_ids.js

@@ -4,8 +4,10 @@ const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
 // persist fallback in order to keep batchedUpdate in-sync
 process.env.BATCH_SIZE = BATCH_SIZE
 
-const { ReadPreference } = require('mongodb')
+const {
-const { db } = require('../app/src/infrastructure/mongodb')
+  db,
+  READ_PREFERENCE_SECONDARY,
+} = require('../app/src/infrastructure/mongodb')
 const { promiseMapWithLimit } = require('../app/src/util/promises')
 const TokenGenerator = require('../app/src/Features/TokenGenerator/TokenGenerator')
 const { batchedUpdate } = require('./helpers/batchedUpdate')
@@ -24,7 +26,7 @@ async function rewriteDuplicates(duplicateReferralIds) {
             { referal_id: referralId },
             {
               projection: { _id: 1 },
-              readPreference: ReadPreference.SECONDARY,
+              readPreference: READ_PREFERENCE_SECONDARY,
             }
           )
           .toArray()
@@ -74,7 +76,7 @@ async function processBatch(users) {
     .find(
       { referal_id: { $in: uniqueReferalIdsInBatch } },
       {
-        readPreference: ReadPreference.SECONDARY,
+        readPreference: READ_PREFERENCE_SECONDARY,
         projection: { _id: true },
       }
     )