diff --git a/libraries/access-token-encryptor/scripts/helpers/re-encrypt-tokens.js b/libraries/access-token-encryptor/scripts/helpers/re-encrypt-tokens.js index abf4d53c12..df65836c23 100644 --- a/libraries/access-token-encryptor/scripts/helpers/re-encrypt-tokens.js +++ b/libraries/access-token-encryptor/scripts/helpers/re-encrypt-tokens.js @@ -56,7 +56,7 @@ async function reEncryptTokensInCollection({ const cursor = collection.find( {}, { - readPreference: ReadPreference.SECONDARY, + readPreference: ReadPreference.secondaryPreferred, projection, } ) diff --git a/services/chat/Makefile b/services/chat/Makefile index b6188d7c25..4ca99b3ed6 100644 --- a/services/chat/Makefile +++ b/services/chat/Makefile @@ -79,6 +79,13 @@ test_acceptance_clean: $(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0 test_acceptance_pre_run: + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \ + while ! mongo --eval "db.version()" > /dev/null; do \ + echo "Waiting for Mongo..."; \ + sleep 1; \ + done; \ + mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"' ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run endif diff --git a/services/chat/docker-compose.ci.yml b/services/chat/docker-compose.ci.yml index cc5d75f7b4..2fcf595760 100644 --- a/services/chat/docker-compose.ci.yml +++ b/services/chat/docker-compose.ci.yml @@ -43,6 +43,7 @@ services: user: root mongo: image: mongo:4.4.16 + command: --replSet overleaf healthcheck: test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" interval: 1s diff --git a/services/chat/docker-compose.yml b/services/chat/docker-compose.yml index 6bea2beb9b..99738494e6 100644 --- a/services/chat/docker-compose.yml +++ b/services/chat/docker-compose.yml @@ -45,6 +45,7 @@ services: mongo: image: mongo:4.4.16 + command: --replSet overleaf healthcheck: test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" interval: 1s diff --git a/services/contacts/Makefile b/services/contacts/Makefile index fea41a26df..205f4ada38 100644 --- a/services/contacts/Makefile +++ b/services/contacts/Makefile @@ -79,6 +79,13 @@ test_acceptance_clean: $(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0 test_acceptance_pre_run: + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \ + while ! mongo --eval "db.version()" > /dev/null; do \ + echo "Waiting for Mongo..."; \ + sleep 1; \ + done; \ + mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"' ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run endif diff --git a/services/contacts/docker-compose.ci.yml b/services/contacts/docker-compose.ci.yml index cc5d75f7b4..2fcf595760 100644 --- a/services/contacts/docker-compose.ci.yml +++ b/services/contacts/docker-compose.ci.yml @@ -43,6 +43,7 @@ services: user: root mongo: image: mongo:4.4.16 + command: --replSet overleaf healthcheck: test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" interval: 1s diff --git a/services/contacts/docker-compose.yml b/services/contacts/docker-compose.yml index 872d708329..745b8ff710 100644 --- a/services/contacts/docker-compose.yml +++ b/services/contacts/docker-compose.yml @@ -45,6 +45,7 @@ services: mongo: image: mongo:4.4.16 + command: --replSet overleaf healthcheck: test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" interval: 1s diff --git a/services/docstore/Makefile b/services/docstore/Makefile index fb20b1ea75..0f4976402e 100644 --- a/services/docstore/Makefile +++ b/services/docstore/Makefile @@ -79,6 +79,13 @@ test_acceptance_clean: $(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0 test_acceptance_pre_run: + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \ + while ! mongo --eval "db.version()" > /dev/null; do \ + echo "Waiting for Mongo..."; \ + sleep 1; \ + done; \ + mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"' ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run endif diff --git a/services/docstore/docker-compose.ci.yml b/services/docstore/docker-compose.ci.yml index 56b6fc9ac7..aad0b1d2c0 100644 --- a/services/docstore/docker-compose.ci.yml +++ b/services/docstore/docker-compose.ci.yml @@ -52,6 +52,7 @@ services: user: root mongo: image: mongo:4.4.16 + command: --replSet overleaf healthcheck: test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" interval: 1s diff --git a/services/docstore/docker-compose.yml b/services/docstore/docker-compose.yml index e493391a30..e95e5f11e6 100644 --- a/services/docstore/docker-compose.yml +++ b/services/docstore/docker-compose.yml @@ -54,6 +54,7 @@ services: mongo: image: mongo:4.4.16 + command: --replSet overleaf healthcheck: test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" interval: 1s diff --git a/services/document-updater/Makefile b/services/document-updater/Makefile index 38d3f09d53..f70b4015be 100644 --- a/services/document-updater/Makefile +++ b/services/document-updater/Makefile @@ -79,6 +79,13 @@ test_acceptance_clean: $(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0 test_acceptance_pre_run: + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \ + while ! mongo --eval "db.version()" > /dev/null; do \ + echo "Waiting for Mongo..."; \ + sleep 1; \ + done; \ + mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"' ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run endif diff --git a/services/document-updater/docker-compose.ci.yml b/services/document-updater/docker-compose.ci.yml index e536ed999d..b4f04f33bd 100644 --- a/services/document-updater/docker-compose.ci.yml +++ b/services/document-updater/docker-compose.ci.yml @@ -52,6 +52,7 @@ services: mongo: image: mongo:4.4.16 + command: --replSet overleaf healthcheck: test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" interval: 1s diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml index 256b7ed9f2..23e2797aa5 100644 --- a/services/document-updater/docker-compose.yml +++ b/services/document-updater/docker-compose.yml @@ -54,6 +54,7 @@ services: mongo: image: mongo:4.4.16 + command: --replSet overleaf healthcheck: test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" interval: 1s diff --git a/services/history-v1/Makefile b/services/history-v1/Makefile index 88dcfdd09e..ae179680d8 100644 --- a/services/history-v1/Makefile +++ b/services/history-v1/Makefile @@ -79,6 +79,13 @@ test_acceptance_clean: $(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0 test_acceptance_pre_run: + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \ + while ! mongo --eval "db.version()" > /dev/null; do \ + echo "Waiting for Mongo..."; \ + sleep 1; \ + done; \ + mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"' ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run endif diff --git a/services/history-v1/docker-compose.ci.yml b/services/history-v1/docker-compose.ci.yml index c6645aeef7..443c479b83 100644 --- a/services/history-v1/docker-compose.ci.yml +++ b/services/history-v1/docker-compose.ci.yml @@ -51,6 +51,7 @@ services: user: root mongo: image: mongo:4.4.16 + command: --replSet overleaf healthcheck: test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" interval: 1s diff --git a/services/history-v1/docker-compose.yml b/services/history-v1/docker-compose.yml index c953644e19..8ba1f14b34 100644 --- a/services/history-v1/docker-compose.yml +++ b/services/history-v1/docker-compose.yml @@ -53,6 +53,7 @@ services: mongo: image: mongo:4.4.16 + command: --replSet overleaf healthcheck: test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" interval: 1s diff --git a/services/history-v1/storage/lib/chunk_store/mongo.js b/services/history-v1/storage/lib/chunk_store/mongo.js index 4025e866c7..29d522564c 100644 --- a/services/history-v1/storage/lib/chunk_store/mongo.js +++ b/services/history-v1/storage/lib/chunk_store/mongo.js @@ -113,7 +113,7 @@ async function insertPendingChunk(projectId, chunk) { /** * Record that a new chunk was created. */ -async function confirmCreate(projectId, chunk, chunkId) { +async function confirmCreate(projectId, chunk, chunkId, mongoOpts = {}) { assert.mongoId(projectId, 'bad projectId') assert.instance(chunk, Chunk, 'bad chunk') assert.mongoId(chunkId, 'bad chunkId') @@ -126,7 +126,8 @@ async function confirmCreate(projectId, chunk, chunkId) { projectId: ObjectId(projectId), state: 'pending', }, - { $set: { state: 'active', updatedAt: new Date() } } + { $set: { state: 'active', updatedAt: new Date() } }, + mongoOpts ) } catch (err) { if (err.code === DUPLICATE_KEY_ERROR_CODE) { @@ -155,8 +156,8 @@ async function confirmUpdate(projectId, oldChunkId, newChunk, newChunkId) { const session = mongodb.client.startSession() try { await session.withTransaction(async () => { - await deleteChunk(projectId, oldChunkId) - await confirmCreate(projectId, newChunk, newChunkId) + await deleteChunk(projectId, oldChunkId, { session }) + await confirmCreate(projectId, newChunk, newChunkId, { session }) }) } finally { await session.endSession() @@ -170,13 +171,14 @@ async function confirmUpdate(projectId, oldChunkId, newChunk, newChunkId) { * @param {number} chunkId * @return {Promise} */ -async function deleteChunk(projectId, chunkId) { +async function deleteChunk(projectId, chunkId, mongoOpts = {}) { assert.mongoId(projectId, 'bad projectId') assert.mongoId(chunkId, 'bad chunkId') await mongodb.chunks.updateOne( { _id: ObjectId(chunkId), projectId: ObjectId(projectId) }, - { $set: { state: 'deleted', updatedAt: new Date() } } + { $set: { state: 'deleted', updatedAt: new Date() } }, + mongoOpts ) } diff --git a/services/notifications/Makefile b/services/notifications/Makefile index 4a34f596d0..4379ee55d0 100644 --- a/services/notifications/Makefile +++ b/services/notifications/Makefile @@ -79,6 +79,13 @@ test_acceptance_clean: $(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0 test_acceptance_pre_run: + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \ + while ! mongo --eval "db.version()" > /dev/null; do \ + echo "Waiting for Mongo..."; \ + sleep 1; \ + done; \ + mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"' ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run endif diff --git a/services/notifications/docker-compose.ci.yml b/services/notifications/docker-compose.ci.yml index cc5d75f7b4..2fcf595760 100644 --- a/services/notifications/docker-compose.ci.yml +++ b/services/notifications/docker-compose.ci.yml @@ -43,6 +43,7 @@ services: user: root mongo: image: mongo:4.4.16 + command: --replSet overleaf healthcheck: test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" interval: 1s diff --git a/services/notifications/docker-compose.yml b/services/notifications/docker-compose.yml index 5e88059baf..d0044e081c 100644 --- a/services/notifications/docker-compose.yml +++ b/services/notifications/docker-compose.yml @@ -45,6 +45,7 @@ services: mongo: image: mongo:4.4.16 + command: --replSet overleaf healthcheck: test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" interval: 1s diff --git a/services/project-history/Makefile b/services/project-history/Makefile index 32019674da..bdcb5faf00 100644 --- a/services/project-history/Makefile +++ b/services/project-history/Makefile @@ -79,6 +79,13 @@ test_acceptance_clean: $(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0 test_acceptance_pre_run: + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \ + while ! mongo --eval "db.version()" > /dev/null; do \ + echo "Waiting for Mongo..."; \ + sleep 1; \ + done; \ + mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"' ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run endif diff --git a/services/project-history/docker-compose.ci.yml b/services/project-history/docker-compose.ci.yml index e536ed999d..b4f04f33bd 100644 --- a/services/project-history/docker-compose.ci.yml +++ b/services/project-history/docker-compose.ci.yml @@ -52,6 +52,7 @@ services: mongo: image: mongo:4.4.16 + command: --replSet overleaf healthcheck: test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" interval: 1s diff --git a/services/project-history/docker-compose.yml b/services/project-history/docker-compose.yml index 9e85657a3a..3444430bcf 100644 --- a/services/project-history/docker-compose.yml +++ b/services/project-history/docker-compose.yml @@ -54,6 +54,7 @@ services: mongo: image: mongo:4.4.16 + command: --replSet overleaf healthcheck: test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" interval: 1s diff --git a/services/track-changes/Makefile b/services/track-changes/Makefile index 489e8f65f2..652a0c37c8 100644 --- a/services/track-changes/Makefile +++ b/services/track-changes/Makefile @@ -79,6 +79,13 @@ test_acceptance_clean: $(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0 test_acceptance_pre_run: + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \ + while ! mongo --eval "db.version()" > /dev/null; do \ + echo "Waiting for Mongo..."; \ + sleep 1; \ + done; \ + mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"' ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run endif diff --git a/services/track-changes/docker-compose.ci.yml b/services/track-changes/docker-compose.ci.yml index 2fb8c9b605..c179b32fae 100644 --- a/services/track-changes/docker-compose.ci.yml +++ b/services/track-changes/docker-compose.ci.yml @@ -59,6 +59,7 @@ services: mongo: image: mongo:4.4.16 + command: --replSet overleaf healthcheck: test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" interval: 1s diff --git a/services/track-changes/docker-compose.yml b/services/track-changes/docker-compose.yml index e9a78a4754..10da1e6a79 100644 --- a/services/track-changes/docker-compose.yml +++ b/services/track-changes/docker-compose.yml @@ -61,6 +61,7 @@ services: mongo: image: mongo:4.4.16 + command: --replSet overleaf healthcheck: test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" interval: 1s diff --git a/services/web/Makefile b/services/web/Makefile index b55ce27f70..d5ae799d44 100644 --- a/services/web/Makefile +++ b/services/web/Makefile @@ -170,6 +170,15 @@ test_frontend_ct_modules: # Acceptance tests # +TEST_ACCEPTANCE_MONGO_INIT := \ + $(DOCKER_COMPOSE) up -d mongo; \ + $(DOCKER_COMPOSE) exec -T mongo sh -c ' \ + while ! mongo --eval "db.version()" > /dev/null; do \ + echo "Waiting for Mongo..."; \ + sleep 1; \ + done; \ + mongo --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"' + test_acceptance: test_acceptance_app test_acceptance_modules test_acceptance_saas: test_acceptance_app_saas test_acceptance_modules_merged_saas test_acceptance_server_ce: test_acceptance_app_server_ce test_acceptance_modules_merged_server_ce @@ -190,6 +199,7 @@ test_acceptance_app_server_pro: export SHARELATEX_CONFIG=$(CFG_SERVER_PRO) $(TEST_ACCEPTANCE_APP): $(DOCKER_COMPOSE) down -v -t 0 + $(TEST_ACCEPTANCE_MONGO_INIT) $(DOCKER_COMPOSE) run --rm test_acceptance $(DOCKER_COMPOSE) down -v -t 0 @@ -338,6 +348,7 @@ TEST_ACCEPTANCE_MODULES_MERGED_VARIANTS = \ $(TEST_ACCEPTANCE_MODULES_MERGED_VARIANTS): $(DOCKER_COMPOSE) down -v -t 0 + $(TEST_ACCEPTANCE_MONGO_INIT) $(DOCKER_COMPOSE) run --rm test_acceptance make test_acceptance_modules_merged_inner $(DOCKER_COMPOSE) down -v -t 0 @@ -360,6 +371,7 @@ $(TEST_ACCEPTANCE_MODULES_MERGED_SPLIT_SAAS): export BASE_CONFIG = $(CFG_SAAS) $(TEST_ACCEPTANCE_MODULES_MERGED_SPLIT_SAAS): test_acceptance_modules_merged_saas_%: $(DOCKER_COMPOSE) down -v -t 0 + $(TEST_ACCEPTANCE_MONGO_INIT) $(DOCKER_COMPOSE) run --rm test_acceptance make test_acceptance_modules_merged_inner_$* $(DOCKER_COMPOSE) down -v -t 0 diff --git a/services/web/app/src/infrastructure/mongodb.js b/services/web/app/src/infrastructure/mongodb.js index db2c8a36e3..7aa44ae76a 100644 --- a/services/web/app/src/infrastructure/mongodb.js +++ b/services/web/app/src/infrastructure/mongodb.js @@ -1,5 +1,6 @@ -const { ObjectId } = require('mongodb') +const { ObjectId, ReadPreference } = require('mongodb') const OError = require('@overleaf/o-error') +const Settings = require('@overleaf/settings') const { getNativeDb } = require('./Mongoose') if ( @@ -11,6 +12,11 @@ if ( ) } +const READ_PREFERENCE_PRIMARY = ReadPreference.primary +const READ_PREFERENCE_SECONDARY = Settings.mongo.hasSecondaries + ? ReadPreference.secondary + : ReadPreference.secondaryPreferred + let setupDbPromise async function waitForDb() { if (!setupDbPromise) { @@ -112,4 +118,6 @@ module.exports = { getCollectionInternal, dropTestDatabase, waitForDb, + READ_PREFERENCE_PRIMARY, + READ_PREFERENCE_SECONDARY, } diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index 898b2b0dc7..6a12236442 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -97,6 +97,7 @@ module.exports = { process.env.MONGO_CONNECTION_STRING || process.env.MONGO_URL || `mongodb://${process.env.MONGO_HOST || '127.0.0.1'}/sharelatex`, + hasSecondaries: process.env.MONGO_HAS_SECONDARIES === 'true', }, redis: { diff --git a/services/web/docker-compose.ci.yml b/services/web/docker-compose.ci.yml index 9958acc88c..f6a14969d3 100644 --- a/services/web/docker-compose.ci.yml +++ b/services/web/docker-compose.ci.yml @@ -94,6 +94,7 @@ services: mongo: image: mongo:4.4.16 + command: --replSet overleaf ldap: restart: always diff --git a/services/web/docker-compose.yml b/services/web/docker-compose.yml index a6ad9ea05d..a298cab572 100644 --- a/services/web/docker-compose.yml +++ b/services/web/docker-compose.yml @@ -90,6 +90,7 @@ services: mongo: image: mongo:4.4.16 + command: --replSet overleaf ldap: restart: always diff --git a/services/web/modules/history-migration/app/src/HistoryUpgradeHelper.js b/services/web/modules/history-migration/app/src/HistoryUpgradeHelper.js index 3d0c950408..2f44a4d9d1 100644 --- a/services/web/modules/history-migration/app/src/HistoryUpgradeHelper.js +++ b/services/web/modules/history-migration/app/src/HistoryUpgradeHelper.js @@ -1,5 +1,8 @@ -const { ReadPreference, ObjectId } = require('mongodb') -const { db } = require('../../../../app/src/infrastructure/mongodb') +const { ObjectId } = require('mongodb') +const { + db, + READ_PREFERENCE_SECONDARY, +} = require('../../../../app/src/infrastructure/mongodb') const Settings = require('@overleaf/settings') const ProjectHistoryHandler = require('../../../../app/src/Features/Project/ProjectHistoryHandler') @@ -311,7 +314,7 @@ async function shouldPreserveHistory(project) { { preserveHistory: { $eq: true } }, ], }, - { readPreference: ReadPreference.SECONDARY } + { readPreference: READ_PREFERENCE_SECONDARY } ) } @@ -320,7 +323,7 @@ async function anyDocHistoryExists(project) { { project_id: { $eq: project._id } }, { projection: { _id: 1 }, - readPreference: ReadPreference.SECONDARY, + readPreference: READ_PREFERENCE_SECONDARY, } ) } @@ -330,7 +333,7 @@ async function anyDocHistoryIndexExists(project) { { project_id: { $eq: project._id } }, { projection: { _id: 1 }, - readPreference: ReadPreference.SECONDARY, + readPreference: READ_PREFERENCE_SECONDARY, } ) } diff --git a/services/web/scripts/back_fill_dummy_doc_meta.js b/services/web/scripts/back_fill_dummy_doc_meta.js index 270cf9d2b3..30a4aa011c 100644 --- a/services/web/scripts/back_fill_dummy_doc_meta.js +++ b/services/web/scripts/back_fill_dummy_doc_meta.js @@ -1,6 +1,10 @@ const { promisify } = require('util') -const { ObjectId, ReadPreference } = require('mongodb') -const { db, waitForDb } = require('../app/src/infrastructure/mongodb') +const { ObjectId } = require('mongodb') +const { + db, + waitForDb, + READ_PREFERENCE_SECONDARY, +} = require('../app/src/infrastructure/mongodb') const sleep = promisify(setTimeout) const _ = require('lodash') @@ -64,7 +68,7 @@ async function main(options) { }, } const docs = await db.docs - .find(query, { readPreference: ReadPreference.SECONDARY }) + .find(query, { readPreference: READ_PREFERENCE_SECONDARY }) .project({ _id: 1, project_id: 1 }) .limit(options.batchSize) .toArray() diff --git a/services/web/scripts/back_fill_staff_access.js b/services/web/scripts/back_fill_staff_access.js index 407c87eb1a..573fc81b81 100644 --- a/services/web/scripts/back_fill_staff_access.js +++ b/services/web/scripts/back_fill_staff_access.js @@ -1,5 +1,8 @@ -const { db, waitForDb } = require('../app/src/infrastructure/mongodb') -const { ReadPreference } = require('mongodb') +const { + db, + waitForDb, + READ_PREFERENCE_SECONDARY, +} = require('../app/src/infrastructure/mongodb') const UserSessionsManager = require('../app/src/Features/User/UserSessionsManager') const COMMIT = process.argv.includes('--commit') @@ -46,7 +49,7 @@ async function main() { email: 1, staffAccess: 1, }, - readPreference: ReadPreference.SECONDARY, + readPreference: READ_PREFERENCE_SECONDARY, } ) .toArray() diff --git a/services/web/scripts/count_encrypted_access_tokens.js b/services/web/scripts/count_encrypted_access_tokens.js index f41bdf38a1..d4f2438880 100644 --- a/services/web/scripts/count_encrypted_access_tokens.js +++ b/services/web/scripts/count_encrypted_access_tokens.js @@ -2,8 +2,11 @@ const TEN_MINUTES = 1000 * 60 * 10 process.env.MONGO_SOCKET_TIMEOUT = process.env.MONGO_SOCKET_TIMEOUT || TEN_MINUTES.toString() -const { ReadPreference } = require('mongodb') -const { db, waitForDb } = require('../app/src/infrastructure/mongodb') +const { + db, + waitForDb, + READ_PREFERENCE_SECONDARY, +} = require('../app/src/infrastructure/mongodb') const _ = require('lodash') const { formatTokenUsageStats, @@ -32,7 +35,7 @@ async function count(collectionName, paths) { const cursor = collection.find( {}, { - readPreference: ReadPreference.SECONDARY, + readPreference: READ_PREFERENCE_SECONDARY, projection, } ) diff --git a/services/web/scripts/count_image_files.js b/services/web/scripts/count_image_files.js index 7b41a836f6..a3c53bb335 100644 --- a/services/web/scripts/count_image_files.js +++ b/services/web/scripts/count_image_files.js @@ -1,5 +1,8 @@ -const { ReadPreference } = require('mongodb') -const { db, waitForDb } = require('../app/src/infrastructure/mongodb') +const { + db, + waitForDb, + READ_PREFERENCE_SECONDARY, +} = require('../app/src/infrastructure/mongodb') const { extname } = require('node:path') const FILE_TYPES = [ @@ -22,7 +25,10 @@ async function main() { await waitForDb() const projects = db.projects.find( {}, - { projection: { rootFolder: 1 }, readPreference: ReadPreference.SECONDARY } + { + projection: { rootFolder: 1 }, + readPreference: READ_PREFERENCE_SECONDARY, + } ) let projectsProcessed = 0 const result = new Map(FILE_TYPES.map(fileType => [fileType, 0])) diff --git a/services/web/scripts/delete_orphaned_data_helper.js b/services/web/scripts/delete_orphaned_data_helper.js index c16cf32f2b..24d9cffebf 100644 --- a/services/web/scripts/delete_orphaned_data_helper.js +++ b/services/web/scripts/delete_orphaned_data_helper.js @@ -1,5 +1,8 @@ -const { ReadPreference } = require('mongodb') -const { db } = require('../app/src/infrastructure/mongodb') +const { + db, + READ_PREFERENCE_PRIMARY, + READ_PREFERENCE_SECONDARY, +} = require('../app/src/infrastructure/mongodb') const { promiseMapWithLimit } = require('../app/src/util/promises') async function getDeletedProject(projectId, readPreference) { @@ -57,14 +60,14 @@ async function checkProjectExistsWithReadPreference(projectId, readPreference) { async function checkProjectExistsOnPrimary(projectId) { return await checkProjectExistsWithReadPreference( projectId, - ReadPreference.PRIMARY + READ_PREFERENCE_PRIMARY ) } async function checkProjectExistsOnSecondary(projectId) { return await checkProjectExistsWithReadPreference( projectId, - ReadPreference.SECONDARY + READ_PREFERENCE_SECONDARY ) } diff --git a/services/web/scripts/delete_orphaned_docs_online_check.js b/services/web/scripts/delete_orphaned_docs_online_check.js index d3b1ab2a7d..06ebb5ca93 100644 --- a/services/web/scripts/delete_orphaned_docs_online_check.js +++ b/services/web/scripts/delete_orphaned_docs_online_check.js @@ -1,7 +1,12 @@ const DocstoreManager = require('../app/src/Features/Docstore/DocstoreManager') const { promisify } = require('util') -const { ObjectId, ReadPreference } = require('mongodb') -const { db, waitForDb } = require('../app/src/infrastructure/mongodb') +const { ObjectId } = require('mongodb') +const { + db, + waitForDb, + READ_PREFERENCE_PRIMARY, + READ_PREFERENCE_SECONDARY, +} = require('../app/src/infrastructure/mongodb') const { promiseMapWithLimit } = require('../app/src/util/promises') const { getHardDeletedProjectIds } = require('./delete_orphaned_data_helper') const sleep = promisify(setTimeout) @@ -52,7 +57,7 @@ async function main() { }, } const docs = await db.docs - .find(query, { readPreference: ReadPreference.SECONDARY }) + .find(query, { readPreference: READ_PREFERENCE_SECONDARY }) .project({ project_id: 1 }) .sort({ project_id: 1 }) .limit(BATCH_SIZE) @@ -97,7 +102,7 @@ async function getProjectDocs(projectId) { { project_id: projectId }, { projection: { _id: 1 }, - readPreference: ReadPreference.PRIMARY, + readPreference: READ_PREFERENCE_PRIMARY, } ) .toArray() diff --git a/services/web/scripts/find_malformed_filetrees.js b/services/web/scripts/find_malformed_filetrees.js index 083685984f..7c581805f3 100644 --- a/services/web/scripts/find_malformed_filetrees.js +++ b/services/web/scripts/find_malformed_filetrees.js @@ -1,11 +1,17 @@ -const { ReadPreference } = require('mongodb') -const { db, waitForDb } = require('../app/src/infrastructure/mongodb') +const { + db, + waitForDb, + READ_PREFERENCE_SECONDARY, +} = require('../app/src/infrastructure/mongodb') async function main() { await waitForDb() const projects = db.projects.find( {}, - { projection: { rootFolder: 1 }, readPreference: ReadPreference.SECONDARY } + { + projection: { rootFolder: 1 }, + readPreference: READ_PREFERENCE_SECONDARY, + } ) let projectsProcessed = 0 for await (const project of projects) { diff --git a/services/web/scripts/helpers/batchedUpdate.js b/services/web/scripts/helpers/batchedUpdate.js index 2d5bd24ac0..e86e0f8115 100644 --- a/services/web/scripts/helpers/batchedUpdate.js +++ b/services/web/scripts/helpers/batchedUpdate.js @@ -1,5 +1,9 @@ -const { ReadPreference, ObjectId } = require('mongodb') -const { db, waitForDb } = require('../../app/src/infrastructure/mongodb') +const { ObjectId } = require('mongodb') +const { + db, + waitForDb, + READ_PREFERENCE_SECONDARY, +} = require('../../app/src/infrastructure/mongodb') const ONE_MONTH_IN_MS = 1000 * 60 * 60 * 24 * 31 let ID_EDGE_PAST @@ -129,7 +133,7 @@ async function batchedUpdate( refreshGlobalOptionsForBatchedUpdate(batchedUpdateOptions) findOptions = findOptions || {} - findOptions.readPreference = ReadPreference.SECONDARY + findOptions.readPreference = READ_PREFERENCE_SECONDARY projection = projection || { _id: 1 } let nextBatch diff --git a/services/web/scripts/history/downgrade_project.js b/services/web/scripts/history/downgrade_project.js index cff3f41b61..b5eec3f297 100644 --- a/services/web/scripts/history/downgrade_project.js +++ b/services/web/scripts/history/downgrade_project.js @@ -5,8 +5,12 @@ process.env.MONGO_SOCKET_TIMEOUT = const PROJECT_ID = process.env.PROJECT_ID -const { ReadPreference, ObjectId } = require('mongodb') -const { db, waitForDb } = require('../../app/src/infrastructure/mongodb') +const { ObjectId } = require('mongodb') +const { + db, + waitForDb, + READ_PREFERENCE_SECONDARY, +} = require('../../app/src/infrastructure/mongodb') const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler') console.log({ @@ -43,7 +47,7 @@ async function shouldPreserveHistory(project) { { preserveHistory: { $eq: true } }, ], }, - { readPreference: ReadPreference.SECONDARY } + { readPreference: READ_PREFERENCE_SECONDARY } ) } diff --git a/services/web/scripts/history/unset_allow_downgrade.js b/services/web/scripts/history/unset_allow_downgrade.js index 39b82ca38e..d6dc5a03a4 100644 --- a/services/web/scripts/history/unset_allow_downgrade.js +++ b/services/web/scripts/history/unset_allow_downgrade.js @@ -1,6 +1,10 @@ const { promisify } = require('util') -const { ObjectId, ReadPreference } = require('mongodb') -const { db, waitForDb } = require('../../app/src/infrastructure/mongodb') +const { ObjectId } = require('mongodb') +const { + db, + waitForDb, + READ_PREFERENCE_SECONDARY, +} = require('../../app/src/infrastructure/mongodb') const sleep = promisify(setTimeout) const _ = require('lodash') @@ -63,7 +67,7 @@ async function main(options) { 'overleaf.history.allowDowngrade': true, } const projects = await db.projects - .find(query, { readPreference: ReadPreference.SECONDARY }) + .find(query, { readPreference: READ_PREFERENCE_SECONDARY }) .project({ _id: 1 }) .limit(options.batchSize) .toArray() diff --git a/services/web/scripts/history/upgrade_none_without_conversion_if_no_sl_history.js b/services/web/scripts/history/upgrade_none_without_conversion_if_no_sl_history.js index 2c9e839dd5..7a74527348 100644 --- a/services/web/scripts/history/upgrade_none_without_conversion_if_no_sl_history.js +++ b/services/web/scripts/history/upgrade_none_without_conversion_if_no_sl_history.js @@ -14,8 +14,10 @@ process.env.BATCH_SIZE = BATCH_SIZE process.env.MONGO_SOCKET_TIMEOUT = parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000 -const { ReadPreference } = require('mongodb') -const { db } = require('../../app/src/infrastructure/mongodb') +const { + db, + READ_PREFERENCE_SECONDARY, +} = require('../../app/src/infrastructure/mongodb') const { promiseMapWithLimit } = require('../../app/src/util/promises') const { batchedUpdate } = require('../helpers/batchedUpdate') const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler') @@ -172,7 +174,7 @@ async function anyDocHistoryExists(project) { { project_id: { $eq: project._id } }, { projection: { _id: 1 }, - readPreference: ReadPreference.SECONDARY, + readPreference: READ_PREFERENCE_SECONDARY, } ) } @@ -182,7 +184,7 @@ async function anyDocHistoryIndexExists(project) { { project_id: { $eq: project._id } }, { projection: { _id: 1 }, - readPreference: ReadPreference.SECONDARY, + readPreference: READ_PREFERENCE_SECONDARY, } ) } diff --git a/services/web/scripts/history/upgrade_project.js b/services/web/scripts/history/upgrade_project.js index 5bf653903a..b36cdb76e4 100644 --- a/services/web/scripts/history/upgrade_project.js +++ b/services/web/scripts/history/upgrade_project.js @@ -1,5 +1,9 @@ -const { ReadPreference, ObjectId } = require('mongodb') -const { db, waitForDb } = require('../../app/src/infrastructure/mongodb') +const { ObjectId } = require('mongodb') +const { + db, + waitForDb, + READ_PREFERENCE_SECONDARY, +} = require('../../app/src/infrastructure/mongodb') const { upgradeProject, } = require('../../modules/history-migration/app/src/HistoryUpgradeHelper') @@ -20,7 +24,7 @@ async function main() { } const options = { projection, - readPreference: ReadPreference.SECONDARY, + readPreference: READ_PREFERENCE_SECONDARY, } const project = await db.projects.findOne(query, options) if (project) { diff --git a/services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js b/services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js index a0604d62a1..7ae69bad24 100644 --- a/services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js +++ b/services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js @@ -16,8 +16,12 @@ process.env.MONGO_SOCKET_TIMEOUT = const PROJECT_ID = process.env.PROJECT_ID -const { ReadPreference, ObjectId } = require('mongodb') -const { db, waitForDb } = require('../../app/src/infrastructure/mongodb') +const { ObjectId } = require('mongodb') +const { + db, + waitForDb, + READ_PREFERENCE_SECONDARY, +} = require('../../app/src/infrastructure/mongodb') const { promiseMapWithLimit } = require('../../app/src/util/promises') const { batchedUpdate } = require('../helpers/batchedUpdate') const ProjectHistoryController = require('../../modules/history-migration/app/src/ProjectHistoryController') @@ -165,7 +169,7 @@ async function shouldPreserveHistory(project) { { preserveHistory: { $eq: true } }, ], }, - { readPreference: ReadPreference.SECONDARY } + { readPreference: READ_PREFERENCE_SECONDARY } ) } @@ -174,7 +178,7 @@ async function anyDocHistoryExists(project) { { project_id: { $eq: project._id } }, { projection: { _id: 1 }, - readPreference: ReadPreference.SECONDARY, + readPreference: READ_PREFERENCE_SECONDARY, } ) } @@ -184,7 +188,7 @@ async function anyDocHistoryIndexExists(project) { { project_id: { $eq: project._id } }, { projection: { _id: 1 }, - readPreference: ReadPreference.SECONDARY, + readPreference: READ_PREFERENCE_SECONDARY, } ) } diff --git a/services/web/scripts/history/upgrade_v1_without_conversion_if_created_after_fph_enabled.js b/services/web/scripts/history/upgrade_v1_without_conversion_if_created_after_fph_enabled.js index d4b9142fad..f059799969 100644 --- a/services/web/scripts/history/upgrade_v1_without_conversion_if_created_after_fph_enabled.js +++ b/services/web/scripts/history/upgrade_v1_without_conversion_if_created_after_fph_enabled.js @@ -10,8 +10,11 @@ process.env.BATCH_SIZE = BATCH_SIZE process.env.MONGO_SOCKET_TIMEOUT = parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000 -const { ReadPreference, ObjectId } = require('mongodb') -const { db } = require('../../app/src/infrastructure/mongodb') +const { ObjectId } = require('mongodb') +const { + db, + READ_PREFERENCE_SECONDARY, +} = require('../../app/src/infrastructure/mongodb') const { promiseMapWithLimit } = require('../../app/src/util/promises') const { batchedUpdate } = require('../helpers/batchedUpdate') @@ -111,7 +114,7 @@ async function anyDocHistoryExists(project) { { project_id: { $eq: project._id } }, { projection: { _id: 1 }, - readPreference: ReadPreference.SECONDARY, + readPreference: READ_PREFERENCE_SECONDARY, } ) } @@ -121,7 +124,7 @@ async function anyDocHistoryIndexExists(project) { { project_id: { $eq: project._id } }, { projection: { _id: 1 }, - readPreference: ReadPreference.SECONDARY, + readPreference: READ_PREFERENCE_SECONDARY, } ) } diff --git a/services/web/scripts/history/upgrade_v1_without_conversion_if_no_sl_history.js b/services/web/scripts/history/upgrade_v1_without_conversion_if_no_sl_history.js index c63d145bd0..6c88c54464 100644 --- a/services/web/scripts/history/upgrade_v1_without_conversion_if_no_sl_history.js +++ b/services/web/scripts/history/upgrade_v1_without_conversion_if_no_sl_history.js @@ -12,8 +12,10 @@ process.env.BATCH_SIZE = BATCH_SIZE process.env.MONGO_SOCKET_TIMEOUT = parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000 -const { ReadPreference } = require('mongodb') -const { db } = require('../../app/src/infrastructure/mongodb') +const { + db, + READ_PREFERENCE_SECONDARY, +} = require('../../app/src/infrastructure/mongodb') const { promiseMapWithLimit } = require('../../app/src/util/promises') const { batchedUpdate } = require('../helpers/batchedUpdate') @@ -99,7 +101,7 @@ async function shouldPreserveHistory(project) { { preserveHistory: { $eq: true } }, ], }, - { readPreference: ReadPreference.SECONDARY } + { readPreference: READ_PREFERENCE_SECONDARY } ) } @@ -108,7 +110,7 @@ async function anyDocHistoryExists(project) { { project_id: { $eq: project._id } }, { projection: { _id: 1 }, - readPreference: ReadPreference.SECONDARY, + readPreference: READ_PREFERENCE_SECONDARY, } ) } @@ -118,7 +120,7 @@ async function anyDocHistoryIndexExists(project) { { project_id: { $eq: project._id } }, { projection: { _id: 1 }, - readPreference: ReadPreference.SECONDARY, + readPreference: READ_PREFERENCE_SECONDARY, } ) } diff --git a/services/web/scripts/oauth/remove_client.js b/services/web/scripts/oauth/remove_client.js index 09edb4d5e2..777de91455 100644 --- a/services/web/scripts/oauth/remove_client.js +++ b/services/web/scripts/oauth/remove_client.js @@ -1,6 +1,9 @@ const minimist = require('minimist') -const { ReadPreference } = require('mongodb') -const { waitForDb, db } = require('../../app/src/infrastructure/mongodb') +const { + waitForDb, + db, + READ_PREFERENCE_SECONDARY, +} = require('../../app/src/infrastructure/mongodb') async function main() { const opts = parseArgs() @@ -49,7 +52,7 @@ async function countAccessTokens(applicationId) { { oauthApplication_id: applicationId, }, - { readPreference: ReadPreference.secondary } + { readPreference: READ_PREFERENCE_SECONDARY } ) } @@ -58,7 +61,7 @@ async function countAuthorizationCodes(applicationId) { { oauthApplication_id: applicationId, }, - { readPreference: ReadPreference.secondary } + { readPreference: READ_PREFERENCE_SECONDARY } ) } diff --git a/services/web/scripts/regenerate_duplicate_referral_ids.js b/services/web/scripts/regenerate_duplicate_referral_ids.js index 081301c82e..946ef8b08a 100644 --- a/services/web/scripts/regenerate_duplicate_referral_ids.js +++ b/services/web/scripts/regenerate_duplicate_referral_ids.js @@ -4,8 +4,10 @@ const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100 // persist fallback in order to keep batchedUpdate in-sync process.env.BATCH_SIZE = BATCH_SIZE -const { ReadPreference } = require('mongodb') -const { db } = require('../app/src/infrastructure/mongodb') +const { + db, + READ_PREFERENCE_SECONDARY, +} = require('../app/src/infrastructure/mongodb') const { promiseMapWithLimit } = require('../app/src/util/promises') const TokenGenerator = require('../app/src/Features/TokenGenerator/TokenGenerator') const { batchedUpdate } = require('./helpers/batchedUpdate') @@ -24,7 +26,7 @@ async function rewriteDuplicates(duplicateReferralIds) { { referal_id: referralId }, { projection: { _id: 1 }, - readPreference: ReadPreference.SECONDARY, + readPreference: READ_PREFERENCE_SECONDARY, } ) .toArray() @@ -74,7 +76,7 @@ async function processBatch(users) { .find( { referal_id: { $in: uniqueReferalIdsInBatch } }, { - readPreference: ReadPreference.SECONDARY, + readPreference: READ_PREFERENCE_SECONDARY, projection: { _id: true }, } )