Merge pull request #21427 from overleaf/jpa-populate-db-on-import

[web] populate db with collections on import, ahead of waitForDb() call

GitOrigin-RevId: 7eb4cd61c2052187acd9947d7060f54d9822d314
This commit is contained in:
Jakob Ackermann 2024-10-31 13:16:54 +01:00 committed by Copybot
parent 7b3e39f63f
commit a7517eefcb
75 changed files with 124 additions and 270 deletions

View file

@ -59,7 +59,7 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {
PlansLocator.ensurePlansAreSetupCorrectly()
Promise.all([mongodb.waitForDb(), mongoose.connectionPromise])
Promise.all([mongodb.connectionPromise, mongoose.connectionPromise])
.then(async () => {
Server.server.listen(port, host, function () {
logger.debug(`web starting up, listening on ${host}:${port}`)
@ -77,7 +77,7 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {
}
// initialise site admin tasks
Promise.all([mongodb.waitForDb(), mongoose.connectionPromise])
Promise.all([mongodb.connectionPromise, mongoose.connectionPromise])
.then(() => SiteAdminHandler.initialise())
.catch(err => {
logger.fatal({ err }, 'Cannot connect to mongo. Exiting.')

View file

@ -30,16 +30,6 @@ const READ_PREFERENCE_SECONDARY = Settings.mongo.hasSecondaries
? ReadPreference.secondary.mode
: ReadPreference.secondaryPreferred.mode
let setupDbPromise
async function waitForDb() {
if (!setupDbPromise) {
setupDbPromise = setupDb()
}
await setupDbPromise
}
const db = {}
const mongoClient = new mongodb.MongoClient(
Settings.mongo.url,
Settings.mongo.options
@ -49,67 +39,60 @@ addConnectionDrainer('mongodb', async () => {
await mongoClient.close()
})
async function setupDb() {
const internalDb = mongoClient.db()
db.contacts = internalDb.collection('contacts')
db.deletedFiles = internalDb.collection('deletedFiles')
db.deletedProjects = internalDb.collection('deletedProjects')
db.deletedSubscriptions = internalDb.collection('deletedSubscriptions')
db.deletedUsers = internalDb.collection('deletedUsers')
db.dropboxEntities = internalDb.collection('dropboxEntities')
db.dropboxProjects = internalDb.collection('dropboxProjects')
db.docHistory = internalDb.collection('docHistory')
db.docHistoryIndex = internalDb.collection('docHistoryIndex')
db.docSnapshots = internalDb.collection('docSnapshots')
db.docs = internalDb.collection('docs')
db.feedbacks = internalDb.collection('feedbacks')
db.githubSyncEntityVersions = internalDb.collection(
'githubSyncEntityVersions'
)
db.githubSyncProjectStates = internalDb.collection('githubSyncProjectStates')
db.githubSyncUserCredentials = internalDb.collection(
'githubSyncUserCredentials'
)
db.globalMetrics = internalDb.collection('globalMetrics')
db.grouppolicies = internalDb.collection('grouppolicies')
db.institutions = internalDb.collection('institutions')
db.messages = internalDb.collection('messages')
db.migrations = internalDb.collection('migrations')
db.notifications = internalDb.collection('notifications')
db.oauthAccessTokens = internalDb.collection('oauthAccessTokens')
db.oauthApplications = internalDb.collection('oauthApplications')
db.oauthAuthorizationCodes = internalDb.collection('oauthAuthorizationCodes')
db.projectAuditLogEntries = internalDb.collection('projectAuditLogEntries')
db.projectHistoryChunks = internalDb.collection('projectHistoryChunks')
db.projectHistoryFailures = internalDb.collection('projectHistoryFailures')
db.projectHistoryLabels = internalDb.collection('projectHistoryLabels')
db.projectHistoryMetaData = internalDb.collection('projectHistoryMetaData')
db.projectHistorySyncState = internalDb.collection('projectHistorySyncState')
db.projectInvites = internalDb.collection('projectInvites')
db.projects = internalDb.collection('projects')
db.publishers = internalDb.collection('publishers')
db.rooms = internalDb.collection('rooms')
db.samlCache = internalDb.collection('samlCache')
db.samlLogs = internalDb.collection('samlLogs')
db.spellingPreferences = internalDb.collection('spellingPreferences')
db.splittests = internalDb.collection('splittests')
db.ssoConfigs = internalDb.collection('ssoConfigs')
db.subscriptions = internalDb.collection('subscriptions')
db.surveys = internalDb.collection('surveys')
db.systemmessages = internalDb.collection('systemmessages')
db.tags = internalDb.collection('tags')
db.teamInvites = internalDb.collection('teamInvites')
db.tokens = internalDb.collection('tokens')
db.userAuditLogEntries = internalDb.collection('userAuditLogEntries')
db.users = internalDb.collection('users')
db.onboardingDataCollection = internalDb.collection(
'onboardingDataCollection'
)
await mongoClient.connect()
const internalDb = mongoClient.db()
const db = {
contacts: internalDb.collection('contacts'),
deletedFiles: internalDb.collection('deletedFiles'),
deletedProjects: internalDb.collection('deletedProjects'),
deletedSubscriptions: internalDb.collection('deletedSubscriptions'),
deletedUsers: internalDb.collection('deletedUsers'),
dropboxEntities: internalDb.collection('dropboxEntities'),
dropboxProjects: internalDb.collection('dropboxProjects'),
docHistory: internalDb.collection('docHistory'),
docHistoryIndex: internalDb.collection('docHistoryIndex'),
docSnapshots: internalDb.collection('docSnapshots'),
docs: internalDb.collection('docs'),
feedbacks: internalDb.collection('feedbacks'),
githubSyncEntityVersions: internalDb.collection('githubSyncEntityVersions'),
githubSyncProjectStates: internalDb.collection('githubSyncProjectStates'),
githubSyncUserCredentials: internalDb.collection('githubSyncUserCredentials'),
globalMetrics: internalDb.collection('globalMetrics'),
grouppolicies: internalDb.collection('grouppolicies'),
institutions: internalDb.collection('institutions'),
messages: internalDb.collection('messages'),
migrations: internalDb.collection('migrations'),
notifications: internalDb.collection('notifications'),
oauthAccessTokens: internalDb.collection('oauthAccessTokens'),
oauthApplications: internalDb.collection('oauthApplications'),
oauthAuthorizationCodes: internalDb.collection('oauthAuthorizationCodes'),
projectAuditLogEntries: internalDb.collection('projectAuditLogEntries'),
projectHistoryChunks: internalDb.collection('projectHistoryChunks'),
projectHistoryFailures: internalDb.collection('projectHistoryFailures'),
projectHistoryLabels: internalDb.collection('projectHistoryLabels'),
projectHistoryMetaData: internalDb.collection('projectHistoryMetaData'),
projectHistorySyncState: internalDb.collection('projectHistorySyncState'),
projectInvites: internalDb.collection('projectInvites'),
projects: internalDb.collection('projects'),
publishers: internalDb.collection('publishers'),
rooms: internalDb.collection('rooms'),
samlCache: internalDb.collection('samlCache'),
samlLogs: internalDb.collection('samlLogs'),
spellingPreferences: internalDb.collection('spellingPreferences'),
splittests: internalDb.collection('splittests'),
ssoConfigs: internalDb.collection('ssoConfigs'),
subscriptions: internalDb.collection('subscriptions'),
surveys: internalDb.collection('surveys'),
systemmessages: internalDb.collection('systemmessages'),
tags: internalDb.collection('tags'),
teamInvites: internalDb.collection('teamInvites'),
tokens: internalDb.collection('tokens'),
userAuditLogEntries: internalDb.collection('userAuditLogEntries'),
users: internalDb.collection('users'),
onboardingDataCollection: internalDb.collection('onboardingDataCollection'),
}
const connectionPromise = mongoClient.connect()
async function getCollectionNames() {
const internalDb = mongoClient.db()
@ -142,10 +125,10 @@ async function getCollectionInternal(name) {
module.exports = {
db,
ObjectId,
connectionPromise,
getCollectionNames,
getCollectionInternal,
dropTestDatabase,
waitForDb,
READ_PREFERENCE_PRIMARY,
READ_PREFERENCE_SECONDARY,
}

View file

@ -5,7 +5,7 @@ import { fileURLToPath } from 'url'
const __filename = fileURLToPath(import.meta.url)
const __dirname = Path.dirname(__filename)
const { db, waitForDb } = mongodb
const { db } = mongodb
const { getNativeDb } = Mongoose
class Adapter {
@ -26,7 +26,6 @@ class Adapter {
}
async connect() {
await waitForDb()
const nativeDb = await getNativeDb()
return { db, nativeDb }
}

View file

@ -1,7 +1,10 @@
// @ts-check
import mongodb from '../../app/src/infrastructure/mongodb.js'
const { db, getCollectionNames, getCollectionInternal, waitForDb } = mongodb
import {
db,
getCollectionNames,
getCollectionInternal,
} from '../../app/src/infrastructure/mongodb.js'
async function addIndexesToCollection(collection, indexes) {
return Promise.all(
@ -29,7 +32,6 @@ async function dropIndexesFromCollection(collection, indexes) {
}
async function dropCollection(collectionName) {
await waitForDb()
if (db[collectionName]) {
throw new Error(`blocking drop of an active collection: ${collectionName}`)
}
@ -46,7 +48,6 @@ async function dropCollection(collectionName) {
* @param {string} migrationName
*/
async function assertDependency(migrationName) {
await waitForDb()
const migrations = await getCollectionInternal('migrations')
const migration = await migrations.findOne({ name: migrationName })
if (migration == null) {

View file

@ -1,13 +1,7 @@
const minimist = require('minimist')
const {
db,
ObjectId,
waitForDb,
} = require('../../../app/src/infrastructure/mongodb')
const { db, ObjectId } = require('../../../app/src/infrastructure/mongodb')
async function main() {
await waitForDb()
const argv = minimist(process.argv.slice(2), {
string: ['user-id', 'compile-timeout'],
})

View file

@ -1,12 +1,15 @@
const { ObjectId } = require('mongodb-legacy')
const { waitForDb, db } = require('../../../app/src/infrastructure/mongodb')
const {
connectionPromise,
db,
} = require('../../../app/src/infrastructure/mongodb')
const { getMongoClient } = require('../../../app/src/infrastructure/Mongoose')
const MIN_MONGO_VERSION = [5, 0]
async function main() {
try {
await waitForDb()
await connectionPromise
} catch (err) {
console.error('Cannot connect to mongodb')
throw err

View file

@ -1,7 +1,6 @@
const { waitForDb, db } = require('../../../app/src/infrastructure/mongodb')
const { db } = require('../../../app/src/infrastructure/mongodb')
async function readImagesInUse() {
await waitForDb()
const projectCount = await db.projects.countDocuments()
if (projectCount === 0) {
return []

View file

@ -1,10 +1,8 @@
const minimist = require('minimist')
const { db, waitForDb } = require('../../../app/src/infrastructure/mongodb')
const { db } = require('../../../app/src/infrastructure/mongodb')
const UserRegistrationHandler = require('../../../app/src/Features/User/UserRegistrationHandler')
async function main() {
await waitForDb()
const argv = minimist(process.argv.slice(2), {
string: ['email'],
boolean: ['admin'],

View file

@ -1,10 +1,7 @@
const { waitForDb } = require('../../../app/src/infrastructure/mongodb')
const UserGetter = require('../../../app/src/Features/User/UserGetter')
const UserDeleter = require('../../../app/src/Features/User/UserDeleter')
async function main() {
await waitForDb()
const email = (process.argv.slice(2).pop() || '').replace(/^--email=/, '')
if (!email) {
console.error(`Usage: node ${__filename} --email=joe@example.com`)

View file

@ -10,7 +10,6 @@
// another.
const minimist = require('minimist')
const { waitForDb } = require('../../../app/src/infrastructure/mongodb')
const os = require('os')
const fs = require('fs')
@ -182,7 +181,6 @@ async function migrateEmails() {
const csvFile = fs.readFileSync(csvFilePath, 'utf8')
const rows = csv.parse(csvFile)
console.log('Number of users to migrate: ', rows.length)
await waitForDb()
const emails = filterEmails(rows)
const existingUserEmails = await checkEmailsAgainstDb(emails)
await doMigration(existingUserEmails)

View file

@ -1,9 +1,7 @@
const minimist = require('minimist')
const { db, waitForDb } = require('../../../app/src/infrastructure/mongodb')
const { db } = require('../../../app/src/infrastructure/mongodb')
async function main() {
await waitForDb()
const argv = minimist(process.argv.slice(2), {
string: ['user-id', 'old-name', 'new_name'],
})

View file

@ -1,6 +1,6 @@
const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
const { db, waitForDb } = require('../../../app/src/infrastructure/mongodb')
const { db } = require('../../../app/src/infrastructure/mongodb')
const {
mergeFeatures,
compareFeatures,
@ -8,8 +8,6 @@ const {
const DRY_RUN = !process.argv.includes('--dry-run=false')
async function main(DRY_RUN, defaultFeatures) {
await waitForDb()
logger.info({ defaultFeatures }, 'default features')
const cursor = db.users.find(

View file

@ -26,7 +26,7 @@
import minimist from 'minimist'
import fs from 'fs'
import { ObjectId, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { ObjectId } from '../app/src/infrastructure/mongodb.js'
import pLimit from 'p-limit'
import FeaturesUpdater from '../app/src/Features/Subscription/FeaturesUpdater.js'
import FeaturesHelper from '../app/src/Features/Subscription/FeaturesHelper.js'
@ -162,8 +162,6 @@ async function processUsers(userIds) {
: 'Every user in file will get feature override'
)
await waitForDb()
_validateUserIdList(userIds)
console.log(`---Starting to process ${userIds.length} users---`)

View file

@ -2,7 +2,7 @@ import fs from 'node:fs'
import minimist from 'minimist'
import { parse } from 'csv'
import Stream from 'stream/promises'
import { ObjectId, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { ObjectId } from '../app/src/infrastructure/mongodb.js'
import { Subscription } from '../app/src/models/Subscription.js'
function usage() {
@ -162,8 +162,6 @@ async function processRows(rows) {
}
async function main() {
await waitForDb()
await Stream.pipeline(
fs.createReadStream(filename),
parse({

View file

@ -7,7 +7,7 @@ import fs from 'fs'
import * as csv from 'csv/sync'
import minimist from 'minimist'
import UserGetter from '../app/src/Features/User/UserGetter.js'
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { db } from '../app/src/infrastructure/mongodb.js'
import _ from 'lodash'
const argv = minimist(process.argv.slice(2), {
@ -45,7 +45,6 @@ if (records.length === 0) {
}
async function main() {
await waitForDb()
for (const record of records) {
const domain = record[argv.domain]
const { domainUserCount, subdomainUserCount } = await getUserCount(domain, {

View file

@ -1,6 +1,5 @@
import GoogleBigQueryHelper from './helpers/GoogleBigQueryHelper.mjs'
import { Subscription } from '../../app/src/models/Subscription.js'
import { waitForDb } from '../../app/src/infrastructure/mongodb.js'
import AnalyticsManager from '../../app/src/Features/Analytics/AnalyticsManager.js'
import { DeletedSubscription } from '../../app/src/models/DeletedSubscription.js'
import minimist from 'minimist'
@ -12,8 +11,6 @@ const { ObjectId } = mongodb
let FETCH_LIMIT, COMMIT, VERBOSE
async function main() {
await waitForDb()
console.log('## Syncing group subscription memberships...')
const subscriptionsCount = await Subscription.countDocuments({

View file

@ -2,7 +2,6 @@ import { promisify } from 'util'
import mongodb from 'mongodb-legacy'
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} from '../app/src/infrastructure/mongodb.js'
import _ from 'lodash'
@ -50,8 +49,6 @@ async function main(options) {
})
await letUserDoubleCheckInputs(options)
await waitForDb()
let startId = options.firstProjectId
let nProcessed = 0

View file

@ -1,6 +1,5 @@
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} from '../app/src/infrastructure/mongodb.js'
import UserSessionsManager from '../app/src/Features/User/UserSessionsManager.js'
@ -39,7 +38,6 @@ function formatUser(user) {
}
async function main() {
await waitForDb()
const adminUsers = await db.users
.find(
{ isAdmin: true },

View file

@ -1,5 +1,5 @@
import NotificationsBuilder from '../app/src/Features/Notifications/NotificationsBuilder.js'
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { db } from '../app/src/infrastructure/mongodb.js'
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
const { batchedUpdate } = BatchedUpdateModule
@ -57,8 +57,6 @@ async function processBatch(groupSubscriptionsBatch) {
}
async function main() {
await waitForDb()
await batchedUpdate('subscriptions', { groupPlan: true }, processBatch, {
member_ids: 1,
})

View file

@ -1,4 +1,4 @@
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { db } from '../app/src/infrastructure/mongodb.js'
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
import minimist from 'minimist'
import CollaboratorsInviteHelper from '../app/src/Features/Collaborators/CollaboratorsInviteHelper.js'
@ -53,7 +53,6 @@ async function addTokenHmacField(DRY_RUN) {
}
async function main(DRY_RUN) {
await waitForDb()
await addTokenHmacField(DRY_RUN)
}

View file

@ -1,4 +1,3 @@
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import InstitutionsManager from '../app/src/Features/Institutions/InstitutionsManager.js'
import { ensureRunningOnMongoSecondaryWithTimeout } from './helpers/env_variable_helper.mjs'
@ -19,7 +18,6 @@ async function main() {
}
try {
await waitForDb()
await main()
} catch (error) {
console.error(error)

View file

@ -1,6 +1,5 @@
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} from '../app/src/infrastructure/mongodb.js'
import UserSessionsManager from '../app/src/Features/User/UserSessionsManager.js'
@ -9,7 +8,6 @@ const COMMIT = process.argv.includes('--commit')
const LOG_SESSIONS = !process.argv.includes('--log-sessions=false')
async function main() {
await waitForDb()
const adminUsers = await db.users
.find(
{ isAdmin: true },

View file

@ -5,11 +5,10 @@
* DRY_RUN=false node scripts/clear_feedback_collection.js 2022-11-01 # deletion mode
*/
import { db, ObjectId, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { db, ObjectId } from '../app/src/infrastructure/mongodb.js'
import { fileURLToPath } from 'url'
const runScript = async (timestamp, dryRun) => {
await waitForDb()
const t = new Date(timestamp)
if (isNaN(t)) {
throw new Error('invalid date ' + timestamp)

View file

@ -1,4 +1,3 @@
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import ProjectDetailsHandler from '../app/src/Features/Project/ProjectDetailsHandler.js'
const projectId = process.argv[2]
@ -22,7 +21,6 @@ function main() {
}
try {
await waitForDb()
await main()
} catch (error) {
console.error(error)

View file

@ -1,5 +1,5 @@
import fs from 'fs'
import { ObjectId, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { ObjectId } from '../app/src/infrastructure/mongodb.js'
import async from 'async'
import UserUpdater from '../app/src/Features/User/UserUpdater.js'
import UserSessionsManager from '../app/src/Features/User/UserSessionsManager.js'
@ -73,8 +73,6 @@ const userIds = usersFile
async function processUsers(userIds) {
console.log('---Starting set_must_reconfirm script---')
await waitForDb()
_validateUserIdList(userIds)
console.log(`---Starting to process ${userIds.length} users---`)
await _loopUsers(userIds)

View file

@ -1,8 +1,7 @@
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { db } from '../app/src/infrastructure/mongodb.js'
import { fileURLToPath } from 'url'
async function updateStringDates() {
await waitForDb()
const users = await db.users.aggregate([
{ $unwind: { path: '$emails' } },
{

View file

@ -1,5 +1,5 @@
import minimist from 'minimist'
import { waitForDb, ObjectId } from '../app/src/infrastructure/mongodb.js'
import { ObjectId } from '../app/src/infrastructure/mongodb.js'
import ProjectEntityUpdateHandler from '../app/src/Features/Project/ProjectEntityUpdateHandler.js'
import Errors from '../app/src/Features/Errors/Errors.js'
@ -16,7 +16,6 @@ async function main() {
}
console.log(`Converting doc ${projectId}/${docId} as user ${userId}`)
await waitForDb()
try {
await ProjectEntityUpdateHandler.promises.convertDocToFile(
projectId,

View file

@ -1,6 +1,5 @@
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} from '../app/src/infrastructure/mongodb.js'
import _ from 'lodash'
@ -56,8 +55,6 @@ async function count(collectionName, paths) {
}
async function main() {
await waitForDb()
const STATS = {}
for (const [collectionName, paths] of Object.entries(CASES)) {
const stats = await count(collectionName, paths)

View file

@ -1,5 +1,4 @@
import readline from 'readline'
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import ProjectEntityHandler from '../app/src/Features/Project/ProjectEntityHandler.js'
import ProjectGetter from '../app/src/Features/Project/ProjectGetter.js'
import Errors from '../app/src/Features/Errors/Errors.js'
@ -35,7 +34,6 @@ async function countFiles() {
}
try {
await waitForDb()
await countFiles()
process.exit(0)
} catch (error) {

View file

@ -1,6 +1,5 @@
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} from '../app/src/infrastructure/mongodb.js'
import { extname } from 'node:path'
@ -22,7 +21,6 @@ const FILE_TYPES = [
const longestFileType = Math.max(...FILE_TYPES.map(fileType => fileType.length))
async function main() {
await waitForDb()
const projects = db.projects.find(
{},
{

View file

@ -1,5 +1,5 @@
import readline from 'readline'
import { waitForDb, ObjectId, db } from '../app/src/infrastructure/mongodb.js'
import { ObjectId, db } from '../app/src/infrastructure/mongodb.js'
import ProjectEntityHandler from '../app/src/Features/Project/ProjectEntityHandler.js'
import ProjectGetter from '../app/src/Features/Project/ProjectGetter.js'
import Errors from '../app/src/Features/Errors/Errors.js'
@ -127,7 +127,6 @@ async function countDocsSizes(docs) {
}
try {
await waitForDb()
await countProjectFiles()
process.exit(0)
} catch (error) {

View file

@ -4,7 +4,6 @@
import parseArgs from 'minimist'
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import OAuthPersonalAccessTokenManager from '../modules/oauth2-server/app/src/OAuthPersonalAccessTokenManager.js'
const argv = parseArgs(process.argv.slice(2), {
@ -19,7 +18,6 @@ if (!userId) {
}
async function createPersonalAccessToken() {
await waitForDb()
const accessToken = await OAuthPersonalAccessTokenManager.createToken(userId)
console.log('Personal Access Token: ' + accessToken)
}

View file

@ -8,7 +8,6 @@ import path from 'path'
import _ from 'lodash'
import parseArgs from 'minimist'
import OError from '@overleaf/o-error'
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import { User } from '../app/src/models/User.js'
import ProjectCreationHandler from '../app/src/Features/Project/ProjectCreationHandler.js'
import ProjectEntityUpdateHandler from '../app/src/Features/Project/ProjectEntityUpdateHandler.js'
@ -207,7 +206,6 @@ async function _applyRandomDocUpdate(ownerId, projectId) {
}
async function createProject() {
await waitForDb()
const user = await User.findById(userId)
console.log('Will create project')
console.log('user_id:', userId, '=>', user.email)

View file

@ -1,4 +1,4 @@
import { db, waitForDb } from '../../app/src/infrastructure/mongodb.js'
import { db } from '../../app/src/infrastructure/mongodb.js'
import minimist from 'minimist'
const argv = minimist(process.argv.slice(2))
@ -9,7 +9,6 @@ if (!commit) {
}
async function getDupes(commit) {
await waitForDb()
const entries = await db.splittests.aggregate([
{
$match: {

View file

@ -1,11 +1,7 @@
import fs from 'fs'
import minimist from 'minimist'
import readline from 'readline'
import {
db,
ObjectId,
waitForDb,
} from '../../app/src/infrastructure/mongodb.js'
import { db, ObjectId } from '../../app/src/infrastructure/mongodb.js'
import DocstoreManagerModule from '../../app/src/Features/Docstore/DocstoreManager.js'
const { promises: DocstoreManager } = DocstoreManagerModule
@ -66,8 +62,6 @@ rl.on('close', async () => {
console.log(`Loaded Data for ${docCount} docs in ${projectCount} Projects`)
await waitForDb()
for (const projectId of Object.keys(orphanedDocs)) {
await deleteOrphanedDocs(projectId, orphanedDocs[projectId])
}

View file

@ -5,7 +5,7 @@
import minimist from 'minimist'
import mongodb from 'mongodb-legacy'
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { db } from '../app/src/infrastructure/mongodb.js'
import Errors from '../app/src/Features/Errors/Errors.js'
import FileStoreHandler from '../app/src/Features/FileStore/FileStoreHandler.js'
import ProjectEntityMongoUpdateHandler from '../app/src/Features/Project/ProjectEntityMongoUpdateHandler.js'
@ -30,7 +30,6 @@ function parseArgs() {
}
async function main() {
await waitForDb()
const projects = await getProjects()
for (const project of projects) {

View file

@ -1,5 +1,4 @@
import minimist from 'minimist'
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import ChatApiHandler from '../app/src/Features/Chat/ChatApiHandler.js'
import DocstoreManager from '../app/src/Features/Docstore/DocstoreManager.js'
import DocumentUpdaterHandler from '../app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js'
@ -14,8 +13,6 @@ const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
* for more detail.
*/
async function main() {
await waitForDb()
const argv = minimist(process.argv.slice(2))
const { projectId, docId } = argv

View file

@ -3,7 +3,6 @@ import { promisify } from 'util'
import mongodb from 'mongodb-legacy'
import {
db,
waitForDb,
READ_PREFERENCE_PRIMARY,
READ_PREFERENCE_SECONDARY,
} from '../app/src/infrastructure/mongodb.js'
@ -41,8 +40,6 @@ function getSecondsFromObjectId(id) {
async function main() {
await letUserDoubleCheckInputs()
await waitForDb()
let lowerProjectId = BATCH_LAST_ID
let nProjectsProcessedTotal = 0

View file

@ -1,7 +1,6 @@
import { Subscription } from '../app/src/models/Subscription.js'
import SubscriptionUpdater from '../app/src/Features/Subscription/SubscriptionUpdater.js'
import minimist from 'minimist'
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import mongodb from 'mongodb-legacy'
const { ObjectId } = mongodb
@ -39,12 +38,10 @@ const setup = () => {
setup()
waitForDb()
.then(run)
.then(() => {
try {
await run()
process.exit(0)
})
.catch(err => {
console.log('Aiee, something went wrong!', err)
} catch (err) {
console.error('Aiee, something went wrong!', err)
process.exit(1)
})
}

View file

@ -1,4 +1,3 @@
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import { User } from '../app/src/models/User.js'
import UserController from '../app/src/Features/User/UserController.js'
import Logger from '@overleaf/logger'
@ -45,7 +44,6 @@ async function run() {
}
try {
await waitForDb()
await run()
process.exit()
} catch (error) {

View file

@ -1,7 +1,6 @@
import {
db,
READ_PREFERENCE_SECONDARY,
waitForDb,
ObjectId,
} from '../app/src/infrastructure/mongodb.js'
import minimist from 'minimist'
@ -76,7 +75,6 @@ function getUserMappings(affiliations) {
async function main() {
const args = parseArgs()
await waitForDb()
const affiliations = await fetchInstitutionAndAffiliations(args.institutionId)
const userMappings = getUserMappings(affiliations)

View file

@ -1,11 +1,9 @@
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} from '../app/src/infrastructure/mongodb.js'
async function main() {
await waitForDb()
const projects = db.projects.find(
{},
{

View file

@ -1,4 +1,4 @@
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { db } from '../app/src/infrastructure/mongodb.js'
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
const { batchedUpdate } = BatchedUpdateModule
@ -46,8 +46,6 @@ async function processBatch(subscriptions) {
}
async function main() {
await waitForDb()
const projection = {
_id: 1,
teamInvites: 1,

View file

@ -6,15 +6,13 @@
* script.
*/
import mongodb from 'mongodb-legacy'
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { db } from '../app/src/infrastructure/mongodb.js'
import ProjectLocator from '../app/src/Features/Project/ProjectLocator.js'
const { ObjectId } = mongodb
async function main() {
const { projectId, mongoPath } = parseArgs()
await waitForDb()
let modifiedCount
if (isRootFolder(mongoPath)) {
modifiedCount = await fixRootFolder(projectId)

View file

@ -1,6 +1,6 @@
import fs from 'fs'
import minimist from 'minimist'
import { waitForDb, ObjectId } from '../app/src/infrastructure/mongodb.js'
import { ObjectId } from '../app/src/infrastructure/mongodb.js'
import DocstoreManager from '../app/src/Features/Docstore/DocstoreManager.js'
import FileStoreHandler from '../app/src/Features/FileStore/FileStoreHandler.js'
import FileWriter from '../app/src/infrastructure/FileWriter.js'
@ -39,7 +39,6 @@ Options:
}
async function main() {
await waitForDb()
for (const projectId of opts.projectIds) {
await processProject(projectId)
}

View file

@ -1,5 +1,5 @@
import mongodb from 'mongodb-legacy'
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { db } from '../app/src/infrastructure/mongodb.js'
import DocumentUpdaterHandler from '../app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js'
const { ObjectId } = mongodb
@ -16,8 +16,6 @@ console.log({
})
async function main() {
await waitForDb()
const { lines, version, ranges } = await getDocument()
const size = lines.reduce((size, line) => size + line.length + 1, 0)

View file

@ -1,7 +1,6 @@
import mongodb from 'mongodb-legacy'
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} from '../../app/src/infrastructure/mongodb.js'
@ -126,7 +125,6 @@ async function batchedUpdate(
findOptions,
batchedUpdateOptions
) {
await waitForDb()
const collection = db[collectionName]
ID_EDGE_PAST = await getIdEdgePast(collection)
if (!ID_EDGE_PAST) {

View file

@ -1,4 +1,4 @@
import { waitForDb, db } from '../../app/src/infrastructure/mongodb.js'
import { db } from '../../app/src/infrastructure/mongodb.js'
import { ensureMongoTimeout } from '../helpers/env_variable_helper.mjs'
// Ensure default mongo query timeout has been increased 1h
if (!process.env.MONGO_SOCKET_TIMEOUT) {
@ -65,9 +65,9 @@ async function gracefullyDropCollection(collection) {
console.log(`removing \`${collectionName}\` data - Done`)
}
waitForDb()
.then(main)
.catch(err => {
try {
await main()
} catch (err) {
console.error(err)
process.exit(1)
})
}

View file

@ -1,9 +1,7 @@
import HistoryRangesSupportMigration from '../../app/src/Features/History/HistoryRangesSupportMigration.mjs'
import { waitForDb } from '../../app/src/infrastructure/mongodb.js'
import minimist from 'minimist'
async function main() {
await waitForDb()
const {
projectIds,
ownerIds,

View file

@ -1,4 +1,4 @@
import { db, ObjectId, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { db, ObjectId } from '../app/src/infrastructure/mongodb.js'
import minimist from 'minimist'
const argv = minimist(process.argv.slice(2))
const commit = argv.commit !== undefined
@ -11,8 +11,6 @@ if (!commit) {
}
console.log('checking', projectIds.length, 'projects')
await waitForDb()
const affectedProjects = await db.projects
.find(
{ _id: { $in: projectIds } },

View file

@ -6,7 +6,7 @@
// node scripts/merge_group_subscription_members \
// --target [targetSubscriptionId] --source [sourceSubscriptionId] --commit
import { db, ObjectId, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { db, ObjectId } from '../app/src/infrastructure/mongodb.js'
import SubscriptionUpdater from '../app/src/Features/Subscription/SubscriptionUpdater.js'
import minimist from 'minimist'
@ -43,8 +43,6 @@ async function main() {
console.log('Doing dry run without --commit')
}
await waitForDb()
const targetSubscription = await getSubscription(new ObjectId(target))
const sourceSubscription = await getSubscription(new ObjectId(source))

View file

@ -1,6 +1,6 @@
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
import { promiseMapWithLimit, promisify } from '@overleaf/promise-utils'
import { db, ObjectId, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { db, ObjectId } from '../app/src/infrastructure/mongodb.js'
import _ from 'lodash'
import { fileURLToPath } from 'url'
@ -153,7 +153,6 @@ export default main
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await waitForDb()
await main()
console.log('Done.')
process.exit(0)

View file

@ -1,12 +1,10 @@
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} from '../../app/src/infrastructure/mongodb.js'
import { hashSecret } from '../../modules/oauth2-server/app/src/SecretsHelper.js'
async function main() {
await waitForDb()
console.log('Hashing client secrets...')
await hashSecrets(db.oauthApplications, 'clientSecret')
console.log('Hashing access tokens...')

View file

@ -1,11 +1,9 @@
import minimist from 'minimist'
import { waitForDb, db } from '../../app/src/infrastructure/mongodb.js'
import { db } from '../../app/src/infrastructure/mongodb.js'
import { hashSecret } from '../../modules/oauth2-server/app/src/SecretsHelper.js'
async function main() {
const opts = parseArgs()
await waitForDb()
if (opts.accessToken == null) {
console.error('Missing --token option')
process.exit(1)

View file

@ -1,13 +1,12 @@
import minimist from 'minimist'
import mongodb from 'mongodb-legacy'
import { waitForDb, db } from '../../app/src/infrastructure/mongodb.js'
import { db } from '../../app/src/infrastructure/mongodb.js'
import { hashSecret } from '../../modules/oauth2-server/app/src/SecretsHelper.js'
const { ObjectId } = mongodb
async function main() {
const opts = parseArgs()
await waitForDb()
const application = await getApplication(opts.id)
if (application == null) {
console.log(

View file

@ -1,13 +1,11 @@
import minimist from 'minimist'
import {
waitForDb,
db,
READ_PREFERENCE_SECONDARY,
} from '../../app/src/infrastructure/mongodb.js'
async function main() {
const opts = parseArgs()
await waitForDb()
const application = await getApplication(opts.clientId)
if (application == null) {
console.error(`Client configuration not found: ${opts.clientId}`)

View file

@ -2,7 +2,7 @@
import minimist from 'minimist'
import { db, waitForDb, ObjectId } from '../app/src/infrastructure/mongodb.js'
import { db, ObjectId } from '../app/src/infrastructure/mongodb.js'
import ProjectEntityUpdateHandler from '../app/src/Features/Project/ProjectEntityUpdateHandler.js'
import ProjectEntityRestoreHandler from '../app/src/Features/Project/ProjectEntityRestoreHandler.js'
import RedisWrapper from '@overleaf/redis-wrapper'
@ -30,7 +30,6 @@ function extractObjectId(s) {
}
async function main() {
await waitForDb()
logger.info({ opts }, 'removing deleted docs')
let cursor = 0
do {

View file

@ -7,7 +7,6 @@ import { setTimeout } from 'node:timers/promises'
import util from 'util'
import pLimit from 'p-limit'
import { waitForDb } from '../../app/src/infrastructure/mongodb.js'
util.inspect.defaultOptions.maxArrayLength = null
@ -147,8 +146,6 @@ const loopForSubscriptions = async skipInitial => {
let retryCounter = 0
const run = async () => {
await waitForDb()
while (true) {
try {
await loopForSubscriptions(

View file

@ -1,4 +1,4 @@
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { db } from '../app/src/infrastructure/mongodb.js'
import minimist from 'minimist'
import _ from 'lodash'
import async from 'async'
@ -191,6 +191,5 @@ const setup = () => {
}
}
await waitForDb()
setup()
run()

View file

@ -1,4 +1,3 @@
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import minimist from 'minimist'
import InstitutionsManager from '../app/src/Features/Institutions/InstitutionsManager.js'
@ -32,7 +31,6 @@ function main() {
}
try {
await waitForDb()
await main()
} catch (error) {
console.error(error)

View file

@ -1,6 +1,5 @@
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} from '../app/src/infrastructure/mongodb.js'
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
@ -192,7 +191,6 @@ async function fixProjectsWithInvalidTokenAccessRefsIds(
}
async function main(DRY_RUN, PROJECTS_LIST) {
await waitForDb()
await fixProjectsWithInvalidTokenAccessRefsIds(DRY_RUN, PROJECTS_LIST)
}

View file

@ -1,4 +1,4 @@
import { ObjectId, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { ObjectId } from '../app/src/infrastructure/mongodb.js'
import UserUpdater from '../app/src/Features/User/UserUpdater.js'
import UserGetter from '../app/src/Features/User/UserGetter.js'
@ -53,7 +53,6 @@ async function removeEmail() {
)
}
try {
await waitForDb()
await removeEmail()
console.log('Done.')
process.exit()

View file

@ -1,14 +1,11 @@
import { OauthApplication } from '../app/src/models/OauthApplication.js'
import parseArgs from 'minimist'
import OError from '@overleaf/o-error'
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
async function _removeOauthApplication(appId) {
if (!appId) {
throw new OError('No app id supplied')
}
console.log('Waiting for db...')
await waitForDb()
console.log(`Removing oauthApplication with id=${appId}`)
const result = await OauthApplication.deleteOne({ id: appId })
console.log('Result', result)

View file

@ -1,4 +1,3 @@
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import ProjectEntityRestoreHandler from '../app/src/Features/Project/ProjectEntityRestoreHandler.js'
import ProjectEntityHandler from '../app/src/Features/Project/ProjectEntityHandler.js'
import DocstoreManager from '../app/src/Features/Docstore/DocstoreManager.js'
@ -36,7 +35,6 @@ async function main() {
}
try {
await waitForDb()
await main()
process.exit(0)
} catch (error) {

View file

@ -1,4 +1,3 @@
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import ProjectEntityRestoreHandler from '../app/src/Features/Project/ProjectEntityRestoreHandler.js'
import DocstoreManager from '../app/src/Features/Docstore/DocstoreManager.js'
@ -25,7 +24,6 @@ async function main() {
}
try {
await waitForDb()
await main()
process.exit(0)
} catch (error) {

View file

@ -1,5 +1,4 @@
import minimist from 'minimist'
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import ProjectDeleter from '../app/src/Features/Project/ProjectDeleter.js'
async function main() {
@ -10,8 +9,6 @@ async function main() {
throw new Error('set --project-id')
}
console.log(`Soft deleting project ${projectId}`)
await waitForDb()
// soft delete, project will be permanently deleted after 90 days
await ProjectDeleter.promises.deleteProject(projectId)
}

View file

@ -1,8 +1,7 @@
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { db } from '../app/src/infrastructure/mongodb.js'
import { fileURLToPath } from 'url'
async function updateStringDates() {
await waitForDb()
const users = db.users.find({
splitTests: { $exists: true },
})

View file

@ -1,4 +1,4 @@
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { db } from '../app/src/infrastructure/mongodb.js'
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
import mongodb from 'mongodb-legacy'
import fs from 'fs'
@ -26,7 +26,6 @@ async function main() {
optedOutList = optedOutFile.split('\n').map(id => new ObjectId(id))
console.log(`preserving opt-outs of ${optedOutList.length} users`)
await waitForDb()
// update all applicable user models
await batchedUpdate(
'users',

View file

@ -1,4 +1,3 @@
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import SAMLUserIdMigrationHandler from '../modules/saas-authentication/app/src/SAML/SAMLUserIdMigrationHandler.mjs'
import { ensureRunningOnMongoSecondaryWithTimeout } from './helpers/env_variable_helper.mjs'
@ -10,12 +9,12 @@ const emitUsers = process.argv.includes('--emit-users')
console.log('Checking SSO user ID migration for institution:', institutionId)
waitForDb()
.then(main)
.catch(error => {
try {
await main()
} catch (error) {
console.error(error)
process.exit(1)
})
}
async function main() {
const result =

View file

@ -1,4 +1,3 @@
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import SAMLUserIdMigrationHandler from '../modules/saas-authentication/app/src/SAML/SAMLUserIdMigrationHandler.mjs'
import { ensureMongoTimeout } from './helpers/env_variable_helper.mjs'
@ -31,7 +30,6 @@ async function main() {
}
try {
await waitForDb()
await main()
} catch (error) {
console.error(error)

View file

@ -432,7 +432,7 @@ async function run() {
}
try {
await Promise.all([mongodb.waitForDb(), mongoose.connectionPromise])
await Promise.all([mongodb.connectionPromise, mongoose.connectionPromise])
await run()
log('Completed')
process.exit(0)

View file

@ -11,7 +11,7 @@
import { Certificate } from '@fidm/x509'
import UKAMFDB from './ukamf-db.js'
import V1ApiModule from '../../app/src/Features/V1/V1Api.js'
import { db, waitForDb } from '../../app/src/infrastructure/mongodb.js'
import { db } from '../../app/src/infrastructure/mongodb.js'
import moment from 'moment'
const { promises: V1Api } = V1ApiModule
@ -91,7 +91,6 @@ async function getActiveProviderIds() {
}
try {
await waitForDb()
await main()
} catch (error) {
console.error(error.stack)

View file

@ -1,5 +1,4 @@
import minimist from 'minimist'
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import ProjectDeleter from '../app/src/Features/Project/ProjectDeleter.js'
async function main() {
@ -11,7 +10,6 @@ async function main() {
throw new Error('set --project-id and --user-id')
}
console.log(`Restoring project ${projectId} to user ${userId}`)
await waitForDb()
await ProjectDeleter.promises.undeleteProject(projectId, { userId })
}

View file

@ -1,4 +1,3 @@
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import minimist from 'minimist'
import ThirdPartyIdentityManager from '../app/src/Features/User/ThirdPartyIdentityManager.js'
import UserGetter from '../app/src/Features/User/UserGetter.js'
@ -42,8 +41,6 @@ async function main() {
throw new Error('No --userId argument provided')
}
await waitForDb()
const auditLog = {
initiatorId: undefined,
ipAddress: '0.0.0.0',

View file

@ -1,6 +1,6 @@
import { execFile } from 'child_process'
import {
waitForDb,
connectionPromise,
db,
dropTestDatabase,
} from '../../../../app/src/infrastructure/mongodb.js'
@ -10,7 +10,9 @@ const DEFAULT_ENV = 'saas'
export default {
initialize() {
before('wait for db', waitForDb)
before('wait for db', async function () {
await connectionPromise
})
if (process.env.CLEANUP_MONGO === 'true') {
before('drop test database', dropTestDatabase)
}