github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-14 20:40:17 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #21097 from overleaf/ls-scripts-to-esm-1

Browse source 
Migrate scripts folder to esm 1/x

GitOrigin-RevId: 4a4bc9a161f144fdb40ce3f2a0a9313b36c6df81
This commit is contained in:
Liangjun Song 2024-10-18 19:04:57 +08:00 committed by Copybot
parent 8bc374c916
commit 26f3f3e2e2
94 changed files with 916 additions and 847 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
server-ce/init_preshutdown_scripts/00_close_site
View file

@ -12,12 +12,12 @@ echo "closed" > "${SITE_MAINTENANCE_FILE}"
sleep 5
# giving a grace period of 5 seconds for users before disconnecting them and start shutting down
cd /overleaf/services/web && node scripts/disconnect_all_users.js --delay-in-seconds=5 >> /var/log/overleaf/web.log 2>&1
cd /overleaf/services/web && node scripts/disconnect_all_users.mjs --delay-in-seconds=5 >> /var/log/overleaf/web.log 2>&1
EXIT_CODE="$?"
if [ $EXIT_CODE -ne 0 ]
then
echo "scripts/disconnect_all_users.js failed with exit code $EXIT_CODE"
echo "scripts/disconnect_all_users.mjs failed with exit code $EXIT_CODE"
exit 1
fi

2
services/web/Makefile
View file

@ -425,7 +425,7 @@ lint_flag_res_send_usage:
lint: lint_overleafModuleImports
lint_overleafModuleImports:
node scripts/check_overleafModuleImports.js
node scripts/check_overleafModuleImports.mjs
lint: typecheck_frontend
typecheck_frontend:

2
services/web/migrations/20210726083523_convert_confirmedAt_strings_to_dates.mjs
View file

@ -1,4 +1,4 @@
import updateStringDates from '../scripts/confirmed_at_to_dates.js'
import updateStringDates from '../scripts/confirmed_at_to_dates.mjs'
const tags = ['saas']

2
services/web/migrations/20210726083523_convert_split_tests_assigned_at_strings_to_dates.mjs
View file

@ -1,4 +1,4 @@
import updateStringDates from '../scripts/confirmed_at_to_dates.js'
import updateStringDates from '../scripts/confirmed_at_to_dates.mjs'
const tags = ['saas']

2
services/web/migrations/20210727123346_ce_sp_backfill_deleted_files.mjs
View file

@ -1,4 +1,4 @@
import runScript from '../scripts/back_fill_deleted_files.js'
import runScript from '../scripts/back_fill_deleted_files.mjs'
const tags = ['server-ce', 'server-pro', 'saas']

2
services/web/migrations/20210727150530_ce_sp_backfill_deleted_docs.mjs
View file

@ -1,4 +1,4 @@
import runScript from '../scripts/back_fill_doc_name_for_deleted_docs.js'
import runScript from '../scripts/back_fill_doc_name_for_deleted_docs.mjs'
const tags = ['server-ce', 'server-pro', 'saas']

2
services/web/migrations/20210728115327_ce_sp_backfill_dummy_doc_meta.mjs
View file

@ -1,4 +1,4 @@
import runScript from '../scripts/back_fill_dummy_doc_meta.js'
import runScript from '../scripts/back_fill_dummy_doc_meta.mjs'
const tags = ['server-ce', 'server-pro', 'saas']

2
services/web/migrations/20220913125500_migrate_auditLog_to_collections.mjs
View file

@ -1,4 +1,4 @@
import runScript from '../scripts/migrate_audit_logs.js'
import runScript from '../scripts/migrate_audit_logs.mjs'
const tags = ['server-ce', 'server-pro', 'saas']

2
services/web/migrations/20221111111111_ce_sp_convert_archived_state.mjs
View file

@ -1,4 +1,4 @@
import runScript from '../scripts/convert_archived_state.js'
import runScript from '../scripts/convert_archived_state.mjs'
const tags = ['server-ce', 'server-pro']

2
services/web/migrations/20230124092607_clear_old_2fa_setup.mjs
View file

@ -1,4 +1,4 @@
import BatchedUpdateScript from '../scripts/helpers/batchedUpdate.js'
import BatchedUpdateScript from '../scripts/helpers/batchedUpdate.mjs'
const { batchedUpdate } = BatchedUpdateScript
const tags = ['saas']

2
services/web/migrations/20230315170739_back_fill_doc_rev.mjs
View file

@ -1,4 +1,4 @@
import runScript from '../scripts/back_fill_doc_rev.js'
import runScript from '../scripts/back_fill_doc_rev.mjs'
const tags = ['server-ce', 'server-pro']

2
services/web/migrations/20240220130452_remove_deleted_users_from_token_access_refs.mjs
View file

@ -1,4 +1,4 @@
import runScript from '../scripts/remove_deleted_users_from_token_access_refs.js'
import runScript from '../scripts/remove_deleted_users_from_token_access_refs.mjs'
const tags = ['server-ce', 'server-pro', 'saas']

2
services/web/migrations/20240524135408_add_token_hmac_project_invite_tokens.mjs
View file

@ -1,7 +1,7 @@
/* eslint-disable no-unused-vars */
import Helpers from './lib/helpers.mjs'
import runScript from '../scripts/backfill_project_invites_token_hmac.js'
import runScript from '../scripts/backfill_project_invites_token_hmac.mjs'
const tags = ['server-ce', 'server-pro', 'saas']

2
services/web/migrations/20240618125145_cleanup_user_features_templates.mjs
View file

@ -1,4 +1,4 @@
import BatchedUpdateScript from '../scripts/helpers/batchedUpdate.js'
import BatchedUpdateScript from '../scripts/helpers/batchedUpdate.mjs'
const { batchedUpdate } = BatchedUpdateScript
const tags = ['server-ce', 'server-pro', 'saas']

21
services/web/scripts/add_feature_override.js → services/web/scripts/add_feature_override.mjs
View file

@ -10,7 +10,7 @@
//
// Usage:
//
// $ node scripts/add_feature_override.js --commit --note 'text description' --expires 2022-01-01 --override JSONFILE --ids IDFILE
// $ node scripts/add_feature_override.mjs --commit --note 'text description' --expires 2022-01-01 --override JSONFILE --ids IDFILE
//
// --commit do the update, remove this option for dry-run testing
// --note text description [optional]
@ -23,14 +23,15 @@
// The feature override is specified with JSON to allow types to be set as string/number/boolean.
// It is contained in a file to avoid any issues with shell quoting.
const minimist = require('minimist')
const fs = require('fs')
const { ObjectId, waitForDb } = require('../app/src/infrastructure/mongodb')
const pLimit = require('p-limit')
const FeaturesUpdater = require('../app/src/Features/Subscription/FeaturesUpdater')
const FeaturesHelper = require('../app/src/Features/Subscription/FeaturesHelper')
const UserFeaturesUpdater = require('../app/src/Features/Subscription/UserFeaturesUpdater')
const UserGetter = require('../app/src/Features/User/UserGetter')
import minimist from 'minimist'
import fs from 'fs'
import { ObjectId, waitForDb } from '../app/src/infrastructure/mongodb.js'
import pLimit from 'p-limit'
import FeaturesUpdater from '../app/src/Features/Subscription/FeaturesUpdater.js'
import FeaturesHelper from '../app/src/Features/Subscription/FeaturesHelper.js'
import UserFeaturesUpdater from '../app/src/Features/Subscription/UserFeaturesUpdater.js'
import UserGetter from '../app/src/Features/User/UserGetter.js'
const processLogger = {
failed: [],
@ -180,4 +181,4 @@ async function processUsers(userIds) {
process.exit()
}
processUsers(userIds)
await processUsers(userIds)

17
services/web/scripts/add_salesforce_data_to_subscriptions.js → services/web/scripts/add_salesforce_data_to_subscriptions.mjs
View file

@ -1,13 +1,13 @@
const fs = require('node:fs')
const minimist = require('minimist')
const { parse } = require('csv')
const Stream = require('stream/promises')
const { ObjectId, waitForDb } = require('../app/src/infrastructure/mongodb')
const { Subscription } = require('../app/src/models/Subscription')
import fs from 'node:fs'
import minimist from 'minimist'
import { parse } from 'csv'
import Stream from 'stream/promises'
import { ObjectId, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { Subscription } from '../app/src/models/Subscription.js'
function usage() {
console.log(
'Usage: node add_salesforce_data_to_subscriptions.js -f <filename> [options]'
'Usage: node add_salesforce_data_to_subscriptions.mjs -f <filename> [options]'
)
console.log(
'Updates the subscriptions collection with external IDs for determining the Salesforce account that goes with the subscription. The file should be a CSV and have columns account_id, v1_id and subscription_id. The account_id column is the Salesforce account ID, the v1_id column is the V1 account ID, and the subscription_id column is the subscription ID.'
@ -201,7 +201,6 @@ if (!commit) {
console.log('Committing changes to the database')
}
main().then(() => {
await main()
showStats()
process.exit()
})

24
services/web/scripts/back_fill_deleted_files.js → services/web/scripts/back_fill_deleted_files.mjs
View file

@ -1,8 +1,11 @@
const { batchedUpdate } = require('./helpers/batchedUpdate')
const { promiseMapWithLimit, promisify } = require('@overleaf/promise-utils')
const { db } = require('../app/src/infrastructure/mongodb')
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
import { promiseMapWithLimit, promisify } from '@overleaf/promise-utils'
import { db } from '../app/src/infrastructure/mongodb.js'
import _ from 'lodash'
import { fileURLToPath } from 'url'
const { batchedUpdate } = BatchedUpdateModule
const sleep = promisify(setTimeout)
const _ = require('lodash')
async function main(options) {
if (!options) {
@ -118,15 +121,14 @@ async function letUserDoubleCheckInputs(options) {
await sleep(options.letUserDoubleCheckInputsFor)
}
module.exports = main
export default main
if (require.main === module) {
main()
.then(() => {
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await main()
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error({ error })
process.exit(1)
})
}
}

24
services/web/scripts/back_fill_doc_name_for_deleted_docs.js → services/web/scripts/back_fill_doc_name_for_deleted_docs.mjs
View file

@ -1,8 +1,11 @@
const { batchedUpdate } = require('./helpers/batchedUpdate')
const { promiseMapWithLimit, promisify } = require('@overleaf/promise-utils')
const { db } = require('../app/src/infrastructure/mongodb')
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
import { promiseMapWithLimit, promisify } from '@overleaf/promise-utils'
import { db } from '../app/src/infrastructure/mongodb.js'
import { fileURLToPath } from 'url'
import _ from 'lodash'
const sleep = promisify(setTimeout)
const _ = require('lodash')
const { batchedUpdate } = BatchedUpdateModule
async function main(options) {
if (!options) {
@ -77,15 +80,14 @@ async function letUserDoubleCheckInputs(options) {
await sleep(options.letUserDoubleCheckInputsFor)
}
module.exports = main
export default main
if (require.main === module) {
main()
.then(() => {
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await main()
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error({ error })
process.exit(1)
})
}
}

20
services/web/scripts/back_fill_doc_rev.js → services/web/scripts/back_fill_doc_rev.mjs
View file

@ -1,5 +1,8 @@
const { db } = require('../app/src/infrastructure/mongodb')
const { batchedUpdate } = require('./helpers/batchedUpdate')
import { db } from '../app/src/infrastructure/mongodb.js'
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
import { fileURLToPath } from 'url'
const { batchedUpdate } = BatchedUpdateModule
const DRY_RUN = !process.argv.includes('--dry-run=false')
const LOG_EVERY_IN_S = parseInt(process.env.LOG_EVERY_IN_S, 10) || 5
@ -44,16 +47,15 @@ async function main(DRY_RUN) {
logProgress()
}
module.exports = main
export default main
if (require.main === module) {
main(DRY_RUN)
.then(() => {
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await main(DRY_RUN)
console.log('Done.')
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error({ error })
process.exit(1)
})
}
}

28
services/web/scripts/back_fill_dummy_doc_meta.js → services/web/scripts/back_fill_dummy_doc_meta.mjs
View file

@ -1,12 +1,16 @@
const { promisify } = require('util')
const { ObjectId } = require('mongodb-legacy')
const {
import { promisify } from 'util'
import mongodb from 'mongodb-legacy'
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
} from '../app/src/infrastructure/mongodb.js'
import _ from 'lodash'
import LRUCache from 'lru-cache'
import { fileURLToPath } from 'url'
const { ObjectId } = mongodb
const sleep = promisify(setTimeout)
const _ = require('lodash')
const NOW_IN_S = Date.now() / 1000
const ONE_WEEK_IN_S = 60 * 60 * 24 * 7
@ -15,7 +19,6 @@ const TEN_SECONDS = 10 * 1000
const DUMMY_NAME = 'unknown.tex'
const DUMMY_TIME = new Date('2021-04-12T00:00:00.000Z')
const LRUCache = require('lru-cache')
let deletedProjectsCache = null
function getSecondsFromObjectId(id) {
@ -147,16 +150,15 @@ async function letUserDoubleCheckInputs(options) {
await sleep(options.letUserDoubleCheckInputsFor)
}
module.exports = main
export default main
if (require.main === module) {
main()
.then(() => {
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await main()
console.error('Done.')
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error({ error })
process.exit(1)
})
}
}

15
services/web/scripts/back_fill_staff_access.js → services/web/scripts/back_fill_staff_access.mjs
View file

@ -1,9 +1,9 @@
const {
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
const UserSessionsManager = require('../app/src/Features/User/UserSessionsManager')
} from '../app/src/infrastructure/mongodb.js'
import UserSessionsManager from '../app/src/Features/User/UserSessionsManager.js'
const COMMIT = process.argv.includes('--commit')
const KEEP_SESSIONS = process.argv.includes('--keep-sessions')
@ -85,12 +85,11 @@ async function main() {
}
}
main()
.then(() => {
try {
await main()
console.error('Done.')
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error({ error })
process.exit(1)
})
}

18
services/web/scripts/back_fill_warning_user_personal_and_group_subscription.js → services/web/scripts/back_fill_warning_user_personal_and_group_subscription.mjs
View file

@ -1,7 +1,8 @@
const NotificationsBuilder = require('../app/src/Features/Notifications/NotificationsBuilder')
const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
const { batchedUpdate } = require('./helpers/batchedUpdate')
import NotificationsBuilder from '../app/src/Features/Notifications/NotificationsBuilder.js'
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
const { batchedUpdate } = BatchedUpdateModule
const DRY_RUN = !process.argv.includes('--dry-run=false')
if (DRY_RUN) {
@ -63,11 +64,10 @@ async function main() {
})
}
main()
.then(() => {
try {
await main()
process.exit(0)
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}

20
services/web/scripts/backfill_mixpanel_user_properties.js → services/web/scripts/backfill_mixpanel_user_properties.mjs
View file

@ -1,15 +1,15 @@
import '../app/src/models/User.js'
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
import { promiseMapWithLimit } from '@overleaf/promise-utils'
import { getQueue } from '../app/src/infrastructure/Queues.js'
import SubscriptionLocator from '../app/src/Features/Subscription/SubscriptionLocator.js'
import PlansLocator from '../app/src/Features/Subscription/PlansLocator.js'
import FeaturesHelper from '../app/src/Features/Subscription/FeaturesHelper.js'
const { batchedUpdateWithResultHandling } = BatchedUpdateModule
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
require('../app/src/models/User')
const { batchedUpdateWithResultHandling } = require('./helpers/batchedUpdate')
const { promiseMapWithLimit } = require('@overleaf/promise-utils')
const { getQueue } = require('../app/src/infrastructure/Queues')
const SubscriptionLocator = require('../app/src/Features/Subscription/SubscriptionLocator')
const PlansLocator = require('../app/src/Features/Subscription/PlansLocator')
const FeaturesHelper = require('../app/src/Features/Subscription/FeaturesHelper')
const mixpanelSinkQueue = getQueue('analytics-mixpanel-sink')
async function processUser(user) {

8
services/web/scripts/backfill_project_image_name.js → services/web/scripts/backfill_project_image_name.mjs
View file

@ -1,5 +1,7 @@
const minimist = require('minimist')
const { batchedUpdateWithResultHandling } = require('./helpers/batchedUpdate')
import minimist from 'minimist'
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
const { batchedUpdateWithResultHandling } = BatchedUpdateModule
const argv = minimist(process.argv.slice(2))
const commit = argv.commit !== undefined
@ -7,7 +9,7 @@ let imageName = argv._[0]
function usage() {
console.log(
'Usage: node backfill_project_image_name.js --commit <texlive_docker_image>'
'Usage: node backfill_project_image_name.mjs --commit <texlive_docker_image>'
)
console.log(
'Argument <texlive_docker_image> is not required when TEX_LIVE_DOCKER_IMAGE is set.'

28
services/web/scripts/backfill_project_invites_token_hmac.js → services/web/scripts/backfill_project_invites_token_hmac.mjs
View file

@ -1,7 +1,10 @@
const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
const { batchedUpdate } = require('./helpers/batchedUpdate')
const minimist = require('minimist')
const CollaboratorsInviteHelper = require('../app/src/Features/Collaborators/CollaboratorsInviteHelper')
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
import minimist from 'minimist'
import CollaboratorsInviteHelper from '../app/src/Features/Collaborators/CollaboratorsInviteHelper.js'
import { fileURLToPath } from 'url'
const { batchedUpdate } = BatchedUpdateModule
const argv = minimist(process.argv.slice(2), {
boolean: ['dry-run', 'help'],
@ -54,11 +57,11 @@ async function main(DRY_RUN) {
await addTokenHmacField(DRY_RUN)
}
module.exports = main
export default main
if (require.main === module) {
if (fileURLToPath(import.meta.url) === process.argv[1]) {
if (argv.help || argv._.length > 1) {
console.error(`Usage: node scripts/backfill_project_invites_token_hmac.js
console.error(`Usage: node scripts/backfill_project_invites_token_hmac.mjs
Adds a "tokenHmac" field (which is a hashed version of the token) to each project invite record.
Options:
@ -68,13 +71,12 @@ if (require.main === module) {
process.exit(1)
}
main(DRY_RUN)
.then(() => {
try {
await main(DRY_RUN)
console.error('Done')
process.exit(0)
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}
}

18
services/web/scripts/backfill_user_properties.js → services/web/scripts/backfill_user_properties.mjs
View file

@ -1,11 +1,13 @@
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
import { promiseMapWithLimit } from '@overleaf/promise-utils'
import SubscriptionLocator from '../app/src/Features/Subscription/SubscriptionLocator.js'
import PlansLocator from '../app/src/Features/Subscription/PlansLocator.js'
import FeaturesHelper from '../app/src/Features/Subscription/FeaturesHelper.js'
import AnalyticsManager from '../app/src/Features/Analytics/AnalyticsManager.js'
const { batchedUpdateWithResultHandling } = require('./helpers/batchedUpdate')
const { promiseMapWithLimit } = require('@overleaf/promise-utils')
const SubscriptionLocator = require('../app/src/Features/Subscription/SubscriptionLocator')
const PlansLocator = require('../app/src/Features/Subscription/PlansLocator')
const FeaturesHelper = require('../app/src/Features/Subscription/FeaturesHelper')
const AnalyticsManager = require('../app/src/Features/Analytics/AnalyticsManager')
const { batchedUpdateWithResultHandling } = BatchedUpdateModule
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
async function getGroupSubscriptionPlanCode(userId) {
const subscriptions =
@ -47,7 +49,7 @@ async function processUser(user) {
}
}
async function processBatch(_, users) {
async function processBatch(users) {
await promiseMapWithLimit(WRITE_CONCURRENCY, users, async user => {
await processUser(user)
})

14
services/web/scripts/backfill_users_sso_attribute.js → services/web/scripts/backfill_users_sso_attribute.mjs
View file

@ -1,4 +1,4 @@
const SAMLUserIdAttributeBatchHandler = require('../modules/saas-authentication/app/src/SAML/SAMLUserIdAttributeBatchHandler')
import SAMLUserIdAttributeBatchHandler from '../modules/saas-authentication/app/src/SAML/SAMLUserIdAttributeBatchHandler.js'
const startInstitutionId = parseInt(process.argv[2])
const endInstitutionId = parseInt(process.argv[3])
@ -20,12 +20,14 @@ console.log(
endInstitutionId || 'none provided, will go to end of ordered list.'
)
SAMLUserIdAttributeBatchHandler.check(startInstitutionId, endInstitutionId)
.then(result => {
try {
const result = await SAMLUserIdAttributeBatchHandler.check(
startInstitutionId,
endInstitutionId
)
console.log(result)
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error(error)
process.exit(1)
})
}

25
services/web/scripts/check_institution_users.js → services/web/scripts/check_institution_users.mjs
View file

@ -1,23 +1,14 @@
process.env.MONGO_SOCKET_TIMEOUT = '300000'
// Run all the mongo queries on secondaries
process.env.MONGO_CONNECTION_STRING =
process.env.READ_ONLY_MONGO_CONNECTION_STRING
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import InstitutionsManager from '../app/src/Features/Institutions/InstitutionsManager.js'
import { ensureRunningOnMongoSecondaryWithTimeout } from './helpers/env_variable_helper.mjs'
const { waitForDb } = require('../app/src/infrastructure/mongodb')
const InstitutionsManager = require('../app/src/Features/Institutions/InstitutionsManager')
ensureRunningOnMongoSecondaryWithTimeout(300000)
const institutionId = parseInt(process.argv[2])
if (isNaN(institutionId)) throw new Error('No institution id')
console.log('Checking users of institution', institutionId)
const emitNonProUserIds = process.argv.includes('--emit-non-pro-user-ids')
waitForDb()
.then(main)
.catch(err => {
console.error(err)
process.exit(1)
})
async function main() {
const usersSummary = await InstitutionsManager.promises.checkInstitutionUsers(
institutionId,
@ -26,3 +17,11 @@ async function main() {
console.log(usersSummary)
process.exit()
}
try {
await waitForDb()
await main()
} catch (error) {
console.error(error)
process.exit(1)
}

8
services/web/scripts/check_overleafModuleImports.js → services/web/scripts/check_overleafModuleImports.mjs
View file

@ -1,11 +1,11 @@
import CE_CONFIG from '../config/settings.defaults.js'
import PRO_CONFIG from '../config/settings.overrides.server-pro.js'
import SAAS_CONFIG from '../config/settings.webpack.js'
function getOverleafModuleImports(settings) {
return Object.keys(settings.overleafModuleImports).sort().join(',')
}
const CE_CONFIG = require('../config/settings.defaults')
const PRO_CONFIG = require('../config/settings.overrides.server-pro')
const SAAS_CONFIG = require('../config/settings.webpack')
function main() {
const CE = getOverleafModuleImports(CE_CONFIG)
const PRO = getOverleafModuleImports(CE_CONFIG.mergeWith(PRO_CONFIG))

19
services/web/scripts/check_saml_emails.js
View file

@ -1,19 +0,0 @@
process.env.MONGO_SOCKET_TIMEOUT = '300000'
// Run all the mongo queries on secondaries
process.env.MONGO_CONNECTION_STRING =
process.env.READ_ONLY_MONGO_CONNECTION_STRING
const SAMLEmailBatchCheck = require('../modules/saas-authentication/app/src/SAML/SAMLEmailBatchCheck')
const startInstitutionId = parseInt(process.argv[2])
const emitDetailedData = process.argv.includes('--detailed-data')
SAMLEmailBatchCheck.promises
.checkEmails(startInstitutionId, emitDetailedData)
.then(result => {
console.table(result)
process.exit()
})
.catch(err => {
console.error(err)
process.exit(1)
})

19
services/web/scripts/check_saml_emails.mjs Normal file
View file

@ -0,0 +1,19 @@
import SAMLEmailBatchCheck from '../modules/saas-authentication/app/src/SAML/SAMLEmailBatchCheck.js'
import { ensureRunningOnMongoSecondaryWithTimeout } from './helpers/env_variable_helper.mjs'
ensureRunningOnMongoSecondaryWithTimeout(300000)
const startInstitutionId = parseInt(process.argv[2])
const emitDetailedData = process.argv.includes('--detailed-data')
try {
const result = await SAMLEmailBatchCheck.promises.checkEmails(
startInstitutionId,
emitDetailedData
)
console.table(result)
process.exit()
} catch (error) {
console.error(error)
process.exit(1)
}

15
services/web/scripts/clear_admin_sessions.js → services/web/scripts/clear_admin_sessions.mjs
View file

@ -1,9 +1,9 @@
const {
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
const UserSessionsManager = require('../app/src/Features/User/UserSessionsManager')
} from '../app/src/infrastructure/mongodb.js'
import UserSessionsManager from '../app/src/Features/User/UserSessionsManager.js'
const COMMIT = process.argv.includes('--commit')
const LOG_SESSIONS = !process.argv.includes('--log-sessions=false')
@ -59,12 +59,11 @@ async function main() {
}
}
main()
.then(() => {
try {
await main()
console.error('Done.')
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error({ error })
process.exit(1)
})
}

7
services/web/scripts/clear_feedback_collection.js → services/web/scripts/clear_feedback_collection.mjs
View file

@ -5,7 +5,8 @@
* DRY_RUN=false node scripts/clear_feedback_collection.js 2022-11-01 # deletion mode
*/
const { db, ObjectId, waitForDb } = require('../app/src/infrastructure/mongodb')
import { db, ObjectId, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { fileURLToPath } from 'url'
const runScript = async (timestamp, dryRun) => {
await waitForDb()
@ -26,7 +27,7 @@ const runScript = async (timestamp, dryRun) => {
}
}
if (!module.parent) {
if (fileURLToPath(import.meta.url) === process.argv[1]) {
// we are in the root module, which means that we're running as a script
const timestamp = process.env.CUTOFF_TIMESTAMP || process.argv[2]
const dryRun = process.env.DRY_RUN !== 'false'
@ -38,4 +39,4 @@ if (!module.parent) {
})
}
module.exports = runScript
export default runScript

18
services/web/scripts/clear_institution_notifications.js → services/web/scripts/clear_institution_notifications.mjs
View file

@ -1,5 +1,6 @@
const { promisify } = require('util')
const InstitutionsManager = require('../app/src/Features/Institutions/InstitutionsManager')
import { promisify } from 'util'
import InstitutionsManager from '../app/src/Features/Institutions/InstitutionsManager.js'
import { fileURLToPath } from 'url'
const sleep = promisify(setTimeout)
async function main() {
@ -36,14 +37,13 @@ async function main() {
console.log('---------------')
}
if (require.main === module) {
main()
.then(() => {
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await main()
console.log('Done.')
process.exit(0)
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}
}

19
services/web/scripts/clear_project_tokens.js → services/web/scripts/clear_project_tokens.mjs
View file

@ -1,5 +1,5 @@
const { waitForDb } = require('../app/src/infrastructure/mongodb')
const ProjectDetailsHandler = require('../app/src/Features/Project/ProjectDetailsHandler')
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import ProjectDetailsHandler from '../app/src/Features/Project/ProjectDetailsHandler.js'
const projectId = process.argv[2]
if (!/^(?=[a-f\d]{24}$)(\d+[a-f]|[a-f]+\d)/.test(projectId)) {
@ -7,13 +7,6 @@ if (!/^(?=[a-f\d]{24}$)(\d+[a-f]|[a-f]+\d)/.test(projectId)) {
process.exit(1)
}
waitForDb()
.then(main)
.catch(err => {
console.error(err)
process.exit(1)
})
function main() {
ProjectDetailsHandler.clearTokens(projectId, err => {
if (err) {
@ -27,3 +20,11 @@ function main() {
process.exit(0)
})
}
try {
await waitForDb()
await main()
} catch (error) {
console.error(error)
process.exit(1)
}

17
services/web/scripts/clear_sessions_2fa.js → services/web/scripts/clear_sessions_2fa.mjs
View file

@ -1,7 +1,10 @@
const { promisify, promiseMapWithLimit } = require('@overleaf/promise-utils')
const rClient = require('../app/src/Features/User/UserSessionsRedis').client()
import { promisify, promiseMapWithLimit } from '@overleaf/promise-utils'
import UserSessionsRedis from '../app/src/Features/User/UserSessionsRedis.js'
import minimist from 'minimist'
const args = require('minimist')(process.argv.slice(2))
const rClient = UserSessionsRedis.client()
const args = minimist(process.argv.slice(2))
const CURSOR = args.cursor
const COMMIT = args.commit === 'true'
const CONCURRENCY = parseInt(args.concurrency, 10) || 50
@ -72,7 +75,9 @@ async function main() {
await rClient.disconnect()
}
main().catch(err => {
console.error(err)
try {
await main()
} catch (error) {
console.error(error)
process.exit(1)
})
}

10
services/web/scripts/clear_sessions_set_must_reconfirm.js → services/web/scripts/clear_sessions_set_must_reconfirm.mjs
View file

@ -1,8 +1,8 @@
const fs = require('fs')
const { ObjectId, waitForDb } = require('../app/src/infrastructure/mongodb')
const async = require('async')
const UserUpdater = require('../app/src/Features/User/UserUpdater')
const UserSessionsManager = require('../app/src/Features/User/UserSessionsManager')
import fs from 'fs'
import { ObjectId, waitForDb } from '../app/src/infrastructure/mongodb.js'
import async from 'async'
import UserUpdater from '../app/src/Features/User/UserUpdater.js'
import UserSessionsManager from '../app/src/Features/User/UserSessionsManager.js'
const ASYNC_LIMIT = 10

16
services/web/scripts/confirmed_at_to_dates.js → services/web/scripts/confirmed_at_to_dates.mjs
View file

@ -1,4 +1,5 @@
const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { fileURLToPath } from 'url'
async function updateStringDates() {
await waitForDb()
@ -40,15 +41,14 @@ async function updateStringDates() {
console.log(`Updated ${count} confirmedAt strings to dates!`)
}
if (require.main === module) {
updateStringDates()
.then(() => {
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await updateStringDates()
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error(error)
process.exit(1)
})
}
}
module.exports = updateStringDates
export default updateStringDates

39
services/web/scripts/convert_archived_state.js → services/web/scripts/convert_archived_state.mjs
View file

@ -1,12 +1,14 @@
const _ = require('lodash')
import _ from 'lodash'
import { db } from '../app/src/infrastructure/mongodb.js'
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
import { promiseMapWithLimit } from '@overleaf/promise-utils'
import { fileURLToPath } from 'url'
const { batchedUpdate } = BatchedUpdateModule
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
const { db } = require('../app/src/infrastructure/mongodb')
const { batchedUpdate } = require('./helpers/batchedUpdate')
const { promiseMapWithLimit } = require('@overleaf/promise-utils')
// $ node scripts/convert_archived_state.js FIRST,SECOND
// $ node scripts/convert_archived_state.mjs FIRST,SECOND
async function main(STAGE) {
for (const FIELD of ['archived', 'trashed']) {
@ -55,19 +57,6 @@ async function main(STAGE) {
}
}
module.exports = main
if (require.main === module) {
main(process.argv.pop())
.then(() => {
process.exit(0)
})
.catch(error => {
console.error({ error })
process.exit(1)
})
}
async function upgradeFieldToArray({ project, FIELD }) {
return db.projects.updateOne(
{ _id: project._id },
@ -92,3 +81,15 @@ function _objectIdEquals(firstVal, secondVal) {
// For use as a comparator for unionWith
return firstVal.toString() === secondVal.toString()
}
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await main(process.argv.pop())
process.exit(0)
} catch (error) {
console.error({ error })
process.exit(1)
}
}
export default main

19
services/web/scripts/convert_doc_to_file.js → services/web/scripts/convert_doc_to_file.mjs
View file

@ -1,7 +1,7 @@
const minimist = require('minimist')
const { waitForDb, ObjectId } = require('../app/src/infrastructure/mongodb')
const ProjectEntityUpdateHandler = require('../app/src/Features/Project/ProjectEntityUpdateHandler')
const Errors = require('../app/src/Features/Errors/Errors')
import minimist from 'minimist'
import { waitForDb, ObjectId } from '../app/src/infrastructure/mongodb.js'
import ProjectEntityUpdateHandler from '../app/src/Features/Project/ProjectEntityUpdateHandler.js'
import Errors from '../app/src/Features/Errors/Errors.js'
async function main() {
const argv = minimist(process.argv.slice(2))
@ -35,12 +35,11 @@ async function main() {
}
}
main()
.then(() => {
try {
await main()
console.log('Done.')
process.exit(0)
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}

31
services/web/scripts/count_encrypted_access_tokens.js → services/web/scripts/count_encrypted_access_tokens.mjs
View file

@ -1,16 +1,16 @@
const TEN_MINUTES = 1000 * 60 * 10
process.env.MONGO_SOCKET_TIMEOUT =
process.env.MONGO_SOCKET_TIMEOUT || TEN_MINUTES.toString()
const {
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
const _ = require('lodash')
const {
formatTokenUsageStats,
} = require('@overleaf/access-token-encryptor/scripts/helpers/format-usage-stats')
} from '../app/src/infrastructure/mongodb.js'
import _ from 'lodash'
import { formatTokenUsageStats } from '@overleaf/access-token-encryptor/scripts/helpers/format-usage-stats.js'
import { ensureMongoTimeout } from './helpers/env_variable_helper.mjs'
if (!process.env.MONGO_SOCKET_TIMEOUT) {
const TEN_MINUTES = 1000 * 60 * 10
ensureMongoTimeout(TEN_MINUTES)
}
const CASES = {
users: {
@ -67,11 +67,10 @@ async function main() {
formatTokenUsageStats(STATS)
}
main()
.then(() => {
try {
await main()
process.exit(0)
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}

25
services/web/scripts/count_files_in_projects.js → services/web/scripts/count_files_in_projects.mjs
View file

@ -1,10 +1,8 @@
const readline = require('readline')
const { waitForDb } = require('../app/src/infrastructure/mongodb')
const ProjectEntityHandler = require('../app/src/Features/Project/ProjectEntityHandler')
const ProjectGetter = require('../app/src/Features/Project/ProjectGetter')
const Errors = require('../app/src/Features/Errors/Errors')
/* eslint-disable no-console */
import readline from 'readline'
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import ProjectEntityHandler from '../app/src/Features/Project/ProjectEntityHandler.js'
import ProjectGetter from '../app/src/Features/Project/ProjectGetter.js'
import Errors from '../app/src/Features/Errors/Errors.js'
async function countFiles() {
const rl = readline.createInterface({
@ -36,12 +34,11 @@ async function countFiles() {
}
}
waitForDb()
.then(countFiles)
.then(() => {
try {
await waitForDb()
await countFiles()
process.exit(0)
})
.catch(err => {
console.log('Aiee, something went wrong!', err)
} catch (error) {
console.log('Aiee, something went wrong!', error)
process.exit(1)
})
}

17
services/web/scripts/count_image_files.js → services/web/scripts/count_image_files.mjs
View file

@ -1,9 +1,9 @@
const {
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
const { extname } = require('node:path')
} from '../app/src/infrastructure/mongodb.js'
import { extname } from 'node:path'
const FILE_TYPES = [
'.jpg',
@ -72,11 +72,10 @@ function countFiles(folder, result) {
return result
}
main()
.then(() => {
try {
await main()
process.exit(0)
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}

29
services/web/scripts/count_project_size.js → services/web/scripts/count_project_size.mjs
View file

@ -1,17 +1,15 @@
const readline = require('readline')
const { waitForDb, ObjectId, db } = require('../app/src/infrastructure/mongodb')
const ProjectEntityHandler = require('../app/src/Features/Project/ProjectEntityHandler')
const ProjectGetter = require('../app/src/Features/Project/ProjectGetter')
const Errors = require('../app/src/Features/Errors/Errors')
const FileStoreHandler = require('../app/src/Features/FileStore/FileStoreHandler')
/* eslint-disable no-console */
import readline from 'readline'
import { waitForDb, ObjectId, db } from '../app/src/infrastructure/mongodb.js'
import ProjectEntityHandler from '../app/src/Features/Project/ProjectEntityHandler.js'
import ProjectGetter from '../app/src/Features/Project/ProjectGetter.js'
import Errors from '../app/src/Features/Errors/Errors.js'
import FileStoreHandler from '../app/src/Features/FileStore/FileStoreHandler.js'
// Handles a list of project IDs from stdin, one per line, and outputs the count of files and docs
// in the project, along with the aggregated size in bytes for all files and docs.
//
// It outputs to stderr, so that the logging junk can be piped elsewhere - e.g., running like:
// node scripts/count_project_size.js < /tmp/project_ids.txt /dev/null 2> /tmp/output.txt
// node scripts/count_project_size.mjs < /tmp/project_ids.txt /dev/null 2> /tmp/output.txt
//
// The output format is line-per-project with data separated by a single space, containing:
// - projectId
@ -128,12 +126,11 @@ async function countDocsSizes(docs) {
return totalDocSize
}
waitForDb()
.then(countProjectFiles)
.then(() => {
try {
await waitForDb()
await countProjectFiles()
process.exit(0)
})
.catch(err => {
console.log('Aiee, something went wrong!', err)
} catch (error) {
console.log('Aiee, something went wrong!', error)
process.exit(1)
})
}

20
services/web/scripts/create_oauth_personal_access_token.js → services/web/scripts/create_oauth_personal_access_token.mjs
View file

@ -1,10 +1,11 @@
// Script to create a Personal Access Token for a given user
// Example:
// node scripts/create_oauth_personal_access_token.js --user-id=643e5b240dc50c83b5bf1127
// node scripts/create_oauth_personal_access_token.mjs --user-id=643e5b240dc50c83b5bf1127
const parseArgs = require('minimist')
const { waitForDb } = require('../app/src/infrastructure/mongodb')
const OAuthPersonalAccessTokenManager = require('../modules/oauth2-server/app/src/OAuthPersonalAccessTokenManager')
import parseArgs from 'minimist'
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import OAuthPersonalAccessTokenManager from '../modules/oauth2-server/app/src/OAuthPersonalAccessTokenManager.js'
const argv = parseArgs(process.argv.slice(2), {
string: ['user-id'],
@ -23,11 +24,10 @@ async function createPersonalAccessToken() {
console.log('Personal Access Token: ' + accessToken)
}
createPersonalAccessToken()
.then(() => {
try {
await createPersonalAccessToken()
process.exit()
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}

37
services/web/scripts/create_project.js → services/web/scripts/create_project.mjs
View file

@ -2,17 +2,21 @@
// Example:
// node scripts/create_project.js --user-id=5dca84e11e71ae002ff73bd4 --name="My Test Project" --old-history
const fs = require('fs')
const path = require('path')
const _ = require('lodash')
const parseArgs = require('minimist')
const OError = require('@overleaf/o-error')
const { waitForDb } = require('../app/src/infrastructure/mongodb')
const { User } = require('../app/src/models/User')
const ProjectCreationHandler = require('../app/src/Features/Project/ProjectCreationHandler')
const ProjectEntityUpdateHandler = require('../app/src/Features/Project/ProjectEntityUpdateHandler')
const ProjectEntityHandler = require('../app/src/Features/Project/ProjectEntityHandler')
const EditorController = require('../app/src/Features/Editor/EditorController')
import fs from 'fs'
import path from 'path'
import _ from 'lodash'
import parseArgs from 'minimist'
import OError from '@overleaf/o-error'
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import { User } from '../app/src/models/User.js'
import ProjectCreationHandler from '../app/src/Features/Project/ProjectCreationHandler.js'
import ProjectEntityUpdateHandler from '../app/src/Features/Project/ProjectEntityUpdateHandler.js'
import ProjectEntityHandler from '../app/src/Features/Project/ProjectEntityHandler.js'
import EditorController from '../app/src/Features/Editor/EditorController.js'
import { fileURLToPath } from 'url'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const argv = parseArgs(process.argv.slice(2), {
string: ['user-id', 'name', 'random-operations', 'extend-project-id'],
@ -226,12 +230,11 @@ async function createProject() {
return projectId
}
createProject()
.then(projectId => {
try {
const projectId = await createProject()
console.log('Created project', projectId)
process.exit()
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}

25
services/web/scripts/delete_dangling_file_refs.js → services/web/scripts/delete_dangling_file_refs.mjs
View file

@ -2,14 +2,16 @@
* This script deletes dangling doc and file refs in projects
*/
const minimist = require('minimist')
const { ObjectId } = require('mongodb-legacy')
import minimist from 'minimist'
const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
const Errors = require('../app/src/Features/Errors/Errors')
const FileStoreHandler = require('../app/src/Features/FileStore/FileStoreHandler')
const ProjectEntityMongoUpdateHandler = require('../app/src/Features/Project/ProjectEntityMongoUpdateHandler')
const { iterablePaths } = require('../app/src/Features/Project/IterablePath')
import mongodb from 'mongodb-legacy'
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import Errors from '../app/src/Features/Errors/Errors.js'
import FileStoreHandler from '../app/src/Features/FileStore/FileStoreHandler.js'
import ProjectEntityMongoUpdateHandler from '../app/src/Features/Project/ProjectEntityMongoUpdateHandler.js'
import { iterablePaths } from '../app/src/Features/Project/IterablePath.js'
const { ObjectId } = mongodb
const OPTIONS = parseArgs()
@ -119,11 +121,10 @@ async function deleteFile(projectId, fileId) {
}
}
main()
.then(() => {
try {
await main()
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error({ error })
process.exit(1)
})
}

24
services/web/scripts/delete_orphaned_chat_threads.js → services/web/scripts/delete_orphaned_chat_threads.mjs
View file

@ -1,3 +1,14 @@
import mongodb from 'mongodb-legacy'
import { promiseMapWithLimit } from '@overleaf/promise-utils'
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
import ChatApiHandler from '../app/src/Features/Chat/ChatApiHandler.js'
import DeleteOrphanedDataHelper from './delete_orphaned_data_helper.mjs'
import { ensureMongoTimeout } from './helpers/env_variable_helper.mjs'
const { batchedUpdate } = BatchedUpdateModule
const { ObjectId } = mongodb
const { getHardDeletedProjectIds } = DeleteOrphanedDataHelper
const READ_CONCURRENCY_SECONDARY =
parseInt(process.env.READ_CONCURRENCY_SECONDARY, 10) || 1000
const READ_CONCURRENCY_PRIMARY =
@ -9,15 +20,10 @@ const MAX_CHATS_TO_DESTROY =
parseInt(process.env.MAX_CHATS_TO_DESTROY, 10) || false
// persist fallback in order to keep batchedUpdate in-sync
process.env.BATCH_SIZE = BATCH_SIZE
// raise mongo timeout to 10mins if otherwise unspecified
process.env.MONGO_SOCKET_TIMEOUT =
parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 600000
const { ObjectId } = require('mongodb-legacy')
const { promiseMapWithLimit } = require('@overleaf/promise-utils')
const { batchedUpdate } = require('./helpers/batchedUpdate')
const ChatApiHandler = require('../app/src/Features/Chat/ChatApiHandler')
const { getHardDeletedProjectIds } = require('./delete_orphaned_data_helper')
// ensure set mongo timeout to 10mins if otherwise unspecified
if (!process.env.MONGO_SOCKET_TIMEOUT) {
ensureMongoTimeout(600000)
}
console.log({
DRY_RUN,

8
services/web/scripts/delete_orphaned_data_helper.js → services/web/scripts/delete_orphaned_data_helper.mjs
View file

@ -1,9 +1,9 @@
const {
import {
db,
READ_PREFERENCE_PRIMARY,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
const { promiseMapWithLimit } = require('@overleaf/promise-utils')
} from '../app/src/infrastructure/mongodb.js'
import { promiseMapWithLimit } from '@overleaf/promise-utils'
async function getDeletedProject(projectId, readPreference) {
return await db.deletedProjects.findOne(
@ -108,6 +108,6 @@ async function getHardDeletedProjectIds({
return hardDeletedProjectIds
}
module.exports = {
export default {
getHardDeletedProjectIds,
}

24
services/web/scripts/delete_orphaned_doc_comment_ranges.js → services/web/scripts/delete_orphaned_doc_comment_ranges.mjs
View file

@ -1,13 +1,12 @@
import minimist from 'minimist'
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import ChatApiHandler from '../app/src/Features/Chat/ChatApiHandler.js'
import DocstoreManager from '../app/src/Features/Docstore/DocstoreManager.js'
import DocumentUpdaterHandler from '../app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js'
import { promiseMapWithLimit } from '@overleaf/promise-utils'
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
const minimist = require('minimist')
const { waitForDb } = require('../app/src/infrastructure/mongodb')
const ChatApiHandler = require('../app/src/Features/Chat/ChatApiHandler')
const DocstoreManager = require('../app/src/Features/Docstore/DocstoreManager')
const DocumentUpdaterHandler = require('../app/src/Features/DocumentUpdater/DocumentUpdaterHandler')
const { promiseMapWithLimit } = require('@overleaf/promise-utils')
/**
* Remove doc comment ranges that are "orphaned" as they do have matching chat
* threads. This can happen when adding comments and the HTTP request fails, but
@ -46,12 +45,11 @@ async function main() {
await DocumentUpdaterHandler.promises.flushDocToMongo(projectId, docId)
}
main()
.then(() => {
try {
await main()
console.log('Done.')
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error({ error })
process.exit(1)
})
}

26
services/web/scripts/delete_orphaned_docs_online_check.js → services/web/scripts/delete_orphaned_docs_online_check.mjs
View file

@ -1,15 +1,18 @@
const DocstoreManager = require('../app/src/Features/Docstore/DocstoreManager')
const { promisify } = require('util')
const { ObjectId } = require('mongodb-legacy')
const {
import DocstoreManager from '../app/src/Features/Docstore/DocstoreManager.js'
import { promisify } from 'util'
import mongodb from 'mongodb-legacy'
import {
db,
waitForDb,
READ_PREFERENCE_PRIMARY,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
const { promiseMapWithLimit } = require('@overleaf/promise-utils')
const { getHardDeletedProjectIds } = require('./delete_orphaned_data_helper')
} from '../app/src/infrastructure/mongodb.js'
import { promiseMapWithLimit } from '@overleaf/promise-utils'
import DeleteOrphanedDataHelper from './delete_orphaned_data_helper.mjs'
const { ObjectId } = mongodb
const sleep = promisify(setTimeout)
const { getHardDeletedProjectIds } = DeleteOrphanedDataHelper
const NOW_IN_S = Date.now() / 1000
const ONE_WEEK_IN_S = 60 * 60 * 24 * 7
@ -169,12 +172,11 @@ async function letUserDoubleCheckInputs() {
await sleep(LET_USER_DOUBLE_CHECK_INPUTS_FOR)
}
main()
.then(() => {
try {
await main()
console.error('Done.')
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error({ error })
process.exit(1)
})
}

11
services/web/scripts/delete_subscriptions.js → services/web/scripts/delete_subscriptions.mjs
View file

@ -1,9 +1,10 @@
const { Subscription } = require('../app/src/models/Subscription')
const SubscriptionUpdater = require('../app/src/Features/Subscription/SubscriptionUpdater')
const minimist = require('minimist')
import { Subscription } from '../app/src/models/Subscription.js'
import SubscriptionUpdater from '../app/src/Features/Subscription/SubscriptionUpdater.js'
import minimist from 'minimist'
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import mongodb from 'mongodb-legacy'
const { waitForDb } = require('../app/src/infrastructure/mongodb')
const { ObjectId } = require('mongodb-legacy')
const { ObjectId } = mongodb
const run = async () => {
for (const id of ids) {

26
services/web/scripts/disconnect_all_users.js → services/web/scripts/disconnect_all_users.mjs
View file

@ -1,21 +1,24 @@
const { promisify } = require('util')
const Settings = require('@overleaf/settings')
const AdminController = require('../app/src/Features/ServerAdmin/AdminController')
import { promisify } from 'util'
import Settings from '@overleaf/settings'
import AdminController from '../app/src/Features/ServerAdmin/AdminController.js'
import minimist from 'minimist'
import { fileURLToPath } from 'url'
const args = require('minimist')(process.argv.slice(2), {
const args = minimist(process.argv.slice(2), {
string: ['confirm-site-url', 'delay-in-seconds'],
default: {
'delay-in-seconds': 10,
'confirm-site-url': '',
},
})
const sleep = promisify(setTimeout)
async function main() {
if (args.help) {
console.error()
console.error(
' usage: node disconnect_all_users.js [--delay-in-seconds=10] --confirm-site-url=https://www....\n'
' usage: node disconnect_all_users.mjs [--delay-in-seconds=10] --confirm-site-url=https://www....\n'
)
process.exit(1)
}
@ -55,14 +58,13 @@ async function main() {
await AdminController._sendDisconnectAllUsersMessage(delay)
}
if (require.main === module) {
main()
.then(() => {
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await main()
console.error('Done.')
process.exit(0)
})
.catch(err => {
console.error('Error', err)
} catch (error) {
console.error('Error', error)
process.exit(1)
})
}
}

23
services/web/scripts/ensure_affiliations.js → services/web/scripts/ensure_affiliations.mjs
View file

@ -1,8 +1,10 @@
const { waitForDb } = require('../app/src/infrastructure/mongodb')
const { User } = require('../app/src/models/User')
const UserController = require('../app/src/Features/User/UserController')
require('@overleaf/logger').logger.level('error')
const pLimit = require('p-limit')
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import { User } from '../app/src/models/User.js'
import UserController from '../app/src/Features/User/UserController.js'
import Logger from '@overleaf/logger'
import pLimit from 'p-limit'
Logger.logger.level('error')
const CONCURRENCY = 10
const failure = []
const success = []
@ -42,12 +44,11 @@ async function run() {
}
}
waitForDb()
.then(run)
.then(() => {
try {
await waitForDb()
await run()
process.exit()
})
.catch(error => {
} catch (error) {
console.log(error)
process.exit(1)
})
}

6
services/web/scripts/esm-check-migration.js → services/web/scripts/esm-check-migration.mjs
View file

@ -1,6 +1,6 @@
const fs = require('fs')
const path = require('path')
const minimist = require('minimist')
import fs from 'fs'
import path from 'path'
import minimist from 'minimist'
const APP_CODE_PATH = ['app', 'modules', 'migrations', 'scripts', 'test']

2
services/web/scripts/export_institution_chat.mjs
View file

@ -5,7 +5,7 @@ import {
ObjectId,
} from '../app/src/infrastructure/mongodb.js'
import minimist from 'minimist'
import InstitutionHubsController from '../modules/metrics/app/src/InstitutionHubsController.js'
import InstitutionHubsController from '../modules/metrics/app/src/InstitutionHubsController.mjs'
function usage() {
console.log(

15
services/web/scripts/find_malformed_filetrees.js → services/web/scripts/find_malformed_filetrees.mjs
View file

@ -1,8 +1,8 @@
const {
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
} from '../app/src/infrastructure/mongodb.js'
async function main() {
await waitForDb()
@ -109,11 +109,10 @@ function findBadPaths(folder) {
return result
}
main()
.then(() => {
try {
await main()
process.exit(0)
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}

16
services/web/scripts/fix_group_invite_emails_to_lowercase.js → services/web/scripts/fix_group_invite_emails_to_lowercase.mjs
View file

@ -1,7 +1,8 @@
const DRY_RUN = process.env.DRY_RUN !== 'false'
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
const { batchedUpdate } = require('./helpers/batchedUpdate')
const { batchedUpdate } = BatchedUpdateModule
const DRY_RUN = process.env.DRY_RUN !== 'false'
console.log({
DRY_RUN,
@ -59,12 +60,11 @@ async function main() {
await batchedUpdate('subscriptions', query, processBatch, projection)
}
main()
.then(() => {
try {
await main()
console.error('Done.')
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error({ error })
process.exit(1)
})
}

19
services/web/scripts/fix_malformed_filetree.js → services/web/scripts/fix_malformed_filetree.mjs
View file

@ -5,9 +5,11 @@
* This is the output format of each line in the find_malformed_filetrees.js
* script.
*/
const { ObjectId } = require('mongodb-legacy')
const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
const ProjectLocator = require('../app/src/Features/Project/ProjectLocator')
import mongodb from 'mongodb-legacy'
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import ProjectLocator from '../app/src/Features/Project/ProjectLocator.js'
const { ObjectId } = mongodb
async function main() {
const { projectId, mongoPath } = parseArgs()
@ -172,11 +174,10 @@ function findUniqueName(existingFilenames) {
return filename
}
main()
.then(() => {
try {
await main()
process.exit(0)
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}

31
services/web/scripts/fix_oversized_docs.js → services/web/scripts/fix_oversized_docs.mjs
View file

@ -1,13 +1,13 @@
const fs = require('fs')
const minimist = require('minimist')
const { waitForDb, ObjectId } = require('../app/src/infrastructure/mongodb')
const DocstoreManager = require('../app/src/Features/Docstore/DocstoreManager')
const FileStoreHandler = require('../app/src/Features/FileStore/FileStoreHandler')
const FileWriter = require('../app/src/infrastructure/FileWriter')
const ProjectEntityMongoUpdateHandler = require('../app/src/Features/Project/ProjectEntityMongoUpdateHandler')
const ProjectLocator = require('../app/src/Features/Project/ProjectLocator')
const RedisWrapper = require('@overleaf/redis-wrapper')
const Settings = require('@overleaf/settings')
import fs from 'fs'
import minimist from 'minimist'
import { waitForDb, ObjectId } from '../app/src/infrastructure/mongodb.js'
import DocstoreManager from '../app/src/Features/Docstore/DocstoreManager.js'
import FileStoreHandler from '../app/src/Features/FileStore/FileStoreHandler.js'
import FileWriter from '../app/src/infrastructure/FileWriter.js'
import ProjectEntityMongoUpdateHandler from '../app/src/Features/Project/ProjectEntityMongoUpdateHandler.js'
import ProjectLocator from '../app/src/Features/Project/ProjectLocator.js'
import RedisWrapper from '@overleaf/redis-wrapper'
import Settings from '@overleaf/settings'
const opts = parseArgs()
const redis = RedisWrapper.createClient(Settings.redis.web)
@ -154,11 +154,10 @@ async function deleteDocFromRedis(projectId, docId) {
await redis.srem(`DocsIn:{${projectId}}`, projectId)
}
main()
.then(() => {
try {
await main()
process.exit(0)
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}

16
services/web/scripts/force_doc_flush.js → services/web/scripts/force_doc_flush.mjs
View file

@ -1,7 +1,8 @@
const { ObjectId } = require('mongodb-legacy')
const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
const DocumentUpdaterHandler = require('../app/src/Features/DocumentUpdater/DocumentUpdaterHandler')
import mongodb from 'mongodb-legacy'
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import DocumentUpdaterHandler from '../app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js'
const { ObjectId } = mongodb
const PROJECT_ID = process.env.PROJECT_ID
const DOC_ID = process.env.DOC_ID
const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
@ -67,12 +68,11 @@ function getDocument() {
})
}
main()
.then(() => {
try {
await main()
console.error('Done.')
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error({ error })
process.exit(1)
})
}

10
services/web/scripts/helpers/batchedUpdate.js → services/web/scripts/helpers/batchedUpdate.mjs
View file

@ -1,9 +1,11 @@
const { ObjectId } = require('mongodb-legacy')
const {
import mongodb from 'mongodb-legacy'
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../../app/src/infrastructure/mongodb')
} from '../../app/src/infrastructure/mongodb.js'
const { ObjectId } = mongodb
const ONE_MONTH_IN_MS = 1000 * 60 * 60 * 24 * 31
let ID_EDGE_PAST
@ -195,7 +197,7 @@ function batchedUpdateWithResultHandling(
})
}
module.exports = {
export default {
batchedUpdate,
batchedUpdateWithResultHandling,
}

31
services/web/scripts/helpers/env_variable_helper.mjs Normal file
View file

@ -0,0 +1,31 @@
/**
* Ensures that the specific MongoDB connection timeout is set.
*
* @param {number} timeoutInMS
* @returns {void}
*/
export function ensureMongoTimeout(timeoutInMS) {
if (process.env.MONGO_SOCKET_TIMEOUT !== timeoutInMS.toString()) {
throw new Error(
`must run with higher mongo timeout: MONGO_SOCKET_TIMEOUT=${timeoutInMS} node ${process.argv[1]}`
)
}
}
/**
* Ensures MongoDB queries are running on secondary and the specific connection timeout is set.
*
* @param {number} timeoutInMS
* @returns {void}
*/
export function ensureRunningOnMongoSecondaryWithTimeout(timeoutInMS) {
if (
process.env.MONGO_SOCKET_TIMEOUT !== timeoutInMS.toString() ||
process.env.MONGO_CONNECTION_STRING !==
process.env.READ_ONLY_MONGO_CONNECTION_STRING
) {
throw new Error(
`must run on secondary with higher mongo timeout: MONGO_SOCKET_TIMEOUT=${timeoutInMS} MONGO_CONNECTION_STRING="$READ_ONLY_MONGO_CONNECTION_STRING" node ${process.argv[1]}`
)
}
}

0
services/web/scripts/inst_table.js → services/web/scripts/inst_table.mjs
View file

53
services/web/scripts/invalidate_tokens.js
View file

@ -1,53 +0,0 @@
const { db, ObjectId, waitForDb } = require('../app/src/infrastructure/mongodb')
const minimist = require('minimist')
const argv = minimist(process.argv.slice(2))
const commit = argv.commit !== undefined
const projectIds = argv._.map(x => {
return new ObjectId(x)
})
if (!commit) {
console.log('Doing dry run without --commit')
}
console.log('checking', projectIds.length, 'projects')
waitForDb().then(async () => {
const affectedProjects = await db.projects
.find(
{ _id: { $in: projectIds } },
{
projection: {
_id: 1,
owner_ref: 1,
tokenAccessReadOnly_refs: 1,
tokenAccessReadAndWrite_refs: 1,
},
}
)
.toArray()
console.log('Found ' + affectedProjects.length + ' affected projects')
affectedProjects.forEach(project => {
console.log(JSON.stringify(project))
})
if (!commit) {
console.log('dry run, not updating')
process.exit(0)
} else {
try {
const result = await db.projects.updateMany(
{ _id: { $in: affectedProjects.map(project => project._id) } },
{
$set: {
publicAccesLevel: 'private', // note the spelling in the db is publicAccesLevel (with one 's')
tokenAccessReadOnly_refs: [],
tokenAccessReadAndWrite_refs: [],
},
}
)
console.log('result', JSON.stringify(result))
process.exit(0)
} catch (err) {
console.error('err', err)
process.exit(1)
}
}
})

54
services/web/scripts/invalidate_tokens.mjs Normal file
View file

@ -0,0 +1,54 @@
import { db, ObjectId, waitForDb } from '../app/src/infrastructure/mongodb.js'
import minimist from 'minimist'
const argv = minimist(process.argv.slice(2))
const commit = argv.commit !== undefined
const projectIds = argv._.map(x => {
return new ObjectId(x)
})
if (!commit) {
console.log('Doing dry run without --commit')
}
console.log('checking', projectIds.length, 'projects')
await waitForDb()
const affectedProjects = await db.projects
.find(
{ _id: { $in: projectIds } },
{
projection: {
_id: 1,
owner_ref: 1,
tokenAccessReadOnly_refs: 1,
tokenAccessReadAndWrite_refs: 1,
},
}
)
.toArray()
console.log('Found ' + affectedProjects.length + ' affected projects')
affectedProjects.forEach(project => {
console.log(JSON.stringify(project))
})
if (!commit) {
console.log('dry run, not updating')
process.exit(0)
} else {
try {
const result = await db.projects.updateMany(
{ _id: { $in: affectedProjects.map(project => project._id) } },
{
$set: {
publicAccesLevel: 'private', // note the spelling in the db is publicAccesLevel (with one 's')
tokenAccessReadOnly_refs: [],
tokenAccessReadAndWrite_refs: [],
},
}
)
console.log('result', JSON.stringify(result))
process.exit(0)
} catch (err) {
console.error('err', err)
process.exit(1)
}
}

26
services/web/scripts/migrate_audit_logs.js → services/web/scripts/migrate_audit_logs.mjs
View file

@ -1,8 +1,11 @@
const { batchedUpdate } = require('./helpers/batchedUpdate')
const { promiseMapWithLimit, promisify } = require('@overleaf/promise-utils')
const { db, ObjectId, waitForDb } = require('../app/src/infrastructure/mongodb')
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
import { promiseMapWithLimit, promisify } from '@overleaf/promise-utils'
import { db, ObjectId, waitForDb } from '../app/src/infrastructure/mongodb.js'
import _ from 'lodash'
import { fileURLToPath } from 'url'
const sleep = promisify(setTimeout)
const _ = require('lodash')
const { batchedUpdate } = BatchedUpdateModule
async function main(options) {
if (!options) {
@ -146,17 +149,16 @@ async function letUserDoubleCheckInputs(options) {
await sleep(options.letUserDoubleCheckInputsFor)
}
module.exports = main
export default main
if (require.main === module) {
waitForDb()
.then(main)
.then(() => {
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await waitForDb()
await main()
console.log('Done.')
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error({ error })
process.exit(1)
})
}
}

2
services/web/scripts/refresh_features.mjs
View file

@ -6,7 +6,7 @@ import FeaturesUpdater from '../app/src/Features/Subscription/FeaturesUpdater.js
import FeaturesHelper from '../app/src/Features/Subscription/FeaturesHelper.js'
import UserFeaturesUpdater from '../app/src/Features/Subscription/UserFeaturesUpdater.js'
import AnalyticsManager from '../app/src/Features/Analytics/AnalyticsManager.js'
import DropboxHandler from '../modules/dropbox/app/src/DropboxHandler.js'
import DropboxHandler from '../modules/dropbox/app/src/DropboxHandler.mjs'
import { OError } from '../app/src/Features/Errors/Errors.js'
const ScriptLogger = {

17
services/web/scripts/regenerate_duplicate_referral_ids.js → services/web/scripts/regenerate_duplicate_referral_ids.mjs
View file

@ -1,17 +1,18 @@
import {
db,
READ_PREFERENCE_SECONDARY,
} from '../app/src/infrastructure/mongodb.js'
import { promiseMapWithLimit } from '@overleaf/promise-utils'
import TokenGenerator from '../app/src/Features/TokenGenerator/TokenGenerator.js'
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
const { batchedUpdate } = BatchedUpdateModule
const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
// persist fallback in order to keep batchedUpdate in-sync
process.env.BATCH_SIZE = BATCH_SIZE
const {
db,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
const { promiseMapWithLimit } = require('@overleaf/promise-utils')
const TokenGenerator = require('../app/src/Features/TokenGenerator/TokenGenerator')
const { batchedUpdate } = require('./helpers/batchedUpdate')
async function rewriteDuplicates(duplicateReferralIds) {
// duplicateReferralIds contains at least one duplicate.
// Find out which is the duplicate in parallel and update

35
services/web/scripts/remove_deleted_users_from_token_access_refs.js → services/web/scripts/remove_deleted_users_from_token_access_refs.mjs
View file

@ -1,11 +1,17 @@
const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
const { batchedUpdate } = require('./helpers/batchedUpdate')
const { ObjectId } = require('mongodb-legacy')
const minimist = require('minimist')
const CollaboratorsHandler = require('../app/src/Features/Collaborators/CollaboratorsHandler')
const {
import {
db,
waitForDb,
READ_PREFERENCE_SECONDARY,
} = require('../app/src/infrastructure/mongodb')
} from '../app/src/infrastructure/mongodb.js'
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
import mongodb from 'mongodb-legacy'
import minimist from 'minimist'
import CollaboratorsHandler from '../app/src/Features/Collaborators/CollaboratorsHandler.js'
import { fileURLToPath } from 'url'
const { ObjectId } = mongodb
const { batchedUpdate } = BatchedUpdateModule
const argv = minimist(process.argv.slice(2), {
string: ['projects'],
@ -190,9 +196,9 @@ async function main(DRY_RUN, PROJECTS_LIST) {
await fixProjectsWithInvalidTokenAccessRefsIds(DRY_RUN, PROJECTS_LIST)
}
module.exports = main
export default main
if (require.main === module) {
if (fileURLToPath(import.meta.url) === process.argv[1]) {
if (argv.help || argv._.length > 1) {
console.error(`Usage: node scripts/remove_deleted_users_from_token_access_refs.js [OPTS]
Finds or removes deleted user ids from token access fields
@ -208,13 +214,12 @@ if (require.main === module) {
process.exit(1)
}
main(DRY_RUN, PROJECTS_LIST)
.then(() => {
try {
await main(DRY_RUN, PROJECTS_LIST)
console.error('Done')
process.exit(0)
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}
}

26
services/web/scripts/remove_email.js → services/web/scripts/remove_email.mjs
View file

@ -1,17 +1,6 @@
const { ObjectId, waitForDb } = require('../app/src/infrastructure/mongodb')
const UserUpdater = require('../app/src/Features/User/UserUpdater')
const UserGetter = require('../app/src/Features/User/UserGetter')
waitForDb()
.then(removeEmail)
.catch(error => {
console.error(error)
process.exit(1)
})
.then(() => {
console.log('Done.')
process.exit()
})
import { ObjectId, waitForDb } from '../app/src/infrastructure/mongodb.js'
import UserUpdater from '../app/src/Features/User/UserUpdater.js'
import UserGetter from '../app/src/Features/User/UserGetter.js'
async function removeEmail() {
const userId = process.argv[2]
@ -63,3 +52,12 @@ async function removeEmail() {
skipParseEmail
)
}
try {
await waitForDb()
await removeEmail()
console.log('Done.')
process.exit()
} catch (error) {
console.error(error)
process.exit(1)
}

19
services/web/scripts/remove_oauth_application.js → services/web/scripts/remove_oauth_application.mjs
View file

@ -1,7 +1,7 @@
const { OauthApplication } = require('../app/src/models/OauthApplication')
const parseArgs = require('minimist')
const OError = require('@overleaf/o-error')
const { waitForDb } = require('../app/src/infrastructure/mongodb')
import { OauthApplication } from '../app/src/models/OauthApplication.js'
import parseArgs from 'minimist'
import OError from '@overleaf/o-error'
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
async function _removeOauthApplication(appId) {
if (!appId) {
@ -26,14 +26,11 @@ async function main() {
await _removeOauthApplication(appId)
}
if (require.main === module) {
main()
.then(() => {
try {
await main()
console.log('Done')
process.exit(0)
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}

19
services/web/scripts/restore_orphaned_docs.js → services/web/scripts/restore_orphaned_docs.mjs
View file

@ -1,7 +1,7 @@
const { waitForDb } = require('../app/src/infrastructure/mongodb')
const ProjectEntityRestoreHandler = require('../app/src/Features/Project/ProjectEntityRestoreHandler')
const ProjectEntityHandler = require('../app/src/Features/Project/ProjectEntityHandler')
const DocstoreManager = require('../app/src/Features/Docstore/DocstoreManager')
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import ProjectEntityRestoreHandler from '../app/src/Features/Project/ProjectEntityRestoreHandler.js'
import ProjectEntityHandler from '../app/src/Features/Project/ProjectEntityHandler.js'
import DocstoreManager from '../app/src/Features/Docstore/DocstoreManager.js'
const ARGV = process.argv.slice(2)
const DEVELOPER_USER_ID = ARGV.shift()
@ -35,12 +35,11 @@ async function main() {
}
}
waitForDb()
.then(main)
.then(() => {
try {
await waitForDb()
await main()
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error(error)
process.exit(1)
})
}

17
services/web/scripts/restore_soft_deleted_docs.js → services/web/scripts/restore_soft_deleted_docs.mjs
View file

@ -1,6 +1,6 @@
const { waitForDb } = require('../app/src/infrastructure/mongodb')
const ProjectEntityRestoreHandler = require('../app/src/Features/Project/ProjectEntityRestoreHandler')
const DocstoreManager = require('../app/src/Features/Docstore/DocstoreManager')
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import ProjectEntityRestoreHandler from '../app/src/Features/Project/ProjectEntityRestoreHandler.js'
import DocstoreManager from '../app/src/Features/Docstore/DocstoreManager.js'
const ARGV = process.argv.slice(2)
const DEVELOPER_USER_ID = ARGV.shift()
@ -24,12 +24,11 @@ async function main() {
}
}
waitForDb()
.then(main)
.then(() => {
try {
await waitForDb()
await main()
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error(error)
process.exit(1)
})
}

19
services/web/scripts/set_tex_live_image.js → services/web/scripts/set_tex_live_image.mjs
View file

@ -1,6 +1,8 @@
const Settings = require('@overleaf/settings')
const { ObjectId } = require('mongodb-legacy')
const { Project } = require('../app/src/models/Project')
import Settings from '@overleaf/settings'
import mongodb from 'mongodb-legacy'
import { Project } from '../app/src/models/Project.js'
const { ObjectId } = mongodb
async function main() {
const { image, projectIds } = parseArgs()
@ -58,11 +60,10 @@ async function updateImage(image, projectIds) {
console.log(`Modified ${res.modifiedCount} projects`)
}
main()
.then(() => {
try {
await main()
process.exit()
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}

17
services/web/scripts/soft_delete_project.js → services/web/scripts/soft_delete_project.mjs
View file

@ -1,6 +1,6 @@
const minimist = require('minimist')
const { waitForDb } = require('../app/src/infrastructure/mongodb')
const ProjectDeleter = require('../app/src/Features/Project/ProjectDeleter')
import minimist from 'minimist'
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import ProjectDeleter from '../app/src/Features/Project/ProjectDeleter.js'
async function main() {
const argv = minimist(process.argv.slice(2))
@ -16,12 +16,11 @@ async function main() {
await ProjectDeleter.promises.deleteProject(projectId)
}
main()
.then(() => {
try {
await main()
console.log('Done.')
process.exit(0)
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}

16
services/web/scripts/split_tests_assigned_at_to_dates.js → services/web/scripts/split_tests_assigned_at_to_dates.mjs
View file

@ -1,4 +1,5 @@
const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import { fileURLToPath } from 'url'
async function updateStringDates() {
await waitForDb()
@ -33,15 +34,14 @@ async function updateStringDates() {
console.log(`Updated ${count} assignedAt strings to dates!`)
}
if (!module.parent) {
updateStringDates()
.then(() => {
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await updateStringDates()
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error(error)
process.exit(1)
})
}
}
module.exports = updateStringDates
export default updateStringDates

24
services/web/scripts/split_writefull_disabled_from_unset.js → services/web/scripts/split_writefull_disabled_from_unset.mjs
View file

@ -1,8 +1,11 @@
const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
const { batchedUpdate } = require('./helpers/batchedUpdate')
const { ObjectId } = require('mongodb-legacy')
const fs = require('fs')
import { db, waitForDb } from '../app/src/infrastructure/mongodb.js'
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
import mongodb from 'mongodb-legacy'
import fs from 'fs'
import { fileURLToPath } from 'url'
const { ObjectId } = mongodb
const { batchedUpdate } = BatchedUpdateModule
const CHUNK_SIZE = 1000
// Function to chunk the array
@ -45,15 +48,14 @@ async function main() {
}
}
module.exports = main
export default main
if (require.main === module) {
main()
.then(() => {
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await main()
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error({ error })
process.exit(1)
})
}
}

9
services/web/scripts/sso_id_migration_check.js → services/web/scripts/sso_id_migration_check.mjs
View file

@ -1,9 +1,8 @@
process.env.MONGO_SOCKET_TIMEOUT = '300000'
process.env.MONGO_CONNECTION_STRING =
process.env.READ_ONLY_MONGO_CONNECTION_STRING
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import SAMLUserIdMigrationHandler from '../modules/saas-authentication/app/src/SAML/SAMLUserIdMigrationHandler.js'
import { ensureRunningOnMongoSecondaryWithTimeout } from './helpers/env_variable_helper.mjs'
const { waitForDb } = require('../app/src/infrastructure/mongodb')
const SAMLUserIdMigrationHandler = require('../modules/saas-authentication/app/src/SAML/SAMLUserIdMigrationHandler')
ensureRunningOnMongoSecondaryWithTimeout(300000)
const institutionId = parseInt(process.argv[2])
if (isNaN(institutionId)) throw new Error('No institution id')

23
services/web/scripts/sso_id_remove_not_migrated.js → services/web/scripts/sso_id_remove_not_migrated.mjs
View file

@ -1,6 +1,8 @@
process.env.MONGO_SOCKET_TIMEOUT = '300000'
const { waitForDb } = require('../app/src/infrastructure/mongodb')
const SAMLUserIdMigrationHandler = require('../modules/saas-authentication/app/src/SAML/SAMLUserIdMigrationHandler')
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import SAMLUserIdMigrationHandler from '../modules/saas-authentication/app/src/SAML/SAMLUserIdMigrationHandler.js'
import { ensureMongoTimeout } from './helpers/env_variable_helper.mjs'
ensureMongoTimeout(300000)
const institutionId = parseInt(process.argv[2])
if (isNaN(institutionId)) throw new Error('No institution id')
@ -11,13 +13,6 @@ console.log(
institutionId
)
waitForDb()
.then(main)
.catch(error => {
console.error(error)
process.exit(1)
})
async function main() {
const result =
await SAMLUserIdMigrationHandler.promises.removeNotMigrated(institutionId)
@ -34,3 +29,11 @@ async function main() {
process.exit()
}
try {
await waitForDb()
await main()
} catch (error) {
console.error(error)
process.exit(1)
}

56
services/web/scripts/stress_test.js → services/web/scripts/stress_test.mjs
View file

@ -1,39 +1,40 @@
const minimist = require('minimist')
const settings = require('@overleaf/settings')
const ProjectDetailsHandler = require('../app/src/Features/Project/ProjectDetailsHandler')
const mongodb = require('../app/src/infrastructure/mongodb')
const mongoose = require('../app/src/infrastructure/Mongoose')
const fs = require('fs')
const path = require('path')
const crypto = require('crypto')
const fetch = require('node-fetch')
const http = require('http')
import minimist from 'minimist'
import settings from '@overleaf/settings'
import ProjectDetailsHandler from '../app/src/Features/Project/ProjectDetailsHandler.js'
import mongodb from '../app/src/infrastructure/mongodb.js'
import mongoose from '../app/src/infrastructure/Mongoose.js'
import fs from 'fs'
import path from 'path'
import crypto from 'crypto'
import fetch from 'node-fetch'
import http from 'http'
import _ from 'lodash'
const { ObjectId } = mongodb
const _ = require('lodash')
// Examples:
//
// Simple usage:
// node stress_test.js --project-id=ID -n 100 --download-zip # download 100 zips from history-v1
// node stress_test.js --project-id=ID -n 100 --create-blob # create 100 blobs in history-v1
// node stress_test.js --project-id=ID -n 100 --fetch-blob # create blob and fetch it 100 times from history-v1
// node stress_test.js --project-id=ID -n 100 --upload-file # upload 100 files to filestore
// node stress_test.js --project-id=ID -n 100 --download-file # create file in filestore and download it 100 times
// node stress_test.mjs --project-id=ID -n 100 --download-zip # download 100 zips from history-v1
// node stress_test.mjs --project-id=ID -n 100 --create-blob # create 100 blobs in history-v1
// node stress_test.mjs --project-id=ID -n 100 --fetch-blob # create blob and fetch it 100 times from history-v1
// node stress_test.mjs --project-id=ID -n 100 --upload-file # upload 100 files to filestore
// node stress_test.mjs --project-id=ID -n 100 --download-file # create file in filestore and download it 100 times
//
// Delay between requests:
// node stress_test.js --project-id=ID -n 100 --download-zip --sleep=0.1 # download 100 zips from history-v1 with 0.1s sleep
// node stress_test.mjs --project-id=ID -n 100 --download-zip --sleep=0.1 # download 100 zips from history-v1 with 0.1s sleep
//
// Abort requests at random times:
// node stress_test.js --project-id=ID -n 100 --download-zip --abort # download 100 zips from history-v1 with aborts
// node stress_test.mjs --project-id=ID -n 100 --download-zip --abort # download 100 zips from history-v1 with aborts
//
// Parallel workers:
// node stress_test.js --project-id=ID -n 1000 -j 10 --upload-file # upload 1000 files in 10 parallel workers
// node stress_test.mjs --project-id=ID -n 1000 -j 10 --upload-file # upload 1000 files in 10 parallel workers
//
// Fixed file size:
// node stress_test.js --project-id=ID -n 1000 --size 1000000 --upload-file # upload 1000 files of 1MB in 10 parallel workers
// node stress_test.mjs --project-id=ID -n 1000 --size 1000000 --upload-file # upload 1000 files of 1MB in 10 parallel workers
//
// Random file size:
// node stress_test.js --project-id=ID -n 1000 --size-min 1024 --size-max 10000000 --upload-file # upload 1000 files of 1KB to 10MB in 10 parallel workers
// node stress_test.mjs --project-id=ID -n 1000 --size-min 1024 --size-max 10000000 --upload-file # upload 1000 files of 1KB to 10MB in 10 parallel workers
const argv = minimist(process.argv.slice(2), {
string: ['n', 'j', 'project-id', 'sleep', 'size', 'size-min', 'size-max'],
@ -58,7 +59,7 @@ const argv = minimist(process.argv.slice(2), {
const projectId = argv['project-id']
if (!projectId) {
console.error(
'Usage: node stress_test.js --project-id ID -n COUNT -j CONCURRENCY --sleep T --size BYTES --use-file --[create-blob|fetch-blob|download-zip|upload-file|download-file]'
'Usage: node stress_test.mjs --project-id ID -n COUNT -j CONCURRENCY --sleep T --size BYTES --use-file --[create-blob|fetch-blob|download-zip|upload-file|download-file]'
)
process.exit(1)
}
@ -430,13 +431,12 @@ async function run() {
log('Stress test done')
}
Promise.all([mongodb.waitForDb(), mongoose.connectionPromise])
.then(() => run())
.then(() => {
try {
await Promise.all([mongodb.waitForDb(), mongoose.connectionPromise])
await run()
log('Completed')
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error(error)
process.exit(1)
})
}

17
services/web/scripts/undelete_project_to_user.js → services/web/scripts/undelete_project_to_user.mjs
View file

@ -1,6 +1,6 @@
const minimist = require('minimist')
const { waitForDb } = require('../app/src/infrastructure/mongodb')
const ProjectDeleter = require('../app/src/Features/Project/ProjectDeleter')
import minimist from 'minimist'
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import ProjectDeleter from '../app/src/Features/Project/ProjectDeleter.js'
async function main() {
const argv = minimist(process.argv.slice(2))
@ -15,12 +15,11 @@ async function main() {
await ProjectDeleter.promises.undeleteProject(projectId, { userId })
}
main()
.then(() => {
try {
await main()
console.log('Done.')
process.exit(0)
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}

23
services/web/scripts/unlink_third_party_id.js → services/web/scripts/unlink_third_party_id.mjs
View file

@ -1,7 +1,7 @@
const { waitForDb } = require('../app/src/infrastructure/mongodb')
const minimist = require('minimist')
const ThirdPartyIdentityManager = require('../app/src/Features/User/ThirdPartyIdentityManager')
const UserGetter = require('../app/src/Features/User/UserGetter')
import { waitForDb } from '../app/src/infrastructure/mongodb.js'
import minimist from 'minimist'
import ThirdPartyIdentityManager from '../app/src/Features/User/ThirdPartyIdentityManager.js'
import UserGetter from '../app/src/Features/User/UserGetter.js'
/**
* This script is used to remove a linked third party identity from a user account.
@ -14,9 +14,9 @@ const UserGetter = require('../app/src/Features/User/UserGetter')
* Usage:
*
* - dry run:
* node scripts/unlink_third_party_id.js --providerId=google --userId=${SOME_USER_ID}
* node scripts/unlink_third_party_id.mjs --providerId=google --userId=${SOME_USER_ID}
* - commit:
* node scripts/unlink_third_party_id.js --providerId=google --userId=${SOME_USER_ID} --commit
* node scripts/unlink_third_party_id.mjs --providerId=google --userId=${SOME_USER_ID} --commit
*/
let COMMIT = false
@ -81,11 +81,10 @@ async function main() {
setup()
main()
.then(() => {
try {
await main()
process.exit(0)
})
.catch(err => {
console.error(err)
} catch (error) {
console.error(error)
process.exit(1)
})
}

12
services/web/scripts/update_project_image_name.js → services/web/scripts/update_project_image_name.mjs
View file

@ -1,5 +1,6 @@
const { batchedUpdate } = require('./helpers/batchedUpdate')
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
const { batchedUpdate } = BatchedUpdateModule
const oldImage = process.argv[2]
const newImage = process.argv[3]
@ -31,16 +32,15 @@ if (!process.env.ALL_TEX_LIVE_DOCKER_IMAGES.split(',').includes(newImage)) {
process.exit(1)
}
batchedUpdate(
try {
await batchedUpdate(
'projects',
{ imageName: oldImage },
{ $set: { imageName: newImage } }
)
.then(() => {
console.log('Done')
process.exit(0)
})
.catch(error => {
} catch (error) {
console.error(error)
process.exit(1)
})
}

12
services/web/scripts/validate-data-of-model.js → services/web/scripts/validate-data-of-model.mjs
View file

@ -1,7 +1,13 @@
const { batchedUpdateWithResultHandling } = require('./helpers/batchedUpdate')
import BatchedUpdateModule from './helpers/batchedUpdate.mjs'
const { batchedUpdateWithResultHandling } = BatchedUpdateModule
const MODEL_NAME = process.argv.pop()
const Model = require(`../app/src/models/${MODEL_NAME}`)[MODEL_NAME]
// Todo: handle mjs file once models have been converted to ES module
const { [MODEL_NAME]: Model } = await import(
`../app/src/models/${MODEL_NAME}.js`
)
function processBatch(batch) {
for (const doc of batch) {
@ -16,7 +22,7 @@ function processBatch(batch) {
batchedUpdateWithResultHandling(
Model.collection.name,
{},
async (_, nextBatch) => {
async nextBatch => {
await processBatch(nextBatch)
},
{}

2
services/web/test/acceptance/src/BackFillDeletedFilesTests.mjs
View file

@ -78,7 +78,7 @@ describe('BackFillDeletedFiles', function () {
try {
result = await promisify(exec)(
['LET_USER_DOUBLE_CHECK_INPUTS_FOR=1', 'VERBOSE_LOGGING=true']
.concat(['node', 'scripts/back_fill_deleted_files'])
.concat(['node', 'scripts/back_fill_deleted_files.mjs'])
.concat(args)
.join(' ')
)

2
services/web/test/acceptance/src/BackFillDocNameForDeletedDocsTests.mjs
View file

@ -67,7 +67,7 @@ describe('BackFillDocNameForDeletedDocs', function () {
try {
result = await promisify(exec)(
['LET_USER_DOUBLE_CHECK_INPUTS_FOR=1']
.concat(['node', 'scripts/back_fill_doc_name_for_deleted_docs'])
.concat(['node', 'scripts/back_fill_doc_name_for_deleted_docs.mjs'])
.concat(args)
.join(' ')
)

2
services/web/test/acceptance/src/BackFillDocRevTests.mjs
View file

@ -24,7 +24,7 @@ describe('BackFillDocRevTests', function () {
[
'VERBOSE_LOGGING=true',
'node',
'scripts/back_fill_doc_rev',
'scripts/back_fill_doc_rev.mjs',
dryRun,
].join(' ')
)

2
services/web/test/acceptance/src/BackFillDummyDocMetaTests.mjs
View file

@ -121,7 +121,7 @@ describe('BackFillDummyDocMeta', function () {
result = await promisify(exec)(
Object.entries(options)
.map(([key, value]) => `${key}=${value}`)
.concat(['node', 'scripts/back_fill_dummy_doc_meta.js'])
.concat(['node', 'scripts/back_fill_dummy_doc_meta.mjs'])
.join(' ')
)
} catch (error) {

3
services/web/test/acceptance/src/BatchedUpdateTests.mjs
View file

@ -22,8 +22,9 @@ describe('BatchedUpdateTests', function () {
})
spawnSync(process.argv0, [
'--input-type=module',
'-e',
'require("./scripts/helpers/batchedUpdate").batchedUpdateWithResultHandling("systemmessages", { content: { $ne: "42" }}, { $set: { content: "42" } })',
'import BatchedUpdateModule from "./scripts/helpers/batchedUpdate.mjs"; BatchedUpdateModule.batchedUpdateWithResultHandling("systemmessages", { content: { $ne: "42" }}, { $set: { content: "42" } })',
])
await expect(

2
services/web/test/acceptance/src/ConvertArchivedState.mjs
View file

@ -120,7 +120,7 @@ describe('ConvertArchivedState', function () {
beforeEach(function (done) {
exec(
'CONNECT_DELAY=1 node scripts/convert_archived_state.js FIRST,SECOND',
'CONNECT_DELAY=1 node scripts/convert_archived_state.mjs FIRST,SECOND',
error => {
if (error) {
return done(error)

2
services/web/test/acceptance/src/DeleteOrphanedDocsOnlineCheckTests.mjs
View file

@ -121,7 +121,7 @@ describe('DeleteOrphanedDocsOnlineCheck', function () {
// Hide deprecation warnings for calling `db.collection.count`
'NODE_OPTIONS=--no-deprecation',
])
.concat(['node', 'scripts/delete_orphaned_docs_online_check.js'])
.concat(['node', 'scripts/delete_orphaned_docs_online_check.mjs'])
.join(' ')
)
} catch (error) {

2
services/web/test/acceptance/src/RegenerateDuplicateReferralIdsTests.mjs
View file

@ -96,7 +96,7 @@ describe('RegenerateDuplicateReferralIds', function () {
// actual command
'node',
'scripts/regenerate_duplicate_referral_ids',
'scripts/regenerate_duplicate_referral_ids.mjs',
].join(' ')
)
} catch (err) {

2
services/web/test/acceptance/src/RemoveDeletedUsersFromTokenAccessRefsTests.mjs
View file

@ -54,7 +54,7 @@ describe('RemoveDeletedUsersFromTokenAccessRefsTests', function () {
[
'VERBOSE_LOGGING=true',
'node',
'scripts/remove_deleted_users_from_token_access_refs',
'scripts/remove_deleted_users_from_token_access_refs.mjs',
dryRun,
projectsList,
].join(' ')