github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-09-16 02:52:31 -04:00
Code Issues Projects Releases Packages Wiki Activity

Add options for controlling hint & retries to history upgrade scripts (#5967)

Browse source 
* Add USE_QUERY_HINT to allow configuring hint usage in history scripts

* Add RETRY_FAILED option to re-attempt previously failed history upgrades

* Add HistoryConversionFailed count to counts script

* Increase default batch size/concurrency on count script

GitOrigin-RevId: a498294719a504b224c04c95363bf098bb80b574
This commit is contained in:
Thomas 2021-12-01 10:29:58 +01:00 committed by Copybot
parent 780dc55761
commit 88dabee6df
5 changed files with 94 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
services/web/scripts/history/count_project_history_categories.js
View file

@ -1,7 +1,8 @@
const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
const VERBOSE_PROJECT_NAMES = process.env.VERBOSE_PROJECT_NAMES === 'true'
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 5
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 50
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 500
const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
// persist fallback in order to keep batchedUpdate in-sync
process.env.BATCH_SIZE = BATCH_SIZE
// raise mongo timeout to 1hr if otherwise unspecified
@ -21,6 +22,7 @@ const COUNT = {
NoneWithConversion: 0,
NoneWithTemporaryHistory: 0,
HistoryUpgradeFailed: 0,
HistoryConversionFailed: 0,
}
// Timestamp of when 'Enable history for SL in background' release
@ -59,6 +61,15 @@ async function processProject(project) {
} has a history upgrade failure recorded`
)
}
} else if (project.overleaf.history.conversionFailed) {
COUNT.HistoryConversionFailed += 1
if (VERBOSE_LOGGING) {
console.log(
`project ${
project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
} has a history upgrade failure recorded`
)
}
} else if (project.overleaf.history.display) {
// v2: full project history, do nothing, (query shoudln't include any, but we should stlll check?)
COUNT.v2 += 1
@ -231,8 +242,9 @@ async function main() {
_id: 1,
overleaf: 1,
}
const options = {
hint: { _id: 1 },
const options = {}
if (USE_QUERY_HINT) {
options.hint = { _id: 1 }
}
if (VERBOSE_PROJECT_NAMES) {
projection.name = 1

35
services/web/scripts/history/upgrade_none_with_conversion_if_sl_history.js
View file

@ -1,8 +1,10 @@
const SCRIPT_VERSION = 1
const SCRIPT_VERSION = 2
const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
const DRY_RUN = process.env.DRY_RUN !== 'false'
const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
const RETRY_FAILED = process.env.RETRY_FAILED === 'true'
const MAX_UPGRADES_TO_ATTEMPT =
parseInt(process.env.MAX_UPGRADES_TO_ATTEMPT, 10) || false
const MAX_FAILURES = parseInt(process.env.MAX_FAILURES, 10) || 50
@ -25,6 +27,8 @@ console.log({
BATCH_SIZE,
MAX_UPGRADES_TO_ATTEMPT,
MAX_FAILURES,
USE_QUERY_HINT,
RETRY_FAILED,
})
const RESULT = {
@ -64,15 +68,17 @@ async function processProject(project) {
if (INTERRUPT) {
return
}
// safety check
if (project.overleaf && project.overleaf.history) {
if (
project.overleaf.history.conversionFailed ||
project.overleaf.history.upgradeFailed
) {
// we don't want to attempt upgrade on projects
// that have been previously attempted and failed
return
// skip safety check if we want to retry failed upgrades
if (!RETRY_FAILED) {
if (project.overleaf && project.overleaf.history) {
if (
project.overleaf.history.conversionFailed ||
project.overleaf.history.upgradeFailed
) {
// we don't want to attempt upgrade on projects
// that have been previously attempted and failed
return
}
}
}
const anyDocHistory = await anyDocHistoryExists(project)
@ -122,6 +128,10 @@ async function doUpgradeForNoneWithConversion(project) {
$set: {
'overleaf.history.upgradeReason': `none-with-conversion/${SCRIPT_VERSION}`,
},
$unset: {
'overleaf.history.upgradeFailed': true,
'overleaf.history.conversionFailed': true,
},
}
)
}
@ -158,8 +168,9 @@ async function main() {
_id: 1,
overleaf: 1,
}
const options = {
hint: { _id: 1 },
const options = {}
if (USE_QUERY_HINT) {
options.hint = { _id: 1 }
}
await batchedUpdate(
'projects',

50
services/web/scripts/history/upgrade_none_without_conversion_if_no_sl_history.js
View file

@ -1,8 +1,10 @@
const SCRIPT_VERSION = 1
const SCRIPT_VERSION = 2
const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
const DRY_RUN = process.env.DRY_RUN !== 'false'
const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
const RETRY_FAILED = process.env.RETRY_FAILED === 'true'
const MAX_UPGRADES_TO_ATTEMPT =
parseInt(process.env.MAX_UPGRADES_TO_ATTEMPT, 10) || false
const MAX_FAILURES = parseInt(process.env.MAX_FAILURES, 10) || 50
@ -26,6 +28,8 @@ console.log({
BATCH_SIZE,
MAX_UPGRADES_TO_ATTEMPT,
MAX_FAILURES,
USE_QUERY_HINT,
RETRY_FAILED,
})
const RESULT = {
@ -65,6 +69,15 @@ async function processProject(project) {
if (INTERRUPT) {
return
}
if (!RETRY_FAILED) {
if (
project.overleaf &&
project.overleaf.history &&
project.overleaf.history.upgradeFailed
) {
return
}
}
const anyDocHistory = await anyDocHistoryExists(project)
if (anyDocHistory) {
return
@ -96,21 +109,20 @@ async function doUpgradeForNoneWithoutConversion(project) {
const historyId = await ProjectHistoryHandler.promises.getHistoryId(
projectId
)
if (historyId != null) {
return
}
const history = await HistoryManager.promises.initializeProject()
if (history && history.overleaf_id) {
await ProjectHistoryHandler.promises.setHistoryId(
projectId,
history.overleaf_id
)
await HistoryManager.promises.resyncProject(projectId, {
force: true,
origin: { kind: 'history-migration' },
})
await HistoryManager.promises.flushProject(projectId)
if (!historyId) {
const history = await HistoryManager.promises.initializeProject()
if (history && history.overleaf_id) {
await ProjectHistoryHandler.promises.setHistoryId(
projectId,
history.overleaf_id
)
}
}
await HistoryManager.promises.resyncProject(projectId, {
force: true,
origin: { kind: 'history-migration' },
})
await HistoryManager.promises.flushProject(projectId)
} catch (err) {
RESULT.failed += 1
console.error(`project ${project._id} FAILED with error: `, err)
@ -132,6 +144,9 @@ async function doUpgradeForNoneWithoutConversion(project) {
'overleaf.history.upgradedAt': new Date(),
'overleaf.history.upgradeReason': `none-without-sl-history/${SCRIPT_VERSION}`,
},
$unset: {
'overleaf.history.upgradeFailed': true,
},
}
)
}
@ -166,8 +181,9 @@ async function main() {
_id: 1,
overleaf: 1,
}
const options = {
hint: { _id: 1 },
const options = {}
if (USE_QUERY_HINT) {
options.hint = { _id: 1 }
}
await batchedUpdate(
'projects',

14
services/web/scripts/history/upgrade_v1_without_conversion_if_created_after_fph_enabled.js
View file

@ -3,6 +3,7 @@ const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
const DRY_RUN = process.env.DRY_RUN !== 'false'
const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
// persist fallback in order to keep batchedUpdate in-sync
process.env.BATCH_SIZE = BATCH_SIZE
// raise mongo timeout to 1hr if otherwise unspecified
@ -14,7 +15,13 @@ const { db } = require('../../app/src/infrastructure/mongodb')
const { promiseMapWithLimit } = require('../../app/src/util/promises')
const { batchedUpdate } = require('../helpers/batchedUpdate')
console.log({ DRY_RUN, VERBOSE_LOGGING, WRITE_CONCURRENCY, BATCH_SIZE })
console.log({
DRY_RUN,
VERBOSE_LOGGING,
WRITE_CONCURRENCY,
BATCH_SIZE,
USE_QUERY_HINT,
})
const RESULT = {
DRY_RUN,
@ -123,8 +130,9 @@ async function main() {
_id: 1,
overleaf: 1,
}
const options = {
hint: { _id: 1 },
const options = {}
if (USE_QUERY_HINT) {
options.hint = { _id: 1 }
}
await batchedUpdate(
'projects',

14
services/web/scripts/history/upgrade_v1_without_conversion_if_no_sl_history.js
View file

@ -3,6 +3,7 @@ const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
const DRY_RUN = process.env.DRY_RUN !== 'false'
const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false'
// persist fallback in order to keep batchedUpdate in-sync
process.env.BATCH_SIZE = BATCH_SIZE
// raise mongo timeout to 1hr if otherwise unspecified
@ -14,7 +15,13 @@ const { db } = require('../../app/src/infrastructure/mongodb')
const { promiseMapWithLimit } = require('../../app/src/util/promises')
const { batchedUpdate } = require('../helpers/batchedUpdate')
console.log({ DRY_RUN, VERBOSE_LOGGING, WRITE_CONCURRENCY, BATCH_SIZE })
console.log({
DRY_RUN,
VERBOSE_LOGGING,
WRITE_CONCURRENCY,
BATCH_SIZE,
USE_QUERY_HINT,
})
const RESULT = {
DRY_RUN,
@ -111,8 +118,9 @@ async function main() {
_id: 1,
overleaf: 1,
}
const options = {
hint: { _id: 1 },
const options = {}
if (USE_QUERY_HINT) {
options.hint = { _id: 1 }
}
await batchedUpdate(
'projects',