github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #5694 from overleaf/tm-history-scripts-mongo-timeouts

Browse source 
Update v1 without preserveHistory, increase default mongo timeouts, add count query hint

GitOrigin-RevId: bfe1a0024258d0c75646544b9646928b2afe33c1
This commit is contained in:
Thomas 2021-11-08 16:04:30 +01:00 committed by Copybot
parent 37018af773
commit bfda92513a
3 changed files with 43 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
services/web/scripts/history/count_project_history_categories.js
View file

@ -4,6 +4,9 @@ const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 5
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
// persist fallback in order to keep batchedUpdate in-sync
process.env.BATCH_SIZE = BATCH_SIZE
// raise mongo timeout to 1hr if otherwise unspecified
process.env.MONGO_SOCKET_TIMEOUT =
parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
const { ReadPreference, ObjectId } = require('mongodb')
const { db } = require('../../app/src/infrastructure/mongodb')
@ -209,6 +212,9 @@ async function main() {
_id: 1,
overleaf: 1,
}
const options = {
hint: { _id: 1 },
}
if (VERBOSE_PROJECT_NAMES) {
projection.name = 1
}
@ -216,7 +222,8 @@ async function main() {
'projects',
{ 'overleaf.history.display': { $ne: true } },
processBatch,
projection
projection,
options
)
console.log('Final')
console.log(COUNT)

3
services/web/scripts/history/upgrade_v1_without_conversion_if_created_after_fph_enabled.js
View file

@ -5,6 +5,9 @@ const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
const DRY_RUN = process.env.DRY_RUN !== 'false'
// persist fallback in order to keep batchedUpdate in-sync
process.env.BATCH_SIZE = BATCH_SIZE
// raise mongo timeout to 1hr if otherwise unspecified
process.env.MONGO_SOCKET_TIMEOUT =
parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
const { ReadPreference, ObjectId } = require('mongodb')
const { db } = require('../../app/src/infrastructure/mongodb')

41
services/web/scripts/history/upgrade_v1_without_conversion_if_no_sl_history.js
View file

@ -1,10 +1,13 @@
const SCRIPT_VERSION = 1
const SCRIPT_VERSION = 2
const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
const DRY_RUN = process.env.DRY_RUN !== 'false'
// persist fallback in order to keep batchedUpdate in-sync
process.env.BATCH_SIZE = BATCH_SIZE
// raise mongo timeout to 1hr if otherwise unspecified
process.env.MONGO_SOCKET_TIMEOUT =
parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
const { ReadPreference } = require('mongodb')
const { db } = require('../../app/src/infrastructure/mongodb')
@ -24,15 +27,23 @@ async function processBatch(_, projects) {
}
async function processProject(project) {
const anyDocHistory = await anyDocHistoryExists(project)
if (anyDocHistory) {
return
const preserveHistory = await shouldPreserveHistory(project)
if (preserveHistory) {
// if we need to preserve history, then we must bail out if history exists
const anyDocHistory = await anyDocHistoryExists(project)
if (anyDocHistory) {
return
}
const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
if (anyDocHistoryIndex) {
return
}
return await doUpgradeForV1WithoutConversion(project)
} else {
// if preserveHistory false, then max 7 days of SL history
// but v1 already record to both histories, so safe to upgrade
return await doUpgradeForV1WithoutConversion(project)
}
const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
if (anyDocHistoryIndex) {
return
}
await doUpgradeForV1WithoutConversion(project)
}
async function doUpgradeForV1WithoutConversion(project) {
@ -54,6 +65,18 @@ async function doUpgradeForV1WithoutConversion(project) {
RESULT.projectsUpgraded += 1
}
async function shouldPreserveHistory(project) {
return await db.projectHistoryMetaData.findOne(
{
$and: [
{ project_id: { $eq: project._id } },
{ preserveHistory: { $eq: true } },
],
},
{ readPreference: ReadPreference.SECONDARY }
)
}
async function anyDocHistoryExists(project) {
return await db.docHistory.findOne(
{ project_id: { $eq: project._id } },