mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-21 20:47:08 -05:00
Merge pull request #8492 from overleaf/msm-history-migration-convert-large-docs
[web] Updated history upgrade scripts to move large docs to filestore GitOrigin-RevId: 11ac9fc879ff03756e3ec41bb156e878c4223014
This commit is contained in:
parent
3a1bba07f7
commit
e7c8329491
2 changed files with 53 additions and 1 deletions
|
@ -1,9 +1,12 @@
|
|||
const { ReadPreference, ObjectId } = require('mongodb')
|
||||
const { db } = require('../../app/src/infrastructure/mongodb')
|
||||
const Settings = require('@overleaf/settings')
|
||||
|
||||
const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler')
|
||||
const HistoryManager = require('../../app/src/Features/History/HistoryManager')
|
||||
const ProjectHistoryController = require('../../modules/admin-panel/app/src/ProjectHistoryController')
|
||||
const ProjectEntityHandler = require('../../app/src/Features/Project/ProjectEntityHandler')
|
||||
const ProjectEntityUpdateHandler = require('../../app/src/Features/Project/ProjectEntityUpdateHandler')
|
||||
|
||||
// Timestamp of when 'Enable history for SL in background' release
|
||||
const ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = '5a8d8a370000000000000000'
|
||||
|
@ -292,8 +295,45 @@ async function anyDocHistoryIndexExists(project) {
|
|||
)
|
||||
}
|
||||
|
||||
async function convertLargeDocsToFile(projectId, userId) {
|
||||
const docs = await ProjectEntityHandler.promises.getAllDocs(projectId)
|
||||
let convertedDocCount = 0
|
||||
for (const doc of Object.values(docs)) {
|
||||
const sizeBound = JSON.stringify(doc.lines)
|
||||
if (docIsTooLarge(sizeBound, doc.lines, Settings.max_doc_length)) {
|
||||
await ProjectEntityUpdateHandler.promises.convertDocToFile(
|
||||
projectId,
|
||||
doc._id,
|
||||
userId
|
||||
)
|
||||
convertedDocCount++
|
||||
}
|
||||
}
|
||||
return convertedDocCount
|
||||
}
|
||||
|
||||
// check whether the total size of the document in characters exceeds the
|
||||
// maxDocLength.
|
||||
//
|
||||
// Copied from document-updater:
|
||||
// https://github.com/overleaf/internal/blob/74adfbebda5f3c2c37d9937f0db5c4106ecde492/services/document-updater/app/js/Limits.js#L18
|
||||
function docIsTooLarge(estimatedSize, lines, maxDocLength) {
|
||||
if (estimatedSize <= maxDocLength) {
|
||||
return false // definitely under the limit, no need to calculate the total size
|
||||
}
|
||||
// calculate the total size, bailing out early if the size limit is reached
|
||||
let size = 0
|
||||
for (const line of lines) {
|
||||
size += line.length + 1 // include the newline
|
||||
if (size > maxDocLength) return true
|
||||
}
|
||||
// since we didn't hit the limit in the loop, the document is within the allowed length
|
||||
return false
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
determineProjectHistoryType,
|
||||
getUpgradeFunctionForType,
|
||||
upgradeProject,
|
||||
convertLargeDocsToFile,
|
||||
}
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
const SCRIPT_VERSION = 3
|
||||
const SCRIPT_VERSION = 4
|
||||
const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true'
|
||||
const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10
|
||||
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100
|
||||
|
@ -16,11 +16,16 @@ process.env.MONGO_SOCKET_TIMEOUT =
|
|||
|
||||
const PROJECT_ID = process.env.PROJECT_ID
|
||||
|
||||
// User id is required to move large documents to filestore
|
||||
const USER_ID = process.env.PROJECT_ID
|
||||
const CONVERT_LARGE_DOCS_TO_FILE = process.env.DRY_RUN === 'true'
|
||||
|
||||
const { ObjectId, ReadPreference } = require('mongodb')
|
||||
const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
|
||||
const { promiseMapWithLimit } = require('../../app/src/util/promises')
|
||||
const { batchedUpdate } = require('../helpers/batchedUpdate')
|
||||
const ProjectHistoryController = require('../../modules/admin-panel/app/src/ProjectHistoryController')
|
||||
const HistoryUpgradeHelper = require('./HistoryUpgradeHelper')
|
||||
|
||||
console.log({
|
||||
DRY_RUN,
|
||||
|
@ -110,6 +115,13 @@ async function doUpgradeForNoneWithConversion(project) {
|
|||
const projectIdString = project._id.toString()
|
||||
if (!DRY_RUN) {
|
||||
try {
|
||||
if (CONVERT_LARGE_DOCS_TO_FILE) {
|
||||
const convertedDocCount =
|
||||
await HistoryUpgradeHelper.convertLargeDocsToFile(projectId, USER_ID)
|
||||
console.log(
|
||||
`converted ${convertedDocCount} large docs to binary files for project ${projectId}`
|
||||
)
|
||||
}
|
||||
await ProjectHistoryController.migrateProjectHistory(projectIdString)
|
||||
} catch (err) {
|
||||
// if migrateProjectHistory fails, it cleans up by deleting
|
||||
|
|
Loading…
Reference in a new issue