Merge pull request #8644 from overleaf/bg-archive-track-changes

archive track changes zip files in project when history migration fails

GitOrigin-RevId: 67f2381b0a8f2fb1de82100752031c65808af885
This commit is contained in:
Brian Gough 2022-06-30 09:21:50 +01:00 committed by Copybot
parent 93c2168ab2
commit da02a4f2be
4 changed files with 44 additions and 1 deletions

View file

@ -148,6 +148,20 @@ async function generateZip(projectId, zipfile) {
Buffer.from(JSON.stringify(manifest, null, 2)),
'manifest.json'
)
zipfile.addBuffer(
Buffer.from(
`History Migration Data
This zip file contains a copy of the raw history for this project.
If this file is present in a project it means the history could not
be fully recovered or migrated.
A new history should have been created starting at the datestamp of
this file.`
),
'README.txt'
)
zipfile.end()
}

View file

@ -125,6 +125,27 @@ const ProjectHistoryHandler = {
)
},
setMigrationArchiveFlag(project_id, callback) {
if (callback == null) {
callback = function () {}
}
Project.updateOne(
{ _id: project_id, version: { $exists: true } },
{
'overleaf.history.zipFileArchivedInProject': true,
},
function (err, result) {
if (err != null) {
return callback(err)
}
if ((result != null ? result.n : undefined) === 0) {
return callback(new Error('migration flag not set'))
}
return callback()
}
)
},
ensureHistoryExistsForProject(project_id, callback) {
// We can only set a history id for a project that doesn't have one. The
// history id is cached in the project history service, and changing an

View file

@ -100,6 +100,7 @@ const ProjectSchema = new Schema({
display: { type: Boolean },
upgradedAt: { type: Date },
allowDowngrade: { type: Boolean },
zipFileArchivedInProject: { type: Boolean },
},
},
collabratecUsers: [

View file

@ -8,6 +8,7 @@ const RETRY_FAILED = process.env.RETRY_FAILED === 'true'
const MAX_UPGRADES_TO_ATTEMPT =
parseInt(process.env.MAX_UPGRADES_TO_ATTEMPT, 10) || false
const MAX_FAILURES = parseInt(process.env.MAX_FAILURES, 10) || 50
const ARCHIVE_ON_FAILURE = process.env.ARCHIVE_ON_FAILURE === 'true'
// persist fallback in order to keep batchedUpdate in-sync
process.env.BATCH_SIZE = BATCH_SIZE
// raise mongo timeout to 1hr if otherwise unspecified
@ -37,6 +38,7 @@ console.log({
MAX_FAILURES,
USE_QUERY_HINT,
RETRY_FAILED,
ARCHIVE_ON_FAILURE,
PROJECT_ID,
})
@ -86,6 +88,9 @@ async function processProject(project) {
project.overleaf.history.conversionFailed ||
project.overleaf.history.upgradeFailed
) {
if (project.overleaf.history.zipFileArchivedInProject) {
return // always give up if we have uploaded the zipfile to the project
}
if (!RETRY_FAILED) {
// we don't want to attempt upgrade on projects
// that have been previously attempted and failed
@ -123,7 +128,9 @@ async function doUpgradeForNoneWithConversion(project) {
`converted ${convertedDocCount} large docs to binary files for project ${projectId}`
)
}
await ProjectHistoryController.migrateProjectHistory(projectIdString)
await ProjectHistoryController.migrateProjectHistory(projectIdString, {
archiveOnFailure: ARCHIVE_ON_FAILURE,
})
} catch (err) {
// if migrateProjectHistory fails, it cleans up by deleting
// the history and unsetting the history id