github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #6180 from overleaf/tm-history-upgrade-individual-projects

Browse source 
Add script for converting individual projects to FPH and history upgrade helper

GitOrigin-RevId: 4cf075e08045869fe87724fb28949fe91b780e93
This commit is contained in:
Thomas 2022-05-30 12:18:11 +02:00 committed by Copybot
parent 96a8f7ffa1
commit 6f55a99ae1
3 changed files with 346 additions and 203 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

299
services/web/scripts/history/HistoryUpgradeHelper.js Normal file
View file

@ -0,0 +1,299 @@
const { ReadPreference, ObjectId } = require('mongodb')
const { db } = require('../../app/src/infrastructure/mongodb')
const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler')
const HistoryManager = require('../../app/src/Features/History/HistoryManager')
const ProjectHistoryController = require('../../modules/admin-panel/app/src/ProjectHistoryController')
// Timestamp of when 'Enable history for SL in background' release
const ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = '5a8d8a370000000000000000'
const OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = new ObjectId(
ID_WHEN_FULL_PROJECT_HISTORY_ENABLED
)
const DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED =
OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED.getTimestamp()
async function determineProjectHistoryType(project) {
if (project.overleaf && project.overleaf.history) {
if (project.overleaf.history.upgradeFailed) {
return 'UpgradeFailed'
}
if (project.overleaf.history.conversionFailed) {
return 'ConversionFailed'
}
}
if (
project.overleaf &&
project.overleaf.history &&
project.overleaf.history.id
) {
if (project.overleaf.history.display) {
// v2: full project history, do nothing
return 'V2'
} else {
if (projectCreatedAfterFullProjectHistoryEnabled(project)) {
// IF project initialised after full project history enabled for all projects
// THEN project history should contain all information we need, without intervention
return 'V1WithoutConversion'
} else {
// ELSE SL history may predate full project history
// THEN delete full project history and convert their SL history to full project history
// --
// TODO: how to verify this, can get rough start date of SL history, but not full project history
const preserveHistory = await shouldPreserveHistory(project)
const anyDocHistory = await anyDocHistoryExists(project)
const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
if (preserveHistory) {
if (anyDocHistory || anyDocHistoryIndex) {
// if SL history exists that we need to preserve, then we must convert
return 'V1WithConversion'
} else {
// otherwise just upgrade without conversion
return 'V1WithoutConversion'
}
} else {
// if preserveHistory false, then max 7 days of SL history
// but v1 already record to both histories, so safe to upgrade
return 'V1WithoutConversion'
}
}
}
} else {
const preserveHistory = await shouldPreserveHistory(project)
const anyDocHistory = await anyDocHistoryExists(project)
const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
if (anyDocHistory || anyDocHistoryIndex) {
// IF there is SL history ->
if (preserveHistory) {
// that needs to be preserved:
// THEN initialise full project history and convert SL history to full project history
return 'NoneWithConversion'
} else {
return 'NoneWithTemporaryHistory'
}
} else {
// ELSE there is not any SL history ->
// THEN initialise full project history and sync with current content
return 'NoneWithoutConversion'
}
}
}
async function upgradeProject(project) {
const historyType = await determineProjectHistoryType(project)
if (historyType === 'V2') {
return { historyType, upgraded: true }
}
const upgradeFn = getUpgradeFunctionForType(historyType)
if (!upgradeFn) {
return { error: 'unsupported history type' }
}
const result = await upgradeFn(project)
result.historyType = historyType
return result
}
// Do upgrades/conversion:
function getUpgradeFunctionForType(historyType) {
return UpgradeFunctionMapping[historyType]
}
const UpgradeFunctionMapping = {
NoneWithoutConversion: doUpgradeForNoneWithoutConversion,
UpgradeFailed: doUpgradeForNoneWithoutConversion,
ConversionFailed: doUpgradeForNoneWithConversion,
V1WithoutConversion: doUpgradeForV1WithoutConversion,
V1WithConversion: doUpgradeForV1WithConversion,
NoneWithConversion: doUpgradeForNoneWithConversion,
NoneWithTemporaryHistory: doUpgradeForNoneWithConversion,
}
async function doUpgradeForV1WithoutConversion(project) {
await db.projects.updateOne(
{ _id: project._id },
{
$set: {
'overleaf.history.display': true,
'overleaf.history.upgradedAt': new Date(),
'overleaf.history.upgradeReason': `v1-without-sl-history`,
},
}
)
return { upgraded: true }
}
async function doUpgradeForV1WithConversion(project) {
const result = {}
const projectId = project._id
// migrateProjectHistory expects project id as a string
const projectIdString = project._id.toString()
try {
// We treat these essentially as None projects, the V1 history is irrelevant,
// so we will delete it, and do a conversion as if we're a None project
await ProjectHistoryController.deleteProjectHistory(projectIdString)
await ProjectHistoryController.migrateProjectHistory(projectIdString)
} catch (err) {
// if migrateProjectHistory fails, it cleans up by deleting
// the history and unsetting the history id
// therefore a failed project will still look like a 'None with conversion' project
result.error = err
await db.projects.updateOne(
{ _id: projectId },
{
$set: {
'overleaf.history.conversionFailed': true,
},
}
)
return result
}
await db.projects.updateOne(
{ _id: projectId },
{
$set: {
'overleaf.history.upgradeReason': `v1-with-conversion`,
},
$unset: {
'overleaf.history.upgradeFailed': true,
'overleaf.history.conversionFailed': true,
},
}
)
result.upgraded = true
return result
}
async function doUpgradeForNoneWithoutConversion(project) {
const result = {}
const projectId = project._id
try {
// Logic originally from ProjectHistoryHandler.ensureHistoryExistsForProject
// However sends a force resync project to project history instead
// of a resync request to doc-updater
const historyId = await ProjectHistoryHandler.promises.getHistoryId(
projectId
)
if (!historyId) {
const history = await HistoryManager.promises.initializeProject()
if (history && history.overleaf_id) {
await ProjectHistoryHandler.promises.setHistoryId(
projectId,
history.overleaf_id
)
}
}
await HistoryManager.promises.resyncProject(projectId, {
force: true,
origin: { kind: 'history-migration' },
})
await HistoryManager.promises.flushProject(projectId)
} catch (err) {
result.error = err
await db.projects.updateOne(
{ _id: project._id },
{
$set: {
'overleaf.history.upgradeFailed': true,
},
}
)
return result
}
await db.projects.updateOne(
{ _id: project._id },
{
$set: {
'overleaf.history.display': true,
'overleaf.history.upgradedAt': new Date(),
'overleaf.history.upgradeReason': `none-without-conversion`,
},
}
)
result.upgraded = true
return result
}
async function doUpgradeForNoneWithConversion(project) {
const result = {}
const projectId = project._id
// migrateProjectHistory expects project id as a string
const projectIdString = project._id.toString()
try {
await ProjectHistoryController.migrateProjectHistory(projectIdString)
} catch (err) {
// if migrateProjectHistory fails, it cleans up by deleting
// the history and unsetting the history id
// therefore a failed project will still look like a 'None with conversion' project
result.error = err
await db.projects.updateOne(
{ _id: projectId },
{
$set: {
'overleaf.history.conversionFailed': true,
},
}
)
return result
}
await db.projects.updateOne(
{ _id: projectId },
{
$set: {
'overleaf.history.upgradeReason': `none-with-conversion`,
},
$unset: {
'overleaf.history.upgradeFailed': true,
'overleaf.history.conversionFailed': true,
},
}
)
result.upgraded = true
return result
}
// Util
function projectCreatedAfterFullProjectHistoryEnabled(project) {
return (
project._id.getTimestamp() >= DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED
)
}
async function shouldPreserveHistory(project) {
return await db.projectHistoryMetaData.findOne(
{
$and: [
{ project_id: { $eq: project._id } },
{ preserveHistory: { $eq: true } },
],
},
{ readPreference: ReadPreference.SECONDARY }
)
}
async function anyDocHistoryExists(project) {
return await db.docHistory.findOne(
{ project_id: { $eq: project._id } },
{
projection: { _id: 1 },
readPreference: ReadPreference.SECONDARY,
}
)
}
async function anyDocHistoryIndexExists(project) {
return await db.docHistoryIndex.findOne(
{ project_id: { $eq: project._id } },
{
projection: { _id: 1 },
readPreference: ReadPreference.SECONDARY,
}
)
}
module.exports = {
determineProjectHistoryType,
getUpgradeFunctionForType,
upgradeProject,
}

212
services/web/scripts/history/count_project_history_categories.js
View file

@ -9,230 +9,36 @@ process.env.BATCH_SIZE = BATCH_SIZE
process.env.MONGO_SOCKET_TIMEOUT =
parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
const { ReadPreference, ObjectId } = require('mongodb')
const { db } = require('../../app/src/infrastructure/mongodb')
const { promiseMapWithLimit } = require('../../app/src/util/promises')
const { batchedUpdate } = require('../helpers/batchedUpdate')
const { determineProjectHistoryType } = require('./HistoryUpgradeHelper')
const COUNT = {
v2: 0,
v1WithoutConversion: 0,
v1WithConversion: 0,
V2: 0,
V1WithoutConversion: 0,
V1WithConversion: 0,
NoneWithoutConversion: 0,
NoneWithConversion: 0,
NoneWithTemporaryHistory: 0,
HistoryUpgradeFailed: 0,
HistoryConversionFailed: 0,
UpgradeFailed: 0,
ConversionFailed: 0,
}
// Timestamp of when 'Enable history for SL in background' release
const ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = '5a8d8a370000000000000000'
const OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = new ObjectId(
ID_WHEN_FULL_PROJECT_HISTORY_ENABLED
)
const DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED =
OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED.getTimestamp()
async function processBatch(_, projects) {
await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
console.log(COUNT)
}
async function processProject(project) {
if (project.overleaf && project.overleaf.history) {
if (project.overleaf.history.upgradeFailed) {
// a failed history upgrade might look like a v1 project, but history may be broken
COUNT.HistoryUpgradeFailed += 1
if (VERBOSE_LOGGING) {
console.log(
`project ${
project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
} has a history upgrade failure recorded`
)
}
return
} else if (project.overleaf.history.conversionFailed) {
COUNT.HistoryConversionFailed += 1
if (VERBOSE_LOGGING) {
console.log(
`project ${
project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
} has a history conversion failure recorded`
)
}
return
}
}
if (
project.overleaf &&
project.overleaf.history &&
project.overleaf.history.id
) {
if (project.overleaf.history.display) {
// v2: full project history, do nothing, (query shoudln't include any, but we should stlll check?)
COUNT.v2 += 1
if (VERBOSE_LOGGING) {
console.log(
`project ${
project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
} is already v2`
)
}
} else {
if (projectCreatedAfterFullProjectHistoryEnabled(project)) {
// IF project initialised after full project history enabled for all projects
// THEN project history should contain all information we need, without intervention
await doUpgradeForV1WithoutConversion(project) // CASE #1
} else {
// ELSE SL history may predate full project history
// THEN delete full project history and convert their SL history to full project history
// --
// TODO: how to verify this, can get rough start date of SL history, but not full project history
const preserveHistory = await shouldPreserveHistory(project)
const anyDocHistory = await anyDocHistoryExists(project)
const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
if (preserveHistory) {
if (anyDocHistory || anyDocHistoryIndex) {
// if SL history exists that we need to preserve, then we must convert
await doUpgradeForV1WithConversion(project) // CASE #4
} else {
// otherwise just upgrade without conversion
await doUpgradeForV1WithoutConversion(project) // CASE #1
}
} else {
// if preserveHistory false, then max 7 days of SL history
// but v1 already record to both histories, so safe to upgrade
await doUpgradeForV1WithoutConversion(project) // CASE #1
}
}
}
} else {
const preserveHistory = await shouldPreserveHistory(project)
const anyDocHistory = await anyDocHistoryExists(project)
const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
if (anyDocHistory || anyDocHistoryIndex) {
// IF there is SL history ->
if (preserveHistory) {
// that needs to be preserved:
// THEN initialise full project history and convert SL history to full project history
await doUpgradeForNoneWithConversion(project) // CASE #3
} else {
await doUpgradeForNoneWithTemporaryHistory(project) // Either #3 or #2
}
} else {
// ELSE there is not any SL history ->
// THEN initialise full project history and sync with current content
await doUpgradeForNoneWithoutConversion(project) // CASE #2
}
}
}
// Helpers:
async function shouldPreserveHistory(project) {
return await db.projectHistoryMetaData.findOne(
{
$and: [
{ project_id: { $eq: project._id } },
{ preserveHistory: { $eq: true } },
],
},
{ readPreference: ReadPreference.SECONDARY }
)
}
async function anyDocHistoryExists(project) {
return await db.docHistory.findOne(
{ project_id: { $eq: project._id } },
{
projection: { _id: 1 },
readPreference: ReadPreference.SECONDARY,
}
)
}
async function anyDocHistoryIndexExists(project) {
return await db.docHistoryIndex.findOne(
{ project_id: { $eq: project._id } },
{
projection: { _id: 1 },
readPreference: ReadPreference.SECONDARY,
}
)
}
function projectCreatedAfterFullProjectHistoryEnabled(project) {
return (
project._id.getTimestamp() >= DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED
)
}
// Do upgrades/conversion:
async function doUpgradeForV1WithoutConversion(project) {
// Simply:
// project.overleaf.history.display = true
// TODO: Sanity check(?)
COUNT.v1WithoutConversion += 1
const historyType = await determineProjectHistoryType(project)
if (VERBOSE_LOGGING) {
console.log(
`project ${
project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
} is v1 and does not require conversion`
)
}
}
async function doUpgradeForV1WithConversion(project) {
// Delete full project history (or create new)
// Use conversion script to convert SL history to full project history
COUNT.v1WithConversion += 1
if (VERBOSE_LOGGING) {
console.log(
`project ${
project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
} is v1 and requires conversion`
)
}
}
async function doUpgradeForNoneWithoutConversion(project) {
// Initialise full project history with current content
COUNT.NoneWithoutConversion += 1
if (VERBOSE_LOGGING) {
console.log(
`project ${
project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
} is None and and does not require conversion`
)
}
}
async function doUpgradeForNoneWithConversion(project) {
// Initialise full project history
// Use conversion script to convert SL history to full project history
COUNT.NoneWithConversion += 1
if (VERBOSE_LOGGING) {
console.log(
`project ${
project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
} is None and and requires conversion`
)
}
}
async function doUpgradeForNoneWithTemporaryHistory(project) {
// If project doesn't have preserveHistory set
// but it has SL history:
// The history is temporary, we could convert, or do a 7 day staged rollout
COUNT.NoneWithTemporaryHistory += 1
if (VERBOSE_LOGGING) {
console.log(
`project ${
project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
} is None and and has temporary history (MAYBE requires conversion)`
} is type ${historyType}`
)
}
COUNT[historyType] += 1
}
async function main() {

38
services/web/scripts/history/upgrade_project.js Normal file
View file

@ -0,0 +1,38 @@
const { ReadPreference, ObjectId } = require('mongodb')
const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
const { upgradeProject } = require('./HistoryUpgradeHelper')
async function processProject(project) {
const result = await upgradeProject(project)
console.log(result)
}
async function main() {
await waitForDb()
const args = process.argv.slice(2)
const projectId = args[0]
const query = { _id: ObjectId(projectId) }
const projection = {
_id: 1,
overleaf: 1,
}
const options = {
projection,
readPreference: ReadPreference.SECONDARY,
}
const project = await db.projects.findOne(query, options)
if (project) {
await processProject(project)
} else {
console.error(`project ${projectId} not found`)
}
}
main()
.then(() => {
process.exit(0)
})
.catch(error => {
console.error({ error })
process.exit(1)
})