overleaf/services/history-v1/storage/tasks/fix_duplicate_versions.js

157 lines
4.4 KiB
JavaScript
Raw Normal View History

#!/usr/bin/env node
'use strict'
const commandLineArgs = require('command-line-args')
const { chunkStore } = require('..')
main()
.then(() => {
process.exit(0)
})
.catch(err => {
console.error(err)
process.exit(1)
})
async function main() {
const opts = commandLineArgs([
{ name: 'project-ids', type: String, multiple: true, defaultOption: true },
{ name: 'save', type: Boolean, defaultValue: false },
{ name: 'help', type: Boolean, defaultValue: false },
])
if (opts.help || opts['project-ids'] == null) {
console.log('Usage: fix_duplicate_versions [--save] PROJECT_ID...')
process.exit()
}
for (const projectId of opts['project-ids']) {
await processProject(projectId, opts.save)
}
if (!opts.save) {
console.log('\nThis was a dry run. Re-run with --save to persist changes.')
}
}
async function processProject(projectId, save) {
console.log(`Project ${projectId}:`)
const chunk = await chunkStore.loadLatest(projectId)
let numChanges = 0
numChanges += removeDuplicateProjectVersions(chunk)
numChanges += removeDuplicateDocVersions(chunk)
console.log(` ${numChanges > 0 ? numChanges : 'no'} changes`)
if (save && numChanges > 0) {
await replaceChunk(projectId, chunk)
}
}
function removeDuplicateProjectVersions(chunk) {
let numChanges = 0
let lastVersion = null
const { snapshot, changes } = chunk.history
if (snapshot.projectVersion != null) {
lastVersion = snapshot.projectVersion
}
for (const change of changes) {
if (change.projectVersion == null) {
// Not a project structure change. Ignore.
continue
}
if (
lastVersion != null &&
!areProjectVersionsIncreasing(lastVersion, change.projectVersion)
) {
// Duplicate. Remove all ops
console.log(
` Removing out-of-order project structure change: ${change.projectVersion} <= ${lastVersion}`
)
change.setOperations([])
delete change.projectVersion
numChanges++
} else {
lastVersion = change.projectVersion
}
}
return numChanges
}
function removeDuplicateDocVersions(chunk) {
let numChanges = 0
const lastVersions = new Map()
const { snapshot, changes } = chunk.history
if (snapshot.v2DocVersions != null) {
for (const { pathname, v } of Object.values(snapshot.v2DocVersions.data)) {
lastVersions.set(pathname, v)
}
}
for (const change of changes) {
if (change.v2DocVersions == null) {
continue
}
// Collect all docs that have problematic versions
const badPaths = []
const badDocIds = []
for (const [docId, { pathname, v }] of Object.entries(
change.v2DocVersions.data
)) {
const lastVersion = lastVersions.get(docId)
if (lastVersion != null && v <= lastVersion) {
// Duplicate. Remove ops related to that doc
console.log(
` Removing out-of-order change for doc ${docId} (${pathname}): ${v} <= ${lastVersion}`
)
badPaths.push(pathname)
badDocIds.push(docId)
numChanges++
} else {
lastVersions.set(docId, v)
}
}
// Remove bad operations
if (badPaths.length > 0) {
change.setOperations(
change.operations.filter(
op => op.pathname == null || !badPaths.includes(op.pathname)
)
)
}
// Remove bad v2 doc versions
for (const docId of badDocIds) {
delete change.v2DocVersions.data[docId]
}
}
return numChanges
}
function areProjectVersionsIncreasing(v1Str, v2Str) {
const v1 = parseProjectVersion(v1Str)
const v2 = parseProjectVersion(v2Str)
return v2.major > v1.major || (v2.major === v1.major && v2.minor > v1.minor)
}
function parseProjectVersion(version) {
const [major, minor] = version.split('.').map(x => parseInt(x, 10))
if (isNaN(major) || isNaN(minor)) {
throw new Error(`Invalid project version: ${version}`)
}
return { major, minor }
}
async function replaceChunk(projectId, chunk) {
const endVersion = chunk.getEndVersion()
const oldChunkId = await chunkStore.getChunkIdForVersion(
projectId,
endVersion
)
console.log(` Replacing chunk ${oldChunkId}`)
// The chunks table has a unique constraint on doc_id and end_version. Because
// we're replacing a chunk with the same end version, we need to destroy the
// chunk first.
await chunkStore.destroy(projectId, oldChunkId)
await chunkStore.create(projectId, chunk)
}