overleaf/services/history-v1/storage/tasks/compress_changes.js
Alf Eaton ee85d948e2 Avoid duplicating a math-closing dollar sign (#11227)
GitOrigin-RevId: ef2ef77e26df59d1af3df6dc664e284d3c70102d
2023-01-16 08:41:42 +00:00

107 lines
2.9 KiB
JavaScript

/**
* Compress changes for projects that have too many text operations.
*
* Usage:
*
* node tasks/compress_changes.js CSV_FILE
*
* where CSV_FILE contains a list of project ids in the first column
*/
const fs = require('fs')
const BPromise = require('bluebird')
const { History } = require('overleaf-editor-core')
const { historyStore, chunkStore } = require('..')
const CONCURRENCY = 10
async function main() {
const filename = process.argv[2]
const projectIds = await readCsv(filename)
const chunks = []
for (const projectId of projectIds) {
const chunkIds = await chunkStore.getProjectChunkIds(projectId)
chunks.push(...chunkIds.map(id => ({ id, projectId })))
}
let totalCompressed = 0
await BPromise.map(
chunks,
async chunk => {
try {
const history = await getHistory(chunk)
const numCompressed = compressChanges(history)
if (numCompressed > 0) {
await storeHistory(chunk, history)
console.log(
`Compressed project ${chunk.projectId}, chunk ${chunk.id}`
)
}
totalCompressed += numCompressed
} catch (err) {
console.log(err)
}
},
{ concurrency: CONCURRENCY }
)
console.log('CHANGES:', totalCompressed)
}
async function readCsv(filename) {
const csv = await fs.promises.readFile(filename, 'utf-8')
const lines = csv.trim().split('\n')
const projectIds = lines.map(line => line.split(',')[0])
return projectIds
}
async function getHistory(chunk) {
const rawHistory = await historyStore.loadRaw(chunk.projectId, chunk.id)
const history = History.fromRaw(rawHistory)
return history
}
async function storeHistory(chunk, history) {
const rawHistory = history.toRaw()
await historyStore.storeRaw(chunk.projectId, chunk.id, rawHistory)
}
function compressChanges(history) {
let numCompressed = 0
for (const change of history.getChanges()) {
const newOperations = compressOperations(change.operations)
if (newOperations.length !== change.operations.length) {
numCompressed++
}
change.setOperations(newOperations)
}
return numCompressed
}
function compressOperations(operations) {
if (!operations.length) return []
const newOperations = []
let currentOperation = operations[0]
for (let operationId = 1; operationId < operations.length; operationId++) {
const nextOperation = operations[operationId]
if (currentOperation.canBeComposedWith(nextOperation)) {
currentOperation = currentOperation.compose(nextOperation)
} else {
// currentOperation and nextOperation cannot be composed. Push the
// currentOperation and start over with nextOperation.
newOperations.push(currentOperation)
currentOperation = nextOperation
}
}
newOperations.push(currentOperation)
return newOperations
}
main()
.then(() => {
process.exit()
})
.catch(err => {
console.error(err)
process.exit(1)
})