overleaf/services/web/modules/history-migration/app/src/ProjectHistoryController.js
Miguel Serrano ef499680aa Merge pull request #11708 from overleaf/bg-reinitialise-project-history
allow re-initialisation of history

GitOrigin-RevId: 85caf506df736917f6870f68b3bfab46d57aa28e
2023-02-09 14:34:56 +00:00

1324 lines
38 KiB
JavaScript

const _ = require('lodash')
const settings = require('@overleaf/settings')
const OError = require('@overleaf/o-error')
const fs = require('fs')
const fse = require('fs-extra')
const { ObjectId } = require('mongodb')
const request = require('request')
const { pipeline } = require('stream')
const unzipper = require('unzipper')
const util = require('util')
const logger = require('@overleaf/logger')
const path = require('path')
const {
FileTooLargeError,
InvalidNameError,
} = require('../../../../app/src/Features/Errors/Errors')
const FilestoreHandler = require('../../../../app/src/Features/FileStore/FileStoreHandler')
const ProjectGetter = require('../../../../app/src/Features/Project/ProjectGetter')
const RedisWrapper = require('../../../../app/src/infrastructure/RedisWrapper')
const HistoryManager = require('../../../../app/src/Features/History/HistoryManager')
const ProjectHistoryHandler = require('../../../../app/src/Features/Project/ProjectHistoryHandler')
const ProjectUpdateHandler = require('../../../../app/src/Features/Project/ProjectUpdateHandler')
const DocumentUpdaterHandler = require('../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler')
const ProjectEntityHandler = require('../../../../app/src/Features/Project/ProjectEntityHandler')
const ProjectEntityUpdateHandler = require('../../../../app/src/Features/Project/ProjectEntityUpdateHandler')
const SafePath = require('../../../../app/src/Features/Project/SafePath')
const { DeletedFile } = require('../../../../app/src/models/DeletedFile')
const { Doc } = require('../../../../app/src/models/Doc')
const {
iterablePaths,
} = require('../../../../app/src/Features/Project/IterablePath')
const rclient = RedisWrapper.client('project_history_migration')
module.exports = { deleteProjectHistory, migrateProjectHistory }
/**
* @typedef {Object} UpdateMeta
* @property {string | null} user_id the id of the user that performed the update
* @property {number} ts the timestamp of the update
*/
/**
* @typedef {UpdateMeta} EditDocUpdateMeta
* @property {string | null} user_id
* @property {number} ts
* @property {string} pathname the doc pathname
* @property {number} doc_length the length of the doc
*/
/**
* @typedef {Object} Update
* @property {string} pathname the path in the file tree
* @property {UpdateMeta} meta
// * @property {string} version a two-part version. The first part is the project version after the updates, as recorded in Mongo. The second part is a counter that increments for each update in this batch.
* @property {string} projectHistoryId the v1 history id for this project
* @property {number} v
*/
/**
* @typedef {Update} FileUpdate
* @property {string} pathname
* @property {UpdateMeta} meta
* @property {string} projectHistoryId
* @property {number} v
* @property {string} file
*/
/**
* @typedef {FileUpdate} AddFileUpdate
* @property {string} pathname
* @property {UpdateMeta} meta
* @property {string} projectHistoryId
* @property {number} v
* @property {string} file
* @property {string} url
*/
/**
* @typedef {Update} DocUpdate
* @property {UpdateMeta} meta
* @property {string} projectHistoryId
* @property {number} v
* @property {string} doc
*/
/**
* @typedef {DocUpdate} AddDocUpdate
* @property {string} pathname
* @property {UpdateMeta} meta
* @property {string} projectHistoryId
* @property {number} v
* @property {string} doc
* @property {string} docLines
* @property {string} docLinesId
* @property {boolean} contentStored
*/
/**
* @typedef {DocUpdate} EditDocUpdate
* @property {EditDocUpdateMeta} meta
* @property {string} projectHistoryId
* @property {number} v
* @property {number} lastV
* @property {string} doc
* @property {Array<Object>} op
*/
/**
* @typedef {AddDocUpdate | AddFileUpdate} AddUpdate
*/
/**
* @typedef {DocUpdate | FileUpdate} DeleteUpdate
* @property {string} pathname
* @property {UpdateMeta} meta
* @property {string} projectHistoryId
* @property {number} v
* @property {string} doc
* @property {string} new_pathname
*/
/**
* @typedef {Update} EditDocUpdateStub
* @property {true} stub
* @property {string} path
* @property {string} pathname
* @property {number} v
* @property {number} doc_length
*/
/**
* @typedef {AddUpdate | DeleteUpdate | EditDocUpdate | EditDocUpdateStub } AnyUpdate
*/
/**
* @typedef {Object} Project
* @property {string} _id the id of the user that performed the update
* @property {Object} overleaf
*/
/**
* @typedef ManifestUpdate
* @property {string} path
* @property {number} doc_length
* @property {number} ts
* @property {number} version
*/
/**
* @typedef ManifestContent
* @property {number} start
*/
/**
* @typedef ManifestDoc
* @property {string} id
* @property {ManifestContent} content
* @property {Array<ManifestUpdate>} updates
*/
/**
* @typedef {Object} Manifest
* @property {string} projectId
* @property {Array<ManifestDoc>} docs
*/
/**
* @typedef Entity
* @property {string} type
* @property {string} path
* @property {string} docLines
* @property {string} deletedAt
* @property {boolean} deleted
*/
/**
* Iterate recursively through the folders in project.rootFolder,
* building a map of all the docs (with content as a docLines string)
* and files (with content as a filestore URL).
*
* @param {Object} project
* @returns {Promise<Map<string, Entity>>}
*/
async function processRootFolder(project) {
const entities = new Map()
async function processFolder(folder, root = '') {
for (const item of iterablePaths(folder, 'docs')) {
const doc = await Doc.findOne(
item._id,
// only read the fields we need to save memory
{ _id: 1, inS3: 1, lines: 1, name: 1 }
).lean()
// skip malformed doc entries
if (!doc?._id) {
logger.warn({ doc }, 'skipping doc with missing id')
continue
}
const id = doc._id.toString()
const docIsInS3 = !!doc.inS3
let docLines
if (docIsInS3) {
const docPeek = await ProjectEntityHandler.promises.getDoc(
project._id,
item._id,
{ peek: true }
)
docLines = docPeek.lines
} else {
docLines = doc.lines
}
if (!docLines) {
throw new Error(`no doc lines for doc ${id} (inS3: ${docIsInS3})`)
}
entities.set(id, {
path: `${root}/${item.name}`, // NOTE: not doc.name, which is "new doc",
type: 'doc',
docLines: docLines.join('\n'),
})
}
for (const item of iterablePaths(folder, 'fileRefs')) {
const path = `${root}/${item.name}`
// skip malformed file entries
if (!item?._id) {
logger.warn({ item }, 'skipping fileRef with missing id')
continue
}
const id = item._id.toString()
entities.set(id, {
path,
type: 'file',
url: FilestoreHandler._buildUrl(project._id.toString(), id),
})
}
for (const subfolder of iterablePaths(folder, 'folders')) {
const path = `${root}/${subfolder.name}`
await processFolder(subfolder, path)
}
}
for (const folder of project.rootFolder) {
await processFolder(folder)
}
return entities
}
/**
* Read docs deleted from a project, from the Doc collection,
* and add them to the entities map with the content in a docLines string.
*
* These entities have a `deleted` property set to `true` and a `deletedAt` date.
*
* @param {Map<string, Object>} entities
* @param {string} projectId
* @returns {Promise<void>}
*/
async function readDeletedDocs(entities, projectId) {
// NOTE: could call DocstoreManager.promises.getAllDeletedDocs(projectId) instead
// Look for all docs, since some deleted docs are found in track-changes manifest,
// but do not have deleted flag set for reasons that are unclear
// (we will not add docs to entities if they were previously added by processRootFolder)
const deletedDocsCursor = Doc.find(
{
project_id: ObjectId(projectId),
},
// only read the fields we need to save memory
{ _id: 1, inS3: 1, lines: 1, name: 1, deletedAt: 1 }
)
.lean()
.cursor()
for await (const doc of deletedDocsCursor) {
// skip malformed deleted doc entries
if (!doc?._id) {
logger.warn({ doc }, 'skipping deleted doc with missing id')
continue
}
const id = doc._id.toString()
// Skip doc if we already have an entry in entities
if (!entities.has(id)) {
const docIsInS3 = !!doc.inS3
let docLines
if (docIsInS3) {
const docPeek = await ProjectEntityHandler.promises.getDoc(
ObjectId(projectId),
doc._id,
{ peek: true }
)
docLines = docPeek.lines
} else {
docLines = doc.lines
}
if (!docLines) {
throw new Error(`no doc lines for doc ${id} (inS3: ${docIsInS3})`)
}
// const ts = Number(
// doc.deletedAt ? new Date(doc.deletedAt) : Date.now()
// )
if (doc.name && !SafePath.isCleanFilename(doc.name)) {
const newName = SafePath.clean(doc.name)
logger.warn(
{ projectId, docId: id, origName: doc.name, newName },
'renaming invalid deleted file'
)
doc.name = newName
}
entities.set(id, {
// NOTE: adding the doc id to the file path to avoid collisions
path: `/_deleted/${id}/${doc.name}`,
name: doc.name || 'unnamed', // fallback for improperly deleted docs
deleted: true,
type: 'doc',
deletedAt: doc.deletedAt,
docLines: docLines.join('\n'),
})
}
}
}
/**
* Read files deleted from a project, from the DeletedFile collection,
* and add them to the entities map.
*
* These entities have a `deleted` property set to `true` and a `deletedAt` date.
* The url is built later, from the project id and file id.
*
* @param {Map<string, Object>} entities
* @param {string} projectId
* @returns {Promise<void>}
*/
async function readDeletedFiles(entities, projectId) {
const deletedFilesCursor = DeletedFile.find(
{
projectId: ObjectId(projectId),
},
// only read the fields we need to save memory
{ _id: 1, name: 1, deletedAt: 1 }
)
.lean()
.cursor()
for await (const file of deletedFilesCursor) {
// skip malformed deleted file entries
if (!file?._id) {
logger.warn({ file }, 'skipping deleted file with missing id')
continue
}
const id = file._id.toString()
// TODO: check if it already exists?
if (!entities.has(id)) {
// const ts = Number(
// file.deletedAt ? new Date(file.deletedAt) : Date.now()
// )
// TODO: would the hash be useful here?
if (file.name && !SafePath.isCleanFilename(file.name)) {
const newName = SafePath.clean(file.name)
logger.warn(
{ projectId, fileId: id, origName: file.name, newName },
'renaming invalid deleted file'
)
file.name = newName
}
entities.set(id, {
// NOTE: adding the doc id to the file path to avoid collisions
path: `/_deleted/${id}/${file.name}`,
name: file.name,
deleted: true,
type: 'file',
deletedAt: file.deletedAt,
})
}
}
}
/**
* Iterate through the sorted array of updates, pushing each one to Redis.
*
* In batches, tell project-history to pull the updates from Redis and process them,
* so the process fails early if something can't be processed.
*
* @param {Array<AnyUpdate>} updates
* @param {string} projectId
* @param {string} projectHistoryId
* @param {Map.<string, Object>} fileMap
* @returns {Promise<void>}
*/
async function sendUpdatesToProjectHistory(
updates,
projectId,
projectHistoryId,
fileMap
) {
let multi = rclient.multi()
let counter = 0
let processed = 0
let size = 0
const projectHistoryKey =
settings.redis.project_history_migration.key_schema.projectHistoryOps({
projectId,
})
// clear out anything in the Redis queue for this project's history
multi.del(projectHistoryKey)
for (let update of updates) {
// read the content for each update stub from the archive
if (update.stub) {
update = await buildEditDocUpdate(projectHistoryId, update, fileMap)
}
// non-edit doc updates need string timestamps, not numbers
if (!('op' in update)) {
update.meta.ts = new Date(update.meta.ts).toISOString()
}
const updateJSON = JSON.stringify(update)
multi.rpush(projectHistoryKey, updateJSON)
counter++
processed++
size += updateJSON.length
// flush the history after every 1000 updates and start a new transaction
if (counter === 1000) {
logger.debug(
{ processed, total: updates.length },
'sending updates to project history'
)
// execute the transaction
await util.promisify(multi.exec)()
// tell project-history to pull the updates from the Redis queue
await HistoryManager.promises.flushProject(projectId) // TODO: roll back if this fails?
counter = 0
size = 0
multi = rclient.multi()
} else if (size > 1024 * 1024) {
// queue entries in redis more frequently to reduce memory usage
await util.promisify(multi.exec)()
size = 0
multi = rclient.multi()
}
}
if (counter > 0) {
// execute the transaction
await util.promisify(multi.exec)()
// tell project-history to pull the updates from the Redis queue
await HistoryManager.promises.flushProject(projectId) // TODO: roll back if this fails?
}
// return the queue length so we can check that it is empty
const queueLength = await rclient.llen(projectHistoryKey)
return queueLength
}
/**
* Compare two arrays of updates, with the earliest timestamp at the end first.
*
* @param {Array<AnyUpdate>} a
* @param {Array<AnyUpdate>} b
* @returns {number}
*/
function earliestTimestampFirst(a, b) {
// both arrays are empty, leave them
if (!a.length && !b.length) {
return 0
}
// a is empty, move b before a
if (!a.length) {
return 1
}
// b is empty, don't move b before a
if (!b.length) {
return -1
}
const tsB = b[b.length - 1].meta.ts
const tsA = a[a.length - 1].meta.ts
// if the last item in b has a lower timestamp that the last item in a, move b above a
if (tsB < tsA) {
return 1
}
if (tsB > tsA) {
return -1
}
// use pathnames as secondary sort key, to make order deterministic for
// updates with the same timestamp
const pathnameB = b[b.length - 1].pathname
const pathnameA = a[a.length - 1].pathname
if (pathnameB < pathnameA) {
return 1
}
if (pathnameB > pathnameA) {
return -1
}
return 0 // shouldn't happen, because pathnames must be distinct
}
/**
* Compare two updates, with the highest version number first
*
* @param {AnyUpdate} a
* @param {AnyUpdate} b
* @returns {number}
*/
function decreasingDocVersion(a, b) {
if (b.v === a.v) {
throw new Error(`Matching version: ${b.v} ${a.v}`)
// return 0
}
// if b.v is greater than a.v, sort b above a
return b.v > a.v ? 1 : -1
}
/**
* Create an array of queued updates for each doc/file, sorted by version
*
* @param {Array<AnyUpdate>} updates
* @returns {Promise<Array<AnyUpdate>>}
*/
async function sortUpdatesByQueue(updates) {
// build a queue of updates for each doc/file
const queues = {}
for (const update of updates) {
const docId = update.doc || update.file
if (!(docId in queues)) {
queues[docId] = []
}
queues[docId].push(update)
}
// convert the map to an array of queues
const values = Object.values(queues)
for (const queue of values) {
// sort each queue in place, with each update in decreasing version ofder
queue.sort(decreasingDocVersion)
}
return values
}
/**
* Fetch all the content and updates for this project from track-changes, as a zip archive.
*
* @param {string} projectId
* @param {string} tempFilePath
* @returns
*/
async function fetchTrackChangesArchive(projectId, tempFilePath) {
const writeStream = fs.createWriteStream(tempFilePath)
const url = `${settings.apis.trackchanges.url}/project/${projectId}/zip`
// exposed for debugging during full-project-history migration
const timeout =
parseInt(process.env.FETCH_TRACK_CHANGES_TIMEOUT, 10) || 2 * 60 * 1000
try {
await util.promisify(pipeline)(request(url, { timeout }), writeStream)
} catch (err) {
logger.error({ err }, 'Error fetching track changes archive')
throw err
}
const { size } = await fs.promises.stat(tempFilePath)
logger.info({ projectId, size }, 'fetched zip file from track-changes')
}
/**
* Open the zip archive and build a Map of each entry in the archive, with the path as the key
*
* @param {string} filePath
* @returns {Promise<Map<string, Object>>}
*/
async function openTrackChangesArchive(filePath) {
const directory = await unzipper.Open.file(filePath)
return new Map(directory.files.map(file => [file.path, file]))
}
/**
* Read the manifest data from the zip archive
*
* @param {Map<string, Object>} fileMap
* @returns {Promise<Manifest>}
*/
async function readTrackChangesManifest(fileMap) {
const manifestBuffer = await fileMap.get('manifest.json').buffer()
return JSON.parse(manifestBuffer.toString())
}
/**
* Check that entities conform to the pathnames allowed by project history
*
* @param {Map<string, Object>} entities
* @param {string} projectId
*/
function validatePaths(entities, projectId) {
const pathErrors = []
for (const [id, entity] of entities) {
if (!SafePath.isCleanPath(entity.path)) {
pathErrors.push(
`${entity.type}:${id}${entity.deleted ? ' (deleted)' : ''} path:${
entity.path
}`
)
}
}
if (pathErrors.length) {
throw new OError('Invalid path in history migration', {
projectId,
pathErrors,
})
}
}
/**
* Build an "add" update for an entity, with docLines or url set for the content.
* This represents a doc or file being added to a project.
*
* @param {Object} entity
* @param {string} entityId
* @param {string} projectId
* @param {string} projectHistoryId
*
* @returns {AddDocUpdate | AddFileUpdate}
*/
function buildAddUpdate(entity, entityId, projectId, projectHistoryId) {
const ts = new ObjectId(entityId).getTimestamp()
const update = {
pathname: entity.path,
v: 0, // NOTE: only for sorting
meta: {
// source?
user_id: null, // TODO: assign the update to a system user?
ts: Number(ts),
origin: { kind: 'history-migration' },
},
projectHistoryId,
}
switch (entity.type) {
case 'doc': {
return {
doc: entityId,
...update,
docLines: entity.docLines,
}
}
case 'file': {
// TODO: set a hash here?
return {
// type: 'external',
file: entityId,
...update,
url: FilestoreHandler._buildUrl(projectId, entityId),
}
}
default:
throw new Error('Unknown entity type')
}
}
/**
* Build a "delete" update for an entity, with new_pathname set to an empty string.
* This represents a doc or file being deleted from a project.
*
* @param {Object} entity
* @param {string} entityId
* @param {string} projectId
* @param {string} projectHistoryId
* @returns DeleteUpdate
*/
function buildDeleteUpdate(entity, entityId, projectId, projectHistoryId) {
const ts = entity.deletedAt || new Date()
const update = {
pathname: entity.path,
new_pathname: '', // empty path = deletion
v: Infinity, // NOTE: only for sorting
meta: {
user_id: null, // TODO: assign this to a system user?
ts: Number(ts),
origin: { kind: 'history-migration' },
},
projectHistoryId,
}
switch (entity.type) {
case 'doc':
return {
doc: entityId,
...update,
}
case 'file':
return {
file: entityId,
...update,
}
default:
throw new Error(`Unknown entity type ${entity.type}`)
}
}
/**
* @typedef TrackedDocUpdateMeta
* @property {string} user_id
* @property {number} start_ts
*/
/**
* @typedef TrackedDocUpdate
* @property {string} doc_id
* @property {Array<Object>} op
* @property {number} v
* @property {TrackedDocUpdateMeta} meta
*/
/**
* Build an "edit" update, with op set to an array of operations from track-changes.
*
* This represents the contents of a doc being edited in a project.
*
* @param {string} projectHistoryId
* @param {EditDocUpdateStub} updateStub
* @param {Map.<string, Object>} fileMap
*
* @returns {Promise<EditDocUpdate>}
*/
async function buildEditDocUpdate(projectHistoryId, updateStub, fileMap) {
const buffer = await fileMap.get(updateStub.path).buffer()
/**
* @type TrackedDocUpdate
*/
const data = JSON.parse(buffer.toString())
let userId = data.meta.user_id
if (userId === 'anonymous-user' || userId === 'null') {
userId = null
}
if (userId != null && !/^[0-9a-f]{24}$/.test(userId)) {
throw new OError('Bad user id in ShareLaTeX history edit update', {
userId,
})
}
return {
doc: data.doc_id,
op: data.op, // NOTE: this is an array of operations
v: data.v,
lastV: data.v - 1,
meta: {
user_id: userId,
ts: data.meta.start_ts, // TODO: use data.meta.end_ts or update.ts?
pathname: updateStub.pathname,
doc_length: updateStub.doc_length,
origin: { kind: 'history-migration' },
},
projectHistoryId,
}
}
/**
* Build a stub for an "edit" update, with all the metadata but not the actual operations.
*
* This represents a doc being edited in a project, with enough information for sorting,
* but avoids loading the actual operations from the zip archive until they're needed,
* so as not to run out of memory if the project's history is large.
*
* @param {ManifestUpdate} update
* @param {Entity} entity
* @param {string} docId
* @returns {EditDocUpdateStub}
*/
function buildEditUpdateStub(update, entity, docId) {
return {
stub: true,
doc: docId,
v: update.version,
path: update.path,
pathname: entity.path,
doc_length: update.doc_length,
meta: {
ts: update.ts,
origin: { kind: 'history-migration' },
},
}
}
/**
* Build the sorted array of updates to be sent to project-history.
*
* 1. Process all the added and edited files from the track-changes archive.
* 2. Process the other files from the project that have been added, and maybe deleted, without any edits.
*
* @param {string} projectId
* @param {string} projectHistoryId
* @param {Manifest} manifest
* @param {Map.<string, Entity>} entities
* @param {Map.<string, Object>} fileMap
* @returns {Promise<Array<AnyUpdate>>}
*/
async function buildUpdates(
projectId,
projectHistoryId,
manifest,
entities,
fileMap
) {
/**
* @type Array<AnyUpdate>
*/
const updates = []
// keep a list of doc ids which have updates in track-changes
const updatedDocs = new Set()
// process the existing docs with updates, from track-changes
for (const doc of manifest.docs) {
const entity = entities.get(doc.id)
if (!entity) {
throw new Error(`Entity not found for ${doc.id}`)
}
if (!entity.path) {
throw new Error(`Path not found for ${doc.id}`)
}
// add the initial content
const contentStart = doc.content.start
const buffer = await fileMap.get(contentStart.path).buffer()
/**
* @type AddDocUpdate
*/
const update = {
doc: doc.id,
pathname: entity.path,
v: contentStart.version - 1,
meta: {
user_id: null, // TODO: assign this to a system user?
ts: Number(ObjectId(doc.id).getTimestamp()),
origin: { kind: 'history-migration' },
},
projectHistoryId,
docLines: buffer.toString(),
}
updates.push(update)
// push the update onto the array of updates
for (const update of doc.updates) {
updates.push(buildEditUpdateStub(update, entity, doc.id))
}
updatedDocs.add(doc.id)
}
// process the docs which have been added/deleted without any updates being recorded
for (const [id, entity] of entities.entries()) {
if (entity.deleted) {
// deleted entity
// add the doc/file
if (!updatedDocs.has(id)) {
updates.push(buildAddUpdate(entity, id, projectId, projectHistoryId))
}
// delete the doc/file again (there may be updates added between adding and deleting)
updates.push(buildDeleteUpdate(entity, id, projectId, projectHistoryId))
} else {
if (!updatedDocs.has(id)) {
// add "not deleted" doc that isn't in the manifest either
updates.push(buildAddUpdate(entity, id, projectId, projectHistoryId))
}
}
}
return updates
}
/**
* Remove the `overleaf.history` object from the project and tell project-history to delete everything for this project.
* (note: project-history may not delete the actual history data yet, but it will at least delete the cached history id)
*
* @param {string} projectId
* @returns {Promise<void>}
*/
async function deleteProjectHistory(projectId) {
// look up the history id from the project
const historyId = await ProjectHistoryHandler.promises.getHistoryId(projectId)
// delete the history from project-history and history-v1
await HistoryManager.promises.deleteProject(projectId, historyId)
// TODO: send a message to document-updater?
await ProjectHistoryHandler.promises.unsetHistory(projectId)
}
/**
* Send the updates from the track changes zip file to project history
*
* @param {string} projectId
* @param {string} projectHistoryId
* @param {Array<AnyUpdate>} updates
* @param {Map.<string, Object>} fileMap
*/
async function migrateTrackChangesUpdates(
projectId,
projectHistoryId,
updates,
fileMap
) {
// Build a queue for each doc, sorted by version (and by timestamp within each version)
const queues = await sortUpdatesByQueue(updates)
const sortedUpdates = []
let item
do {
// Find the earliest item from the tail of all queues
queues.sort(earliestTimestampFirst)
item = queues[0].pop()
if (item) {
sortedUpdates.push(item)
}
} while (item)
// NOTE: leaving the version string code commented out, in case it ends up being needed
// let majorVersion = 0
// let minorVersion = 0
for (const update of sortedUpdates) {
// increment majorVersion if this is a file change
if (!('op' in update)) {
// remove v (only used for sorting)
delete update.v
// set version
// majorVersion++
// // minorVersion = 0
// update.version = `${majorVersion}.${minorVersion}` // NOTE: not set as project-history doesn't need it and could cause problems if it gets higher than project.version
}
// increment minorVersion after every update
// minorVersion++
}
// add each update to the Redis queue for project-history to process
logger.debug(
{ projectId, projectHistoryId },
'Sending updates for project to Redis'
)
const remainingQueueLength = await sendUpdatesToProjectHistory(
sortedUpdates,
projectId,
projectHistoryId,
fileMap
)
// Failure will cause queued updates to be deleted (in the catch below)
logger.debug(
{
projectId,
projectHistoryId,
remainingQueueLength,
},
'Updates sent to project-history'
)
if (remainingQueueLength > 0) {
throw new Error('flush to project-history did not complete')
}
// TODO: roll back if any of the following fail?
// TODO: check that the Redis queue is empty?
// Clear any old entries in the main project history queue (these will not
// have a history id)
await HistoryManager.promises.flushProject(projectId)
}
/**
* Add the zip file from track changes to the project file tree.
* We may be able to recover a failed history from the zip file in future.
*
* @param {string} projectId
* @param {string} rootFolderId
* @param {string} tempFilePath
*/
async function uploadTrackChangesArchiveToProject(
projectId,
rootFolderId,
tempFilePath
) {
const { size } = await fs.promises.stat(tempFilePath)
if (size > settings.maxUploadSize) {
throw new FileTooLargeError({
message: 'track-changes archive exceeds maximum size for archiving',
info: { size },
})
}
const { fileRef } = await ProjectEntityUpdateHandler.promises.addFile(
projectId,
rootFolderId, // project.rootFolder[0]._id,
`OverleafHistory-${new Date().toISOString().substring(0, 10)}.zip`,
tempFilePath,
null,
null, // no owner
null // no source
)
logger.debug(
{ projectId, fileRef },
'Uploaded track-changes zip archive to project due to error in migration'
)
}
/**
* Check all updates for invalid characters (nonBMP or null) and substitute
* the unicode replacement character if options.fixInvalidCharacters is true,
* otherwise throw an exception.
* @param {Array<AnyUpdate>} updates
* @param {string} projectId
* @param {Object} options
*/
function validateUpdates(updates, projectId, options) {
const replace = options.fixInvalidCharacters
// check for invalid characters
function containsBadChars(str) {
return /[\uD800-\uDBFF]/.test(str) || str.indexOf('\x00') !== -1
}
// Replace invalid characters so that they will be accepted by history_v1.
function sanitise(str) {
if (replace) {
return str.replace(/[\uD800-\uDFFF]/g, '\uFFFD').replace('\x00', '\uFFFD')
} else {
throw new Error('invalid character in content')
}
}
// Check size of doclines in update against max size allowed by history_v1.
// This catches docs which are too large when created, but not when they
// go over the limit due to edits.
function checkSize(update) {
if (update?.docLines?.length > settings.max_doc_length) {
throw new FileTooLargeError({
message: 'docLines exceeds maximum size for history',
info: { docId: update.doc, size: update.docLines.length },
})
}
}
let latestTimestamp = 0
// Iterate over the all the updates and their doclines or ops
for (const update of updates) {
checkSize(update)
// Find the timestamp of the most recent edit (either adding a doc or editing a doc)
// we exclude deletions as these are created in the migration and we didn't record
// the deletion time for older files.
const isDeleteUpdate = update.new_pathname === ''
if (
update.doc &&
!isDeleteUpdate &&
update.meta.ts &&
update.meta.ts > latestTimestamp
) {
latestTimestamp = update.meta.ts
}
if (update.docLines && containsBadChars(update.docLines)) {
logger.debug({ update, replace }, 'invalid character in docLines')
update.docLines = sanitise(update.docLines)
}
if (update.op) {
for (const op of update.op) {
if (op.i && containsBadChars(op.i)) {
logger.debug({ update, replace }, 'invalid character in insert op')
op.i = sanitise(op.i)
}
if (op.d && containsBadChars(op.d)) {
logger.debug({ update, replace }, 'invalid character in delete op')
op.d = sanitise(op.d)
}
}
}
}
logger.debug(
{ projectId, latestTimestamp, date: new Date(latestTimestamp) },
'timestamp of most recent edit'
)
if (options.cutoffDate && new Date(latestTimestamp) > options.cutoffDate) {
throw new Error('project was edited after cutoff date')
}
}
/**
* Migrate a project's history from track-changes to project-history
*
* @param {string} projectId
*
* @returns {Promise<void>}
*/
async function migrateProjectHistory(projectId, options = {}) {
await fse.ensureDir(settings.path.projectHistories)
const projectHistoriesDir = await fs.promises.realpath(
settings.path.projectHistories
)
const tempDir = await fs.promises.mkdtemp(projectHistoriesDir + path.sep)
const tempFilePath = path.join(tempDir, 'project.zip')
try {
// fetch the zip archive of rewound content and updates from track-changes
// store the zip archive to disk, open it and build a Map of the entries
if (options.importZipFilePath) {
// use an existing track-changes archive on disk
logger.debug(
{ src: options.importZipFilePath, dst: tempFilePath },
'importing zip file'
)
await fs.promises.copyFile(options.importZipFilePath, tempFilePath)
const { size } = await fs.promises.stat(tempFilePath)
logger.info({ projectId, size }, 'imported zip file from disk')
} else {
await fetchTrackChangesArchive(projectId, tempFilePath)
}
const fileMap = await openTrackChangesArchive(tempFilePath)
// read the manifest from the zip archive
const manifest = await readTrackChangesManifest(fileMap)
// check that the project id in the manifest matches
// to be sure we are using the correct zip file
if (manifest.projectId !== projectId) {
throw new Error(`Incorrect projectId: ${manifest.projectId}`)
}
// load the Project from MongoDB
const project = await ProjectGetter.promises.getProject(projectId)
// create a history id for this project
const oldProjectHistoryId = _.get(project, 'overleaf.history.id')
// throw an error if there is already a history associated with the project
if (oldProjectHistoryId) {
throw new Error(
`Project ${projectId} already has history ${oldProjectHistoryId}`
)
}
try {
// initialize a new project history and use the history id
// NOTE: not setting the history id on the project yet
const projectHistoryId = await HistoryManager.promises.initializeProject(
projectId
)
try {
// build a Map of the entities (docs and fileRefs) currently in the project,
// with _id as the key
const entities = await processRootFolder(project)
// find all the deleted docs for this project and add them to the entity map
await readDeletedDocs(entities, projectId)
// find all the deleted files for this project and add them to the entity map
await readDeletedFiles(entities, projectId)
// check that the paths will not be rejected
validatePaths(entities, projectId)
// build the array of updates that make up the new history for this project
const updates = await buildUpdates(
projectId,
projectHistoryId,
manifest,
entities,
fileMap
)
// check that the updates don't contain any characters that will be rejected by history_v1.
validateUpdates(updates, projectId, options)
if (updates.length) {
await migrateTrackChangesUpdates(
projectId,
projectHistoryId,
updates,
fileMap
)
}
} catch (error) {
if (options?.archiveOnFailure) {
// on error, optionally store the zip file in the project for future reference
logger.debug(
{ projectId, error },
'Error sending track-changes updates to project history, attempting to archive zip file in project'
)
try {
await uploadTrackChangesArchiveToProject(
projectId,
project.rootFolder[0]._id,
tempFilePath
)
} catch (error) {
if (error instanceof InvalidNameError) {
logger.info({ projectId }, 'zip file already archived in project')
} else {
throw error
}
} finally {
// roll back the last updated timestamp and user
logger.debug(
{ projectId },
'rolling back last updated time after uploading zip file'
)
await ProjectUpdateHandler.promises.resetUpdated(
projectId,
project.lastUpdated,
project.lastUpdatedBy
)
}
// set the overleaf.history.zipFileArchivedInProject flag for future reference
await ProjectHistoryHandler.promises.setMigrationArchiveFlag(
projectId
)
// we consider archiving the zip file as "success" (at least we've given up on attempting
// to migrate the history) so we don't rethrow the error and continue to initialise the new
// empty history below.
} else {
// if we're not archiving the zip file then we rethrown the error to fail the migration
throw error
}
}
// set the project's history id once the updates have been successfully processed
// (or we have given up and archived the zip file in the project).
logger.debug(
{ projectId, projectHistoryId },
'Setting history id on project'
)
await ProjectHistoryHandler.promises.setHistoryId(
projectId,
projectHistoryId
)
try {
// tell document updater to reload docs with the new history id
logger.debug({ projectId }, 'Asking document-updater to clear project')
await DocumentUpdaterHandler.promises.flushProjectToMongoAndDelete(
projectId
)
// run a project history resync in case any changes have arrived since the migration
logger.debug(
{ projectId },
'Asking project-history to force resync project'
)
await HistoryManager.promises.resyncProject(projectId, {
force: true,
origin: { kind: 'history-migration' },
})
} catch (error) {
if (options.forceNewHistoryOnFailure) {
logger.warn(
{ projectId },
'failed to resync project, forcing new history'
)
} else {
throw error
}
}
logger.debug(
{ projectId },
'Switching on full project history display for project'
)
// Set the display to v2 history but allow downgrading (second argument allowDowngrade = true)
await ProjectHistoryHandler.promises.upgradeHistory(projectId, true)
} catch (error) {
// delete the history id again if something failed?
logger.warn(
OError.tag(
error,
'Something went wrong flushing and resyncing project; clearing full project history for project',
{ projectId }
)
)
await deleteProjectHistory(projectId)
throw error
}
} finally {
// clean up the temporary directory
await fse.remove(tempDir)
}
}