import logger from '@overleaf/logger' import OError from '@overleaf/o-error' import request from 'request' import * as UpdatesProcessor from './UpdatesProcessor.js' import * as SummarizedUpdatesManager from './SummarizedUpdatesManager.js' import * as DiffManager from './DiffManager.js' import * as HistoryStoreManager from './HistoryStoreManager.js' import * as WebApiManager from './WebApiManager.js' import * as SnapshotManager from './SnapshotManager.js' import * as HealthChecker from './HealthChecker.js' import * as SyncManager from './SyncManager.js' import * as ErrorRecorder from './ErrorRecorder.js' import * as RedisManager from './RedisManager.js' import * as LabelsManager from './LabelsManager.js' import * as HistoryApiManager from './HistoryApiManager.js' import * as RetryManager from './RetryManager.js' import * as FlushManager from './FlushManager.js' import { pipeline } from 'stream' const ONE_DAY_IN_SECONDS = 24 * 60 * 60 export function getProjectBlob(req, res, next) { const historyId = req.params.history_id const blobHash = req.params.hash HistoryStoreManager.getProjectBlobStream( historyId, blobHash, (err, stream) => { if (err != null) { return next(OError.tag(err)) } res.setHeader('Cache-Control', `private, max-age=${ONE_DAY_IN_SECONDS}`) pipeline(stream, res, err => { if (err) next(err) // res.end() is already called via 'end' event by pipeline. }) } ) } export function initializeProject(req, res, next) { const { historyId } = req.body HistoryStoreManager.initializeProject(historyId, (error, id) => { if (error != null) { return next(OError.tag(error)) } res.json({ project: { id } }) }) } export function flushProject(req, res, next) { const projectId = req.params.project_id if (req.query.debug) { logger.debug( { projectId }, 'compressing project history in single-step mode' ) UpdatesProcessor.processSingleUpdateForProject(projectId, error => { if (error != null) { return next(OError.tag(error)) } res.sendStatus(204) }) } else if (req.query.bisect) { logger.debug({ projectId }, 'compressing project history in bisect mode') UpdatesProcessor.processUpdatesForProjectUsingBisect( projectId, UpdatesProcessor.REDIS_READ_BATCH_SIZE, error => { if (error != null) { return next(OError.tag(error)) } res.sendStatus(204) } ) } else { logger.debug({ projectId }, 'compressing project history') UpdatesProcessor.processUpdatesForProject(projectId, error => { if (error != null) { return next(OError.tag(error)) } res.sendStatus(204) }) } } export function dumpProject(req, res, next) { const projectId = req.params.project_id const batchSize = req.query.count || UpdatesProcessor.REDIS_READ_BATCH_SIZE logger.debug({ projectId }, 'retrieving raw updates') UpdatesProcessor.getRawUpdates(projectId, batchSize, (error, rawUpdates) => { if (error != null) { return next(OError.tag(error)) } res.json(rawUpdates) }) } export function flushOld(req, res, next) { const { maxAge, queueDelay, limit, timeout, background } = req.query const options = { maxAge, queueDelay, limit, timeout, background } FlushManager.flushOldOps(options, (error, results) => { if (error != null) { return next(OError.tag(error)) } res.send(results) }) } export function getDiff(req, res, next) { const projectId = req.params.project_id const { pathname, from, to } = req.query if (pathname == null) { return res.sendStatus(400) } logger.debug({ projectId, pathname, from, to }, 'getting diff') DiffManager.getDiff(projectId, pathname, from, to, (error, diff) => { if (error != null) { return next(OError.tag(error)) } res.json({ diff }) }) } export function getFileTreeDiff(req, res, next) { const projectId = req.params.project_id const { to, from } = req.query DiffManager.getFileTreeDiff(projectId, from, to, (error, diff) => { if (error != null) { return next(OError.tag(error)) } res.json({ diff }) }) } export function getUpdates(req, res, next) { const projectId = req.params.project_id const { before, min_count: minCount } = req.query SummarizedUpdatesManager.getSummarizedProjectUpdates( projectId, { before, min_count: minCount }, (error, updates, nextBeforeTimestamp) => { if (error != null) { return next(OError.tag(error)) } for (const update of updates) { // Sets don't JSONify, so convert to arrays update.pathnames = Array.from(update.pathnames || []).sort() } res.json({ updates, nextBeforeTimestamp, }) } ) } export function latestVersion(req, res, next) { const projectId = req.params.project_id logger.debug({ projectId }, 'compressing project history and getting version') UpdatesProcessor.processUpdatesForProject(projectId, error => { if (error != null) { return next(OError.tag(error)) } WebApiManager.getHistoryId(projectId, (error, historyId) => { if (error != null) { return next(OError.tag(error)) } HistoryStoreManager.getMostRecentVersion( projectId, historyId, (error, version, projectStructureAndDocVersions, lastChange) => { if (error != null) { return next(OError.tag(error)) } res.json({ version, timestamp: lastChange != null ? lastChange.timestamp : undefined, v2Authors: lastChange != null ? lastChange.v2Authors : undefined, }) } ) }) }) } export function getFileSnapshot(req, res, next) { const { project_id: projectId, version, pathname } = req.params SnapshotManager.getFileSnapshotStream( projectId, version, pathname, (error, stream) => { if (error != null) { return next(OError.tag(error)) } pipeline(stream, res, err => { if (err) next(err) // res.end() is already called via 'end' event by pipeline. }) } ) } export function getRangesSnapshot(req, res, next) { const { project_id: projectId, version, pathname } = req.params SnapshotManager.getRangesSnapshot( projectId, version, pathname, (err, ranges) => { if (err) { return next(OError.tag(err)) } res.json(ranges) } ) } export function getFileMetadataSnapshot(req, res, next) { const { project_id: projectId, version, pathname } = req.params SnapshotManager.getFileMetadataSnapshot( projectId, version, pathname, (err, data) => { if (err) { return next(OError.tag(err)) } res.json(data) } ) } export function getMostRecentChunk(req, res, next) { const { project_id: projectId } = req.params WebApiManager.getHistoryId(projectId, (error, historyId) => { if (error) return next(OError.tag(error)) HistoryStoreManager.getMostRecentChunk( projectId, historyId, (err, data) => { if (err) return next(OError.tag(err)) res.json(data) } ) }) } export function getLatestSnapshot(req, res, next) { const { project_id: projectId } = req.params WebApiManager.getHistoryId(projectId, (error, historyId) => { if (error) return next(OError.tag(error)) SnapshotManager.getLatestSnapshot( projectId, historyId, (error, details) => { if (error != null) { return next(error) } const { snapshot, version } = details res.json({ snapshot: snapshot.toRaw(), version }) } ) }) } export function getChangesSince(req, res, next) { const { project_id: projectId } = req.params const { since } = req.query WebApiManager.getHistoryId(projectId, (error, historyId) => { if (error) return next(OError.tag(error)) SnapshotManager.getChangesSince( projectId, historyId, since, (error, changes) => { if (error != null) { return next(error) } res.json(changes.map(c => c.toRaw())) } ) }) } export function getChangesInChunkSince(req, res, next) { const { project_id: projectId } = req.params const { since } = req.query WebApiManager.getHistoryId(projectId, (error, historyId) => { if (error) return next(OError.tag(error)) SnapshotManager.getChangesInChunkSince( projectId, historyId, since, (error, details) => { if (error != null) { return next(error) } const { latestStartVersion, changes } = details res.json({ latestStartVersion, changes: changes.map(c => c.toRaw()), }) } ) }) } export function getProjectSnapshot(req, res, next) { const { project_id: projectId, version } = req.params SnapshotManager.getProjectSnapshot( projectId, version, (error, snapshotData) => { if (error != null) { return next(error) } res.json(snapshotData) } ) } export function getPathsAtVersion(req, res, next) { const { project_id: projectId, version } = req.params SnapshotManager.getPathsAtVersion(projectId, version, (error, result) => { if (error != null) { return next(error) } res.json(result) }) } export function healthCheck(req, res) { HealthChecker.check(err => { if (err != null) { logger.err({ err }, 'error performing health check') res.sendStatus(500) } else { res.sendStatus(200) } }) } export function checkLock(req, res) { HealthChecker.checkLock(err => { if (err != null) { logger.err({ err }, 'error performing lock check') res.sendStatus(500) } else { res.sendStatus(200) } }) } export function resyncProject(req, res, next) { const projectId = req.params.project_id const options = {} if (req.body.origin) { options.origin = req.body.origin } if (req.body.historyRangesMigration) { options.historyRangesMigration = req.body.historyRangesMigration } if (req.query.force || req.body.force) { // this will delete the queue and clear the sync state // use if the project is completely broken SyncManager.startHardResync(projectId, options, error => { if (error != null) { return next(error) } // flush the sync operations UpdatesProcessor.processUpdatesForProject(projectId, error => { if (error != null) { return next(error) } res.sendStatus(204) }) }) } else { SyncManager.startResync(projectId, options, error => { if (error != null) { return next(error) } // flush the sync operations UpdatesProcessor.processUpdatesForProject(projectId, error => { if (error != null) { return next(error) } res.sendStatus(204) }) }) } } export function forceDebugProject(req, res, next) { const projectId = req.params.project_id // set the debug flag to true unless we see ?clear=true const state = !req.query.clear ErrorRecorder.setForceDebug(projectId, state, error => { if (error != null) { return next(error) } // display the failure record to help debugging ErrorRecorder.getFailureRecord(projectId, (error, result) => { if (error != null) { return next(error) } res.send(result) }) }) } export function getFailures(req, res, next) { ErrorRecorder.getFailures((error, result) => { if (error != null) { return next(error) } res.send({ failures: result }) }) } export function getQueueCounts(req, res, next) { RedisManager.getProjectIdsWithHistoryOpsCount((err, queuedProjectsCount) => { if (err != null) { return next(err) } res.send({ queuedProjects: queuedProjectsCount }) }) } export function getLabels(req, res, next) { const projectId = req.params.project_id HistoryApiManager.shouldUseProjectHistory( projectId, (error, shouldUseProjectHistory) => { if (error != null) { return next(error) } if (shouldUseProjectHistory) { LabelsManager.getLabels(projectId, (error, labels) => { if (error != null) { return next(error) } res.json(labels) }) } else { res.sendStatus(409) } } ) } export function createLabel(req, res, next) { const { project_id: projectId, user_id: userId } = req.params const { version, comment, created_at: createdAt, validate_exists: validateExists, } = req.body HistoryApiManager.shouldUseProjectHistory( projectId, (error, shouldUseProjectHistory) => { if (error != null) { return next(error) } if (shouldUseProjectHistory) { LabelsManager.createLabel( projectId, userId, version, comment, createdAt, validateExists, (error, label) => { if (error != null) { return next(error) } res.json(label) } ) } else { logger.error( { projectId, userId, version, comment, createdAt, validateExists, }, 'not using v2 history' ) res.sendStatus(409) } } ) } /** * This will delete a label if it is owned by the current user. If you wish to * delete a label regardless of the current user, then use `deleteLabel` instead. */ export function deleteLabelForUser(req, res, next) { const { project_id: projectId, user_id: userId, label_id: labelId, } = req.params LabelsManager.deleteLabelForUser(projectId, userId, labelId, error => { if (error != null) { return next(error) } res.sendStatus(204) }) } export function deleteLabel(req, res, next) { const { project_id: projectId, label_id: labelId } = req.params LabelsManager.deleteLabel(projectId, labelId, error => { if (error != null) { return next(error) } res.sendStatus(204) }) } export function retryFailures(req, res, next) { const { failureType, timeout, limit, callbackUrl } = req.query if (callbackUrl) { // send response but run in background when callbackUrl provided res.send({ retryStatus: 'running retryFailures in background' }) } RetryManager.retryFailures( { failureType, timeout, limit }, (error, result) => { if (callbackUrl) { // if present, notify the callbackUrl on success if (!error) { // Needs Node 12 // const callbackHeaders = Object.fromEntries(Object.entries(req.headers || {}).filter(([k,v]) => k.match(/^X-CALLBACK-/i))) const callbackHeaders = {} for (const key of Object.getOwnPropertyNames( req.headers || {} ).filter(key => key.match(/^X-CALLBACK-/i))) { const found = key.match(/^X-CALLBACK-(.*)/i) callbackHeaders[found[1]] = req.headers[key] } request({ url: callbackUrl, headers: callbackHeaders }) } } else { if (error != null) { return next(error) } res.send({ retryStatus: result }) } } ) } export function transferLabels(req, res, next) { const { from_user: fromUser, to_user: toUser } = req.params LabelsManager.transferLabels(fromUser, toUser, error => { if (error != null) { return next(error) } res.sendStatus(204) }) } export function deleteProject(req, res, next) { const { project_id: projectId } = req.params // clear the timestamp before clearing the queue, // because the queue location is used in the migration RedisManager.clearFirstOpTimestamp(projectId, err => { if (err) { return next(err) } RedisManager.clearCachedHistoryId(projectId, err => { if (err) { return next(err) } RedisManager.destroyDocUpdatesQueue(projectId, err => { if (err) { return next(err) } SyncManager.clearResyncState(projectId, err => { if (err) { return next(err) } // The third parameter to the following call is the error. Calling it // with null will remove any failure record for this project. ErrorRecorder.record(projectId, 0, null, err => { if (err) { return next(err) } res.sendStatus(204) }) }) }) }) }) }