// @ts-check import { callbackify } from 'util' import Core from 'overleaf-editor-core' import { Readable as StringStream } from 'stream' import OError from '@overleaf/o-error' import * as HistoryStoreManager from './HistoryStoreManager.js' import * as WebApiManager from './WebApiManager.js' import * as Errors from './Errors.js' import _ from 'lodash' /** * @typedef {import('stream').Readable} ReadableStream * @typedef {import('overleaf-editor-core').Snapshot} Snapshot * @typedef {import('./types').RangesSnapshot} RangesSnapshot */ StringStream.prototype._read = function () {} const MAX_REQUESTS = 4 // maximum number of parallel requests to v1 history service /** * * @param {string} projectId * @param {number} version * @param {string} pathname */ async function getFileSnapshotStream(projectId, version, pathname) { const snapshot = await _getSnapshotAtVersion(projectId, version) const file = snapshot.getFile(pathname) if (file == null) { throw new Errors.NotFoundError(`${pathname} not found`, { projectId, version, pathname, }) } const historyId = await WebApiManager.promises.getHistoryId(projectId) if (file.isEditable()) { await file.load('eager', HistoryStoreManager.getBlobStore(historyId)) const stream = new StringStream() stream.push(file.getContent({ filterTrackedDeletes: true })) stream.push(null) return stream } else { return await HistoryStoreManager.promises.getProjectBlobStream( historyId, file.getHash() ) } } /** * Constructs a snapshot of the ranges in a document-updater compatible format. * Positions will be relative to a document where tracked deletes have been * removed from the string. This also means that if a tracked delete overlaps * a comment range, the comment range will be truncated. * * @param {string} projectId * @param {number} version * @param {string} pathname * @returns {Promise} */ async function getRangesSnapshot(projectId, version, pathname) { const snapshot = await _getSnapshotAtVersion(projectId, version) const file = snapshot.getFile(pathname) if (!file) { throw new Errors.NotFoundError(`${pathname} not found`, { projectId, version, pathname, }) } if (!file.isEditable()) { throw new Error('File is not editable') } const historyId = await WebApiManager.promises.getHistoryId(projectId) await file.load('eager', HistoryStoreManager.getBlobStore(historyId)) const content = file.getContent() if (content == null) { throw new Error('Unable to read file contents') } const trackedChanges = file.getTrackedChanges().asSorted() const comments = file.getComments().toArray() const docUpdaterCompatibleTrackedChanges = [] let trackedDeletionOffset = 0 for (const trackedChange of trackedChanges) { const isTrackedDeletion = trackedChange.tracking.type === 'delete' const trackedChangeContent = content.slice( trackedChange.range.start, trackedChange.range.end ) const tcContent = isTrackedDeletion ? { d: trackedChangeContent } : { i: trackedChangeContent } docUpdaterCompatibleTrackedChanges.push({ op: { p: trackedChange.range.start - trackedDeletionOffset, ...tcContent, }, metadata: { ts: trackedChange.tracking.ts.toISOString(), user_id: trackedChange.tracking.userId, }, }) if (isTrackedDeletion) { trackedDeletionOffset += trackedChange.range.length } } // Comments are shifted left by the length of any previous tracked deletions. // If they overlap with a tracked deletion, they are truncated. // // Example: // { } comment // [ ] tracked deletion // the quic[k {b]rown [fox] jum[ps} ove]r the lazy dog // => rown jum // starting at position 8 const trackedDeletions = trackedChanges.filter( tc => tc.tracking.type === 'delete' ) const docUpdaterCompatibleComments = [] for (const comment of comments) { trackedDeletionOffset = 0 let trackedDeletionIndex = 0 if (comment.ranges.length === 0) { // Translate detached comments into zero length comments at position 0 docUpdaterCompatibleComments.push({ op: { p: 0, c: '', t: comment.id, resolved: comment.resolved, }, }) continue } for (const commentRange of comment.ranges) { let commentRangeContent = '' let offsetFromOverlappingRangeAtStart = 0 while ( trackedDeletionIndex < trackedDeletions.length && trackedDeletions[trackedDeletionIndex].range.start < commentRange.start && trackedDeletions[trackedDeletionIndex].range.end <= commentRange.start ) { // Skip over tracked deletions that are before the current comment range trackedDeletionOffset += trackedDeletions[trackedDeletionIndex].range.length trackedDeletionIndex++ } if ( trackedDeletions[trackedDeletionIndex]?.range.start < commentRange.start ) { // There's overlap with a tracked deletion, move the position left and // truncate the overlap offsetFromOverlappingRangeAtStart = commentRange.start - trackedDeletions[trackedDeletionIndex].range.start } // The position of the comment in the document after tracked deletions const position = commentRange.start - trackedDeletionOffset - offsetFromOverlappingRangeAtStart let cursor = commentRange.start while (cursor < commentRange.end) { const trackedDeletion = trackedDeletions[trackedDeletionIndex] if ( !trackedDeletion || trackedDeletion.range.start >= commentRange.end ) { // We've run out of relevant tracked changes commentRangeContent += content.slice(cursor, commentRange.end) break } if (trackedDeletion.range.start > cursor) { // There's a gap between the current cursor and the tracked deletion commentRangeContent += content.slice( cursor, trackedDeletion.range.start ) } // Skip to the end of the tracked delete cursor = trackedDeletion.range.end trackedDeletionIndex++ trackedDeletionOffset += trackedDeletion.range.length } docUpdaterCompatibleComments.push({ op: { p: position, c: commentRangeContent, t: comment.id, resolved: comment.resolved, }, }) } } return { changes: docUpdaterCompatibleTrackedChanges, comments: docUpdaterCompatibleComments, } } /** * Gets the file metadata at a specific version. * * @param {string} projectId * @param {number} version * @param {string} pathname * @returns {Promise<{metadata: any}>} */ async function getFileMetadataSnapshot(projectId, version, pathname) { const snapshot = await _getSnapshotAtVersion(projectId, version) const file = snapshot.getFile(pathname) if (!file) { throw new Errors.NotFoundError(`${pathname} not found`, { projectId, version, pathname, }) } const rawMetadata = file.getMetadata() const metadata = _.isEmpty(rawMetadata) ? undefined : rawMetadata return { metadata } } // Returns project snapshot containing the document content for files with // text operations in the relevant chunk, and hashes for unmodified/binary // files. Used by git bridge to get the state of the project. async function getProjectSnapshot(projectId, version) { const snapshot = await _getSnapshotAtVersion(projectId, version) const historyId = await WebApiManager.promises.getHistoryId(projectId) await _loadFilesLimit( snapshot, 'eager', HistoryStoreManager.getBlobStore(historyId) ) return { projectId, files: snapshot.getFileMap().map(file => { if (!file) { return null } const content = file.getContent({ filterTrackedDeletes: true, }) if (content === null) { return { data: { hash: file.getHash() } } } return { data: { content } } }), } } async function getPathsAtVersion(projectId, version) { const snapshot = await _getSnapshotAtVersion(projectId, version) return { paths: snapshot.getFilePathnames(), } } /** * * @param {string} projectId * @param {number} version */ async function _getSnapshotAtVersion(projectId, version) { const historyId = await WebApiManager.promises.getHistoryId(projectId) const data = await HistoryStoreManager.promises.getChunkAtVersion( projectId, historyId, version ) const chunk = Core.Chunk.fromRaw(data.chunk) const snapshot = chunk.getSnapshot() const changes = chunk.getChanges().slice(0, version - chunk.getStartVersion()) snapshot.applyAll(changes) return snapshot } async function getLatestSnapshotFiles(projectId, historyId) { const { snapshot } = await getLatestSnapshot(projectId, historyId) const snapshotFiles = await snapshot.loadFiles( 'lazy', HistoryStoreManager.getBlobStore(historyId) ) return snapshotFiles } async function getLatestSnapshot(projectId, historyId) { const data = await HistoryStoreManager.promises.getMostRecentChunk( projectId, historyId ) if (data == null || data.chunk == null) { throw new OError('undefined chunk') } // apply all the changes in the chunk to get the current snapshot const chunk = Core.Chunk.fromRaw(data.chunk) const snapshot = chunk.getSnapshot() const changes = chunk.getChanges() snapshot.applyAll(changes) return { snapshot, version: chunk.getEndVersion(), } } async function getChangesSince(projectId, historyId, sinceVersion) { const allChanges = [] let nextVersion while (true) { let data if (nextVersion) { data = await HistoryStoreManager.promises.getChunkAtVersion( projectId, historyId, nextVersion ) } else { data = await HistoryStoreManager.promises.getMostRecentChunk( projectId, historyId ) } if (data == null || data.chunk == null) { throw new OError('undefined chunk') } const chunk = Core.Chunk.fromRaw(data.chunk) if (sinceVersion > chunk.getEndVersion()) { throw new OError('requested version past the end') } const changes = chunk.getChanges() if (chunk.getStartVersion() > sinceVersion) { allChanges.unshift(...changes) nextVersion = chunk.getStartVersion() } else { allChanges.unshift( ...changes.slice(sinceVersion - chunk.getStartVersion()) ) break } } return allChanges } async function _loadFilesLimit(snapshot, kind, blobStore) { await snapshot.fileMap.mapAsync(async file => { // only load changed files or files with tracked changes, others can be // dereferenced from their blobs (this method is only used by the git // bridge which understands how to load blobs). if (!file.isEditable() || (file.getHash() && !file.getRangesHash())) { return } await file.load(kind, blobStore) }, MAX_REQUESTS) } // EXPORTS const getChangesSinceCb = callbackify(getChangesSince) const getFileSnapshotStreamCb = callbackify(getFileSnapshotStream) const getProjectSnapshotCb = callbackify(getProjectSnapshot) const getLatestSnapshotCb = callbackify(getLatestSnapshot) const getLatestSnapshotFilesCb = callbackify(getLatestSnapshotFiles) const getRangesSnapshotCb = callbackify(getRangesSnapshot) const getFileMetadataSnapshotCb = callbackify(getFileMetadataSnapshot) const getPathsAtVersionCb = callbackify(getPathsAtVersion) export { getChangesSinceCb as getChangesSince, getFileSnapshotStreamCb as getFileSnapshotStream, getProjectSnapshotCb as getProjectSnapshot, getFileMetadataSnapshotCb as getFileMetadataSnapshot, getLatestSnapshotCb as getLatestSnapshot, getLatestSnapshotFilesCb as getLatestSnapshotFiles, getRangesSnapshotCb as getRangesSnapshot, getPathsAtVersionCb as getPathsAtVersion, } export const promises = { getChangesSince, getFileSnapshotStream, getProjectSnapshot, getLatestSnapshot, getLatestSnapshotFiles, getRangesSnapshot, getPathsAtVersion, getFileMetadataSnapshot, }