Merge pull request #17729 from overleaf/em-promisify-sync-manager

Promisify SyncManager

GitOrigin-RevId: 134770d812a493e39410debb370ed4a58ffff4bf
This commit is contained in:
Eric Mc Sween 2024-04-11 08:32:51 -04:00 committed by Copybot
parent c9373c25f4
commit f03e3fd51e
10 changed files with 780 additions and 805 deletions

2
package-lock.json generated
View file

@ -43037,6 +43037,7 @@
"@overleaf/logger": "*", "@overleaf/logger": "*",
"@overleaf/metrics": "*", "@overleaf/metrics": "*",
"@overleaf/o-error": "*", "@overleaf/o-error": "*",
"@overleaf/promise-utils": "*",
"@overleaf/redis-wrapper": "*", "@overleaf/redis-wrapper": "*",
"@overleaf/settings": "*", "@overleaf/settings": "*",
"async": "^3.2.2", "async": "^3.2.2",
@ -51636,6 +51637,7 @@
"@overleaf/logger": "*", "@overleaf/logger": "*",
"@overleaf/metrics": "*", "@overleaf/metrics": "*",
"@overleaf/o-error": "*", "@overleaf/o-error": "*",
"@overleaf/promise-utils": "*",
"@overleaf/redis-wrapper": "*", "@overleaf/redis-wrapper": "*",
"@overleaf/settings": "*", "@overleaf/settings": "*",
"async": "^3.2.2", "async": "^3.2.2",

View file

@ -305,6 +305,11 @@ export function getFailures(callback) {
export const promises = { export const promises = {
getFailedProjects: promisify(getFailedProjects), getFailedProjects: promisify(getFailedProjects),
record: promisify(record),
getFailureRecord: promisify(getFailureRecord), getFailureRecord: promisify(getFailureRecord),
getLastFailure: promisify(getLastFailure),
getFailuresByType: promisify(getFailuresByType),
getFailures: promisify(getFailures),
record: promisify(record),
recordSyncStart: promisify(recordSyncStart),
setForceDebug: promisify(setForceDebug),
} }

View file

@ -10,6 +10,7 @@
* DS207: Consider shorter variations of null checks * DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/ */
import { promisify } from 'util'
import fs from 'fs' import fs from 'fs'
import crypto from 'crypto' import crypto from 'crypto'
import OError from '@overleaf/o-error' import OError from '@overleaf/o-error'
@ -51,3 +52,7 @@ export function _getBlobHash(fsPath, callback) {
}) })
}) })
} }
export const promises = {
_getBlobHash: promisify(_getBlobHash),
}

View file

@ -1,9 +1,9 @@
import { promisify } from 'util'
import fs from 'fs' import fs from 'fs'
import request from 'request' import request from 'request'
import stream from 'stream' import stream from 'stream'
import logger from '@overleaf/logger' import logger from '@overleaf/logger'
import _ from 'lodash' import _ from 'lodash'
import BPromise from 'bluebird'
import { URL } from 'url' import { URL } from 'url'
import OError from '@overleaf/o-error' import OError from '@overleaf/o-error'
import Settings from '@overleaf/settings' import Settings from '@overleaf/settings'
@ -354,7 +354,7 @@ export function deleteProject(projectId, callback) {
) )
} }
const getProjectBlobAsync = BPromise.promisify(getProjectBlob) const getProjectBlobAsync = promisify(getProjectBlob)
class BlobStore { class BlobStore {
constructor(projectId) { constructor(projectId) {
@ -435,3 +435,15 @@ function _requestHistoryService(options, callback) {
} }
}) })
} }
export const promises = {
getMostRecentChunk: promisify(getMostRecentChunk),
getChunkAtVersion: promisify(getChunkAtVersion),
getMostRecentVersion: promisify(getMostRecentVersion),
getProjectBlob: promisify(getProjectBlob),
getProjectBlobStream: promisify(getProjectBlobStream),
sendChanges: promisify(sendChanges),
createBlobForUpdate: promisify(createBlobForUpdate),
initializeProject: promisify(initializeProject),
deleteProject: promisify(deleteProject),
}

View file

@ -7,6 +7,7 @@
* DS207: Consider shorter variations of null checks * DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/ */
import { promisify } from 'util'
import async from 'async' import async from 'async'
import metrics from '@overleaf/metrics' import metrics from '@overleaf/metrics'
import Settings from '@overleaf/settings' import Settings from '@overleaf/settings'
@ -273,3 +274,41 @@ class Lock {
}) })
} }
} }
/**
* Promisified version of runWithLock.
*
* @param {string} key
* @param {(extendLock: Function) => Promise<any>} runner
*/
async function runWithLockPromises(key, runner) {
const runnerCb = (extendLock, callback) => {
const extendLockPromises = promisify(extendLock)
runner(extendLockPromises)
.then(result => {
callback(null, result)
})
.catch(err => {
callback(err)
})
}
return new Promise((resolve, reject) => {
runWithLock(key, runnerCb, (err, result) => {
if (err) {
reject(err)
} else {
resolve(result)
}
})
})
}
export const promises = {
tryLock: promisify(tryLock),
extendLock: promisify(extendLock),
getLock: promisify(getLock),
checkLock: promisify(checkLock),
releaseLock: promisify(releaseLock),
runWithLock: runWithLockPromises,
}

View file

@ -1,4 +1,6 @@
// @ts-check // @ts-check
import { promisify } from 'util'
import Core from 'overleaf-editor-core' import Core from 'overleaf-editor-core'
import { Readable as StringStream } from 'stream' import { Readable as StringStream } from 'stream'
import BPromise from 'bluebird' import BPromise from 'bluebird'
@ -171,3 +173,9 @@ function _loadFilesLimit(snapshot, kind, blobStore) {
{ concurrency: MAX_REQUESTS } { concurrency: MAX_REQUESTS }
) )
} }
export const promises = {
getFileSnapshotStream: promisify(getFileSnapshotStream),
getProjectSnapshot: promisify(getProjectSnapshot),
getLatestSnapshot: promisify(getLatestSnapshot),
}

View file

@ -1,6 +1,6 @@
import _ from 'lodash' import _ from 'lodash'
import async from 'async' import { callbackify, promisify } from 'util'
import { promisify } from 'util' import { callbackifyMultiResult } from '@overleaf/promise-utils'
import Settings from '@overleaf/settings' import Settings from '@overleaf/settings'
import logger from '@overleaf/logger' import logger from '@overleaf/logger'
import Metrics from '@overleaf/metrics' import Metrics from '@overleaf/metrics'
@ -27,7 +27,7 @@ const keys = Settings.redis.lock.key_schema
// To add expiresAt field to existing entries in collection (choose a suitable future expiry date): // To add expiresAt field to existing entries in collection (choose a suitable future expiry date):
// db.projectHistorySyncState.updateMany({resyncProjectStructure: false, resyncDocContents: [], expiresAt: {$exists:false}}, {$set: {expiresAt: new Date("2019-07-01")}}) // db.projectHistorySyncState.updateMany({resyncProjectStructure: false, resyncDocContents: [], expiresAt: {$exists:false}}, {$set: {expiresAt: new Date("2019-07-01")}})
export function startResync(projectId, options, callback) { async function startResync(projectId, options = {}) {
// We have three options here // We have three options here
// //
// 1. If we update mongo before making the call to web then there's a // 1. If we update mongo before making the call to web then there's a
@ -39,114 +39,71 @@ export function startResync(projectId, options, callback) {
// after, causing all updates to be ignored from then on // after, causing all updates to be ignored from then on
// //
// 3. We can wrap everything in a project lock // 3. We can wrap everything in a project lock
if (typeof options === 'function') {
callback = options
options = {}
}
Metrics.inc('project_history_resync') Metrics.inc('project_history_resync')
LockManager.runWithLock( try {
await LockManager.promises.runWithLock(
keys.projectHistoryLock({ project_id: projectId }), keys.projectHistoryLock({ project_id: projectId }),
(extendLock, releaseLock) => async extendLock => {
_startResyncWithoutLock(projectId, options, releaseLock), await _startResyncWithoutLock(projectId, options)
function (error) {
if (error) {
OError.tag(error)
// record error in starting sync ("sync ongoing")
ErrorRecorder.record(projectId, -1, error, () => callback(error))
} else {
callback()
}
} }
) )
} catch (error) {
// record error in starting sync ("sync ongoing")
try {
await ErrorRecorder.promises.record(projectId, -1, error)
} catch (err) {
// swallow any error thrown by ErrorRecorder.record()
}
throw error
}
} }
export function startHardResync(projectId, options, callback) { async function startHardResync(projectId, options = {}) {
if (typeof options === 'function') {
callback = options
options = {}
}
Metrics.inc('project_history_hard_resync') Metrics.inc('project_history_hard_resync')
LockManager.runWithLock( try {
await LockManager.promises.runWithLock(
keys.projectHistoryLock({ project_id: projectId }), keys.projectHistoryLock({ project_id: projectId }),
(extendLock, releaseLock) => async extendLock => {
clearResyncState(projectId, function (err) { await clearResyncState(projectId)
if (err) { await RedisManager.promises.clearFirstOpTimestamp(projectId)
return releaseLock(OError.tag(err)) await RedisManager.promises.destroyDocUpdatesQueue(projectId)
} await _startResyncWithoutLock(projectId, options)
RedisManager.clearFirstOpTimestamp(projectId, function (err) {
if (err) {
return releaseLock(OError.tag(err))
}
RedisManager.destroyDocUpdatesQueue(projectId, function (err) {
if (err) {
return releaseLock(OError.tag(err))
}
_startResyncWithoutLock(projectId, options, releaseLock)
})
})
}),
function (error) {
if (error) {
OError.tag(error)
// record error in starting sync ("sync ongoing")
ErrorRecorder.record(projectId, -1, error, () => callback(error))
} else {
callback()
}
} }
) )
} catch (error) {
// record error in starting sync ("sync ongoing")
await ErrorRecorder.promises.record(projectId, -1, error)
throw error
}
} }
function _startResyncWithoutLock(projectId, options, callback) { async function _startResyncWithoutLock(projectId, options) {
ErrorRecorder.recordSyncStart(projectId, function (error) { await ErrorRecorder.promises.recordSyncStart(projectId)
if (error) {
return callback(OError.tag(error))
}
_getResyncState(projectId, function (error, syncState) {
if (error) {
return callback(OError.tag(error))
}
const syncState = await _getResyncState(projectId)
if (syncState.isSyncOngoing()) { if (syncState.isSyncOngoing()) {
return callback(new OError('sync ongoing')) throw new OError('sync ongoing')
} }
syncState.setOrigin(options.origin || { kind: 'history-resync' }) syncState.setOrigin(options.origin || { kind: 'history-resync' })
syncState.startProjectStructureSync() syncState.startProjectStructureSync()
WebApiManager.requestResync(projectId, function (error) { await WebApiManager.promises.requestResync(projectId)
if (error) { await setResyncState(projectId, syncState)
return callback(OError.tag(error))
} }
setResyncState(projectId, syncState, callback) async function _getResyncState(projectId) {
}) const rawSyncState = await db.projectHistorySyncState.findOne({
})
})
}
function _getResyncState(projectId, callback) {
db.projectHistorySyncState.findOne(
{
project_id: new ObjectId(projectId.toString()), project_id: new ObjectId(projectId.toString()),
}, })
function (error, rawSyncState) {
if (error) {
return callback(OError.tag(error))
}
const syncState = SyncState.fromRaw(projectId, rawSyncState) const syncState = SyncState.fromRaw(projectId, rawSyncState)
callback(null, syncState) return syncState
}
)
} }
export function setResyncState(projectId, syncState, callback) { async function setResyncState(projectId, syncState) {
// skip if syncState is null (i.e. unchanged)
if (syncState == null) { if (syncState == null) {
return callback() return
} // skip if syncState is null (i.e. unchanged) }
const update = { const update = {
$set: syncState.toRaw(), $set: syncState.toRaw(),
$push: { $push: {
@ -158,64 +115,46 @@ export function setResyncState(projectId, syncState, callback) {
}, },
$currentDate: { lastUpdated: true }, $currentDate: { lastUpdated: true },
} }
// handle different cases // handle different cases
if (syncState.isSyncOngoing()) { if (syncState.isSyncOngoing()) {
// starting a new sync // starting a new sync; prevent the entry expiring while sync is in ongoing
update.$inc = { resyncCount: 1 } update.$inc = { resyncCount: 1 }
update.$unset = { expiresAt: true } // prevent the entry expiring while sync is in ongoing update.$unset = { expiresAt: true }
} else { } else {
// successful completion of existing sync // successful completion of existing sync; set the entry to expire in the
// future
update.$set.expiresAt = new Date( update.$set.expiresAt = new Date(
Date.now() + EXPIRE_RESYNC_HISTORY_INTERVAL_MS Date.now() + EXPIRE_RESYNC_HISTORY_INTERVAL_MS
) // set the entry to expire in the future )
} }
// apply the update // apply the update
db.projectHistorySyncState.updateOne( await db.projectHistorySyncState.updateOne(
{ { project_id: new ObjectId(projectId) },
project_id: new ObjectId(projectId),
},
update, update,
{ { upsert: true }
upsert: true,
},
callback
) )
} }
export function clearResyncState(projectId, callback) { async function clearResyncState(projectId) {
db.projectHistorySyncState.deleteOne( await db.projectHistorySyncState.deleteOne({
{
project_id: new ObjectId(projectId.toString()), project_id: new ObjectId(projectId.toString()),
}, })
callback
)
}
export function skipUpdatesDuringSync(projectId, updates, callback) {
_getResyncState(projectId, function (error, syncState) {
if (error) {
return callback(OError.tag(error))
} }
async function skipUpdatesDuringSync(projectId, updates) {
const syncState = await _getResyncState(projectId)
if (!syncState.isSyncOngoing()) { if (!syncState.isSyncOngoing()) {
logger.debug({ projectId }, 'not skipping updates: no resync in progress') logger.debug({ projectId }, 'not skipping updates: no resync in progress')
return callback(null, updates) // don't return synsState when unchanged // don't return syncState when unchanged
return { updates, syncState: null }
} }
const filteredUpdates = [] const filteredUpdates = []
for (const update of updates) { for (const update of updates) {
try {
syncState.updateState(update) syncState.updateState(update)
} catch (error1) {
error = OError.tag(error1)
if (error instanceof SyncError) {
return callback(error)
} else {
throw error
}
}
const shouldSkipUpdate = syncState.shouldSkipUpdate(update) const shouldSkipUpdate = syncState.shouldSkipUpdate(update)
if (!shouldSkipUpdate) { if (!shouldSkipUpdate) {
filteredUpdates.push(update) filteredUpdates.push(update)
@ -223,77 +162,56 @@ export function skipUpdatesDuringSync(projectId, updates, callback) {
logger.debug({ projectId, update }, 'skipping update due to resync') logger.debug({ projectId, update }, 'skipping update due to resync')
} }
} }
return { updates: filteredUpdates, syncState }
callback(null, filteredUpdates, syncState)
})
} }
export function expandSyncUpdates( async function expandSyncUpdates(
projectId, projectId,
projectHistoryId, projectHistoryId,
updates, updates,
extendLock, extendLock
callback
) { ) {
const areSyncUpdatesQueued = const areSyncUpdatesQueued =
_.some(updates, 'resyncProjectStructure') || _.some(updates, 'resyncProjectStructure') ||
_.some(updates, 'resyncDocContent') _.some(updates, 'resyncDocContent')
if (!areSyncUpdatesQueued) { if (!areSyncUpdatesQueued) {
logger.debug({ projectId }, 'no resync updates to expand') logger.debug({ projectId }, 'no resync updates to expand')
return callback(null, updates) return updates
} }
_getResyncState(projectId, (error, syncState) => { const syncState = await _getResyncState(projectId)
if (error) {
return callback(OError.tag(error))
}
// compute the current snapshot from the most recent chunk // compute the current snapshot from the most recent chunk
SnapshotManager.getLatestSnapshot( const snapshotFiles = await SnapshotManager.promises.getLatestSnapshot(
projectId, projectId,
projectHistoryId, projectHistoryId
(error, snapshotFiles) => { )
if (error) {
return callback(OError.tag(error))
}
// check if snapshot files are valid // check if snapshot files are valid
const invalidFiles = _.pickBy( const invalidFiles = _.pickBy(
snapshotFiles, snapshotFiles,
(v, k) => v == null || typeof v.isEditable !== 'function' (v, k) => v == null || typeof v.isEditable !== 'function'
) )
if (_.size(invalidFiles) > 0) { if (_.size(invalidFiles) > 0) {
return callback( throw new SyncError('file is missing isEditable method', {
new SyncError('file is missing isEditable method', {
projectId, projectId,
invalidFiles, invalidFiles,
}) })
)
} }
const expander = new SyncUpdateExpander( const expander = new SyncUpdateExpander(
projectId, projectId,
snapshotFiles, snapshotFiles,
syncState.origin syncState.origin
) )
// expand updates asynchronously to avoid blocking // expand updates asynchronously to avoid blocking
const handleUpdate = ( for (const update of updates) {
update, await expander.expandUpdate(update)
cb // n.b. lock manager calls cb asynchronously await extendLock()
) =>
expander.expandUpdate(update, error => {
if (error) {
return cb(OError.tag(error))
} }
extendLock(cb)
}) return expander.getExpandedUpdates()
async.eachSeries(updates, handleUpdate, error => {
if (error) {
return callback(OError.tag(error))
}
callback(null, expander.getExpandedUpdates())
})
}
)
})
} }
class SyncState { class SyncState {
@ -456,7 +374,7 @@ class SyncUpdateExpander {
return !matchedExpectedFile return !matchedExpectedFile
} }
expandUpdate(update, cb) { async expandUpdate(update) {
if (update.resyncProjectStructure != null) { if (update.resyncProjectStructure != null) {
logger.debug( logger.debug(
{ projectId: this.projectId, update }, { projectId: this.projectId, update },
@ -523,16 +441,14 @@ class SyncUpdateExpander {
expectedBinaryFiles, expectedBinaryFiles,
persistedBinaryFiles persistedBinaryFiles
) )
cb()
} else if (update.resyncDocContent != null) { } else if (update.resyncDocContent != null) {
logger.debug( logger.debug(
{ projectId: this.projectId, update }, { projectId: this.projectId, update },
'expanding resyncDocContent update' 'expanding resyncDocContent update'
) )
this.queueTextOpForOutOfSyncContents(update, cb) await this.queueTextOpForOutOfSyncContents(update)
} else { } else {
this.expandedUpdates.push(update) this.expandedUpdates.push(update)
cb()
} }
} }
@ -637,13 +553,13 @@ class SyncUpdateExpander {
} }
} }
queueTextOpForOutOfSyncContents(update, cb) { async queueTextOpForOutOfSyncContents(update) {
const pathname = UpdateTranslator._convertPathname(update.path) const pathname = UpdateTranslator._convertPathname(update.path)
const snapshotFile = this.files[pathname] const snapshotFile = this.files[pathname]
const expectedFile = update.resyncDocContent const expectedFile = update.resyncDocContent
if (!snapshotFile) { if (!snapshotFile) {
return cb(new OError('unrecognised file: not in snapshot')) throw new OError('unrecognised file: not in snapshot')
} }
// Compare hashes to see if the persisted file matches the expected content. // Compare hashes to see if the persisted file matches the expected content.
@ -664,7 +580,7 @@ class SyncUpdateExpander {
{ projectId: this.projectId, persistedHash, expectedHash }, { projectId: this.projectId, persistedHash, expectedHash },
'skipping diff because hashes match and persisted file has no ops' 'skipping diff because hashes match and persisted file has no ops'
) )
return cb() return
} }
} else { } else {
logger.debug('cannot compare hashes, will retrieve content') logger.debug('cannot compare hashes, will retrieve content')
@ -672,28 +588,40 @@ class SyncUpdateExpander {
const expectedContent = update.resyncDocContent.content const expectedContent = update.resyncDocContent.content
const computeDiff = (persistedContent, cb) => { let persistedContent
// compute the difference between the expected and persisted content
if (snapshotFile.load != null) {
const historyId = await WebApiManager.promises.getHistoryId(
this.projectId
)
const file = await snapshotFile.load(
'eager',
HistoryStoreManager.getBlobStore(historyId)
)
persistedContent = file.getContent()
} else if (snapshotFile.content != null) {
// use dummy content from queueAddOpsForMissingFiles for added missing files
persistedContent = snapshotFile.content
} else {
throw new OError('unrecognised file')
}
let op let op
logger.debug( logger.debug(
{ projectId: this.projectId, persistedContent, expectedContent }, { projectId: this.projectId, persistedContent, expectedContent },
'diffing doc contents' 'diffing doc contents'
) )
try { try {
op = UpdateCompressor.diffAsShareJsOps( op = UpdateCompressor.diffAsShareJsOps(persistedContent, expectedContent)
persistedContent,
expectedContent
)
} catch (error) { } catch (error) {
return cb( throw OError.tag(error, 'error from diffAsShareJsOps', {
OError.tag(error, 'error from diffAsShareJsOps', {
projectId: this.projectId, projectId: this.projectId,
persistedContent, persistedContent,
expectedContent, expectedContent,
}) })
)
} }
if (op.length === 0) { if (op.length === 0) {
return cb() return
} }
update = { update = {
doc: update.doc, doc: update.doc,
@ -714,37 +642,50 @@ class SyncUpdateExpander {
Metrics.inc('project_history_resync_operation', 1, { Metrics.inc('project_history_resync_operation', 1, {
status: 'update text file contents', status: 'update text file contents',
}) })
cb() }
} }
// compute the difference between the expected and persisted content // EXPORTS
if (snapshotFile.load != null) {
WebApiManager.getHistoryId(this.projectId, (err, historyId) => { const startResyncCb = callbackify(startResync)
if (err) { const startHardResyncCb = callbackify(startHardResync)
return cb(OError.tag(err)) const setResyncStateCb = callbackify(setResyncState)
} const clearResyncStateCb = callbackify(clearResyncState)
const loadFile = snapshotFile.load( const skipUpdatesDuringSyncCb = callbackifyMultiResult(skipUpdatesDuringSync, [
'eager', 'updates',
HistoryStoreManager.getBlobStore(historyId) 'syncState',
) ])
loadFile const expandSyncUpdatesCb = (
.then(file => computeDiff(file.getContent(), cb)) projectId,
.catch(err => cb(err)) // error loading file or computing diff projectHistoryId,
updates,
extendLock,
callback
) => {
const extendLockPromises = promisify(extendLock)
expandSyncUpdates(projectId, projectHistoryId, updates, extendLockPromises)
.then(result => {
callback(null, result)
})
.catch(err => {
callback(err)
}) })
} else if (snapshotFile.content != null) {
// use dummy content from queueAddOpsForMissingFiles for added missing files
computeDiff(snapshotFile.content, cb)
} else {
cb(new OError('unrecognised file'))
}
} }
export {
startResyncCb as startResync,
startHardResyncCb as startHardResync,
setResyncStateCb as setResyncState,
clearResyncStateCb as clearResyncState,
skipUpdatesDuringSyncCb as skipUpdatesDuringSync,
expandSyncUpdatesCb as expandSyncUpdates,
} }
export const promises = { export const promises = {
startResync: promisify(startResync), startResync,
startHardResync: promisify(startHardResync), startHardResync,
setResyncState: promisify(setResyncState), setResyncState,
clearResyncState: promisify(clearResyncState), clearResyncState,
skipUpdatesDuringSync: promisify(skipUpdatesDuringSync), skipUpdatesDuringSync,
expandSyncUpdates: promisify(expandSyncUpdates), expandSyncUpdates,
} }

View file

@ -22,6 +22,7 @@
"@overleaf/logger": "*", "@overleaf/logger": "*",
"@overleaf/metrics": "*", "@overleaf/metrics": "*",
"@overleaf/o-error": "*", "@overleaf/o-error": "*",
"@overleaf/promise-utils": "*",
"@overleaf/redis-wrapper": "*", "@overleaf/redis-wrapper": "*",
"@overleaf/settings": "*", "@overleaf/settings": "*",
"async": "^3.2.2", "async": "^3.2.2",

View file

@ -162,7 +162,7 @@ describe('Syncing with web and doc-updater', function () {
], ],
error => { error => {
if (error) { if (error) {
throw error return done(error)
} }
assert( assert(
createBlob.isDone(), createBlob.isDone(),

View file

@ -39,47 +39,65 @@ describe('SyncManager', function () {
this.syncState = { origin: { kind: 'history-resync' } } this.syncState = { origin: { kind: 'history-resync' } }
this.db = { this.db = {
projectHistorySyncState: { projectHistorySyncState: {
findOne: sinon.stub().yields(null, this.syncState), findOne: sinon.stub().resolves(this.syncState),
updateOne: sinon.stub().yields(), updateOne: sinon.stub().resolves(),
}, },
} }
this.callback = sinon.stub() this.extendLock = sinon.stub().resolves()
this.extendLock = sinon.stub().yields()
this.LockManager = { this.LockManager = {
runWithLock: sinon.spy((key, runner, callback) => { promises: {
runner(this.extendLock, callback) runWithLock: sinon.stub().callsFake(async (key, runner) => {
await runner(this.extendLock)
}), }),
},
} }
this.UpdateCompressor = { this.UpdateCompressor = {
diffAsShareJsOps: sinon.stub(), diffAsShareJsOps: sinon.stub(),
} }
this.UpdateTranslator = { this.UpdateTranslator = {
isTextUpdate: sinon.stub(), isTextUpdate: sinon.stub(),
_convertPathname: sinon.stub(), _convertPathname: sinon.stub(),
} }
this.WebApiManager = { this.WebApiManager = {
promises: {
getHistoryId: sinon.stub(), getHistoryId: sinon.stub(),
requestResync: sinon.stub().yields(), requestResync: sinon.stub().resolves(),
},
} }
this.WebApiManager.getHistoryId this.WebApiManager.promises.getHistoryId
.withArgs(this.projectId) .withArgs(this.projectId)
.yields(null, this.historyId) .resolves(this.historyId)
this.ErrorRecorder = { this.ErrorRecorder = {
record: sinon.stub().yields(), promises: {
recordSyncStart: sinon.stub().yields(), record: sinon.stub().resolves(),
recordSyncStart: sinon.stub().resolves(),
},
} }
this.RedisManager = {} this.RedisManager = {}
this.SnapshotManager = { this.SnapshotManager = {
promises: {
getLatestSnapshot: sinon.stub(), getLatestSnapshot: sinon.stub(),
},
} }
this.HistoryStoreManager = { this.HistoryStoreManager = {
getBlobStore: sinon.stub(), getBlobStore: sinon.stub(),
_getBlobHashFromString: sinon.stub().returns('random-hash'), _getBlobHashFromString: sinon.stub().returns('random-hash'),
} }
this.HashManager = { this.HashManager = {
_getBlobHashFromString: sinon.stub(), _getBlobHashFromString: sinon.stub(),
} }
this.Metrics = { inc: sinon.stub() } this.Metrics = { inc: sinon.stub() }
this.Settings = { this.Settings = {
redis: { redis: {
lock: { lock: {
@ -91,6 +109,7 @@ describe('SyncManager', function () {
}, },
}, },
} }
this.SyncManager = await esmock(MODULE_PATH, { this.SyncManager = await esmock(MODULE_PATH, {
'../../../../app/js/LockManager.js': this.LockManager, '../../../../app/js/LockManager.js': this.LockManager,
'../../../../app/js/UpdateCompressor.js': this.UpdateCompressor, '../../../../app/js/UpdateCompressor.js': this.UpdateCompressor,
@ -113,13 +132,13 @@ describe('SyncManager', function () {
describe('startResync', function () { describe('startResync', function () {
describe('if a sync is not in progress', function () { describe('if a sync is not in progress', function () {
beforeEach(function () { beforeEach(async function () {
this.db.projectHistorySyncState.findOne.yields(null, {}) this.db.projectHistorySyncState.findOne.resolves({})
this.SyncManager.startResync(this.projectId, this.callback) await this.SyncManager.promises.startResync(this.projectId)
}) })
it('takes the project lock', function () { it('takes the project lock', function () {
expect(this.LockManager.runWithLock).to.have.been.calledWith( expect(this.LockManager.promises.runWithLock).to.have.been.calledWith(
`ProjectHistoryLock:${this.projectId}` `ProjectHistoryLock:${this.projectId}`
) )
}) })
@ -131,9 +150,9 @@ describe('SyncManager', function () {
}) })
it('requests a resync from web', function () { it('requests a resync from web', function () {
expect(this.WebApiManager.requestResync).to.have.been.calledWith( expect(
this.projectId this.WebApiManager.promises.requestResync
) ).to.have.been.calledWith(this.projectId)
}) })
it('sets the sync state in mongo and prevents it expiring', function () { it('sets the sync state in mongo and prevents it expiring', function () {
@ -158,37 +177,31 @@ describe('SyncManager', function () {
} }
) )
}) })
it('calls the callback', function () {
expect(this.callback).to.have.been.called
})
}) })
describe('if project structure sync is in progress', function () { describe('if project structure sync is in progress', function () {
beforeEach(function () { beforeEach(function () {
const syncState = { resyncProjectStructure: true } const syncState = { resyncProjectStructure: true }
this.db.projectHistorySyncState.findOne.yields(null, syncState) this.db.projectHistorySyncState.findOne.resolves(syncState)
this.SyncManager.startResync(this.projectId, this.callback)
}) })
it('returns an error if already syncing', function () { it('returns an error if already syncing', async function () {
expect(this.callback).to.have.been.calledWithMatch({ await expect(
message: 'sync ongoing', this.SyncManager.promises.startResync(this.projectId)
}) ).to.be.rejectedWith('sync ongoing')
}) })
}) })
describe('if doc content sync in is progress', function () { describe('if doc content sync in is progress', function () {
beforeEach(function () { beforeEach(async function () {
const syncState = { resyncDocContents: ['/foo.tex'] } const syncState = { resyncDocContents: ['/foo.tex'] }
this.db.projectHistorySyncState.findOne.yields(null, syncState) this.db.projectHistorySyncState.findOne.resolves(syncState)
this.SyncManager.startResync(this.projectId, this.callback)
}) })
it('returns an error if already syncing', function () { it('returns an error if already syncing', async function () {
expect(this.callback).to.have.been.calledWithMatch({ await expect(
message: 'sync ongoing', this.SyncManager.promises.startResync(this.projectId)
}) ).to.be.rejectedWith('sync ongoing')
}) })
}) })
}) })
@ -208,32 +221,25 @@ describe('SyncManager', function () {
} }
}) })
it('sets the sync state in mongo and prevents it expiring', function (done) { it('sets the sync state in mongo and prevents it expiring', async function () {
// SyncState is a private class of SyncManager // SyncState is a private class of SyncManager
// we know the interface however: // we know the interface however:
this.SyncManager.setResyncState( await this.SyncManager.promises.setResyncState(
this.projectId, this.projectId,
this.syncState, this.syncState
error => { )
expect(error).to.be.undefined
expect( expect(
this.db.projectHistorySyncState.updateOne this.db.projectHistorySyncState.updateOne
).to.have.been.calledWith( ).to.have.been.calledWith(
{ { project_id: new ObjectId(this.projectId) },
project_id: new ObjectId(this.projectId),
},
sinon.match({ sinon.match({
$set: this.syncState.toRaw(), $set: this.syncState.toRaw(),
$currentDate: { lastUpdated: true }, $currentDate: { lastUpdated: true },
$inc: { resyncCount: 1 }, $inc: { resyncCount: 1 },
$unset: { expiresAt: true }, $unset: { expiresAt: true },
}), }),
{ { upsert: true }
upsert: true,
}
)
done()
}
) )
}) })
}) })
@ -252,71 +258,58 @@ describe('SyncManager', function () {
} }
}) })
it('sets the sync state entry in mongo to expire', function (done) { it('sets the sync state entry in mongo to expire', async function () {
this.SyncManager.setResyncState( await this.SyncManager.promises.setResyncState(
this.projectId, this.projectId,
this.syncState, this.syncState
error => { )
expect(error).to.be.undefined
expect( expect(
this.db.projectHistorySyncState.updateOne this.db.projectHistorySyncState.updateOne
).to.have.been.calledWith( ).to.have.been.calledWith(
{ { project_id: new ObjectId(this.projectId) },
project_id: new ObjectId(this.projectId),
},
sinon.match({ sinon.match({
$set: { $set: {
resyncProjectStructure: false, resyncProjectStructure: false,
resyncDocContents: [], resyncDocContents: [],
origin: { kind: 'history-resync' }, origin: { kind: 'history-resync' },
expiresAt: new Date( expiresAt: new Date(this.now.getTime() + 90 * 24 * 3600 * 1000),
this.now.getTime() + 90 * 24 * 3600 * 1000
),
}, },
$currentDate: { lastUpdated: true }, $currentDate: { lastUpdated: true },
}), }),
{ { upsert: true }
upsert: true,
}
)
done()
}
) )
}) })
}) })
describe('when the new sync state is null', function () { describe('when the new sync state is null', function () {
it('does not update the sync state in mongo', function (done) { it('does not update the sync state in mongo', async function () {
// SyncState is a private class of SyncManager // SyncState is a private class of SyncManager
// we know the interface however: // we know the interface however:
this.SyncManager.setResyncState(this.projectId, null, error => { await this.SyncManager.promises.setResyncState(this.projectId, null)
expect(error).to.be.undefined
expect(this.db.projectHistorySyncState.updateOne).to.not.have.been expect(this.db.projectHistorySyncState.updateOne).to.not.have.been
.called .called
done()
})
}) })
}) })
}) })
describe('skipUpdatesDuringSync', function () { describe('skipUpdatesDuringSync', function () {
describe('if a sync is not in progress', function () { describe('if a sync is not in progress', function () {
beforeEach(function () { beforeEach(async function () {
this.db.projectHistorySyncState.findOne.yields(null, {}) this.db.projectHistorySyncState.findOne.resolves({})
this.updates = ['some', 'mock', 'updates'] this.updates = ['some', 'mock', 'updates']
this.SyncManager.skipUpdatesDuringSync( this.result = await this.SyncManager.promises.skipUpdatesDuringSync(
this.projectId, this.projectId,
this.updates, this.updates
this.callback
) )
}) })
it('returns all updates', function () { it('returns all updates', function () {
expect(this.callback).to.have.been.calledWith(null, this.updates) expect(this.result.updates).to.deep.equal(this.updates)
}) })
it('should not return any newSyncState', function () { it('should not return any newSyncState', function () {
expect(this.callback).to.have.been.calledWithExactly(null, this.updates) expect(this.result.syncState).to.be.null
}) })
}) })
@ -364,39 +357,37 @@ describe('SyncManager', function () {
resyncDocContents: [], resyncDocContents: [],
origin: { kind: 'history-resync' }, origin: { kind: 'history-resync' },
} }
this.db.projectHistorySyncState.findOne.yields(null, syncState) this.db.projectHistorySyncState.findOne.resolves(syncState)
}) })
it('remove updates before a project structure sync update', function () { it('remove updates before a project structure sync update', async function () {
const updates = [ const updates = [
this.renameUpdate, this.renameUpdate,
this.textUpdate, this.textUpdate,
this.projectStructureSyncUpdate, this.projectStructureSyncUpdate,
] ]
this.SyncManager.skipUpdatesDuringSync( const { updates: filteredUpdates, syncState } =
await this.SyncManager.promises.skipUpdatesDuringSync(
this.projectId, this.projectId,
updates, updates
(error, filteredUpdates, syncState) => { )
expect(error).to.be.null
expect(filteredUpdates).to.deep.equal([ expect(filteredUpdates).to.deep.equal([this.projectStructureSyncUpdate])
this.projectStructureSyncUpdate,
])
expect(syncState.toRaw()).to.deep.equal({ expect(syncState.toRaw()).to.deep.equal({
resyncProjectStructure: false, resyncProjectStructure: false,
resyncDocContents: ['new.tex'], resyncDocContents: ['new.tex'],
origin: { kind: 'history-resync' }, origin: { kind: 'history-resync' },
}) })
}
)
}) })
it('allow project structure updates after project structure sync update', function () { it('allow project structure updates after project structure sync update', async function () {
const updates = [this.projectStructureSyncUpdate, this.renameUpdate] const updates = [this.projectStructureSyncUpdate, this.renameUpdate]
this.SyncManager.skipUpdatesDuringSync( const { updates: filteredUpdates, syncState } =
await this.SyncManager.promises.skipUpdatesDuringSync(
this.projectId, this.projectId,
updates, updates
(error, filteredUpdates, syncState) => { )
expect(error).to.be.null
expect(filteredUpdates).to.deep.equal([ expect(filteredUpdates).to.deep.equal([
this.projectStructureSyncUpdate, this.projectStructureSyncUpdate,
this.renameUpdate, this.renameUpdate,
@ -406,21 +397,20 @@ describe('SyncManager', function () {
resyncDocContents: ['new.tex'], resyncDocContents: ['new.tex'],
origin: { kind: 'history-resync' }, origin: { kind: 'history-resync' },
}) })
}
)
}) })
it('remove text updates for a doc before doc sync update', function () { it('remove text updates for a doc before doc sync update', async function () {
const updates = [ const updates = [
this.projectStructureSyncUpdate, this.projectStructureSyncUpdate,
this.textUpdate, this.textUpdate,
this.docContentSyncUpdate, this.docContentSyncUpdate,
] ]
this.SyncManager.skipUpdatesDuringSync( const { updates: filteredUpdates, syncState } =
await this.SyncManager.promises.skipUpdatesDuringSync(
this.projectId, this.projectId,
updates, updates
(error, filteredUpdates, syncState) => { )
expect(error).to.be.null
expect(filteredUpdates).to.deep.equal([ expect(filteredUpdates).to.deep.equal([
this.projectStructureSyncUpdate, this.projectStructureSyncUpdate,
this.docContentSyncUpdate, this.docContentSyncUpdate,
@ -430,21 +420,20 @@ describe('SyncManager', function () {
resyncDocContents: [], resyncDocContents: [],
origin: { kind: 'history-resync' }, origin: { kind: 'history-resync' },
}) })
}
)
}) })
it('allow text updates for a doc after doc sync update', function () { it('allow text updates for a doc after doc sync update', async function () {
const updates = [ const updates = [
this.projectStructureSyncUpdate, this.projectStructureSyncUpdate,
this.docContentSyncUpdate, this.docContentSyncUpdate,
this.textUpdate, this.textUpdate,
] ]
this.SyncManager.skipUpdatesDuringSync( const { updates: filteredUpdates, syncState } =
await this.SyncManager.promises.skipUpdatesDuringSync(
this.projectId, this.projectId,
updates, updates
(error, filteredUpdates, syncState) => { )
expect(error).to.be.null
expect(filteredUpdates).to.deep.equal([ expect(filteredUpdates).to.deep.equal([
this.projectStructureSyncUpdate, this.projectStructureSyncUpdate,
this.docContentSyncUpdate, this.docContentSyncUpdate,
@ -455,8 +444,6 @@ describe('SyncManager', function () {
resyncDocContents: [], resyncDocContents: [],
origin: { kind: 'history-resync' }, origin: { kind: 'history-resync' },
}) })
}
)
}) })
}) })
}) })
@ -493,55 +480,53 @@ describe('SyncManager', function () {
.returns('another.tex') .returns('another.tex')
this.UpdateTranslator._convertPathname.withArgs('1.png').returns('1.png') this.UpdateTranslator._convertPathname.withArgs('1.png').returns('1.png')
this.UpdateTranslator._convertPathname.withArgs('2.png').returns('2.png') this.UpdateTranslator._convertPathname.withArgs('2.png').returns('2.png')
this.SnapshotManager.getLatestSnapshot.yields(null, this.fileMap) this.SnapshotManager.promises.getLatestSnapshot.resolves(this.fileMap)
}) })
it('returns updates if no sync updates are queued', function () { it('returns updates if no sync updates are queued', async function () {
const updates = ['some', 'mock', 'updates'] const updates = ['some', 'mock', 'updates']
this.SyncManager.expandSyncUpdates( const expandedUpdates = await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => {
expect(error).to.be.null
expect(expandedUpdates).to.equal(updates)
expect(this.SnapshotManager.getLatestSnapshot).to.not.have.been.called
expect(this.extendLock).to.not.have.been.called
}
) )
expect(expandedUpdates).to.equal(updates)
expect(this.SnapshotManager.promises.getLatestSnapshot).to.not.have.been
.called
expect(this.extendLock).to.not.have.been.called
}) })
describe('expanding project structure sync updates', function () { describe('expanding project structure sync updates', function () {
it('queues nothing for expected docs and files', function () { it('queues nothing for expected docs and files', async function () {
const updates = [ const updates = [
resyncProjectStructureUpdate( resyncProjectStructureUpdate(
[this.persistedDoc], [this.persistedDoc],
[this.persistedFile] [this.persistedFile]
), ),
] ]
this.SyncManager.expandSyncUpdates( const expandedUpdates =
await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => { )
expect(error).to.be.null
expect(expandedUpdates).to.deep.equal([]) expect(expandedUpdates).to.deep.equal([])
expect(this.extendLock).to.have.been.called expect(this.extendLock).to.have.been.called
}
)
}) })
it('queues file removes for unexpected files', function () { it('queues file removes for unexpected files', async function () {
const updates = [resyncProjectStructureUpdate([this.persistedDoc], [])] const updates = [resyncProjectStructureUpdate([this.persistedDoc], [])]
this.SyncManager.expandSyncUpdates( const expandedUpdates =
await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => { )
expect(error).to.be.null
expect(expandedUpdates).to.deep.equal([ expect(expandedUpdates).to.deep.equal([
{ {
pathname: this.persistedFile.path, pathname: this.persistedFile.path,
@ -554,19 +539,18 @@ describe('SyncManager', function () {
}, },
]) ])
expect(this.extendLock).to.have.been.called expect(this.extendLock).to.have.been.called
}
)
}) })
it('queues doc removes for unexpected docs', function () { it('queues doc removes for unexpected docs', async function () {
const updates = [resyncProjectStructureUpdate([], [this.persistedFile])] const updates = [resyncProjectStructureUpdate([], [this.persistedFile])]
this.SyncManager.expandSyncUpdates( const expandedUpdates =
await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => { )
expect(error).to.be.null
expect(expandedUpdates).to.deep.equal([ expect(expandedUpdates).to.deep.equal([
{ {
pathname: this.persistedDoc.path, pathname: this.persistedDoc.path,
@ -579,11 +563,9 @@ describe('SyncManager', function () {
}, },
]) ])
expect(this.extendLock).to.have.been.called expect(this.extendLock).to.have.been.called
}
)
}) })
it('queues file additions for missing files', function () { it('queues file additions for missing files', async function () {
const newFile = { const newFile = {
path: '2.png', path: '2.png',
file: {}, file: {},
@ -595,13 +577,14 @@ describe('SyncManager', function () {
[this.persistedFile, newFile] [this.persistedFile, newFile]
), ),
] ]
this.SyncManager.expandSyncUpdates( const expandedUpdates =
await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => { )
expect(error).to.be.null
expect(expandedUpdates).to.deep.equal([ expect(expandedUpdates).to.deep.equal([
{ {
pathname: newFile.path, pathname: newFile.path,
@ -615,11 +598,9 @@ describe('SyncManager', function () {
}, },
]) ])
expect(this.extendLock).to.have.been.called expect(this.extendLock).to.have.been.called
}
)
}) })
it('queues blank doc additions for missing docs', function () { it('queues blank doc additions for missing docs', async function () {
const newDoc = { const newDoc = {
path: 'another.tex', path: 'another.tex',
doc: new ObjectId().toString(), doc: new ObjectId().toString(),
@ -630,13 +611,14 @@ describe('SyncManager', function () {
[this.persistedFile] [this.persistedFile]
), ),
] ]
this.SyncManager.expandSyncUpdates( const expandedUpdates =
await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => { )
expect(error).to.be.null
expect(expandedUpdates).to.deep.equal([ expect(expandedUpdates).to.deep.equal([
{ {
pathname: newDoc.path, pathname: newDoc.path,
@ -650,11 +632,9 @@ describe('SyncManager', function () {
}, },
]) ])
expect(this.extendLock).to.have.been.called expect(this.extendLock).to.have.been.called
}
)
}) })
it('removes and re-adds files if whether they are binary differs', function () { it('removes and re-adds files if whether they are binary differs', async function () {
const fileWichWasADoc = { const fileWichWasADoc = {
path: this.persistedDoc.path, path: this.persistedDoc.path,
url: 'filestore/2.png', url: 'filestore/2.png',
@ -667,13 +647,14 @@ describe('SyncManager', function () {
[fileWichWasADoc, this.persistedFile] [fileWichWasADoc, this.persistedFile]
), ),
] ]
this.SyncManager.expandSyncUpdates( const expandedUpdates =
await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => { )
expect(error).to.be.null
expect(expandedUpdates).to.deep.equal([ expect(expandedUpdates).to.deep.equal([
{ {
pathname: fileWichWasADoc.path, pathname: fileWichWasADoc.path,
@ -696,11 +677,9 @@ describe('SyncManager', function () {
}, },
]) ])
expect(this.extendLock).to.have.been.called expect(this.extendLock).to.have.been.called
}
)
}) })
it("does not remove and re-add files if the expected file doesn't have a hash", function () { it("does not remove and re-add files if the expected file doesn't have a hash", async function () {
const fileWichWasADoc = { const fileWichWasADoc = {
path: this.persistedDoc.path, path: this.persistedDoc.path,
url: 'filestore/2.png', url: 'filestore/2.png',
@ -712,20 +691,19 @@ describe('SyncManager', function () {
[fileWichWasADoc, this.persistedFile] [fileWichWasADoc, this.persistedFile]
), ),
] ]
this.SyncManager.expandSyncUpdates( const expandedUpdates =
await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => { )
expect(error).to.be.null
expect(expandedUpdates).to.deep.equal([]) expect(expandedUpdates).to.deep.equal([])
expect(this.extendLock).to.have.been.called expect(this.extendLock).to.have.been.called
}
)
}) })
it('does not remove and re-add editable files if there is a binary file with same hash', function () { it('does not remove and re-add editable files if there is a binary file with same hash', async function () {
const binaryFile = { const binaryFile = {
file: Object().toString(), file: Object().toString(),
// The paths in the resyncProjectStructureUpdate must have a leading slash ('/') // The paths in the resyncProjectStructureUpdate must have a leading slash ('/')
@ -741,20 +719,19 @@ describe('SyncManager', function () {
const updates = [ const updates = [
resyncProjectStructureUpdate([], [binaryFile, this.persistedFile]), resyncProjectStructureUpdate([], [binaryFile, this.persistedFile]),
] ]
this.SyncManager.expandSyncUpdates( const expandedUpdates =
await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => { )
expect(error).to.be.null
expect(expandedUpdates).to.deep.equal([]) expect(expandedUpdates).to.deep.equal([])
expect(this.extendLock).to.have.been.called expect(this.extendLock).to.have.been.called
}
)
}) })
it('removes and re-adds binary files if they do not have same hash', function () { it('removes and re-adds binary files if they do not have same hash', async function () {
const persistedFileWithNewContent = { const persistedFileWithNewContent = {
_hash: 'anotherhashvalue', _hash: 'anotherhashvalue',
hello: 'world', hello: 'world',
@ -767,13 +744,14 @@ describe('SyncManager', function () {
[persistedFileWithNewContent] [persistedFileWithNewContent]
), ),
] ]
this.SyncManager.expandSyncUpdates( const expandedUpdates =
await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => { )
expect(error).to.be.null
expect(expandedUpdates).to.deep.equal([ expect(expandedUpdates).to.deep.equal([
{ {
pathname: persistedFileWithNewContent.path, pathname: persistedFileWithNewContent.path,
@ -796,11 +774,9 @@ describe('SyncManager', function () {
}, },
]) ])
expect(this.extendLock).to.have.been.called expect(this.extendLock).to.have.been.called
}
)
}) })
it('preserves other updates', function () { it('preserves other updates', async function () {
const update = 'mock-update' const update = 'mock-update'
const updates = [ const updates = [
update, update,
@ -809,22 +785,21 @@ describe('SyncManager', function () {
[this.persistedFile] [this.persistedFile]
), ),
] ]
this.SyncManager.expandSyncUpdates( const expandedUpdates =
await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => { )
expect(error).to.be.null
expect(expandedUpdates).to.deep.equal([update]) expect(expandedUpdates).to.deep.equal([update])
expect(this.extendLock).to.have.been.called expect(this.extendLock).to.have.been.called
}
)
}) })
}) })
describe('expanding doc contents sync updates', function () { describe('expanding doc contents sync updates', function () {
it('returns errors from diffAsShareJsOps', function () { it('returns errors from diffAsShareJsOps', async function () {
const diffError = new Error('test') const diffError = new Error('test')
this.UpdateCompressor.diffAsShareJsOps.throws(diffError) this.UpdateCompressor.diffAsShareJsOps.throws(diffError)
const updates = [ const updates = [
@ -834,33 +809,30 @@ describe('SyncManager', function () {
), ),
docContentSyncUpdate(this.persistedDoc, this.persistedDoc.content), docContentSyncUpdate(this.persistedDoc, this.persistedDoc.content),
] ]
this.SyncManager.expandSyncUpdates( await expect(
this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => { )
expect(error).to.equal(diffError) ).to.be.rejectedWith(diffError)
expect(this.extendLock).to.have.been.called expect(this.extendLock).to.have.been.called
}
)
}) })
it('handles an update for a file that is missing from the snapshot', function () { it('handles an update for a file that is missing from the snapshot', async function () {
const updates = [docContentSyncUpdate('not-in-snapshot.txt', 'test')] const updates = [docContentSyncUpdate('not-in-snapshot.txt', 'test')]
this.SyncManager.expandSyncUpdates( await expect(
this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => {
expect(error).to.exist
expect(error.message).to.equal('unrecognised file: not in snapshot')
}
) )
).to.be.rejectedWith('unrecognised file: not in snapshot')
}) })
it('queues nothing for in docs whose contents is in sync', function () { it('queues nothing for in docs whose contents is in sync', async function () {
const updates = [ const updates = [
resyncProjectStructureUpdate( resyncProjectStructureUpdate(
[this.persistedDoc], [this.persistedDoc],
@ -869,20 +841,19 @@ describe('SyncManager', function () {
docContentSyncUpdate(this.persistedDoc, this.persistedDoc.content), docContentSyncUpdate(this.persistedDoc, this.persistedDoc.content),
] ]
this.UpdateCompressor.diffAsShareJsOps.returns([]) this.UpdateCompressor.diffAsShareJsOps.returns([])
this.SyncManager.expandSyncUpdates( const expandedUpdates =
await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => { )
expect(error).to.be.null
expect(expandedUpdates).to.deep.equal([]) expect(expandedUpdates).to.deep.equal([])
expect(this.extendLock).to.have.been.called expect(this.extendLock).to.have.been.called
}
)
}) })
it('queues text updates for docs whose contents is out of sync', function () { it('queues text updates for docs whose contents is out of sync', async function () {
const updates = [ const updates = [
resyncProjectStructureUpdate( resyncProjectStructureUpdate(
[this.persistedDoc], [this.persistedDoc],
@ -891,13 +862,14 @@ describe('SyncManager', function () {
docContentSyncUpdate(this.persistedDoc, 'a'), docContentSyncUpdate(this.persistedDoc, 'a'),
] ]
this.UpdateCompressor.diffAsShareJsOps.returns([{ d: 'sdf', p: 1 }]) this.UpdateCompressor.diffAsShareJsOps.returns([{ d: 'sdf', p: 1 }])
this.SyncManager.expandSyncUpdates( const expandedUpdates =
await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => { )
expect(error).to.be.null
expect(expandedUpdates).to.deep.equal([ expect(expandedUpdates).to.deep.equal([
{ {
doc: this.persistedDoc.doc, doc: this.persistedDoc.doc,
@ -912,11 +884,9 @@ describe('SyncManager', function () {
}, },
]) ])
expect(this.extendLock).to.have.been.called expect(this.extendLock).to.have.been.called
}
)
}) })
it('queues text updates for docs created by project structure sync', function () { it('queues text updates for docs created by project structure sync', async function () {
this.UpdateCompressor.diffAsShareJsOps.returns([{ i: 'a', p: 0 }]) this.UpdateCompressor.diffAsShareJsOps.returns([{ i: 'a', p: 0 }])
const newDoc = { const newDoc = {
path: 'another.tex', path: 'another.tex',
@ -930,13 +900,14 @@ describe('SyncManager', function () {
), ),
docContentSyncUpdate(newDoc, newDoc.content), docContentSyncUpdate(newDoc, newDoc.content),
] ]
this.SyncManager.expandSyncUpdates( const expandedUpdates =
await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => { )
expect(error).to.be.null
expect(expandedUpdates).to.deep.equal([ expect(expandedUpdates).to.deep.equal([
{ {
pathname: newDoc.path, pathname: newDoc.path,
@ -961,11 +932,9 @@ describe('SyncManager', function () {
}, },
]) ])
expect(this.extendLock).to.have.been.called expect(this.extendLock).to.have.been.called
}
)
}) })
it('skips text updates for docs when hashes match', function () { it('skips text updates for docs when hashes match', async function () {
this.fileMap['main.tex'].getHash = sinon.stub().returns('special-hash') this.fileMap['main.tex'].getHash = sinon.stub().returns('special-hash')
this.HashManager._getBlobHashFromString.returns('special-hash') this.HashManager._getBlobHashFromString.returns('special-hash')
const updates = [ const updates = [
@ -975,20 +944,19 @@ describe('SyncManager', function () {
), ),
docContentSyncUpdate(this.persistedDoc, 'hello'), docContentSyncUpdate(this.persistedDoc, 'hello'),
] ]
this.SyncManager.expandSyncUpdates( const expandedUpdates =
await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => { )
expect(error).to.be.null
expect(expandedUpdates).to.deep.equal([]) expect(expandedUpdates).to.deep.equal([])
expect(this.extendLock).to.have.been.called expect(this.extendLock).to.have.been.called
}
)
}) })
it('computes text updates for docs when hashes differ', function () { it('computes text updates for docs when hashes differ', async function () {
this.fileMap['main.tex'].getHash = sinon.stub().returns('first-hash') this.fileMap['main.tex'].getHash = sinon.stub().returns('first-hash')
this.HashManager._getBlobHashFromString.returns('second-hash') this.HashManager._getBlobHashFromString.returns('second-hash')
this.UpdateCompressor.diffAsShareJsOps.returns([ this.UpdateCompressor.diffAsShareJsOps.returns([
@ -1001,13 +969,14 @@ describe('SyncManager', function () {
), ),
docContentSyncUpdate(this.persistedDoc, 'hello'), docContentSyncUpdate(this.persistedDoc, 'hello'),
] ]
this.SyncManager.expandSyncUpdates( const expandedUpdates =
await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => { )
expect(error).to.be.null
expect(expandedUpdates).to.deep.equal([ expect(expandedUpdates).to.deep.equal([
{ {
doc: this.persistedDoc.doc, doc: this.persistedDoc.doc,
@ -1022,12 +991,10 @@ describe('SyncManager', function () {
}, },
]) ])
expect(this.extendLock).to.have.been.called expect(this.extendLock).to.have.been.called
}
)
}) })
describe('for docs whose contents is out of sync', function () { describe('for docs whose contents is out of sync', function () {
beforeEach(function (done) { beforeEach(async function () {
const updates = [ const updates = [
resyncProjectStructureUpdate( resyncProjectStructureUpdate(
[this.persistedDoc], [this.persistedDoc],
@ -1038,21 +1005,16 @@ describe('SyncManager', function () {
const file = { getContent: sinon.stub().returns('stored content') } const file = { getContent: sinon.stub().returns('stored content') }
this.fileMap['main.tex'].load = sinon.stub().resolves(file) this.fileMap['main.tex'].load = sinon.stub().resolves(file)
this.UpdateCompressor.diffAsShareJsOps.returns([{ d: 'sdf', p: 1 }]) this.UpdateCompressor.diffAsShareJsOps.returns([{ d: 'sdf', p: 1 }])
this.SyncManager.expandSyncUpdates( this.expandedUpdates =
await this.SyncManager.promises.expandSyncUpdates(
this.projectId, this.projectId,
this.historyId, this.historyId,
updates, updates,
this.extendLock, this.extendLock
(error, expandedUpdates) => {
this.error = error
this.expandedUpdates = expandedUpdates
done()
}
) )
}) })
it('loads content from the history service when needed', function () { it('loads content from the history service when needed', function () {
expect(this.error).to.be.null
expect(this.expandedUpdates).to.deep.equal([ expect(this.expandedUpdates).to.deep.equal([
{ {
doc: this.persistedDoc.doc, doc: this.persistedDoc.doc,