mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-21 20:47:08 -05:00
Merge pull request #17729 from overleaf/em-promisify-sync-manager
Promisify SyncManager GitOrigin-RevId: 134770d812a493e39410debb370ed4a58ffff4bf
This commit is contained in:
parent
c9373c25f4
commit
f03e3fd51e
10 changed files with 780 additions and 805 deletions
2
package-lock.json
generated
2
package-lock.json
generated
|
@ -43037,6 +43037,7 @@
|
||||||
"@overleaf/logger": "*",
|
"@overleaf/logger": "*",
|
||||||
"@overleaf/metrics": "*",
|
"@overleaf/metrics": "*",
|
||||||
"@overleaf/o-error": "*",
|
"@overleaf/o-error": "*",
|
||||||
|
"@overleaf/promise-utils": "*",
|
||||||
"@overleaf/redis-wrapper": "*",
|
"@overleaf/redis-wrapper": "*",
|
||||||
"@overleaf/settings": "*",
|
"@overleaf/settings": "*",
|
||||||
"async": "^3.2.2",
|
"async": "^3.2.2",
|
||||||
|
@ -51636,6 +51637,7 @@
|
||||||
"@overleaf/logger": "*",
|
"@overleaf/logger": "*",
|
||||||
"@overleaf/metrics": "*",
|
"@overleaf/metrics": "*",
|
||||||
"@overleaf/o-error": "*",
|
"@overleaf/o-error": "*",
|
||||||
|
"@overleaf/promise-utils": "*",
|
||||||
"@overleaf/redis-wrapper": "*",
|
"@overleaf/redis-wrapper": "*",
|
||||||
"@overleaf/settings": "*",
|
"@overleaf/settings": "*",
|
||||||
"async": "^3.2.2",
|
"async": "^3.2.2",
|
||||||
|
|
|
@ -305,6 +305,11 @@ export function getFailures(callback) {
|
||||||
|
|
||||||
export const promises = {
|
export const promises = {
|
||||||
getFailedProjects: promisify(getFailedProjects),
|
getFailedProjects: promisify(getFailedProjects),
|
||||||
record: promisify(record),
|
|
||||||
getFailureRecord: promisify(getFailureRecord),
|
getFailureRecord: promisify(getFailureRecord),
|
||||||
|
getLastFailure: promisify(getLastFailure),
|
||||||
|
getFailuresByType: promisify(getFailuresByType),
|
||||||
|
getFailures: promisify(getFailures),
|
||||||
|
record: promisify(record),
|
||||||
|
recordSyncStart: promisify(recordSyncStart),
|
||||||
|
setForceDebug: promisify(setForceDebug),
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,6 +10,7 @@
|
||||||
* DS207: Consider shorter variations of null checks
|
* DS207: Consider shorter variations of null checks
|
||||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||||
*/
|
*/
|
||||||
|
import { promisify } from 'util'
|
||||||
import fs from 'fs'
|
import fs from 'fs'
|
||||||
import crypto from 'crypto'
|
import crypto from 'crypto'
|
||||||
import OError from '@overleaf/o-error'
|
import OError from '@overleaf/o-error'
|
||||||
|
@ -51,3 +52,7 @@ export function _getBlobHash(fsPath, callback) {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const promises = {
|
||||||
|
_getBlobHash: promisify(_getBlobHash),
|
||||||
|
}
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
|
import { promisify } from 'util'
|
||||||
import fs from 'fs'
|
import fs from 'fs'
|
||||||
import request from 'request'
|
import request from 'request'
|
||||||
import stream from 'stream'
|
import stream from 'stream'
|
||||||
import logger from '@overleaf/logger'
|
import logger from '@overleaf/logger'
|
||||||
import _ from 'lodash'
|
import _ from 'lodash'
|
||||||
import BPromise from 'bluebird'
|
|
||||||
import { URL } from 'url'
|
import { URL } from 'url'
|
||||||
import OError from '@overleaf/o-error'
|
import OError from '@overleaf/o-error'
|
||||||
import Settings from '@overleaf/settings'
|
import Settings from '@overleaf/settings'
|
||||||
|
@ -354,7 +354,7 @@ export function deleteProject(projectId, callback) {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
const getProjectBlobAsync = BPromise.promisify(getProjectBlob)
|
const getProjectBlobAsync = promisify(getProjectBlob)
|
||||||
|
|
||||||
class BlobStore {
|
class BlobStore {
|
||||||
constructor(projectId) {
|
constructor(projectId) {
|
||||||
|
@ -435,3 +435,15 @@ function _requestHistoryService(options, callback) {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const promises = {
|
||||||
|
getMostRecentChunk: promisify(getMostRecentChunk),
|
||||||
|
getChunkAtVersion: promisify(getChunkAtVersion),
|
||||||
|
getMostRecentVersion: promisify(getMostRecentVersion),
|
||||||
|
getProjectBlob: promisify(getProjectBlob),
|
||||||
|
getProjectBlobStream: promisify(getProjectBlobStream),
|
||||||
|
sendChanges: promisify(sendChanges),
|
||||||
|
createBlobForUpdate: promisify(createBlobForUpdate),
|
||||||
|
initializeProject: promisify(initializeProject),
|
||||||
|
deleteProject: promisify(deleteProject),
|
||||||
|
}
|
||||||
|
|
|
@ -7,6 +7,7 @@
|
||||||
* DS207: Consider shorter variations of null checks
|
* DS207: Consider shorter variations of null checks
|
||||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||||
*/
|
*/
|
||||||
|
import { promisify } from 'util'
|
||||||
import async from 'async'
|
import async from 'async'
|
||||||
import metrics from '@overleaf/metrics'
|
import metrics from '@overleaf/metrics'
|
||||||
import Settings from '@overleaf/settings'
|
import Settings from '@overleaf/settings'
|
||||||
|
@ -273,3 +274,41 @@ class Lock {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Promisified version of runWithLock.
|
||||||
|
*
|
||||||
|
* @param {string} key
|
||||||
|
* @param {(extendLock: Function) => Promise<any>} runner
|
||||||
|
*/
|
||||||
|
async function runWithLockPromises(key, runner) {
|
||||||
|
const runnerCb = (extendLock, callback) => {
|
||||||
|
const extendLockPromises = promisify(extendLock)
|
||||||
|
runner(extendLockPromises)
|
||||||
|
.then(result => {
|
||||||
|
callback(null, result)
|
||||||
|
})
|
||||||
|
.catch(err => {
|
||||||
|
callback(err)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
runWithLock(key, runnerCb, (err, result) => {
|
||||||
|
if (err) {
|
||||||
|
reject(err)
|
||||||
|
} else {
|
||||||
|
resolve(result)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export const promises = {
|
||||||
|
tryLock: promisify(tryLock),
|
||||||
|
extendLock: promisify(extendLock),
|
||||||
|
getLock: promisify(getLock),
|
||||||
|
checkLock: promisify(checkLock),
|
||||||
|
releaseLock: promisify(releaseLock),
|
||||||
|
runWithLock: runWithLockPromises,
|
||||||
|
}
|
||||||
|
|
|
@ -1,4 +1,6 @@
|
||||||
// @ts-check
|
// @ts-check
|
||||||
|
|
||||||
|
import { promisify } from 'util'
|
||||||
import Core from 'overleaf-editor-core'
|
import Core from 'overleaf-editor-core'
|
||||||
import { Readable as StringStream } from 'stream'
|
import { Readable as StringStream } from 'stream'
|
||||||
import BPromise from 'bluebird'
|
import BPromise from 'bluebird'
|
||||||
|
@ -171,3 +173,9 @@ function _loadFilesLimit(snapshot, kind, blobStore) {
|
||||||
{ concurrency: MAX_REQUESTS }
|
{ concurrency: MAX_REQUESTS }
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const promises = {
|
||||||
|
getFileSnapshotStream: promisify(getFileSnapshotStream),
|
||||||
|
getProjectSnapshot: promisify(getProjectSnapshot),
|
||||||
|
getLatestSnapshot: promisify(getLatestSnapshot),
|
||||||
|
}
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import _ from 'lodash'
|
import _ from 'lodash'
|
||||||
import async from 'async'
|
import { callbackify, promisify } from 'util'
|
||||||
import { promisify } from 'util'
|
import { callbackifyMultiResult } from '@overleaf/promise-utils'
|
||||||
import Settings from '@overleaf/settings'
|
import Settings from '@overleaf/settings'
|
||||||
import logger from '@overleaf/logger'
|
import logger from '@overleaf/logger'
|
||||||
import Metrics from '@overleaf/metrics'
|
import Metrics from '@overleaf/metrics'
|
||||||
|
@ -27,7 +27,7 @@ const keys = Settings.redis.lock.key_schema
|
||||||
// To add expiresAt field to existing entries in collection (choose a suitable future expiry date):
|
// To add expiresAt field to existing entries in collection (choose a suitable future expiry date):
|
||||||
// db.projectHistorySyncState.updateMany({resyncProjectStructure: false, resyncDocContents: [], expiresAt: {$exists:false}}, {$set: {expiresAt: new Date("2019-07-01")}})
|
// db.projectHistorySyncState.updateMany({resyncProjectStructure: false, resyncDocContents: [], expiresAt: {$exists:false}}, {$set: {expiresAt: new Date("2019-07-01")}})
|
||||||
|
|
||||||
export function startResync(projectId, options, callback) {
|
async function startResync(projectId, options = {}) {
|
||||||
// We have three options here
|
// We have three options here
|
||||||
//
|
//
|
||||||
// 1. If we update mongo before making the call to web then there's a
|
// 1. If we update mongo before making the call to web then there's a
|
||||||
|
@ -39,114 +39,71 @@ export function startResync(projectId, options, callback) {
|
||||||
// after, causing all updates to be ignored from then on
|
// after, causing all updates to be ignored from then on
|
||||||
//
|
//
|
||||||
// 3. We can wrap everything in a project lock
|
// 3. We can wrap everything in a project lock
|
||||||
if (typeof options === 'function') {
|
|
||||||
callback = options
|
|
||||||
options = {}
|
|
||||||
}
|
|
||||||
|
|
||||||
Metrics.inc('project_history_resync')
|
Metrics.inc('project_history_resync')
|
||||||
LockManager.runWithLock(
|
try {
|
||||||
|
await LockManager.promises.runWithLock(
|
||||||
keys.projectHistoryLock({ project_id: projectId }),
|
keys.projectHistoryLock({ project_id: projectId }),
|
||||||
(extendLock, releaseLock) =>
|
async extendLock => {
|
||||||
_startResyncWithoutLock(projectId, options, releaseLock),
|
await _startResyncWithoutLock(projectId, options)
|
||||||
function (error) {
|
|
||||||
if (error) {
|
|
||||||
OError.tag(error)
|
|
||||||
// record error in starting sync ("sync ongoing")
|
|
||||||
ErrorRecorder.record(projectId, -1, error, () => callback(error))
|
|
||||||
} else {
|
|
||||||
callback()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
} catch (error) {
|
||||||
|
// record error in starting sync ("sync ongoing")
|
||||||
|
try {
|
||||||
|
await ErrorRecorder.promises.record(projectId, -1, error)
|
||||||
|
} catch (err) {
|
||||||
|
// swallow any error thrown by ErrorRecorder.record()
|
||||||
|
}
|
||||||
|
throw error
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export function startHardResync(projectId, options, callback) {
|
async function startHardResync(projectId, options = {}) {
|
||||||
if (typeof options === 'function') {
|
|
||||||
callback = options
|
|
||||||
options = {}
|
|
||||||
}
|
|
||||||
|
|
||||||
Metrics.inc('project_history_hard_resync')
|
Metrics.inc('project_history_hard_resync')
|
||||||
LockManager.runWithLock(
|
try {
|
||||||
|
await LockManager.promises.runWithLock(
|
||||||
keys.projectHistoryLock({ project_id: projectId }),
|
keys.projectHistoryLock({ project_id: projectId }),
|
||||||
(extendLock, releaseLock) =>
|
async extendLock => {
|
||||||
clearResyncState(projectId, function (err) {
|
await clearResyncState(projectId)
|
||||||
if (err) {
|
await RedisManager.promises.clearFirstOpTimestamp(projectId)
|
||||||
return releaseLock(OError.tag(err))
|
await RedisManager.promises.destroyDocUpdatesQueue(projectId)
|
||||||
}
|
await _startResyncWithoutLock(projectId, options)
|
||||||
RedisManager.clearFirstOpTimestamp(projectId, function (err) {
|
|
||||||
if (err) {
|
|
||||||
return releaseLock(OError.tag(err))
|
|
||||||
}
|
|
||||||
RedisManager.destroyDocUpdatesQueue(projectId, function (err) {
|
|
||||||
if (err) {
|
|
||||||
return releaseLock(OError.tag(err))
|
|
||||||
}
|
|
||||||
_startResyncWithoutLock(projectId, options, releaseLock)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}),
|
|
||||||
function (error) {
|
|
||||||
if (error) {
|
|
||||||
OError.tag(error)
|
|
||||||
// record error in starting sync ("sync ongoing")
|
|
||||||
ErrorRecorder.record(projectId, -1, error, () => callback(error))
|
|
||||||
} else {
|
|
||||||
callback()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
} catch (error) {
|
||||||
|
// record error in starting sync ("sync ongoing")
|
||||||
|
await ErrorRecorder.promises.record(projectId, -1, error)
|
||||||
|
throw error
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function _startResyncWithoutLock(projectId, options, callback) {
|
async function _startResyncWithoutLock(projectId, options) {
|
||||||
ErrorRecorder.recordSyncStart(projectId, function (error) {
|
await ErrorRecorder.promises.recordSyncStart(projectId)
|
||||||
if (error) {
|
|
||||||
return callback(OError.tag(error))
|
|
||||||
}
|
|
||||||
|
|
||||||
_getResyncState(projectId, function (error, syncState) {
|
|
||||||
if (error) {
|
|
||||||
return callback(OError.tag(error))
|
|
||||||
}
|
|
||||||
|
|
||||||
|
const syncState = await _getResyncState(projectId)
|
||||||
if (syncState.isSyncOngoing()) {
|
if (syncState.isSyncOngoing()) {
|
||||||
return callback(new OError('sync ongoing'))
|
throw new OError('sync ongoing')
|
||||||
}
|
}
|
||||||
|
|
||||||
syncState.setOrigin(options.origin || { kind: 'history-resync' })
|
syncState.setOrigin(options.origin || { kind: 'history-resync' })
|
||||||
syncState.startProjectStructureSync()
|
syncState.startProjectStructureSync()
|
||||||
|
|
||||||
WebApiManager.requestResync(projectId, function (error) {
|
await WebApiManager.promises.requestResync(projectId)
|
||||||
if (error) {
|
await setResyncState(projectId, syncState)
|
||||||
return callback(OError.tag(error))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
setResyncState(projectId, syncState, callback)
|
async function _getResyncState(projectId) {
|
||||||
})
|
const rawSyncState = await db.projectHistorySyncState.findOne({
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
function _getResyncState(projectId, callback) {
|
|
||||||
db.projectHistorySyncState.findOne(
|
|
||||||
{
|
|
||||||
project_id: new ObjectId(projectId.toString()),
|
project_id: new ObjectId(projectId.toString()),
|
||||||
},
|
})
|
||||||
function (error, rawSyncState) {
|
|
||||||
if (error) {
|
|
||||||
return callback(OError.tag(error))
|
|
||||||
}
|
|
||||||
const syncState = SyncState.fromRaw(projectId, rawSyncState)
|
const syncState = SyncState.fromRaw(projectId, rawSyncState)
|
||||||
callback(null, syncState)
|
return syncState
|
||||||
}
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function setResyncState(projectId, syncState, callback) {
|
async function setResyncState(projectId, syncState) {
|
||||||
|
// skip if syncState is null (i.e. unchanged)
|
||||||
if (syncState == null) {
|
if (syncState == null) {
|
||||||
return callback()
|
return
|
||||||
} // skip if syncState is null (i.e. unchanged)
|
}
|
||||||
const update = {
|
const update = {
|
||||||
$set: syncState.toRaw(),
|
$set: syncState.toRaw(),
|
||||||
$push: {
|
$push: {
|
||||||
|
@ -158,64 +115,46 @@ export function setResyncState(projectId, syncState, callback) {
|
||||||
},
|
},
|
||||||
$currentDate: { lastUpdated: true },
|
$currentDate: { lastUpdated: true },
|
||||||
}
|
}
|
||||||
|
|
||||||
// handle different cases
|
// handle different cases
|
||||||
if (syncState.isSyncOngoing()) {
|
if (syncState.isSyncOngoing()) {
|
||||||
// starting a new sync
|
// starting a new sync; prevent the entry expiring while sync is in ongoing
|
||||||
update.$inc = { resyncCount: 1 }
|
update.$inc = { resyncCount: 1 }
|
||||||
update.$unset = { expiresAt: true } // prevent the entry expiring while sync is in ongoing
|
update.$unset = { expiresAt: true }
|
||||||
} else {
|
} else {
|
||||||
// successful completion of existing sync
|
// successful completion of existing sync; set the entry to expire in the
|
||||||
|
// future
|
||||||
update.$set.expiresAt = new Date(
|
update.$set.expiresAt = new Date(
|
||||||
Date.now() + EXPIRE_RESYNC_HISTORY_INTERVAL_MS
|
Date.now() + EXPIRE_RESYNC_HISTORY_INTERVAL_MS
|
||||||
) // set the entry to expire in the future
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// apply the update
|
// apply the update
|
||||||
db.projectHistorySyncState.updateOne(
|
await db.projectHistorySyncState.updateOne(
|
||||||
{
|
{ project_id: new ObjectId(projectId) },
|
||||||
project_id: new ObjectId(projectId),
|
|
||||||
},
|
|
||||||
update,
|
update,
|
||||||
{
|
{ upsert: true }
|
||||||
upsert: true,
|
|
||||||
},
|
|
||||||
callback
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
export function clearResyncState(projectId, callback) {
|
async function clearResyncState(projectId) {
|
||||||
db.projectHistorySyncState.deleteOne(
|
await db.projectHistorySyncState.deleteOne({
|
||||||
{
|
|
||||||
project_id: new ObjectId(projectId.toString()),
|
project_id: new ObjectId(projectId.toString()),
|
||||||
},
|
})
|
||||||
callback
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
export function skipUpdatesDuringSync(projectId, updates, callback) {
|
|
||||||
_getResyncState(projectId, function (error, syncState) {
|
|
||||||
if (error) {
|
|
||||||
return callback(OError.tag(error))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function skipUpdatesDuringSync(projectId, updates) {
|
||||||
|
const syncState = await _getResyncState(projectId)
|
||||||
if (!syncState.isSyncOngoing()) {
|
if (!syncState.isSyncOngoing()) {
|
||||||
logger.debug({ projectId }, 'not skipping updates: no resync in progress')
|
logger.debug({ projectId }, 'not skipping updates: no resync in progress')
|
||||||
return callback(null, updates) // don't return synsState when unchanged
|
// don't return syncState when unchanged
|
||||||
|
return { updates, syncState: null }
|
||||||
}
|
}
|
||||||
|
|
||||||
const filteredUpdates = []
|
const filteredUpdates = []
|
||||||
|
|
||||||
for (const update of updates) {
|
for (const update of updates) {
|
||||||
try {
|
|
||||||
syncState.updateState(update)
|
syncState.updateState(update)
|
||||||
} catch (error1) {
|
|
||||||
error = OError.tag(error1)
|
|
||||||
if (error instanceof SyncError) {
|
|
||||||
return callback(error)
|
|
||||||
} else {
|
|
||||||
throw error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const shouldSkipUpdate = syncState.shouldSkipUpdate(update)
|
const shouldSkipUpdate = syncState.shouldSkipUpdate(update)
|
||||||
if (!shouldSkipUpdate) {
|
if (!shouldSkipUpdate) {
|
||||||
filteredUpdates.push(update)
|
filteredUpdates.push(update)
|
||||||
|
@ -223,77 +162,56 @@ export function skipUpdatesDuringSync(projectId, updates, callback) {
|
||||||
logger.debug({ projectId, update }, 'skipping update due to resync')
|
logger.debug({ projectId, update }, 'skipping update due to resync')
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return { updates: filteredUpdates, syncState }
|
||||||
callback(null, filteredUpdates, syncState)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function expandSyncUpdates(
|
async function expandSyncUpdates(
|
||||||
projectId,
|
projectId,
|
||||||
projectHistoryId,
|
projectHistoryId,
|
||||||
updates,
|
updates,
|
||||||
extendLock,
|
extendLock
|
||||||
callback
|
|
||||||
) {
|
) {
|
||||||
const areSyncUpdatesQueued =
|
const areSyncUpdatesQueued =
|
||||||
_.some(updates, 'resyncProjectStructure') ||
|
_.some(updates, 'resyncProjectStructure') ||
|
||||||
_.some(updates, 'resyncDocContent')
|
_.some(updates, 'resyncDocContent')
|
||||||
if (!areSyncUpdatesQueued) {
|
if (!areSyncUpdatesQueued) {
|
||||||
logger.debug({ projectId }, 'no resync updates to expand')
|
logger.debug({ projectId }, 'no resync updates to expand')
|
||||||
return callback(null, updates)
|
return updates
|
||||||
}
|
}
|
||||||
|
|
||||||
_getResyncState(projectId, (error, syncState) => {
|
const syncState = await _getResyncState(projectId)
|
||||||
if (error) {
|
|
||||||
return callback(OError.tag(error))
|
|
||||||
}
|
|
||||||
|
|
||||||
// compute the current snapshot from the most recent chunk
|
// compute the current snapshot from the most recent chunk
|
||||||
SnapshotManager.getLatestSnapshot(
|
const snapshotFiles = await SnapshotManager.promises.getLatestSnapshot(
|
||||||
projectId,
|
projectId,
|
||||||
projectHistoryId,
|
projectHistoryId
|
||||||
(error, snapshotFiles) => {
|
)
|
||||||
if (error) {
|
|
||||||
return callback(OError.tag(error))
|
|
||||||
}
|
|
||||||
// check if snapshot files are valid
|
// check if snapshot files are valid
|
||||||
const invalidFiles = _.pickBy(
|
const invalidFiles = _.pickBy(
|
||||||
snapshotFiles,
|
snapshotFiles,
|
||||||
(v, k) => v == null || typeof v.isEditable !== 'function'
|
(v, k) => v == null || typeof v.isEditable !== 'function'
|
||||||
)
|
)
|
||||||
if (_.size(invalidFiles) > 0) {
|
if (_.size(invalidFiles) > 0) {
|
||||||
return callback(
|
throw new SyncError('file is missing isEditable method', {
|
||||||
new SyncError('file is missing isEditable method', {
|
|
||||||
projectId,
|
projectId,
|
||||||
invalidFiles,
|
invalidFiles,
|
||||||
})
|
})
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const expander = new SyncUpdateExpander(
|
const expander = new SyncUpdateExpander(
|
||||||
projectId,
|
projectId,
|
||||||
snapshotFiles,
|
snapshotFiles,
|
||||||
syncState.origin
|
syncState.origin
|
||||||
)
|
)
|
||||||
|
|
||||||
// expand updates asynchronously to avoid blocking
|
// expand updates asynchronously to avoid blocking
|
||||||
const handleUpdate = (
|
for (const update of updates) {
|
||||||
update,
|
await expander.expandUpdate(update)
|
||||||
cb // n.b. lock manager calls cb asynchronously
|
await extendLock()
|
||||||
) =>
|
|
||||||
expander.expandUpdate(update, error => {
|
|
||||||
if (error) {
|
|
||||||
return cb(OError.tag(error))
|
|
||||||
}
|
}
|
||||||
extendLock(cb)
|
|
||||||
})
|
return expander.getExpandedUpdates()
|
||||||
async.eachSeries(updates, handleUpdate, error => {
|
|
||||||
if (error) {
|
|
||||||
return callback(OError.tag(error))
|
|
||||||
}
|
|
||||||
callback(null, expander.getExpandedUpdates())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
class SyncState {
|
class SyncState {
|
||||||
|
@ -456,7 +374,7 @@ class SyncUpdateExpander {
|
||||||
return !matchedExpectedFile
|
return !matchedExpectedFile
|
||||||
}
|
}
|
||||||
|
|
||||||
expandUpdate(update, cb) {
|
async expandUpdate(update) {
|
||||||
if (update.resyncProjectStructure != null) {
|
if (update.resyncProjectStructure != null) {
|
||||||
logger.debug(
|
logger.debug(
|
||||||
{ projectId: this.projectId, update },
|
{ projectId: this.projectId, update },
|
||||||
|
@ -523,16 +441,14 @@ class SyncUpdateExpander {
|
||||||
expectedBinaryFiles,
|
expectedBinaryFiles,
|
||||||
persistedBinaryFiles
|
persistedBinaryFiles
|
||||||
)
|
)
|
||||||
cb()
|
|
||||||
} else if (update.resyncDocContent != null) {
|
} else if (update.resyncDocContent != null) {
|
||||||
logger.debug(
|
logger.debug(
|
||||||
{ projectId: this.projectId, update },
|
{ projectId: this.projectId, update },
|
||||||
'expanding resyncDocContent update'
|
'expanding resyncDocContent update'
|
||||||
)
|
)
|
||||||
this.queueTextOpForOutOfSyncContents(update, cb)
|
await this.queueTextOpForOutOfSyncContents(update)
|
||||||
} else {
|
} else {
|
||||||
this.expandedUpdates.push(update)
|
this.expandedUpdates.push(update)
|
||||||
cb()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -637,13 +553,13 @@ class SyncUpdateExpander {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
queueTextOpForOutOfSyncContents(update, cb) {
|
async queueTextOpForOutOfSyncContents(update) {
|
||||||
const pathname = UpdateTranslator._convertPathname(update.path)
|
const pathname = UpdateTranslator._convertPathname(update.path)
|
||||||
const snapshotFile = this.files[pathname]
|
const snapshotFile = this.files[pathname]
|
||||||
const expectedFile = update.resyncDocContent
|
const expectedFile = update.resyncDocContent
|
||||||
|
|
||||||
if (!snapshotFile) {
|
if (!snapshotFile) {
|
||||||
return cb(new OError('unrecognised file: not in snapshot'))
|
throw new OError('unrecognised file: not in snapshot')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Compare hashes to see if the persisted file matches the expected content.
|
// Compare hashes to see if the persisted file matches the expected content.
|
||||||
|
@ -664,7 +580,7 @@ class SyncUpdateExpander {
|
||||||
{ projectId: this.projectId, persistedHash, expectedHash },
|
{ projectId: this.projectId, persistedHash, expectedHash },
|
||||||
'skipping diff because hashes match and persisted file has no ops'
|
'skipping diff because hashes match and persisted file has no ops'
|
||||||
)
|
)
|
||||||
return cb()
|
return
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
logger.debug('cannot compare hashes, will retrieve content')
|
logger.debug('cannot compare hashes, will retrieve content')
|
||||||
|
@ -672,28 +588,40 @@ class SyncUpdateExpander {
|
||||||
|
|
||||||
const expectedContent = update.resyncDocContent.content
|
const expectedContent = update.resyncDocContent.content
|
||||||
|
|
||||||
const computeDiff = (persistedContent, cb) => {
|
let persistedContent
|
||||||
|
// compute the difference between the expected and persisted content
|
||||||
|
if (snapshotFile.load != null) {
|
||||||
|
const historyId = await WebApiManager.promises.getHistoryId(
|
||||||
|
this.projectId
|
||||||
|
)
|
||||||
|
const file = await snapshotFile.load(
|
||||||
|
'eager',
|
||||||
|
HistoryStoreManager.getBlobStore(historyId)
|
||||||
|
)
|
||||||
|
persistedContent = file.getContent()
|
||||||
|
} else if (snapshotFile.content != null) {
|
||||||
|
// use dummy content from queueAddOpsForMissingFiles for added missing files
|
||||||
|
persistedContent = snapshotFile.content
|
||||||
|
} else {
|
||||||
|
throw new OError('unrecognised file')
|
||||||
|
}
|
||||||
|
|
||||||
let op
|
let op
|
||||||
logger.debug(
|
logger.debug(
|
||||||
{ projectId: this.projectId, persistedContent, expectedContent },
|
{ projectId: this.projectId, persistedContent, expectedContent },
|
||||||
'diffing doc contents'
|
'diffing doc contents'
|
||||||
)
|
)
|
||||||
try {
|
try {
|
||||||
op = UpdateCompressor.diffAsShareJsOps(
|
op = UpdateCompressor.diffAsShareJsOps(persistedContent, expectedContent)
|
||||||
persistedContent,
|
|
||||||
expectedContent
|
|
||||||
)
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
return cb(
|
throw OError.tag(error, 'error from diffAsShareJsOps', {
|
||||||
OError.tag(error, 'error from diffAsShareJsOps', {
|
|
||||||
projectId: this.projectId,
|
projectId: this.projectId,
|
||||||
persistedContent,
|
persistedContent,
|
||||||
expectedContent,
|
expectedContent,
|
||||||
})
|
})
|
||||||
)
|
|
||||||
}
|
}
|
||||||
if (op.length === 0) {
|
if (op.length === 0) {
|
||||||
return cb()
|
return
|
||||||
}
|
}
|
||||||
update = {
|
update = {
|
||||||
doc: update.doc,
|
doc: update.doc,
|
||||||
|
@ -714,37 +642,50 @@ class SyncUpdateExpander {
|
||||||
Metrics.inc('project_history_resync_operation', 1, {
|
Metrics.inc('project_history_resync_operation', 1, {
|
||||||
status: 'update text file contents',
|
status: 'update text file contents',
|
||||||
})
|
})
|
||||||
cb()
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// compute the difference between the expected and persisted content
|
// EXPORTS
|
||||||
if (snapshotFile.load != null) {
|
|
||||||
WebApiManager.getHistoryId(this.projectId, (err, historyId) => {
|
const startResyncCb = callbackify(startResync)
|
||||||
if (err) {
|
const startHardResyncCb = callbackify(startHardResync)
|
||||||
return cb(OError.tag(err))
|
const setResyncStateCb = callbackify(setResyncState)
|
||||||
}
|
const clearResyncStateCb = callbackify(clearResyncState)
|
||||||
const loadFile = snapshotFile.load(
|
const skipUpdatesDuringSyncCb = callbackifyMultiResult(skipUpdatesDuringSync, [
|
||||||
'eager',
|
'updates',
|
||||||
HistoryStoreManager.getBlobStore(historyId)
|
'syncState',
|
||||||
)
|
])
|
||||||
loadFile
|
const expandSyncUpdatesCb = (
|
||||||
.then(file => computeDiff(file.getContent(), cb))
|
projectId,
|
||||||
.catch(err => cb(err)) // error loading file or computing diff
|
projectHistoryId,
|
||||||
|
updates,
|
||||||
|
extendLock,
|
||||||
|
callback
|
||||||
|
) => {
|
||||||
|
const extendLockPromises = promisify(extendLock)
|
||||||
|
expandSyncUpdates(projectId, projectHistoryId, updates, extendLockPromises)
|
||||||
|
.then(result => {
|
||||||
|
callback(null, result)
|
||||||
|
})
|
||||||
|
.catch(err => {
|
||||||
|
callback(err)
|
||||||
})
|
})
|
||||||
} else if (snapshotFile.content != null) {
|
|
||||||
// use dummy content from queueAddOpsForMissingFiles for added missing files
|
|
||||||
computeDiff(snapshotFile.content, cb)
|
|
||||||
} else {
|
|
||||||
cb(new OError('unrecognised file'))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export {
|
||||||
|
startResyncCb as startResync,
|
||||||
|
startHardResyncCb as startHardResync,
|
||||||
|
setResyncStateCb as setResyncState,
|
||||||
|
clearResyncStateCb as clearResyncState,
|
||||||
|
skipUpdatesDuringSyncCb as skipUpdatesDuringSync,
|
||||||
|
expandSyncUpdatesCb as expandSyncUpdates,
|
||||||
}
|
}
|
||||||
|
|
||||||
export const promises = {
|
export const promises = {
|
||||||
startResync: promisify(startResync),
|
startResync,
|
||||||
startHardResync: promisify(startHardResync),
|
startHardResync,
|
||||||
setResyncState: promisify(setResyncState),
|
setResyncState,
|
||||||
clearResyncState: promisify(clearResyncState),
|
clearResyncState,
|
||||||
skipUpdatesDuringSync: promisify(skipUpdatesDuringSync),
|
skipUpdatesDuringSync,
|
||||||
expandSyncUpdates: promisify(expandSyncUpdates),
|
expandSyncUpdates,
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,6 +22,7 @@
|
||||||
"@overleaf/logger": "*",
|
"@overleaf/logger": "*",
|
||||||
"@overleaf/metrics": "*",
|
"@overleaf/metrics": "*",
|
||||||
"@overleaf/o-error": "*",
|
"@overleaf/o-error": "*",
|
||||||
|
"@overleaf/promise-utils": "*",
|
||||||
"@overleaf/redis-wrapper": "*",
|
"@overleaf/redis-wrapper": "*",
|
||||||
"@overleaf/settings": "*",
|
"@overleaf/settings": "*",
|
||||||
"async": "^3.2.2",
|
"async": "^3.2.2",
|
||||||
|
|
|
@ -162,7 +162,7 @@ describe('Syncing with web and doc-updater', function () {
|
||||||
],
|
],
|
||||||
error => {
|
error => {
|
||||||
if (error) {
|
if (error) {
|
||||||
throw error
|
return done(error)
|
||||||
}
|
}
|
||||||
assert(
|
assert(
|
||||||
createBlob.isDone(),
|
createBlob.isDone(),
|
||||||
|
|
|
@ -39,47 +39,65 @@ describe('SyncManager', function () {
|
||||||
this.syncState = { origin: { kind: 'history-resync' } }
|
this.syncState = { origin: { kind: 'history-resync' } }
|
||||||
this.db = {
|
this.db = {
|
||||||
projectHistorySyncState: {
|
projectHistorySyncState: {
|
||||||
findOne: sinon.stub().yields(null, this.syncState),
|
findOne: sinon.stub().resolves(this.syncState),
|
||||||
updateOne: sinon.stub().yields(),
|
updateOne: sinon.stub().resolves(),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
this.callback = sinon.stub()
|
this.extendLock = sinon.stub().resolves()
|
||||||
this.extendLock = sinon.stub().yields()
|
|
||||||
this.LockManager = {
|
this.LockManager = {
|
||||||
runWithLock: sinon.spy((key, runner, callback) => {
|
promises: {
|
||||||
runner(this.extendLock, callback)
|
runWithLock: sinon.stub().callsFake(async (key, runner) => {
|
||||||
|
await runner(this.extendLock)
|
||||||
}),
|
}),
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
this.UpdateCompressor = {
|
this.UpdateCompressor = {
|
||||||
diffAsShareJsOps: sinon.stub(),
|
diffAsShareJsOps: sinon.stub(),
|
||||||
}
|
}
|
||||||
|
|
||||||
this.UpdateTranslator = {
|
this.UpdateTranslator = {
|
||||||
isTextUpdate: sinon.stub(),
|
isTextUpdate: sinon.stub(),
|
||||||
_convertPathname: sinon.stub(),
|
_convertPathname: sinon.stub(),
|
||||||
}
|
}
|
||||||
|
|
||||||
this.WebApiManager = {
|
this.WebApiManager = {
|
||||||
|
promises: {
|
||||||
getHistoryId: sinon.stub(),
|
getHistoryId: sinon.stub(),
|
||||||
requestResync: sinon.stub().yields(),
|
requestResync: sinon.stub().resolves(),
|
||||||
|
},
|
||||||
}
|
}
|
||||||
this.WebApiManager.getHistoryId
|
this.WebApiManager.promises.getHistoryId
|
||||||
.withArgs(this.projectId)
|
.withArgs(this.projectId)
|
||||||
.yields(null, this.historyId)
|
.resolves(this.historyId)
|
||||||
|
|
||||||
this.ErrorRecorder = {
|
this.ErrorRecorder = {
|
||||||
record: sinon.stub().yields(),
|
promises: {
|
||||||
recordSyncStart: sinon.stub().yields(),
|
record: sinon.stub().resolves(),
|
||||||
|
recordSyncStart: sinon.stub().resolves(),
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
this.RedisManager = {}
|
this.RedisManager = {}
|
||||||
|
|
||||||
this.SnapshotManager = {
|
this.SnapshotManager = {
|
||||||
|
promises: {
|
||||||
getLatestSnapshot: sinon.stub(),
|
getLatestSnapshot: sinon.stub(),
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
this.HistoryStoreManager = {
|
this.HistoryStoreManager = {
|
||||||
getBlobStore: sinon.stub(),
|
getBlobStore: sinon.stub(),
|
||||||
_getBlobHashFromString: sinon.stub().returns('random-hash'),
|
_getBlobHashFromString: sinon.stub().returns('random-hash'),
|
||||||
}
|
}
|
||||||
|
|
||||||
this.HashManager = {
|
this.HashManager = {
|
||||||
_getBlobHashFromString: sinon.stub(),
|
_getBlobHashFromString: sinon.stub(),
|
||||||
}
|
}
|
||||||
|
|
||||||
this.Metrics = { inc: sinon.stub() }
|
this.Metrics = { inc: sinon.stub() }
|
||||||
|
|
||||||
this.Settings = {
|
this.Settings = {
|
||||||
redis: {
|
redis: {
|
||||||
lock: {
|
lock: {
|
||||||
|
@ -91,6 +109,7 @@ describe('SyncManager', function () {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
this.SyncManager = await esmock(MODULE_PATH, {
|
this.SyncManager = await esmock(MODULE_PATH, {
|
||||||
'../../../../app/js/LockManager.js': this.LockManager,
|
'../../../../app/js/LockManager.js': this.LockManager,
|
||||||
'../../../../app/js/UpdateCompressor.js': this.UpdateCompressor,
|
'../../../../app/js/UpdateCompressor.js': this.UpdateCompressor,
|
||||||
|
@ -113,13 +132,13 @@ describe('SyncManager', function () {
|
||||||
|
|
||||||
describe('startResync', function () {
|
describe('startResync', function () {
|
||||||
describe('if a sync is not in progress', function () {
|
describe('if a sync is not in progress', function () {
|
||||||
beforeEach(function () {
|
beforeEach(async function () {
|
||||||
this.db.projectHistorySyncState.findOne.yields(null, {})
|
this.db.projectHistorySyncState.findOne.resolves({})
|
||||||
this.SyncManager.startResync(this.projectId, this.callback)
|
await this.SyncManager.promises.startResync(this.projectId)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('takes the project lock', function () {
|
it('takes the project lock', function () {
|
||||||
expect(this.LockManager.runWithLock).to.have.been.calledWith(
|
expect(this.LockManager.promises.runWithLock).to.have.been.calledWith(
|
||||||
`ProjectHistoryLock:${this.projectId}`
|
`ProjectHistoryLock:${this.projectId}`
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
@ -131,9 +150,9 @@ describe('SyncManager', function () {
|
||||||
})
|
})
|
||||||
|
|
||||||
it('requests a resync from web', function () {
|
it('requests a resync from web', function () {
|
||||||
expect(this.WebApiManager.requestResync).to.have.been.calledWith(
|
expect(
|
||||||
this.projectId
|
this.WebApiManager.promises.requestResync
|
||||||
)
|
).to.have.been.calledWith(this.projectId)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('sets the sync state in mongo and prevents it expiring', function () {
|
it('sets the sync state in mongo and prevents it expiring', function () {
|
||||||
|
@ -158,37 +177,31 @@ describe('SyncManager', function () {
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('calls the callback', function () {
|
|
||||||
expect(this.callback).to.have.been.called
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('if project structure sync is in progress', function () {
|
describe('if project structure sync is in progress', function () {
|
||||||
beforeEach(function () {
|
beforeEach(function () {
|
||||||
const syncState = { resyncProjectStructure: true }
|
const syncState = { resyncProjectStructure: true }
|
||||||
this.db.projectHistorySyncState.findOne.yields(null, syncState)
|
this.db.projectHistorySyncState.findOne.resolves(syncState)
|
||||||
this.SyncManager.startResync(this.projectId, this.callback)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('returns an error if already syncing', function () {
|
it('returns an error if already syncing', async function () {
|
||||||
expect(this.callback).to.have.been.calledWithMatch({
|
await expect(
|
||||||
message: 'sync ongoing',
|
this.SyncManager.promises.startResync(this.projectId)
|
||||||
})
|
).to.be.rejectedWith('sync ongoing')
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('if doc content sync in is progress', function () {
|
describe('if doc content sync in is progress', function () {
|
||||||
beforeEach(function () {
|
beforeEach(async function () {
|
||||||
const syncState = { resyncDocContents: ['/foo.tex'] }
|
const syncState = { resyncDocContents: ['/foo.tex'] }
|
||||||
this.db.projectHistorySyncState.findOne.yields(null, syncState)
|
this.db.projectHistorySyncState.findOne.resolves(syncState)
|
||||||
this.SyncManager.startResync(this.projectId, this.callback)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('returns an error if already syncing', function () {
|
it('returns an error if already syncing', async function () {
|
||||||
expect(this.callback).to.have.been.calledWithMatch({
|
await expect(
|
||||||
message: 'sync ongoing',
|
this.SyncManager.promises.startResync(this.projectId)
|
||||||
})
|
).to.be.rejectedWith('sync ongoing')
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
@ -208,32 +221,25 @@ describe('SyncManager', function () {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
it('sets the sync state in mongo and prevents it expiring', function (done) {
|
it('sets the sync state in mongo and prevents it expiring', async function () {
|
||||||
// SyncState is a private class of SyncManager
|
// SyncState is a private class of SyncManager
|
||||||
// we know the interface however:
|
// we know the interface however:
|
||||||
this.SyncManager.setResyncState(
|
await this.SyncManager.promises.setResyncState(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.syncState,
|
this.syncState
|
||||||
error => {
|
)
|
||||||
expect(error).to.be.undefined
|
|
||||||
expect(
|
expect(
|
||||||
this.db.projectHistorySyncState.updateOne
|
this.db.projectHistorySyncState.updateOne
|
||||||
).to.have.been.calledWith(
|
).to.have.been.calledWith(
|
||||||
{
|
{ project_id: new ObjectId(this.projectId) },
|
||||||
project_id: new ObjectId(this.projectId),
|
|
||||||
},
|
|
||||||
sinon.match({
|
sinon.match({
|
||||||
$set: this.syncState.toRaw(),
|
$set: this.syncState.toRaw(),
|
||||||
$currentDate: { lastUpdated: true },
|
$currentDate: { lastUpdated: true },
|
||||||
$inc: { resyncCount: 1 },
|
$inc: { resyncCount: 1 },
|
||||||
$unset: { expiresAt: true },
|
$unset: { expiresAt: true },
|
||||||
}),
|
}),
|
||||||
{
|
{ upsert: true }
|
||||||
upsert: true,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
done()
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
@ -252,71 +258,58 @@ describe('SyncManager', function () {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
it('sets the sync state entry in mongo to expire', function (done) {
|
it('sets the sync state entry in mongo to expire', async function () {
|
||||||
this.SyncManager.setResyncState(
|
await this.SyncManager.promises.setResyncState(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.syncState,
|
this.syncState
|
||||||
error => {
|
)
|
||||||
expect(error).to.be.undefined
|
|
||||||
expect(
|
expect(
|
||||||
this.db.projectHistorySyncState.updateOne
|
this.db.projectHistorySyncState.updateOne
|
||||||
).to.have.been.calledWith(
|
).to.have.been.calledWith(
|
||||||
{
|
{ project_id: new ObjectId(this.projectId) },
|
||||||
project_id: new ObjectId(this.projectId),
|
|
||||||
},
|
|
||||||
sinon.match({
|
sinon.match({
|
||||||
$set: {
|
$set: {
|
||||||
resyncProjectStructure: false,
|
resyncProjectStructure: false,
|
||||||
resyncDocContents: [],
|
resyncDocContents: [],
|
||||||
origin: { kind: 'history-resync' },
|
origin: { kind: 'history-resync' },
|
||||||
expiresAt: new Date(
|
expiresAt: new Date(this.now.getTime() + 90 * 24 * 3600 * 1000),
|
||||||
this.now.getTime() + 90 * 24 * 3600 * 1000
|
|
||||||
),
|
|
||||||
},
|
},
|
||||||
$currentDate: { lastUpdated: true },
|
$currentDate: { lastUpdated: true },
|
||||||
}),
|
}),
|
||||||
{
|
{ upsert: true }
|
||||||
upsert: true,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
done()
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('when the new sync state is null', function () {
|
describe('when the new sync state is null', function () {
|
||||||
it('does not update the sync state in mongo', function (done) {
|
it('does not update the sync state in mongo', async function () {
|
||||||
// SyncState is a private class of SyncManager
|
// SyncState is a private class of SyncManager
|
||||||
// we know the interface however:
|
// we know the interface however:
|
||||||
this.SyncManager.setResyncState(this.projectId, null, error => {
|
await this.SyncManager.promises.setResyncState(this.projectId, null)
|
||||||
expect(error).to.be.undefined
|
|
||||||
expect(this.db.projectHistorySyncState.updateOne).to.not.have.been
|
expect(this.db.projectHistorySyncState.updateOne).to.not.have.been
|
||||||
.called
|
.called
|
||||||
done()
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('skipUpdatesDuringSync', function () {
|
describe('skipUpdatesDuringSync', function () {
|
||||||
describe('if a sync is not in progress', function () {
|
describe('if a sync is not in progress', function () {
|
||||||
beforeEach(function () {
|
beforeEach(async function () {
|
||||||
this.db.projectHistorySyncState.findOne.yields(null, {})
|
this.db.projectHistorySyncState.findOne.resolves({})
|
||||||
this.updates = ['some', 'mock', 'updates']
|
this.updates = ['some', 'mock', 'updates']
|
||||||
this.SyncManager.skipUpdatesDuringSync(
|
this.result = await this.SyncManager.promises.skipUpdatesDuringSync(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.updates,
|
this.updates
|
||||||
this.callback
|
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('returns all updates', function () {
|
it('returns all updates', function () {
|
||||||
expect(this.callback).to.have.been.calledWith(null, this.updates)
|
expect(this.result.updates).to.deep.equal(this.updates)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should not return any newSyncState', function () {
|
it('should not return any newSyncState', function () {
|
||||||
expect(this.callback).to.have.been.calledWithExactly(null, this.updates)
|
expect(this.result.syncState).to.be.null
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -364,39 +357,37 @@ describe('SyncManager', function () {
|
||||||
resyncDocContents: [],
|
resyncDocContents: [],
|
||||||
origin: { kind: 'history-resync' },
|
origin: { kind: 'history-resync' },
|
||||||
}
|
}
|
||||||
this.db.projectHistorySyncState.findOne.yields(null, syncState)
|
this.db.projectHistorySyncState.findOne.resolves(syncState)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('remove updates before a project structure sync update', function () {
|
it('remove updates before a project structure sync update', async function () {
|
||||||
const updates = [
|
const updates = [
|
||||||
this.renameUpdate,
|
this.renameUpdate,
|
||||||
this.textUpdate,
|
this.textUpdate,
|
||||||
this.projectStructureSyncUpdate,
|
this.projectStructureSyncUpdate,
|
||||||
]
|
]
|
||||||
this.SyncManager.skipUpdatesDuringSync(
|
const { updates: filteredUpdates, syncState } =
|
||||||
|
await this.SyncManager.promises.skipUpdatesDuringSync(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
updates,
|
updates
|
||||||
(error, filteredUpdates, syncState) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(filteredUpdates).to.deep.equal([
|
expect(filteredUpdates).to.deep.equal([this.projectStructureSyncUpdate])
|
||||||
this.projectStructureSyncUpdate,
|
|
||||||
])
|
|
||||||
expect(syncState.toRaw()).to.deep.equal({
|
expect(syncState.toRaw()).to.deep.equal({
|
||||||
resyncProjectStructure: false,
|
resyncProjectStructure: false,
|
||||||
resyncDocContents: ['new.tex'],
|
resyncDocContents: ['new.tex'],
|
||||||
origin: { kind: 'history-resync' },
|
origin: { kind: 'history-resync' },
|
||||||
})
|
})
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('allow project structure updates after project structure sync update', function () {
|
it('allow project structure updates after project structure sync update', async function () {
|
||||||
const updates = [this.projectStructureSyncUpdate, this.renameUpdate]
|
const updates = [this.projectStructureSyncUpdate, this.renameUpdate]
|
||||||
this.SyncManager.skipUpdatesDuringSync(
|
const { updates: filteredUpdates, syncState } =
|
||||||
|
await this.SyncManager.promises.skipUpdatesDuringSync(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
updates,
|
updates
|
||||||
(error, filteredUpdates, syncState) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(filteredUpdates).to.deep.equal([
|
expect(filteredUpdates).to.deep.equal([
|
||||||
this.projectStructureSyncUpdate,
|
this.projectStructureSyncUpdate,
|
||||||
this.renameUpdate,
|
this.renameUpdate,
|
||||||
|
@ -406,21 +397,20 @@ describe('SyncManager', function () {
|
||||||
resyncDocContents: ['new.tex'],
|
resyncDocContents: ['new.tex'],
|
||||||
origin: { kind: 'history-resync' },
|
origin: { kind: 'history-resync' },
|
||||||
})
|
})
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('remove text updates for a doc before doc sync update', function () {
|
it('remove text updates for a doc before doc sync update', async function () {
|
||||||
const updates = [
|
const updates = [
|
||||||
this.projectStructureSyncUpdate,
|
this.projectStructureSyncUpdate,
|
||||||
this.textUpdate,
|
this.textUpdate,
|
||||||
this.docContentSyncUpdate,
|
this.docContentSyncUpdate,
|
||||||
]
|
]
|
||||||
this.SyncManager.skipUpdatesDuringSync(
|
const { updates: filteredUpdates, syncState } =
|
||||||
|
await this.SyncManager.promises.skipUpdatesDuringSync(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
updates,
|
updates
|
||||||
(error, filteredUpdates, syncState) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(filteredUpdates).to.deep.equal([
|
expect(filteredUpdates).to.deep.equal([
|
||||||
this.projectStructureSyncUpdate,
|
this.projectStructureSyncUpdate,
|
||||||
this.docContentSyncUpdate,
|
this.docContentSyncUpdate,
|
||||||
|
@ -430,21 +420,20 @@ describe('SyncManager', function () {
|
||||||
resyncDocContents: [],
|
resyncDocContents: [],
|
||||||
origin: { kind: 'history-resync' },
|
origin: { kind: 'history-resync' },
|
||||||
})
|
})
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('allow text updates for a doc after doc sync update', function () {
|
it('allow text updates for a doc after doc sync update', async function () {
|
||||||
const updates = [
|
const updates = [
|
||||||
this.projectStructureSyncUpdate,
|
this.projectStructureSyncUpdate,
|
||||||
this.docContentSyncUpdate,
|
this.docContentSyncUpdate,
|
||||||
this.textUpdate,
|
this.textUpdate,
|
||||||
]
|
]
|
||||||
this.SyncManager.skipUpdatesDuringSync(
|
const { updates: filteredUpdates, syncState } =
|
||||||
|
await this.SyncManager.promises.skipUpdatesDuringSync(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
updates,
|
updates
|
||||||
(error, filteredUpdates, syncState) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(filteredUpdates).to.deep.equal([
|
expect(filteredUpdates).to.deep.equal([
|
||||||
this.projectStructureSyncUpdate,
|
this.projectStructureSyncUpdate,
|
||||||
this.docContentSyncUpdate,
|
this.docContentSyncUpdate,
|
||||||
|
@ -455,8 +444,6 @@ describe('SyncManager', function () {
|
||||||
resyncDocContents: [],
|
resyncDocContents: [],
|
||||||
origin: { kind: 'history-resync' },
|
origin: { kind: 'history-resync' },
|
||||||
})
|
})
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
@ -493,55 +480,53 @@ describe('SyncManager', function () {
|
||||||
.returns('another.tex')
|
.returns('another.tex')
|
||||||
this.UpdateTranslator._convertPathname.withArgs('1.png').returns('1.png')
|
this.UpdateTranslator._convertPathname.withArgs('1.png').returns('1.png')
|
||||||
this.UpdateTranslator._convertPathname.withArgs('2.png').returns('2.png')
|
this.UpdateTranslator._convertPathname.withArgs('2.png').returns('2.png')
|
||||||
this.SnapshotManager.getLatestSnapshot.yields(null, this.fileMap)
|
this.SnapshotManager.promises.getLatestSnapshot.resolves(this.fileMap)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('returns updates if no sync updates are queued', function () {
|
it('returns updates if no sync updates are queued', async function () {
|
||||||
const updates = ['some', 'mock', 'updates']
|
const updates = ['some', 'mock', 'updates']
|
||||||
this.SyncManager.expandSyncUpdates(
|
const expandedUpdates = await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
|
||||||
expect(error).to.be.null
|
|
||||||
expect(expandedUpdates).to.equal(updates)
|
|
||||||
expect(this.SnapshotManager.getLatestSnapshot).to.not.have.been.called
|
|
||||||
expect(this.extendLock).to.not.have.been.called
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
expect(expandedUpdates).to.equal(updates)
|
||||||
|
expect(this.SnapshotManager.promises.getLatestSnapshot).to.not.have.been
|
||||||
|
.called
|
||||||
|
expect(this.extendLock).to.not.have.been.called
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('expanding project structure sync updates', function () {
|
describe('expanding project structure sync updates', function () {
|
||||||
it('queues nothing for expected docs and files', function () {
|
it('queues nothing for expected docs and files', async function () {
|
||||||
const updates = [
|
const updates = [
|
||||||
resyncProjectStructureUpdate(
|
resyncProjectStructureUpdate(
|
||||||
[this.persistedDoc],
|
[this.persistedDoc],
|
||||||
[this.persistedFile]
|
[this.persistedFile]
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
this.SyncManager.expandSyncUpdates(
|
const expandedUpdates =
|
||||||
|
await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(expandedUpdates).to.deep.equal([])
|
expect(expandedUpdates).to.deep.equal([])
|
||||||
expect(this.extendLock).to.have.been.called
|
expect(this.extendLock).to.have.been.called
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('queues file removes for unexpected files', function () {
|
it('queues file removes for unexpected files', async function () {
|
||||||
const updates = [resyncProjectStructureUpdate([this.persistedDoc], [])]
|
const updates = [resyncProjectStructureUpdate([this.persistedDoc], [])]
|
||||||
this.SyncManager.expandSyncUpdates(
|
const expandedUpdates =
|
||||||
|
await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(expandedUpdates).to.deep.equal([
|
expect(expandedUpdates).to.deep.equal([
|
||||||
{
|
{
|
||||||
pathname: this.persistedFile.path,
|
pathname: this.persistedFile.path,
|
||||||
|
@ -554,19 +539,18 @@ describe('SyncManager', function () {
|
||||||
},
|
},
|
||||||
])
|
])
|
||||||
expect(this.extendLock).to.have.been.called
|
expect(this.extendLock).to.have.been.called
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('queues doc removes for unexpected docs', function () {
|
it('queues doc removes for unexpected docs', async function () {
|
||||||
const updates = [resyncProjectStructureUpdate([], [this.persistedFile])]
|
const updates = [resyncProjectStructureUpdate([], [this.persistedFile])]
|
||||||
this.SyncManager.expandSyncUpdates(
|
const expandedUpdates =
|
||||||
|
await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(expandedUpdates).to.deep.equal([
|
expect(expandedUpdates).to.deep.equal([
|
||||||
{
|
{
|
||||||
pathname: this.persistedDoc.path,
|
pathname: this.persistedDoc.path,
|
||||||
|
@ -579,11 +563,9 @@ describe('SyncManager', function () {
|
||||||
},
|
},
|
||||||
])
|
])
|
||||||
expect(this.extendLock).to.have.been.called
|
expect(this.extendLock).to.have.been.called
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('queues file additions for missing files', function () {
|
it('queues file additions for missing files', async function () {
|
||||||
const newFile = {
|
const newFile = {
|
||||||
path: '2.png',
|
path: '2.png',
|
||||||
file: {},
|
file: {},
|
||||||
|
@ -595,13 +577,14 @@ describe('SyncManager', function () {
|
||||||
[this.persistedFile, newFile]
|
[this.persistedFile, newFile]
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
this.SyncManager.expandSyncUpdates(
|
const expandedUpdates =
|
||||||
|
await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(expandedUpdates).to.deep.equal([
|
expect(expandedUpdates).to.deep.equal([
|
||||||
{
|
{
|
||||||
pathname: newFile.path,
|
pathname: newFile.path,
|
||||||
|
@ -615,11 +598,9 @@ describe('SyncManager', function () {
|
||||||
},
|
},
|
||||||
])
|
])
|
||||||
expect(this.extendLock).to.have.been.called
|
expect(this.extendLock).to.have.been.called
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('queues blank doc additions for missing docs', function () {
|
it('queues blank doc additions for missing docs', async function () {
|
||||||
const newDoc = {
|
const newDoc = {
|
||||||
path: 'another.tex',
|
path: 'another.tex',
|
||||||
doc: new ObjectId().toString(),
|
doc: new ObjectId().toString(),
|
||||||
|
@ -630,13 +611,14 @@ describe('SyncManager', function () {
|
||||||
[this.persistedFile]
|
[this.persistedFile]
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
this.SyncManager.expandSyncUpdates(
|
const expandedUpdates =
|
||||||
|
await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(expandedUpdates).to.deep.equal([
|
expect(expandedUpdates).to.deep.equal([
|
||||||
{
|
{
|
||||||
pathname: newDoc.path,
|
pathname: newDoc.path,
|
||||||
|
@ -650,11 +632,9 @@ describe('SyncManager', function () {
|
||||||
},
|
},
|
||||||
])
|
])
|
||||||
expect(this.extendLock).to.have.been.called
|
expect(this.extendLock).to.have.been.called
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('removes and re-adds files if whether they are binary differs', function () {
|
it('removes and re-adds files if whether they are binary differs', async function () {
|
||||||
const fileWichWasADoc = {
|
const fileWichWasADoc = {
|
||||||
path: this.persistedDoc.path,
|
path: this.persistedDoc.path,
|
||||||
url: 'filestore/2.png',
|
url: 'filestore/2.png',
|
||||||
|
@ -667,13 +647,14 @@ describe('SyncManager', function () {
|
||||||
[fileWichWasADoc, this.persistedFile]
|
[fileWichWasADoc, this.persistedFile]
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
this.SyncManager.expandSyncUpdates(
|
const expandedUpdates =
|
||||||
|
await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(expandedUpdates).to.deep.equal([
|
expect(expandedUpdates).to.deep.equal([
|
||||||
{
|
{
|
||||||
pathname: fileWichWasADoc.path,
|
pathname: fileWichWasADoc.path,
|
||||||
|
@ -696,11 +677,9 @@ describe('SyncManager', function () {
|
||||||
},
|
},
|
||||||
])
|
])
|
||||||
expect(this.extendLock).to.have.been.called
|
expect(this.extendLock).to.have.been.called
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it("does not remove and re-add files if the expected file doesn't have a hash", function () {
|
it("does not remove and re-add files if the expected file doesn't have a hash", async function () {
|
||||||
const fileWichWasADoc = {
|
const fileWichWasADoc = {
|
||||||
path: this.persistedDoc.path,
|
path: this.persistedDoc.path,
|
||||||
url: 'filestore/2.png',
|
url: 'filestore/2.png',
|
||||||
|
@ -712,20 +691,19 @@ describe('SyncManager', function () {
|
||||||
[fileWichWasADoc, this.persistedFile]
|
[fileWichWasADoc, this.persistedFile]
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
this.SyncManager.expandSyncUpdates(
|
const expandedUpdates =
|
||||||
|
await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(expandedUpdates).to.deep.equal([])
|
expect(expandedUpdates).to.deep.equal([])
|
||||||
expect(this.extendLock).to.have.been.called
|
expect(this.extendLock).to.have.been.called
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('does not remove and re-add editable files if there is a binary file with same hash', function () {
|
it('does not remove and re-add editable files if there is a binary file with same hash', async function () {
|
||||||
const binaryFile = {
|
const binaryFile = {
|
||||||
file: Object().toString(),
|
file: Object().toString(),
|
||||||
// The paths in the resyncProjectStructureUpdate must have a leading slash ('/')
|
// The paths in the resyncProjectStructureUpdate must have a leading slash ('/')
|
||||||
|
@ -741,20 +719,19 @@ describe('SyncManager', function () {
|
||||||
const updates = [
|
const updates = [
|
||||||
resyncProjectStructureUpdate([], [binaryFile, this.persistedFile]),
|
resyncProjectStructureUpdate([], [binaryFile, this.persistedFile]),
|
||||||
]
|
]
|
||||||
this.SyncManager.expandSyncUpdates(
|
const expandedUpdates =
|
||||||
|
await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(expandedUpdates).to.deep.equal([])
|
expect(expandedUpdates).to.deep.equal([])
|
||||||
expect(this.extendLock).to.have.been.called
|
expect(this.extendLock).to.have.been.called
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('removes and re-adds binary files if they do not have same hash', function () {
|
it('removes and re-adds binary files if they do not have same hash', async function () {
|
||||||
const persistedFileWithNewContent = {
|
const persistedFileWithNewContent = {
|
||||||
_hash: 'anotherhashvalue',
|
_hash: 'anotherhashvalue',
|
||||||
hello: 'world',
|
hello: 'world',
|
||||||
|
@ -767,13 +744,14 @@ describe('SyncManager', function () {
|
||||||
[persistedFileWithNewContent]
|
[persistedFileWithNewContent]
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
this.SyncManager.expandSyncUpdates(
|
const expandedUpdates =
|
||||||
|
await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(expandedUpdates).to.deep.equal([
|
expect(expandedUpdates).to.deep.equal([
|
||||||
{
|
{
|
||||||
pathname: persistedFileWithNewContent.path,
|
pathname: persistedFileWithNewContent.path,
|
||||||
|
@ -796,11 +774,9 @@ describe('SyncManager', function () {
|
||||||
},
|
},
|
||||||
])
|
])
|
||||||
expect(this.extendLock).to.have.been.called
|
expect(this.extendLock).to.have.been.called
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('preserves other updates', function () {
|
it('preserves other updates', async function () {
|
||||||
const update = 'mock-update'
|
const update = 'mock-update'
|
||||||
const updates = [
|
const updates = [
|
||||||
update,
|
update,
|
||||||
|
@ -809,22 +785,21 @@ describe('SyncManager', function () {
|
||||||
[this.persistedFile]
|
[this.persistedFile]
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
this.SyncManager.expandSyncUpdates(
|
const expandedUpdates =
|
||||||
|
await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(expandedUpdates).to.deep.equal([update])
|
expect(expandedUpdates).to.deep.equal([update])
|
||||||
expect(this.extendLock).to.have.been.called
|
expect(this.extendLock).to.have.been.called
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('expanding doc contents sync updates', function () {
|
describe('expanding doc contents sync updates', function () {
|
||||||
it('returns errors from diffAsShareJsOps', function () {
|
it('returns errors from diffAsShareJsOps', async function () {
|
||||||
const diffError = new Error('test')
|
const diffError = new Error('test')
|
||||||
this.UpdateCompressor.diffAsShareJsOps.throws(diffError)
|
this.UpdateCompressor.diffAsShareJsOps.throws(diffError)
|
||||||
const updates = [
|
const updates = [
|
||||||
|
@ -834,33 +809,30 @@ describe('SyncManager', function () {
|
||||||
),
|
),
|
||||||
docContentSyncUpdate(this.persistedDoc, this.persistedDoc.content),
|
docContentSyncUpdate(this.persistedDoc, this.persistedDoc.content),
|
||||||
]
|
]
|
||||||
this.SyncManager.expandSyncUpdates(
|
await expect(
|
||||||
|
this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
)
|
||||||
expect(error).to.equal(diffError)
|
).to.be.rejectedWith(diffError)
|
||||||
expect(this.extendLock).to.have.been.called
|
expect(this.extendLock).to.have.been.called
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('handles an update for a file that is missing from the snapshot', function () {
|
it('handles an update for a file that is missing from the snapshot', async function () {
|
||||||
const updates = [docContentSyncUpdate('not-in-snapshot.txt', 'test')]
|
const updates = [docContentSyncUpdate('not-in-snapshot.txt', 'test')]
|
||||||
this.SyncManager.expandSyncUpdates(
|
await expect(
|
||||||
|
this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
|
||||||
expect(error).to.exist
|
|
||||||
expect(error.message).to.equal('unrecognised file: not in snapshot')
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
|
).to.be.rejectedWith('unrecognised file: not in snapshot')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('queues nothing for in docs whose contents is in sync', function () {
|
it('queues nothing for in docs whose contents is in sync', async function () {
|
||||||
const updates = [
|
const updates = [
|
||||||
resyncProjectStructureUpdate(
|
resyncProjectStructureUpdate(
|
||||||
[this.persistedDoc],
|
[this.persistedDoc],
|
||||||
|
@ -869,20 +841,19 @@ describe('SyncManager', function () {
|
||||||
docContentSyncUpdate(this.persistedDoc, this.persistedDoc.content),
|
docContentSyncUpdate(this.persistedDoc, this.persistedDoc.content),
|
||||||
]
|
]
|
||||||
this.UpdateCompressor.diffAsShareJsOps.returns([])
|
this.UpdateCompressor.diffAsShareJsOps.returns([])
|
||||||
this.SyncManager.expandSyncUpdates(
|
const expandedUpdates =
|
||||||
|
await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(expandedUpdates).to.deep.equal([])
|
expect(expandedUpdates).to.deep.equal([])
|
||||||
expect(this.extendLock).to.have.been.called
|
expect(this.extendLock).to.have.been.called
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('queues text updates for docs whose contents is out of sync', function () {
|
it('queues text updates for docs whose contents is out of sync', async function () {
|
||||||
const updates = [
|
const updates = [
|
||||||
resyncProjectStructureUpdate(
|
resyncProjectStructureUpdate(
|
||||||
[this.persistedDoc],
|
[this.persistedDoc],
|
||||||
|
@ -891,13 +862,14 @@ describe('SyncManager', function () {
|
||||||
docContentSyncUpdate(this.persistedDoc, 'a'),
|
docContentSyncUpdate(this.persistedDoc, 'a'),
|
||||||
]
|
]
|
||||||
this.UpdateCompressor.diffAsShareJsOps.returns([{ d: 'sdf', p: 1 }])
|
this.UpdateCompressor.diffAsShareJsOps.returns([{ d: 'sdf', p: 1 }])
|
||||||
this.SyncManager.expandSyncUpdates(
|
const expandedUpdates =
|
||||||
|
await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(expandedUpdates).to.deep.equal([
|
expect(expandedUpdates).to.deep.equal([
|
||||||
{
|
{
|
||||||
doc: this.persistedDoc.doc,
|
doc: this.persistedDoc.doc,
|
||||||
|
@ -912,11 +884,9 @@ describe('SyncManager', function () {
|
||||||
},
|
},
|
||||||
])
|
])
|
||||||
expect(this.extendLock).to.have.been.called
|
expect(this.extendLock).to.have.been.called
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('queues text updates for docs created by project structure sync', function () {
|
it('queues text updates for docs created by project structure sync', async function () {
|
||||||
this.UpdateCompressor.diffAsShareJsOps.returns([{ i: 'a', p: 0 }])
|
this.UpdateCompressor.diffAsShareJsOps.returns([{ i: 'a', p: 0 }])
|
||||||
const newDoc = {
|
const newDoc = {
|
||||||
path: 'another.tex',
|
path: 'another.tex',
|
||||||
|
@ -930,13 +900,14 @@ describe('SyncManager', function () {
|
||||||
),
|
),
|
||||||
docContentSyncUpdate(newDoc, newDoc.content),
|
docContentSyncUpdate(newDoc, newDoc.content),
|
||||||
]
|
]
|
||||||
this.SyncManager.expandSyncUpdates(
|
const expandedUpdates =
|
||||||
|
await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(expandedUpdates).to.deep.equal([
|
expect(expandedUpdates).to.deep.equal([
|
||||||
{
|
{
|
||||||
pathname: newDoc.path,
|
pathname: newDoc.path,
|
||||||
|
@ -961,11 +932,9 @@ describe('SyncManager', function () {
|
||||||
},
|
},
|
||||||
])
|
])
|
||||||
expect(this.extendLock).to.have.been.called
|
expect(this.extendLock).to.have.been.called
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('skips text updates for docs when hashes match', function () {
|
it('skips text updates for docs when hashes match', async function () {
|
||||||
this.fileMap['main.tex'].getHash = sinon.stub().returns('special-hash')
|
this.fileMap['main.tex'].getHash = sinon.stub().returns('special-hash')
|
||||||
this.HashManager._getBlobHashFromString.returns('special-hash')
|
this.HashManager._getBlobHashFromString.returns('special-hash')
|
||||||
const updates = [
|
const updates = [
|
||||||
|
@ -975,20 +944,19 @@ describe('SyncManager', function () {
|
||||||
),
|
),
|
||||||
docContentSyncUpdate(this.persistedDoc, 'hello'),
|
docContentSyncUpdate(this.persistedDoc, 'hello'),
|
||||||
]
|
]
|
||||||
this.SyncManager.expandSyncUpdates(
|
const expandedUpdates =
|
||||||
|
await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(expandedUpdates).to.deep.equal([])
|
expect(expandedUpdates).to.deep.equal([])
|
||||||
expect(this.extendLock).to.have.been.called
|
expect(this.extendLock).to.have.been.called
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('computes text updates for docs when hashes differ', function () {
|
it('computes text updates for docs when hashes differ', async function () {
|
||||||
this.fileMap['main.tex'].getHash = sinon.stub().returns('first-hash')
|
this.fileMap['main.tex'].getHash = sinon.stub().returns('first-hash')
|
||||||
this.HashManager._getBlobHashFromString.returns('second-hash')
|
this.HashManager._getBlobHashFromString.returns('second-hash')
|
||||||
this.UpdateCompressor.diffAsShareJsOps.returns([
|
this.UpdateCompressor.diffAsShareJsOps.returns([
|
||||||
|
@ -1001,13 +969,14 @@ describe('SyncManager', function () {
|
||||||
),
|
),
|
||||||
docContentSyncUpdate(this.persistedDoc, 'hello'),
|
docContentSyncUpdate(this.persistedDoc, 'hello'),
|
||||||
]
|
]
|
||||||
this.SyncManager.expandSyncUpdates(
|
const expandedUpdates =
|
||||||
|
await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
)
|
||||||
expect(error).to.be.null
|
|
||||||
expect(expandedUpdates).to.deep.equal([
|
expect(expandedUpdates).to.deep.equal([
|
||||||
{
|
{
|
||||||
doc: this.persistedDoc.doc,
|
doc: this.persistedDoc.doc,
|
||||||
|
@ -1022,12 +991,10 @@ describe('SyncManager', function () {
|
||||||
},
|
},
|
||||||
])
|
])
|
||||||
expect(this.extendLock).to.have.been.called
|
expect(this.extendLock).to.have.been.called
|
||||||
}
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('for docs whose contents is out of sync', function () {
|
describe('for docs whose contents is out of sync', function () {
|
||||||
beforeEach(function (done) {
|
beforeEach(async function () {
|
||||||
const updates = [
|
const updates = [
|
||||||
resyncProjectStructureUpdate(
|
resyncProjectStructureUpdate(
|
||||||
[this.persistedDoc],
|
[this.persistedDoc],
|
||||||
|
@ -1038,21 +1005,16 @@ describe('SyncManager', function () {
|
||||||
const file = { getContent: sinon.stub().returns('stored content') }
|
const file = { getContent: sinon.stub().returns('stored content') }
|
||||||
this.fileMap['main.tex'].load = sinon.stub().resolves(file)
|
this.fileMap['main.tex'].load = sinon.stub().resolves(file)
|
||||||
this.UpdateCompressor.diffAsShareJsOps.returns([{ d: 'sdf', p: 1 }])
|
this.UpdateCompressor.diffAsShareJsOps.returns([{ d: 'sdf', p: 1 }])
|
||||||
this.SyncManager.expandSyncUpdates(
|
this.expandedUpdates =
|
||||||
|
await this.SyncManager.promises.expandSyncUpdates(
|
||||||
this.projectId,
|
this.projectId,
|
||||||
this.historyId,
|
this.historyId,
|
||||||
updates,
|
updates,
|
||||||
this.extendLock,
|
this.extendLock
|
||||||
(error, expandedUpdates) => {
|
|
||||||
this.error = error
|
|
||||||
this.expandedUpdates = expandedUpdates
|
|
||||||
done()
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('loads content from the history service when needed', function () {
|
it('loads content from the history service when needed', function () {
|
||||||
expect(this.error).to.be.null
|
|
||||||
expect(this.expandedUpdates).to.deep.equal([
|
expect(this.expandedUpdates).to.deep.equal([
|
||||||
{
|
{
|
||||||
doc: this.persistedDoc.doc,
|
doc: this.persistedDoc.doc,
|
||||||
|
|
Loading…
Reference in a new issue