github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #12218 from overleaf/em-camel-case-track-changes

Browse source 
Camel case variables in track-changes

GitOrigin-RevId: 92878e2b7dfa051069e0baaf604e96f4d2e0a501
This commit is contained in:
Eric Mc Sween 2023-03-21 08:21:51 -04:00 committed by Copybot
parent 3831416c2f
commit 90a921cbe6
26 changed files with 538 additions and 640 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
services/track-changes/app/js/DiffGenerator.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
no-proto,
no-unused-vars,
*/
@ -59,10 +58,10 @@ module.exports = DiffGenerator = {
// is the case with this op, and shift p back appropriately to match
// ShareJS if so.
;({ p } = op)
const max_p = content.length - op.i.length
if (p > max_p) {
logger.warn({ max_p, p }, 'truncating position to content length')
p = max_p
const maxP = content.length - op.i.length
if (p > maxP) {
logger.warn({ maxP, p }, 'truncating position to content length')
p = maxP
op.p = p // fix out of range offsets to avoid invalid history exports in ZipManager
}

35
services/track-changes/app/js/DiffManager.js
View file

@ -1,6 +1,3 @@
/* eslint-disable
camelcase,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
@ -17,15 +14,15 @@ const DiffGenerator = require('./DiffGenerator')
const logger = require('@overleaf/logger')
module.exports = DiffManager = {
getLatestDocAndUpdates(project_id, doc_id, fromVersion, callback) {
getLatestDocAndUpdates(projectId, docId, fromVersion, callback) {
// Get updates last, since then they must be ahead and it
// might be possible to rewind to the same version as the doc.
if (callback == null) {
callback = function () {}
}
return DocumentUpdaterManager.getDocument(
project_id,
doc_id,
projectId,
docId,
function (error, content, version) {
if (error != null) {
return callback(error)
@ -35,8 +32,8 @@ module.exports = DiffManager = {
return callback(null, content, version, [])
}
return UpdatesManager.getDocUpdatesWithUserInfo(
project_id,
doc_id,
projectId,
docId,
{ from: fromVersion },
function (error, updates) {
if (error != null) {
@ -49,13 +46,13 @@ module.exports = DiffManager = {
)
},
getDiff(project_id, doc_id, fromVersion, toVersion, callback) {
getDiff(projectId, docId, fromVersion, toVersion, callback) {
if (callback == null) {
callback = function () {}
}
return DiffManager.getDocumentBeforeVersion(
project_id,
doc_id,
projectId,
docId,
fromVersion,
function (error, startingContent, updates) {
let diff
@ -85,7 +82,7 @@ module.exports = DiffManager = {
)
},
getDocumentBeforeVersion(project_id, doc_id, version, _callback) {
getDocumentBeforeVersion(projectId, docId, version, _callback) {
// Whichever order we get the latest document and the latest updates,
// there is potential for updates to be applied between them so that
// they do not return the same 'latest' versions.
@ -100,7 +97,7 @@ module.exports = DiffManager = {
if (error != null) {
if (error.retry && retries > 0) {
logger.warn(
{ error, project_id, doc_id, version, retries },
{ error, projectId, docId, version, retries },
'retrying getDocumentBeforeVersion'
)
return retry()
@ -115,25 +112,25 @@ module.exports = DiffManager = {
return (retry = function () {
retries--
return DiffManager._tryGetDocumentBeforeVersion(
project_id,
doc_id,
projectId,
docId,
version,
callback
)
})()
},
_tryGetDocumentBeforeVersion(project_id, doc_id, version, callback) {
_tryGetDocumentBeforeVersion(projectId, docId, version, callback) {
if (callback == null) {
callback = function () {}
}
logger.debug(
{ project_id, doc_id, version },
{ projectId, docId, version },
'getting document before version'
)
return DiffManager.getLatestDocAndUpdates(
project_id,
doc_id,
projectId,
docId,
version,
function (error, content, version, updates) {
let startingContent

7
services/track-changes/app/js/DocstoreManager.js
View file

@ -5,10 +5,7 @@ const Errors = require('./Errors')
function peekDocument(projectId, docId, callback) {
const url = `${Settings.apis.docstore.url}/project/${projectId}/doc/${docId}/peek`
logger.debug(
{ project_id: projectId, doc_id: docId },
'getting doc from docstore'
)
logger.debug({ projectId, docId }, 'getting doc from docstore')
request.get(url, function (error, res, body) {
if (error != null) {
return callback(error)
@ -20,7 +17,7 @@ function peekDocument(projectId, docId, callback) {
return callback(error)
}
logger.debug(
{ project_id: projectId, doc_id: docId, version: body.version },
{ projectId, docId, version: body.version },
'got doc from docstore'
)
return callback(null, body.lines.join('\n'), body.version)

41
services/track-changes/app/js/DocumentUpdaterManager.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
@ -17,12 +16,12 @@ const Settings = require('@overleaf/settings')
const Errors = require('./Errors')
module.exports = DocumentUpdaterManager = {
_requestDocument(project_id, doc_id, url, callback) {
_requestDocument(projectId, docId, url, callback) {
if (callback == null) {
callback = function () {}
}
logger.debug({ project_id, doc_id }, 'getting doc from document updater')
logger.debug({ projectId, docId }, 'getting doc from document updater')
return request.get(url, function (error, res, body) {
if (error != null) {
return callback(error)
@ -35,7 +34,7 @@ module.exports = DocumentUpdaterManager = {
return callback(error)
}
logger.debug(
{ project_id, doc_id, version: body.version },
{ projectId, docId, version: body.version },
'got doc from document updater'
)
return callback(null, body.lines.join('\n'), body.version)
@ -44,14 +43,14 @@ module.exports = DocumentUpdaterManager = {
`doc updater returned a non-success status code: ${res.statusCode}`
)
logger.error(
{ err: error, project_id, doc_id, url },
{ err: error, projectId, docId, url },
'error accessing doc updater'
)
if (res.statusCode === 404) {
return callback(
new Errors.NotFoundError('doc not found', {
projectId: project_id,
docId: doc_id,
projectId,
docId,
})
)
} else {
@ -61,29 +60,29 @@ module.exports = DocumentUpdaterManager = {
})
},
getDocument(project_id, doc_id, callback) {
const url = `${Settings.apis.documentupdater.url}/project/${project_id}/doc/${doc_id}`
DocumentUpdaterManager._requestDocument(project_id, doc_id, url, callback)
getDocument(projectId, docId, callback) {
const url = `${Settings.apis.documentupdater.url}/project/${projectId}/doc/${docId}`
DocumentUpdaterManager._requestDocument(projectId, docId, url, callback)
},
peekDocument(project_id, doc_id, callback) {
const url = `${Settings.apis.documentupdater.url}/project/${project_id}/doc/${doc_id}/peek`
DocumentUpdaterManager._requestDocument(project_id, doc_id, url, callback)
peekDocument(projectId, docId, callback) {
const url = `${Settings.apis.documentupdater.url}/project/${projectId}/doc/${docId}/peek`
DocumentUpdaterManager._requestDocument(projectId, docId, url, callback)
},
setDocument(project_id, doc_id, content, user_id, callback) {
setDocument(projectId, docId, content, userId, callback) {
if (callback == null) {
callback = function () {}
}
const url = `${Settings.apis.documentupdater.url}/project/${project_id}/doc/${doc_id}`
logger.debug({ project_id, doc_id }, 'setting doc in document updater')
const url = `${Settings.apis.documentupdater.url}/project/${projectId}/doc/${docId}`
logger.debug({ projectId, docId }, 'setting doc in document updater')
return request.post(
{
url,
json: {
lines: content.split('\n'),
source: 'restore',
user_id,
user_id: userId,
undoing: true,
},
},
@ -98,7 +97,7 @@ module.exports = DocumentUpdaterManager = {
`doc updater returned a non-success status code: ${res.statusCode}`
)
logger.error(
{ err: error, project_id, doc_id, url },
{ err: error, projectId, docId, url },
'error accessing doc updater'
)
return callback(error)
@ -111,11 +110,11 @@ module.exports = DocumentUpdaterManager = {
module.exports.promises = {
// peekDocument returns two arguments so we can't use util.promisify, which only handles a single argument, we need
// to treat this it as a special case.
peekDocument: (project_id, doc_id) => {
peekDocument: (projectId, docId) => {
return new Promise((resolve, reject) => {
DocumentUpdaterManager.peekDocument(
project_id,
doc_id,
projectId,
docId,
(err, content, version) => {
if (err) {
reject(err)

15
services/track-changes/app/js/HealthChecker.js
View file

@ -1,6 +1,3 @@
/* eslint-disable
camelcase,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
@ -19,9 +16,9 @@ const LockManager = require('./LockManager')
module.exports = {
check(callback) {
const project_id = ObjectId(settings.trackchanges.healthCheck.project_id)
const url = `http://localhost:${port}/project/${project_id}`
logger.debug({ project_id }, 'running health check')
const projectId = ObjectId(settings.trackchanges.healthCheck.project_id)
const url = `http://localhost:${port}/project/${projectId}`
logger.debug({ projectId }, 'running health check')
const jobs = [
cb =>
request.get(
@ -29,7 +26,7 @@ module.exports = {
function (err, res, body) {
if (err != null) {
logger.err(
{ err, project_id },
{ err, projectId },
'error checking lock for health check'
)
return cb(err)
@ -47,7 +44,7 @@ module.exports = {
{ url: `${url}/flush`, timeout: 10000 },
function (err, res, body) {
if (err != null) {
logger.err({ err, project_id }, 'error flushing for health check')
logger.err({ err, projectId }, 'error flushing for health check')
return cb(err)
} else if ((res != null ? res.statusCode : undefined) !== 204) {
return cb(
@ -64,7 +61,7 @@ module.exports = {
function (err, res, body) {
if (err != null) {
logger.err(
{ err, project_id },
{ err, projectId },
'error getting updates for health check'
)
return cb(err)

81
services/track-changes/app/js/HttpController.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
@ -28,12 +27,12 @@ module.exports = HttpController = {
if (next == null) {
next = function () {}
}
const { doc_id } = req.params
const { project_id } = req.params
logger.debug({ project_id, doc_id }, 'compressing doc history')
const { doc_id: docId } = req.params
const { project_id: projectId } = req.params
logger.debug({ projectId, docId }, 'compressing doc history')
return UpdatesManager.processUncompressedUpdatesWithLock(
project_id,
doc_id,
projectId,
docId,
function (error) {
if (error != null) {
return next(error)
@ -47,10 +46,10 @@ module.exports = HttpController = {
if (next == null) {
next = function () {}
}
const { project_id } = req.params
logger.debug({ project_id }, 'compressing project history')
const { project_id: projectId } = req.params
logger.debug({ projectId }, 'compressing project history')
return UpdatesManager.processUncompressedUpdatesForProject(
project_id,
projectId,
function (error) {
if (error != null) {
return next(error)
@ -114,12 +113,12 @@ module.exports = HttpController = {
if (next == null) {
next = function () {}
}
const { doc_id } = req.params
const { project_id } = req.params
logger.debug({ project_id, doc_id }, 'checking doc history')
const { doc_id: docId } = req.params
const { project_id: projectId } = req.params
logger.debug({ projectId, docId }, 'checking doc history')
return DiffManager.getDocumentBeforeVersion(
project_id,
doc_id,
projectId,
docId,
1,
function (error, document, rewoundUpdates) {
if (error != null) {
@ -147,8 +146,8 @@ module.exports = HttpController = {
if (next == null) {
next = function () {}
}
const { doc_id } = req.params
const { project_id } = req.params
const { doc_id: docId } = req.params
const { project_id: projectId } = req.params
if (req.query.from != null) {
from = parseInt(req.query.from, 10)
@ -161,10 +160,10 @@ module.exports = HttpController = {
to = null
}
logger.debug({ project_id, doc_id, from, to }, 'getting diff')
logger.debug({ projectId, docId, from, to }, 'getting diff')
return DiffManager.getDiff(
project_id,
doc_id,
projectId,
docId,
from,
to,
function (error, diff) {
@ -177,22 +176,22 @@ module.exports = HttpController = {
},
getUpdates(req, res, next) {
let before, min_count
let before, minCount
if (next == null) {
next = function () {}
}
const { project_id } = req.params
const { project_id: projectId } = req.params
if (req.query.before != null) {
before = parseInt(req.query.before, 10)
}
if (req.query.min_count != null) {
min_count = parseInt(req.query.min_count, 10)
minCount = parseInt(req.query.min_count, 10)
}
return UpdatesManager.getSummarizedProjectUpdates(
project_id,
{ before, min_count },
projectId,
{ before, min_count: minCount },
function (error, updates, nextBeforeTimestamp) {
if (error != null) {
return next(error)
@ -233,10 +232,10 @@ module.exports = HttpController = {
// - updates can weight MBs for insert/delete of full doc
// - multiple updates form a pack
// Flush updates per pack onto the wire.
const { project_id } = req.params
logger.debug({ project_id }, 'exporting project history')
const { project_id: projectId } = req.params
logger.debug({ projectId }, 'exporting project history')
UpdatesManager.exportProject(
project_id,
projectId,
function (err, { updates, userIds }, confirmWrite) {
const abortStreaming = req.destroyed || res.finished || res.destroyed
if (abortStreaming) {
@ -246,7 +245,7 @@ module.exports = HttpController = {
}
const hasStartedStreamingResponse = res.headersSent
if (err) {
logger.error({ project_id, err }, 'export failed')
logger.error({ projectId, err }, 'export failed')
if (!hasStartedStreamingResponse) {
// Generate a nice 500
return next(err)
@ -294,14 +293,14 @@ module.exports = HttpController = {
if (next == null) {
next = function () {}
}
let { doc_id, project_id, version } = req.params
const user_id = req.headers['x-user-id']
let { doc_id: docId, project_id: projectId, version } = req.params
const userId = req.headers['x-user-id']
version = parseInt(version, 10)
return RestoreManager.restoreToBeforeVersion(
project_id,
doc_id,
projectId,
docId,
version,
user_id,
userId,
function (error) {
if (error != null) {
return next(error)
@ -315,10 +314,10 @@ module.exports = HttpController = {
if (next == null) {
next = function () {}
}
const { project_id } = req.params
const { doc_id } = req.params
logger.debug({ project_id, doc_id }, 'pushing all finalised changes to s3')
return PackManager.pushOldPacks(project_id, doc_id, function (error) {
const { project_id: projectId } = req.params
const { doc_id: docId } = req.params
logger.debug({ projectId, docId }, 'pushing all finalised changes to s3')
return PackManager.pushOldPacks(projectId, docId, function (error) {
if (error != null) {
return next(error)
}
@ -330,10 +329,10 @@ module.exports = HttpController = {
if (next == null) {
next = function () {}
}
const { project_id } = req.params
const { doc_id } = req.params
logger.debug({ project_id, doc_id }, 'pulling all packs from s3')
return PackManager.pullOldPacks(project_id, doc_id, function (error) {
const { project_id: projectId } = req.params
const { doc_id: docId } = req.params
logger.debug({ projectId, docId }, 'pulling all packs from s3')
return PackManager.pullOldPacks(projectId, docId, function (error) {
if (error != null) {
return next(error)
}

2
services/track-changes/app/js/LockManager.js
View file

@ -115,7 +115,7 @@ module.exports = LockManager = {
if (result != null && result !== 1) {
// successful unlock should release exactly one key
logger.error(
{ key, lockValue, redis_err: err, redis_result: result },
{ key, lockValue, redisErr: err, redisResult: result },
'unlocking error'
)
return callback(new Error('tried to release timed out lock'))

66
services/track-changes/app/js/MongoAWS.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
no-return-assign,
no-unused-vars,
*/
@ -25,7 +24,7 @@ const Metrics = require('@overleaf/metrics')
const DAYS = 24 * 3600 * 1000 // one day in milliseconds
const createStream = function (streamConstructor, project_id, doc_id, pack_id) {
const createStream = function (streamConstructor, projectId, docId, packId) {
const AWS_CONFIG = {
accessKeyId: settings.trackchanges.s3.key,
secretAccessKey: settings.trackchanges.s3.secret,
@ -35,12 +34,12 @@ const createStream = function (streamConstructor, project_id, doc_id, pack_id) {
return streamConstructor(new AWS.S3(AWS_CONFIG), {
Bucket: settings.trackchanges.stores.doc_history,
Key: project_id + '/changes-' + doc_id + '/pack-' + pack_id,
Key: projectId + '/changes-' + docId + '/pack-' + packId,
})
}
module.exports = MongoAWS = {
archivePack(project_id, doc_id, pack_id, _callback) {
archivePack(projectId, docId, packId, _callback) {
if (_callback == null) {
_callback = function () {}
}
@ -50,23 +49,23 @@ module.exports = MongoAWS = {
}
const query = {
_id: ObjectId(pack_id),
doc_id: ObjectId(doc_id),
_id: ObjectId(packId),
doc_id: ObjectId(docId),
}
if (project_id == null) {
if (projectId == null) {
return callback(new Error('invalid project id'))
}
if (doc_id == null) {
if (docId == null) {
return callback(new Error('invalid doc id'))
}
if (pack_id == null) {
if (packId == null) {
return callback(new Error('invalid pack id'))
}
logger.debug({ project_id, doc_id, pack_id }, 'uploading data to s3')
logger.debug({ projectId, docId, packId }, 'uploading data to s3')
const upload = createStream(S3S.WriteStream, project_id, doc_id, pack_id)
const upload = createStream(S3S.WriteStream, projectId, docId, packId)
return db.docHistory.findOne(query, function (err, result) {
if (err != null) {
@ -81,15 +80,15 @@ module.exports = MongoAWS = {
const uncompressedData = JSON.stringify(result)
if (uncompressedData.indexOf('\u0000') !== -1) {
const error = new Error('null bytes found in upload')
logger.error({ err: error, project_id, doc_id, pack_id }, error.message)
logger.error({ err: error, projectId, docId, packId }, error.message)
return callback(error)
}
return zlib.gzip(uncompressedData, function (err, buf) {
logger.debug(
{
project_id,
doc_id,
pack_id,
projectId,
docId,
packId,
origSize: uncompressedData.length,
newSize: buf.length,
},
@ -101,10 +100,7 @@ module.exports = MongoAWS = {
upload.on('error', err => callback(err))
upload.on('finish', function () {
Metrics.inc('archive-pack')
logger.debug(
{ project_id, doc_id, pack_id },
'upload to s3 completed'
)
logger.debug({ projectId, docId, packId }, 'upload to s3 completed')
return callback(null)
})
upload.write(buf)
@ -113,7 +109,7 @@ module.exports = MongoAWS = {
})
},
readArchivedPack(project_id, doc_id, pack_id, _callback) {
readArchivedPack(projectId, docId, packId, _callback) {
if (_callback == null) {
_callback = function () {}
}
@ -122,19 +118,19 @@ module.exports = MongoAWS = {
return (_callback = function () {})
}
if (project_id == null) {
if (projectId == null) {
return callback(new Error('invalid project id'))
}
if (doc_id == null) {
if (docId == null) {
return callback(new Error('invalid doc id'))
}
if (pack_id == null) {
if (packId == null) {
return callback(new Error('invalid pack id'))
}
logger.debug({ project_id, doc_id, pack_id }, 'downloading data from s3')
logger.debug({ projectId, docId, packId }, 'downloading data from s3')
const download = createStream(S3S.ReadStream, project_id, doc_id, pack_id)
const download = createStream(S3S.ReadStream, projectId, docId, packId)
const inputStream = download
.on('open', obj => 1)
@ -144,7 +140,7 @@ module.exports = MongoAWS = {
gunzip.setEncoding('utf8')
gunzip.on('error', function (err) {
logger.debug(
{ project_id, doc_id, pack_id, err },
{ projectId, docId, packId, err },
'error uncompressing gzip stream'
)
return callback(err)
@ -155,10 +151,7 @@ module.exports = MongoAWS = {
outputStream.on('error', err => callback(err))
outputStream.on('end', function () {
let object
logger.debug(
{ project_id, doc_id, pack_id },
'download from s3 completed'
)
logger.debug({ projectId, docId, packId }, 'download from s3 completed')
try {
object = JSON.parse(parts.join(''))
} catch (e) {
@ -177,14 +170,14 @@ module.exports = MongoAWS = {
return outputStream.on('data', data => parts.push(data))
},
unArchivePack(project_id, doc_id, pack_id, callback) {
unArchivePack(projectId, docId, packId, callback) {
if (callback == null) {
callback = function () {}
}
return MongoAWS.readArchivedPack(
project_id,
doc_id,
pack_id,
projectId,
docId,
packId,
function (err, object) {
if (err != null) {
return callback(err)
@ -192,10 +185,7 @@ module.exports = MongoAWS = {
Metrics.inc('unarchive-pack')
// allow the object to expire, we can always retrieve it again
object.expiresAt = new Date(Date.now() + 7 * DAYS)
logger.debug(
{ project_id, doc_id, pack_id },
'inserting object from s3'
)
logger.debug({ projectId, docId, packId }, 'inserting object from s3')
return db.docHistory.insertOne(object, (err, confirmation) => {
if (err) return callback(err)
object._id = confirmation.insertedId

29
services/track-changes/app/js/MongoManager.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
@ -19,13 +18,13 @@ const metrics = require('@overleaf/metrics')
const logger = require('@overleaf/logger')
module.exports = MongoManager = {
getLastCompressedUpdate(doc_id, callback) {
getLastCompressedUpdate(docId, callback) {
if (callback == null) {
callback = function () {}
}
return db.docHistory
.find(
{ doc_id: ObjectId(doc_id.toString()) },
{ doc_id: ObjectId(docId.toString()) },
// only return the last entry in a pack
{ projection: { pack: { $slice: -1 } } }
)
@ -39,7 +38,7 @@ module.exports = MongoManager = {
})
},
peekLastCompressedUpdate(doc_id, callback) {
peekLastCompressedUpdate(docId, callback) {
// under normal use we pass back the last update as
// callback(null,update,version).
//
@ -50,7 +49,7 @@ module.exports = MongoManager = {
callback = function () {}
}
return MongoManager.getLastCompressedUpdate(
doc_id,
docId,
function (error, update) {
if (error != null) {
return callback(error)
@ -79,7 +78,7 @@ module.exports = MongoManager = {
}
} else {
return PackManager.getLastPackFromIndex(
doc_id,
docId,
function (error, pack) {
if (error != null) {
return callback(error)
@ -98,41 +97,41 @@ module.exports = MongoManager = {
)
},
backportProjectId(project_id, doc_id, callback) {
backportProjectId(projectId, docId, callback) {
if (callback == null) {
callback = function () {}
}
return db.docHistory.updateMany(
{
doc_id: ObjectId(doc_id.toString()),
doc_id: ObjectId(docId.toString()),
project_id: { $exists: false },
},
{
$set: { project_id: ObjectId(project_id.toString()) },
$set: { project_id: ObjectId(projectId.toString()) },
},
callback
)
},
getProjectMetaData(project_id, callback) {
getProjectMetaData(projectId, callback) {
if (callback == null) {
callback = function () {}
}
return db.projectHistoryMetaData.findOne(
{
project_id: ObjectId(project_id.toString()),
project_id: ObjectId(projectId.toString()),
},
callback
)
},
setProjectMetaData(project_id, metadata, callback) {
setProjectMetaData(projectId, metadata, callback) {
if (callback == null) {
callback = function () {}
}
return db.projectHistoryMetaData.updateOne(
{
project_id: ObjectId(project_id),
project_id: ObjectId(projectId),
},
{
$set: metadata,
@ -144,14 +143,14 @@ module.exports = MongoManager = {
)
},
upgradeHistory(project_id, callback) {
upgradeHistory(projectId, callback) {
// preserve the project's existing history
if (callback == null) {
callback = function () {}
}
return db.docHistory.updateMany(
{
project_id: ObjectId(project_id),
project_id: ObjectId(projectId),
temporary: true,
expiresAt: { $exists: true },
},

407
services/track-changes/app/js/PackManager.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
@ -69,8 +68,8 @@ module.exports = PackManager = {
MAX_COUNT: 1024,
insertCompressedUpdates(
project_id,
doc_id,
projectId,
docId,
lastUpdate,
newUpdates,
temporary,
@ -113,8 +112,8 @@ module.exports = PackManager = {
}
return PackManager.flushCompressedUpdates(
project_id,
doc_id,
projectId,
docId,
lastUpdate,
updatesToFlush,
temporary,
@ -123,8 +122,8 @@ module.exports = PackManager = {
return callback(error)
}
return PackManager.insertCompressedUpdates(
project_id,
doc_id,
projectId,
docId,
null,
updatesRemaining,
temporary,
@ -135,8 +134,8 @@ module.exports = PackManager = {
},
flushCompressedUpdates(
project_id,
doc_id,
projectId,
docId,
lastUpdate,
newUpdates,
temporary,
@ -167,8 +166,8 @@ module.exports = PackManager = {
if (canAppend) {
return PackManager.appendUpdatesToExistingPack(
project_id,
doc_id,
projectId,
docId,
lastUpdate,
newUpdates,
temporary,
@ -176,8 +175,8 @@ module.exports = PackManager = {
)
} else {
return PackManager.insertUpdatesIntoNewPack(
project_id,
doc_id,
projectId,
docId,
newUpdates,
temporary,
callback
@ -185,13 +184,7 @@ module.exports = PackManager = {
}
},
insertUpdatesIntoNewPack(
project_id,
doc_id,
newUpdates,
temporary,
callback
) {
insertUpdatesIntoNewPack(projectId, docId, newUpdates, temporary, callback) {
if (callback == null) {
callback = function () {}
}
@ -200,8 +193,8 @@ module.exports = PackManager = {
const n = newUpdates.length
const sz = BSON.calculateObjectSize(newUpdates)
const newPack = {
project_id: ObjectId(project_id.toString()),
doc_id: ObjectId(doc_id.toString()),
project_id: ObjectId(projectId.toString()),
doc_id: ObjectId(docId.toString()),
pack: newUpdates,
n,
sz,
@ -218,7 +211,7 @@ module.exports = PackManager = {
newPack.last_checked = new Date(Date.now() + 30 * DAYS) // never check temporary packs
}
logger.debug(
{ project_id, doc_id, newUpdates },
{ projectId, docId, newUpdates },
'inserting updates into new pack'
)
return db.docHistory.insertOne(newPack, function (err) {
@ -229,14 +222,14 @@ module.exports = PackManager = {
if (temporary) {
return callback()
} else {
return PackManager.updateIndex(project_id, doc_id, callback)
return PackManager.updateIndex(projectId, docId, callback)
}
})
},
appendUpdatesToExistingPack(
project_id,
doc_id,
projectId,
docId,
lastUpdate,
newUpdates,
temporary,
@ -251,8 +244,8 @@ module.exports = PackManager = {
const sz = BSON.calculateObjectSize(newUpdates)
const query = {
_id: lastUpdate._id,
project_id: ObjectId(project_id.toString()),
doc_id: ObjectId(doc_id.toString()),
project_id: ObjectId(projectId.toString()),
doc_id: ObjectId(docId.toString()),
pack: { $exists: true },
}
const update = {
@ -272,7 +265,7 @@ module.exports = PackManager = {
update.$set.expiresAt = new Date(Date.now() + 7 * DAYS)
}
logger.debug(
{ project_id, doc_id, lastUpdate, newUpdates },
{ projectId, docId, lastUpdate, newUpdates },
'appending updates to existing pack'
)
Metrics.inc(`append-pack-${temporary ? 'temporary' : 'permanent'}`)
@ -281,18 +274,18 @@ module.exports = PackManager = {
// Retrieve all changes for a document
getOpsByVersionRange(project_id, doc_id, fromVersion, toVersion, callback) {
getOpsByVersionRange(projectId, docId, fromVersion, toVersion, callback) {
if (callback == null) {
callback = function () {}
}
return PackManager.loadPacksByVersionRange(
project_id,
doc_id,
projectId,
docId,
fromVersion,
toVersion,
function (error) {
if (error) return callback(error)
const query = { doc_id: ObjectId(doc_id.toString()) }
const query = { doc_id: ObjectId(docId.toString()) }
if (toVersion != null) {
query.v = { $lte: toVersion }
}
@ -335,14 +328,8 @@ module.exports = PackManager = {
)
},
loadPacksByVersionRange(
project_id,
doc_id,
fromVersion,
toVersion,
callback
) {
return PackManager.getIndex(doc_id, function (err, indexResult) {
loadPacksByVersionRange(projectId, docId, fromVersion, toVersion, callback) {
return PackManager.getIndex(docId, function (err, indexResult) {
let pack
if (err != null) {
return callback(err)
@ -369,8 +356,8 @@ module.exports = PackManager = {
})()
if (neededIds.length) {
return PackManager.fetchPacksIfNeeded(
project_id,
doc_id,
projectId,
docId,
neededIds,
callback
)
@ -380,20 +367,17 @@ module.exports = PackManager = {
})
},
fetchPacksIfNeeded(project_id, doc_id, pack_ids, callback) {
fetchPacksIfNeeded(projectId, docId, packIds, callback) {
let id
return db.docHistory
.find(
{ _id: { $in: pack_ids.map(ObjectId) } },
{ projection: { _id: 1 } }
)
.find({ _id: { $in: packIds.map(ObjectId) } }, { projection: { _id: 1 } })
.toArray(function (err, loadedPacks) {
if (err != null) {
return callback(err)
}
const allPackIds = (() => {
const result1 = []
for (id of Array.from(pack_ids)) {
for (id of Array.from(packIds)) {
result1.push(id.toString())
}
return result1
@ -403,7 +387,7 @@ module.exports = PackManager = {
)
const packIdsToFetch = _.difference(allPackIds, loadedPackIds)
logger.debug(
{ project_id, doc_id, loadedPackIds, allPackIds, packIdsToFetch },
{ projectId, docId, loadedPackIds, allPackIds, packIdsToFetch },
'analysed packs'
)
if (packIdsToFetch.length === 0) {
@ -412,27 +396,26 @@ module.exports = PackManager = {
return async.eachLimit(
packIdsToFetch,
4,
(pack_id, cb) =>
MongoAWS.unArchivePack(project_id, doc_id, pack_id, cb),
(packId, cb) => MongoAWS.unArchivePack(projectId, docId, packId, cb),
function (err) {
if (err != null) {
return callback(err)
}
logger.debug({ project_id, doc_id }, 'done unarchiving')
logger.debug({ projectId, docId }, 'done unarchiving')
return callback()
}
)
})
},
findAllDocsInProject(project_id, callback) {
findAllDocsInProject(projectId, callback) {
const docIdSet = new Set()
async.series(
[
cb => {
db.docHistory
.find(
{ project_id: ObjectId(project_id) },
{ project_id: ObjectId(projectId) },
{ projection: { pack: false } }
)
.toArray((err, packs) => {
@ -445,7 +428,7 @@ module.exports = PackManager = {
},
cb => {
db.docHistoryIndex
.find({ project_id: ObjectId(project_id) })
.find({ project_id: ObjectId(projectId) })
.toArray((err, indexes) => {
if (err) return callback(err)
indexes.forEach(index => {
@ -514,9 +497,9 @@ module.exports = PackManager = {
})
},
makeProjectIterator(project_id, before, callback) {
makeProjectIterator(projectId, before, callback) {
PackManager._findPacks(
{ project_id: ObjectId(project_id) },
{ project_id: ObjectId(projectId) },
{ 'meta.end_ts': -1 },
function (err, allPacks) {
if (err) return callback(err)
@ -528,9 +511,9 @@ module.exports = PackManager = {
)
},
makeDocIterator(doc_id, callback) {
makeDocIterator(docId, callback) {
PackManager._findPacks(
{ doc_id: ObjectId(doc_id) },
{ doc_id: ObjectId(docId) },
{ v: -1 },
function (err, allPacks) {
if (err) return callback(err)
@ -539,13 +522,13 @@ module.exports = PackManager = {
)
},
getPackById(project_id, doc_id, pack_id, callback) {
return db.docHistory.findOne({ _id: pack_id }, function (err, pack) {
getPackById(projectId, docId, packId, callback) {
return db.docHistory.findOne({ _id: packId }, function (err, pack) {
if (err != null) {
return callback(err)
}
if (pack == null) {
return MongoAWS.unArchivePack(project_id, doc_id, pack_id, callback)
return MongoAWS.unArchivePack(projectId, docId, packId, callback)
} else if (pack.expiresAt != null && pack.temporary === false) {
// we only need to touch the TTL when listing the changes in the project
// because diffs on individual documents are always done after that
@ -573,24 +556,24 @@ module.exports = PackManager = {
// Manage docHistoryIndex collection
getIndex(doc_id, callback) {
getIndex(docId, callback) {
return db.docHistoryIndex.findOne(
{ _id: ObjectId(doc_id.toString()) },
{ _id: ObjectId(docId.toString()) },
callback
)
},
getPackFromIndex(doc_id, pack_id, callback) {
getPackFromIndex(docId, packId, callback) {
return db.docHistoryIndex.findOne(
{ _id: ObjectId(doc_id.toString()), 'packs._id': pack_id },
{ _id: ObjectId(docId.toString()), 'packs._id': packId },
{ projection: { 'packs.$': 1 } },
callback
)
},
getLastPackFromIndex(doc_id, callback) {
getLastPackFromIndex(docId, callback) {
return db.docHistoryIndex.findOne(
{ _id: ObjectId(doc_id.toString()) },
{ _id: ObjectId(docId.toString()) },
{ projection: { packs: { $slice: -1 } } },
function (err, indexPack) {
if (err != null) {
@ -604,8 +587,8 @@ module.exports = PackManager = {
)
},
getIndexWithKeys(doc_id, callback) {
return PackManager.getIndex(doc_id, function (err, index) {
getIndexWithKeys(docId, callback) {
return PackManager.getIndex(docId, function (err, index) {
if (err != null) {
return callback(err)
}
@ -621,10 +604,10 @@ module.exports = PackManager = {
})
},
initialiseIndex(project_id, doc_id, callback) {
initialiseIndex(projectId, docId, callback) {
return PackManager.findCompletedPacks(
project_id,
doc_id,
projectId,
docId,
function (err, packs) {
// console.log 'err', err, 'packs', packs, packs?.length
if (err != null) {
@ -634,8 +617,8 @@ module.exports = PackManager = {
return callback()
}
return PackManager.insertPacksIntoIndexWithLock(
project_id,
doc_id,
projectId,
docId,
packs,
callback
)
@ -643,11 +626,11 @@ module.exports = PackManager = {
)
},
updateIndex(project_id, doc_id, callback) {
updateIndex(projectId, docId, callback) {
// find all packs prior to current pack
return PackManager.findUnindexedPacks(
project_id,
doc_id,
projectId,
docId,
function (err, newPacks) {
if (err != null) {
return callback(err)
@ -656,15 +639,15 @@ module.exports = PackManager = {
return callback()
}
return PackManager.insertPacksIntoIndexWithLock(
project_id,
doc_id,
projectId,
docId,
newPacks,
function (err) {
if (err != null) {
return callback(err)
}
logger.debug(
{ project_id, doc_id, newPacks },
{ projectId, docId, newPacks },
'added new packs to index'
)
return callback()
@ -674,9 +657,9 @@ module.exports = PackManager = {
)
},
findCompletedPacks(project_id, doc_id, callback) {
findCompletedPacks(projectId, docId, callback) {
const query = {
doc_id: ObjectId(doc_id.toString()),
doc_id: ObjectId(docId.toString()),
expiresAt: { $exists: false },
}
return db.docHistory
@ -700,9 +683,9 @@ module.exports = PackManager = {
})
},
findPacks(project_id, doc_id, callback) {
findPacks(projectId, docId, callback) {
const query = {
doc_id: ObjectId(doc_id.toString()),
doc_id: ObjectId(docId.toString()),
expiresAt: { $exists: false },
}
return db.docHistory
@ -722,14 +705,14 @@ module.exports = PackManager = {
})
},
findUnindexedPacks(project_id, doc_id, callback) {
return PackManager.getIndexWithKeys(doc_id, function (err, indexResult) {
findUnindexedPacks(projectId, docId, callback) {
return PackManager.getIndexWithKeys(docId, function (err, indexResult) {
if (err != null) {
return callback(err)
}
return PackManager.findCompletedPacks(
project_id,
doc_id,
projectId,
docId,
function (err, historyPacks) {
let pack
if (err != null) {
@ -770,7 +753,7 @@ module.exports = PackManager = {
})()
if (newPacks.length) {
logger.debug(
{ project_id, doc_id, n: newPacks.length },
{ projectId, docId, n: newPacks.length },
'found new packs'
)
}
@ -780,13 +763,13 @@ module.exports = PackManager = {
})
},
insertPacksIntoIndexWithLock(project_id, doc_id, newPacks, callback) {
insertPacksIntoIndexWithLock(projectId, docId, newPacks, callback) {
return LockManager.runWithLock(
keys.historyIndexLock({ doc_id }),
keys.historyIndexLock({ doc_id: docId }),
releaseLock =>
PackManager._insertPacksIntoIndex(
project_id,
doc_id,
projectId,
docId,
newPacks,
releaseLock
),
@ -794,11 +777,11 @@ module.exports = PackManager = {
)
},
_insertPacksIntoIndex(project_id, doc_id, newPacks, callback) {
_insertPacksIntoIndex(projectId, docId, newPacks, callback) {
return db.docHistoryIndex.updateOne(
{ _id: ObjectId(doc_id.toString()) },
{ _id: ObjectId(docId.toString()) },
{
$setOnInsert: { project_id: ObjectId(project_id.toString()) },
$setOnInsert: { project_id: ObjectId(projectId.toString()) },
$push: {
packs: { $each: newPacks, $sort: { v: 1 } },
},
@ -812,14 +795,14 @@ module.exports = PackManager = {
// Archiving packs to S3
archivePack(project_id, doc_id, pack_id, callback) {
archivePack(projectId, docId, packId, callback) {
const clearFlagOnError = function (err, cb) {
if (err != null) {
// clear the inS3 flag on error
return PackManager.clearPackAsArchiveInProgress(
project_id,
doc_id,
pack_id,
projectId,
docId,
packId,
function (err2) {
if (err2 != null) {
return cb(err2)
@ -834,42 +817,27 @@ module.exports = PackManager = {
return async.series(
[
cb =>
PackManager.checkArchiveNotInProgress(
project_id,
doc_id,
pack_id,
cb
),
PackManager.checkArchiveNotInProgress(projectId, docId, packId, cb),
cb =>
PackManager.markPackAsArchiveInProgress(
project_id,
doc_id,
pack_id,
cb
),
PackManager.markPackAsArchiveInProgress(projectId, docId, packId, cb),
cb =>
MongoAWS.archivePack(project_id, doc_id, pack_id, err =>
MongoAWS.archivePack(projectId, docId, packId, err =>
clearFlagOnError(err, cb)
),
cb =>
PackManager.checkArchivedPack(project_id, doc_id, pack_id, err =>
PackManager.checkArchivedPack(projectId, docId, packId, err =>
clearFlagOnError(err, cb)
),
cb => PackManager.markPackAsArchived(project_id, doc_id, pack_id, cb),
cb => PackManager.markPackAsArchived(projectId, docId, packId, cb),
cb =>
PackManager.setTTLOnArchivedPack(
project_id,
doc_id,
pack_id,
callback
),
PackManager.setTTLOnArchivedPack(projectId, docId, packId, callback),
],
callback
)
},
checkArchivedPack(project_id, doc_id, pack_id, callback) {
return db.docHistory.findOne({ _id: pack_id }, function (err, pack) {
checkArchivedPack(projectId, docId, packId, callback) {
return db.docHistory.findOne({ _id: packId }, function (err, pack) {
if (err != null) {
return callback(err)
}
@ -877,9 +845,9 @@ module.exports = PackManager = {
return callback(new Error('pack not found'))
}
return MongoAWS.readArchivedPack(
project_id,
doc_id,
pack_id,
projectId,
docId,
packId,
function (err, result) {
if (err) return callback(err)
delete result.last_checked
@ -917,8 +885,8 @@ module.exports = PackManager = {
},
// Extra methods to test archive/unarchive for a doc_id
pushOldPacks(project_id, doc_id, callback) {
return PackManager.findPacks(project_id, doc_id, function (err, packs) {
pushOldPacks(projectId, docId, callback) {
return PackManager.findPacks(projectId, docId, function (err, packs) {
if (err != null) {
return callback(err)
}
@ -926,18 +894,18 @@ module.exports = PackManager = {
return callback()
}
return PackManager.processOldPack(
project_id,
doc_id,
projectId,
docId,
packs[0]._id,
callback
)
})
},
pullOldPacks(project_id, doc_id, callback) {
pullOldPacks(projectId, docId, callback) {
return PackManager.loadPacksByVersionRange(
project_id,
doc_id,
projectId,
docId,
null,
null,
callback
@ -946,21 +914,16 @@ module.exports = PackManager = {
// Processing old packs via worker
processOldPack(project_id, doc_id, pack_id, callback) {
processOldPack(projectId, docId, packId, callback) {
const markAsChecked = err =>
PackManager.markPackAsChecked(
project_id,
doc_id,
pack_id,
function (err2) {
if (err2 != null) {
return callback(err2)
}
return callback(err)
PackManager.markPackAsChecked(projectId, docId, packId, function (err2) {
if (err2 != null) {
return callback(err2)
}
)
logger.debug({ project_id, doc_id }, 'processing old packs')
return db.docHistory.findOne({ _id: pack_id }, function (err, pack) {
return callback(err)
})
logger.debug({ projectId, docId }, 'processing old packs')
return db.docHistory.findOne({ _id: packId }, function (err, pack) {
if (err != null) {
return markAsChecked(err)
}
@ -971,8 +934,8 @@ module.exports = PackManager = {
return callback()
} // return directly
return PackManager.finaliseIfNeeded(
project_id,
doc_id,
projectId,
docId,
pack._id,
pack,
function (err) {
@ -980,15 +943,15 @@ module.exports = PackManager = {
return markAsChecked(err)
}
return PackManager.updateIndexIfNeeded(
project_id,
doc_id,
projectId,
docId,
function (err) {
if (err != null) {
return markAsChecked(err)
}
return PackManager.findUnarchivedPacks(
project_id,
doc_id,
projectId,
docId,
function (err, unarchivedPacks) {
if (err != null) {
return markAsChecked(err)
@ -999,7 +962,7 @@ module.exports = PackManager = {
: undefined)
) {
logger.debug(
{ project_id, doc_id },
{ projectId, docId },
'no packs need archiving'
)
return markAsChecked()
@ -1007,12 +970,12 @@ module.exports = PackManager = {
return async.eachSeries(
unarchivedPacks,
(pack, cb) =>
PackManager.archivePack(project_id, doc_id, pack._id, cb),
PackManager.archivePack(projectId, docId, pack._id, cb),
function (err) {
if (err != null) {
return markAsChecked(err)
}
logger.debug({ project_id, doc_id }, 'done processing')
logger.debug({ projectId, docId }, 'done processing')
return markAsChecked()
}
)
@ -1025,88 +988,80 @@ module.exports = PackManager = {
})
},
finaliseIfNeeded(project_id, doc_id, pack_id, pack, callback) {
finaliseIfNeeded(projectId, docId, packId, pack, callback) {
const sz = pack.sz / (1024 * 1024) // in fractions of a megabyte
const n = pack.n / 1024 // in fraction of 1024 ops
const age = (Date.now() - pack.meta.end_ts) / DAYS
if (age < 30) {
// always keep if less than 1 month old
logger.debug(
{ project_id, doc_id, pack_id, age },
'less than 30 days old'
)
logger.debug({ projectId, docId, packId, age }, 'less than 30 days old')
return callback()
}
// compute an archiving threshold which decreases for each month of age
const archive_threshold = 30 / age
if (sz > archive_threshold || n > archive_threshold || age > 90) {
const archiveThreshold = 30 / age
if (sz > archiveThreshold || n > archiveThreshold || age > 90) {
logger.debug(
{ project_id, doc_id, pack_id, age, archive_threshold, sz, n },
{ projectId, docId, packId, age, archiveThreshold, sz, n },
'meets archive threshold'
)
return PackManager.markPackAsFinalisedWithLock(
project_id,
doc_id,
pack_id,
projectId,
docId,
packId,
callback
)
} else {
logger.debug(
{ project_id, doc_id, pack_id, age, archive_threshold, sz, n },
{ projectId, docId, packId, age, archiveThreshold, sz, n },
'does not meet archive threshold'
)
return callback()
}
},
markPackAsFinalisedWithLock(project_id, doc_id, pack_id, callback) {
markPackAsFinalisedWithLock(projectId, docId, packId, callback) {
return LockManager.runWithLock(
keys.historyLock({ doc_id }),
keys.historyLock({ doc_id: docId }),
releaseLock =>
PackManager._markPackAsFinalised(
project_id,
doc_id,
pack_id,
releaseLock
),
PackManager._markPackAsFinalised(projectId, docId, packId, releaseLock),
callback
)
},
_markPackAsFinalised(project_id, doc_id, pack_id, callback) {
logger.debug({ project_id, doc_id, pack_id }, 'marking pack as finalised')
_markPackAsFinalised(projectId, docId, packId, callback) {
logger.debug({ projectId, docId, packId }, 'marking pack as finalised')
return db.docHistory.updateOne(
{ _id: pack_id },
{ _id: packId },
{ $set: { finalised: true } },
callback
)
},
updateIndexIfNeeded(project_id, doc_id, callback) {
logger.debug({ project_id, doc_id }, 'archiving old packs')
return PackManager.getIndexWithKeys(doc_id, function (err, index) {
updateIndexIfNeeded(projectId, docId, callback) {
logger.debug({ projectId, docId }, 'archiving old packs')
return PackManager.getIndexWithKeys(docId, function (err, index) {
if (err != null) {
return callback(err)
}
if (index == null) {
return PackManager.initialiseIndex(project_id, doc_id, callback)
return PackManager.initialiseIndex(projectId, docId, callback)
} else {
return PackManager.updateIndex(project_id, doc_id, callback)
return PackManager.updateIndex(projectId, docId, callback)
}
})
},
markPackAsChecked(project_id, doc_id, pack_id, callback) {
logger.debug({ project_id, doc_id, pack_id }, 'marking pack as checked')
markPackAsChecked(projectId, docId, packId, callback) {
logger.debug({ projectId, docId, packId }, 'marking pack as checked')
return db.docHistory.updateOne(
{ _id: pack_id },
{ _id: packId },
{ $currentDate: { last_checked: true } },
callback
)
},
findUnarchivedPacks(project_id, doc_id, callback) {
return PackManager.getIndex(doc_id, function (err, indexResult) {
findUnarchivedPacks(projectId, docId, callback) {
return PackManager.getIndex(docId, function (err, indexResult) {
if (err != null) {
return callback(err)
}
@ -1123,7 +1078,7 @@ module.exports = PackManager = {
})()
if (unArchivedPacks.length) {
logger.debug(
{ project_id, doc_id, n: unArchivedPacks.length },
{ projectId, docId, n: unArchivedPacks.length },
'find unarchived packs'
)
}
@ -1133,41 +1088,37 @@ module.exports = PackManager = {
// Archive locking flags
checkArchiveNotInProgress(project_id, doc_id, pack_id, callback) {
checkArchiveNotInProgress(projectId, docId, packId, callback) {
logger.debug(
{ project_id, doc_id, pack_id },
{ projectId, docId, packId },
'checking if archive in progress'
)
return PackManager.getPackFromIndex(
doc_id,
pack_id,
function (err, result) {
if (err != null) {
return callback(err)
}
if (result == null) {
return callback(new Error('pack not found in index'))
}
if (result.inS3) {
return callback(new Error('pack archiving already done'))
} else if (result.inS3 != null) {
return callback(new Error('pack archiving already in progress'))
} else {
return callback()
}
return PackManager.getPackFromIndex(docId, packId, function (err, result) {
if (err != null) {
return callback(err)
}
)
if (result == null) {
return callback(new Error('pack not found in index'))
}
if (result.inS3) {
return callback(new Error('pack archiving already done'))
} else if (result.inS3 != null) {
return callback(new Error('pack archiving already in progress'))
} else {
return callback()
}
})
},
markPackAsArchiveInProgress(project_id, doc_id, pack_id, callback) {
markPackAsArchiveInProgress(projectId, docId, packId, callback) {
logger.debug(
{ project_id, doc_id },
{ projectId, docId },
'marking pack as archive in progress status'
)
return db.docHistoryIndex.findOneAndUpdate(
{
_id: ObjectId(doc_id.toString()),
packs: { $elemMatch: { _id: pack_id, inS3: { $exists: false } } },
_id: ObjectId(docId.toString()),
packs: { $elemMatch: { _id: packId, inS3: { $exists: false } } },
},
{ $set: { 'packs.$.inS3': false } },
{ projection: { 'packs.$': 1 } },
@ -1179,7 +1130,7 @@ module.exports = PackManager = {
return callback(new Error('archive is already in progress'))
}
logger.debug(
{ project_id, doc_id, pack_id },
{ projectId, docId, packId },
'marked as archive in progress'
)
return callback()
@ -1187,27 +1138,27 @@ module.exports = PackManager = {
)
},
clearPackAsArchiveInProgress(project_id, doc_id, pack_id, callback) {
clearPackAsArchiveInProgress(projectId, docId, packId, callback) {
logger.debug(
{ project_id, doc_id, pack_id },
{ projectId, docId, packId },
'clearing as archive in progress'
)
return db.docHistoryIndex.updateOne(
{
_id: ObjectId(doc_id.toString()),
packs: { $elemMatch: { _id: pack_id, inS3: false } },
_id: ObjectId(docId.toString()),
packs: { $elemMatch: { _id: packId, inS3: false } },
},
{ $unset: { 'packs.$.inS3': true } },
callback
)
},
markPackAsArchived(project_id, doc_id, pack_id, callback) {
logger.debug({ project_id, doc_id, pack_id }, 'marking pack as archived')
markPackAsArchived(projectId, docId, packId, callback) {
logger.debug({ projectId, docId, packId }, 'marking pack as archived')
return db.docHistoryIndex.findOneAndUpdate(
{
_id: ObjectId(doc_id.toString()),
packs: { $elemMatch: { _id: pack_id, inS3: false } },
_id: ObjectId(docId.toString()),
packs: { $elemMatch: { _id: packId, inS3: false } },
},
{ $set: { 'packs.$.inS3': true } },
{ projection: { 'packs.$': 1 } },
@ -1218,21 +1169,21 @@ module.exports = PackManager = {
if (!result.value) {
return callback(new Error('archive is not marked as progress'))
}
logger.debug({ project_id, doc_id, pack_id }, 'marked as archived')
logger.debug({ projectId, docId, packId }, 'marked as archived')
return callback()
}
)
},
setTTLOnArchivedPack(project_id, doc_id, pack_id, callback) {
setTTLOnArchivedPack(projectId, docId, packId, callback) {
return db.docHistory.updateOne(
{ _id: pack_id },
{ _id: packId },
{ $set: { expiresAt: new Date(Date.now() + 1 * DAYS) } },
function (err) {
if (err) {
return callback(err)
}
logger.debug({ project_id, doc_id, pack_id }, 'set expiry on pack')
logger.debug({ projectId, docId, packId }, 'set expiry on pack')
return callback()
}
)

19
services/track-changes/app/js/PackWorker.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
@ -14,7 +13,7 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let LIMIT, pending
let project_id, doc_id
let projectId, docId
const { callbackify } = require('util')
const Settings = require('@overleaf/settings')
const async = require('async')
@ -48,8 +47,8 @@ if (!source.match(/^[0-9]+$/)) {
const result = (() => {
const result1 = []
for (const line of Array.from(file.toString().split('\n'))) {
;[project_id, doc_id] = Array.from(line.split(' '))
result1.push({ doc_id, project_id })
;[projectId, docId] = Array.from(line.split(' '))
result1.push({ doc_id: docId, project_id: projectId })
}
return result1
})()
@ -108,12 +107,12 @@ const processUpdates = pending =>
pending,
function (result, callback) {
let _id
;({ _id, project_id, doc_id } = result)
;({ _id, project_id: projectId, doc_id: docId } = result)
COUNT++
logger.debug({ project_id, doc_id }, `processing ${COUNT}/${TOTAL}`)
if (project_id == null || doc_id == null) {
logger.debug({ projectId, docId }, `processing ${COUNT}/${TOTAL}`)
if (projectId == null || docId == null) {
logger.debug(
{ project_id, doc_id },
{ projectId, docId },
'skipping pack, missing project/doc id'
)
return callback()
@ -138,9 +137,9 @@ const processUpdates = pending =>
return setTimeout(() => callback(err, result), DOCUMENT_PACK_DELAY)
}
if (_id == null) {
return PackManager.pushOldPacks(project_id, doc_id, handler)
return PackManager.pushOldPacks(projectId, docId, handler)
} else {
return PackManager.processOldPack(project_id, doc_id, _id, handler)
return PackManager.processOldPack(projectId, docId, _id, handler)
}
},
function (err, results) {

34
services/track-changes/app/js/RedisManager.js
View file

@ -1,6 +1,3 @@
/* eslint-disable
camelcase,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
@ -19,11 +16,11 @@ const Keys = Settings.redis.history.key_schema
const async = require('async')
module.exports = RedisManager = {
getOldestDocUpdates(doc_id, batchSize, callback) {
getOldestDocUpdates(docId, batchSize, callback) {
if (callback == null) {
callback = function () {}
}
const key = Keys.uncompressedHistoryOps({ doc_id })
const key = Keys.uncompressedHistoryOps({ doc_id: docId })
return rclient.lrange(key, 0, batchSize - 1, callback)
},
@ -42,14 +39,14 @@ module.exports = RedisManager = {
return callback(null, rawUpdates)
},
deleteAppliedDocUpdates(project_id, doc_id, docUpdates, callback) {
deleteAppliedDocUpdates(projectId, docId, docUpdates, callback) {
if (callback == null) {
callback = function () {}
}
const multi = rclient.multi()
// Delete all the updates which have been applied (exact match)
for (const update of Array.from(docUpdates || [])) {
multi.lrem(Keys.uncompressedHistoryOps({ doc_id }), 1, update)
multi.lrem(Keys.uncompressedHistoryOps({ doc_id: docId }), 1, update)
}
return multi.exec(function (error, results) {
if (error != null) {
@ -58,8 +55,8 @@ module.exports = RedisManager = {
// It's ok to delete the doc_id from the set here. Even though the list
// of updates may not be empty, we will continue to process it until it is.
return rclient.srem(
Keys.docsWithHistoryOps({ project_id }),
doc_id,
Keys.docsWithHistoryOps({ project_id: projectId }),
docId,
function (error) {
if (error != null) {
return callback(error)
@ -70,11 +67,14 @@ module.exports = RedisManager = {
})
},
getDocIdsWithHistoryOps(project_id, callback) {
getDocIdsWithHistoryOps(projectId, callback) {
if (callback == null) {
callback = function () {}
}
return rclient.smembers(Keys.docsWithHistoryOps({ project_id }), callback)
return rclient.smembers(
Keys.docsWithHistoryOps({ project_id: projectId }),
callback
)
},
// iterate over keys asynchronously using redis scan (non-blocking)
@ -139,12 +139,12 @@ module.exports = RedisManager = {
}
return RedisManager._getKeys(
Keys.docsWithHistoryOps({ project_id: '*' }),
function (error, project_keys) {
function (error, projectKeys) {
if (error != null) {
return callback(error)
}
const project_ids = RedisManager._extractIds(project_keys)
return callback(error, project_ids)
const projectIds = RedisManager._extractIds(projectKeys)
return callback(error, projectIds)
}
)
},
@ -157,12 +157,12 @@ module.exports = RedisManager = {
}
return RedisManager._getKeys(
Keys.uncompressedHistoryOps({ doc_id: '*' }),
function (error, doc_keys) {
function (error, docKeys) {
if (error != null) {
return callback(error)
}
const doc_ids = RedisManager._extractIds(doc_keys)
return callback(error, doc_ids)
const docIds = RedisManager._extractIds(docKeys)
return callback(error, docIds)
}
)
},

15
services/track-changes/app/js/RestoreManager.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
@ -16,24 +15,24 @@ const DiffManager = require('./DiffManager')
const logger = require('@overleaf/logger')
module.exports = RestoreManager = {
restoreToBeforeVersion(project_id, doc_id, version, user_id, callback) {
restoreToBeforeVersion(projectId, docId, version, userId, callback) {
if (callback == null) {
callback = function () {}
}
logger.debug({ project_id, doc_id, version, user_id }, 'restoring document')
logger.debug({ projectId, docId, version, userId }, 'restoring document')
return DiffManager.getDocumentBeforeVersion(
project_id,
doc_id,
projectId,
docId,
version,
function (error, content) {
if (error != null) {
return callback(error)
}
return DocumentUpdaterManager.setDocument(
project_id,
doc_id,
projectId,
docId,
content,
user_id,
userId,
function (error) {
if (error != null) {
return callback(error)

11
services/track-changes/app/js/UpdateCompressor.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
new-cap,
no-throw-literal,
no-unused-vars,
@ -16,8 +15,8 @@ let oneMinute, twoMegabytes, UpdateCompressor
const strInject = (s1, pos, s2) => s1.slice(0, pos) + s2 + s1.slice(pos)
const strRemove = (s1, pos, length) => s1.slice(0, pos) + s1.slice(pos + length)
const { diff_match_patch } = require('../lib/diff_match_patch')
const dmp = new diff_match_patch()
const { diff_match_patch: diffMatchPatch } = require('../lib/diff_match_patch')
const dmp = new diffMatchPatch()
module.exports = UpdateCompressor = {
NOOP: 'noop',
@ -254,8 +253,8 @@ module.exports = UpdateCompressor = {
firstOp.p === secondOp.p
) {
offset = firstOp.p
const diff_ops = this.diffAsShareJsOps(firstOp.d, secondOp.i)
if (diff_ops.length === 0) {
const diffOps = this.diffAsShareJsOps(firstOp.d, secondOp.i)
if (diffOps.length === 0) {
return [
{
// Noop
@ -272,7 +271,7 @@ module.exports = UpdateCompressor = {
},
]
} else {
return diff_ops.map(function (op) {
return diffOps.map(function (op) {
op.p += offset
return {
meta: {

13
services/track-changes/app/js/UpdateTrimmer.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
@ -16,12 +15,12 @@ const WebApiManager = require('./WebApiManager')
const logger = require('@overleaf/logger')
module.exports = UpdateTrimmer = {
shouldTrimUpdates(project_id, callback) {
shouldTrimUpdates(projectId, callback) {
if (callback == null) {
callback = function () {}
}
return MongoManager.getProjectMetaData(
project_id,
projectId,
function (error, metadata) {
if (error != null) {
return callback(error)
@ -30,22 +29,22 @@ module.exports = UpdateTrimmer = {
return callback(null, false)
} else {
return WebApiManager.getProjectDetails(
project_id,
projectId,
function (error, details) {
if (error != null) {
return callback(error)
}
logger.debug({ project_id, details }, 'got details')
logger.debug({ projectId, details }, 'got details')
if (details?.features?.versioning) {
return MongoManager.setProjectMetaData(
project_id,
projectId,
{ preserveHistory: true },
function (error) {
if (error != null) {
return callback(error)
}
return MongoManager.upgradeHistory(
project_id,
projectId,
function (error) {
if (error != null) {
return callback(error)

234
services/track-changes/app/js/UpdatesManager.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
@ -29,13 +28,7 @@ const keys = Settings.redis.lock.key_schema
const util = require('util')
module.exports = UpdatesManager = {
compressAndSaveRawUpdates(
project_id,
doc_id,
rawUpdates,
temporary,
callback
) {
compressAndSaveRawUpdates(projectId, docId, rawUpdates, temporary, callback) {
let i
if (callback == null) {
callback = function () {}
@ -54,8 +47,8 @@ module.exports = UpdatesManager = {
if (!(prevVersion < thisVersion)) {
logger.error(
{
project_id,
doc_id,
projectId,
docId,
rawUpdates,
temporary,
thisVersion,
@ -70,7 +63,7 @@ module.exports = UpdatesManager = {
// FIXME: we no longer need the lastCompressedUpdate, so change functions not to need it
// CORRECTION: we do use it to log the time in case of error
return MongoManager.peekLastCompressedUpdate(
doc_id,
docId,
function (error, lastCompressedUpdate, lastVersion) {
// lastCompressedUpdate is the most recent update in Mongo, and
// lastVersion is its sharejs version number.
@ -93,23 +86,23 @@ module.exports = UpdatesManager = {
}
if (discardedUpdates.length) {
logger.error(
{ project_id, doc_id, discardedUpdates, temporary, lastVersion },
{ projectId, docId, discardedUpdates, temporary, lastVersion },
'discarded updates already present'
)
}
if (rawUpdates[0] != null && rawUpdates[0].v !== lastVersion + 1) {
const ts = lastCompressedUpdate?.meta?.end_ts
const last_timestamp = ts != null ? new Date(ts) : 'unknown time'
const lastTimestamp = ts != null ? new Date(ts) : 'unknown time'
error = new Error(
`Tried to apply raw op at version ${rawUpdates[0].v} to last compressed update with version ${lastVersion} from ${last_timestamp}`
`Tried to apply raw op at version ${rawUpdates[0].v} to last compressed update with version ${lastVersion} from ${lastTimestamp}`
)
logger.error(
{
err: error,
doc_id,
project_id,
prev_end_ts: ts,
docId,
projectId,
prevEndTs: ts,
temporary,
lastCompressedUpdate,
},
@ -155,7 +148,7 @@ module.exports = UpdatesManager = {
`dropped op exceeding maximum allowed size of ${REJECT_LARGE_OP_SIZE}`
)
logger.error(
{ err: error, doc_id, project_id, size, rawUpdate },
{ err: error, docId, projectId, size, rawUpdate },
'dropped op - too big'
)
rawUpdate.op = []
@ -167,8 +160,8 @@ module.exports = UpdatesManager = {
rawUpdates
)
return PackManager.insertCompressedUpdates(
project_id,
doc_id,
projectId,
docId,
lastCompressedUpdate,
compressedUpdates,
temporary,
@ -179,13 +172,13 @@ module.exports = UpdatesManager = {
if (result != null) {
logger.debug(
{
project_id,
doc_id,
orig_v:
projectId,
docId,
origV:
lastCompressedUpdate != null
? lastCompressedUpdate.v
: undefined,
new_v: result.v,
newV: result.v,
},
'inserted updates into pack'
)
@ -198,12 +191,12 @@ module.exports = UpdatesManager = {
},
// Check whether the updates are temporary (per-project property)
_prepareProjectForUpdates(project_id, callback) {
_prepareProjectForUpdates(projectId, callback) {
if (callback == null) {
callback = function () {}
}
return UpdateTrimmer.shouldTrimUpdates(
project_id,
projectId,
function (error, temporary) {
if (error != null) {
return callback(error)
@ -214,11 +207,11 @@ module.exports = UpdatesManager = {
},
// Check for project id on document history (per-document property)
_prepareDocForUpdates(project_id, doc_id, callback) {
_prepareDocForUpdates(projectId, docId, callback) {
if (callback == null) {
callback = function () {}
}
return MongoManager.backportProjectId(project_id, doc_id, function (error) {
return MongoManager.backportProjectId(projectId, docId, function (error) {
if (error != null) {
return callback(error)
}
@ -228,13 +221,13 @@ module.exports = UpdatesManager = {
// Apply updates for specific project/doc after preparing at project and doc level
REDIS_READ_BATCH_SIZE: 100,
processUncompressedUpdates(project_id, doc_id, temporary, callback) {
processUncompressedUpdates(projectId, docId, temporary, callback) {
// get the updates as strings from redis (so we can delete them after they are applied)
if (callback == null) {
callback = function () {}
}
return RedisManager.getOldestDocUpdates(
doc_id,
docId,
UpdatesManager.REDIS_READ_BATCH_SIZE,
function (error, docUpdates) {
if (error != null) {
@ -247,18 +240,18 @@ module.exports = UpdatesManager = {
function (error, rawUpdates) {
if (error != null) {
logger.err(
{ project_id, doc_id, docUpdates },
{ projectId, docId, docUpdates },
'failed to parse docUpdates'
)
return callback(error)
}
logger.debug(
{ project_id, doc_id, rawUpdates },
{ projectId, docId, rawUpdates },
'retrieved raw updates from redis'
)
return UpdatesManager.compressAndSaveRawUpdates(
project_id,
doc_id,
projectId,
docId,
rawUpdates,
temporary,
function (error) {
@ -266,13 +259,13 @@ module.exports = UpdatesManager = {
return callback(error)
}
logger.debug(
{ project_id, doc_id },
{ projectId, docId },
'compressed and saved doc updates'
)
// delete the applied updates from redis
return RedisManager.deleteAppliedDocUpdates(
project_id,
doc_id,
projectId,
docId,
docUpdates,
function (error) {
if (error != null) {
@ -281,14 +274,14 @@ module.exports = UpdatesManager = {
if (length === UpdatesManager.REDIS_READ_BATCH_SIZE) {
// There might be more updates
logger.debug(
{ project_id, doc_id },
{ projectId, docId },
'continuing processing updates'
)
return setTimeout(
() =>
UpdatesManager.processUncompressedUpdates(
project_id,
doc_id,
projectId,
docId,
temporary,
callback
),
@ -296,7 +289,7 @@ module.exports = UpdatesManager = {
)
} else {
logger.debug(
{ project_id, doc_id },
{ projectId, docId },
'all raw updates processed'
)
return callback()
@ -312,19 +305,19 @@ module.exports = UpdatesManager = {
},
// Process updates for a doc when we flush it individually
processUncompressedUpdatesWithLock(project_id, doc_id, callback) {
processUncompressedUpdatesWithLock(projectId, docId, callback) {
if (callback == null) {
callback = function () {}
}
return UpdatesManager._prepareProjectForUpdates(
project_id,
projectId,
function (error, temporary) {
if (error != null) {
return callback(error)
}
return UpdatesManager._processUncompressedUpdatesForDocWithLock(
project_id,
doc_id,
projectId,
docId,
temporary,
callback
)
@ -334,8 +327,8 @@ module.exports = UpdatesManager = {
// Process updates for a doc when the whole project is flushed (internal method)
_processUncompressedUpdatesForDocWithLock(
project_id,
doc_id,
projectId,
docId,
temporary,
callback
) {
@ -343,18 +336,18 @@ module.exports = UpdatesManager = {
callback = function () {}
}
return UpdatesManager._prepareDocForUpdates(
project_id,
doc_id,
projectId,
docId,
function (error) {
if (error != null) {
return callback(error)
}
return LockManager.runWithLock(
keys.historyLock({ doc_id }),
keys.historyLock({ doc_id: docId }),
releaseLock =>
UpdatesManager.processUncompressedUpdates(
project_id,
doc_id,
projectId,
docId,
temporary,
releaseLock
),
@ -365,31 +358,31 @@ module.exports = UpdatesManager = {
},
// Process all updates for a project, only check project-level information once
processUncompressedUpdatesForProject(project_id, callback) {
processUncompressedUpdatesForProject(projectId, callback) {
if (callback == null) {
callback = function () {}
}
return RedisManager.getDocIdsWithHistoryOps(
project_id,
function (error, doc_ids) {
projectId,
function (error, docIds) {
if (error != null) {
return callback(error)
}
return UpdatesManager._prepareProjectForUpdates(
project_id,
projectId,
function (error, temporary) {
if (error) return callback(error)
const jobs = []
for (const doc_id of Array.from(doc_ids)) {
;(doc_id =>
for (const docId of Array.from(docIds)) {
;(docId =>
jobs.push(cb =>
UpdatesManager._processUncompressedUpdatesForDocWithLock(
project_id,
doc_id,
projectId,
docId,
temporary,
cb
)
))(doc_id)
))(docId)
}
return async.parallelLimit(jobs, 5, callback)
}
@ -405,31 +398,31 @@ module.exports = UpdatesManager = {
}
return RedisManager.getProjectIdsWithHistoryOps(function (
error,
project_ids
projectIds
) {
let project_id
let projectId
if (error != null) {
return callback(error)
}
logger.debug(
{
count: project_ids != null ? project_ids.length : undefined,
project_ids,
count: projectIds != null ? projectIds.length : undefined,
projectIds,
},
'found projects'
)
const jobs = []
project_ids = _.shuffle(project_ids) // randomise to avoid hitting same projects each time
projectIds = _.shuffle(projectIds) // randomise to avoid hitting same projects each time
const selectedProjects =
limit < 0 ? project_ids : project_ids.slice(0, limit)
for (project_id of Array.from(selectedProjects)) {
;(project_id =>
limit < 0 ? projectIds : projectIds.slice(0, limit)
for (projectId of Array.from(selectedProjects)) {
;(projectId =>
jobs.push(cb =>
UpdatesManager.processUncompressedUpdatesForProject(
project_id,
err => cb(null, { failed: err != null, project_id })
projectId,
err => cb(null, { failed: err != null, project_id: projectId })
)
))(project_id)
))(projectId)
}
return async.series(jobs, function (error, result) {
let x
@ -457,7 +450,7 @@ module.exports = UpdatesManager = {
return callback(null, {
failed: failedProjects,
succeeded: succeededProjects,
all: project_ids,
all: projectIds,
})
})
})
@ -467,16 +460,13 @@ module.exports = UpdatesManager = {
if (callback == null) {
callback = function () {}
}
return RedisManager.getAllDocIdsWithHistoryOps(function (
error,
all_doc_ids
) {
return RedisManager.getAllDocIdsWithHistoryOps(function (error, allDocIds) {
if (error != null) {
return callback(error)
}
return RedisManager.getProjectIdsWithHistoryOps(function (
error,
all_project_ids
allProjectIds
) {
if (error != null) {
return callback(error)
@ -484,25 +474,25 @@ module.exports = UpdatesManager = {
// function to get doc_ids for each project
const task = cb =>
async.concatSeries(
all_project_ids,
allProjectIds,
RedisManager.getDocIdsWithHistoryOps,
cb
)
// find the dangling doc ids
return task(function (error, project_doc_ids) {
return task(function (error, projectDocIds) {
if (error) return callback(error)
const dangling_doc_ids = _.difference(all_doc_ids, project_doc_ids)
const danglingDocIds = _.difference(allDocIds, projectDocIds)
logger.debug(
{ all_doc_ids, all_project_ids, project_doc_ids, dangling_doc_ids },
{ allDocIds, allProjectIds, projectDocIds, danglingDocIds },
'checking for dangling doc ids'
)
return callback(null, dangling_doc_ids)
return callback(null, danglingDocIds)
})
})
})
},
getDocUpdates(project_id, doc_id, options, callback) {
getDocUpdates(projectId, docId, options, callback) {
if (options == null) {
options = {}
}
@ -510,16 +500,16 @@ module.exports = UpdatesManager = {
callback = function () {}
}
return UpdatesManager.processUncompressedUpdatesWithLock(
project_id,
doc_id,
projectId,
docId,
function (error) {
if (error != null) {
return callback(error)
}
// console.log "options", options
return PackManager.getOpsByVersionRange(
project_id,
doc_id,
projectId,
docId,
options.from,
options.to,
function (error, updates) {
@ -533,7 +523,7 @@ module.exports = UpdatesManager = {
)
},
getDocUpdatesWithUserInfo(project_id, doc_id, options, callback) {
getDocUpdatesWithUserInfo(projectId, docId, options, callback) {
if (options == null) {
options = {}
}
@ -541,8 +531,8 @@ module.exports = UpdatesManager = {
callback = function () {}
}
return UpdatesManager.getDocUpdates(
project_id,
doc_id,
projectId,
docId,
options,
function (error, updates) {
if (error != null) {
@ -558,7 +548,7 @@ module.exports = UpdatesManager = {
)
},
getSummarizedProjectUpdates(project_id, options, callback) {
getSummarizedProjectUpdates(projectId, options, callback) {
if (options == null) {
options = {}
}
@ -572,13 +562,13 @@ module.exports = UpdatesManager = {
const { before } = options
let nextBeforeTimestamp = null
return UpdatesManager.processUncompressedUpdatesForProject(
project_id,
projectId,
function (error) {
if (error != null) {
return callback(error)
}
return PackManager.makeProjectIterator(
project_id,
projectId,
before,
function (err, iterator) {
if (err != null) {
@ -692,17 +682,17 @@ module.exports = UpdatesManager = {
}
const jobs = []
const fetchedUserInfo = {}
for (const user_id in users) {
;(user_id =>
for (const userId in users) {
;(userId =>
jobs.push(callback =>
WebApiManager.getUserInfo(user_id, function (error, userInfo) {
WebApiManager.getUserInfo(userId, function (error, userInfo) {
if (error != null) {
return callback(error)
}
fetchedUserInfo[user_id] = userInfo
fetchedUserInfo[userId] = userInfo
return callback()
})
))(user_id)
))(userId)
}
return async.series(jobs, function (err) {
@ -714,15 +704,15 @@ module.exports = UpdatesManager = {
},
fillUserInfo(updates, callback) {
let update, user_id
let update, userId
if (callback == null) {
callback = function () {}
}
const users = {}
for (update of Array.from(updates)) {
;({ user_id } = update.meta)
if (UpdatesManager._validUserId(user_id)) {
users[user_id] = true
;({ user_id: userId } = update.meta)
if (UpdatesManager._validUserId(userId)) {
users[userId] = true
}
}
@ -733,10 +723,10 @@ module.exports = UpdatesManager = {
return callback(error)
}
for (update of Array.from(updates)) {
;({ user_id } = update.meta)
;({ user_id: userId } = update.meta)
delete update.meta.user_id
if (UpdatesManager._validUserId(user_id)) {
update.meta.user = fetchedUserInfo[user_id]
if (UpdatesManager._validUserId(userId)) {
update.meta.user = fetchedUserInfo[userId]
}
}
return callback(null, updates)
@ -745,16 +735,16 @@ module.exports = UpdatesManager = {
},
fillSummarizedUserInfo(updates, callback) {
let update, user_id, user_ids
let update, userId, userIds
if (callback == null) {
callback = function () {}
}
const users = {}
for (update of Array.from(updates)) {
user_ids = update.meta.user_ids || []
for (user_id of Array.from(user_ids)) {
if (UpdatesManager._validUserId(user_id)) {
users[user_id] = true
userIds = update.meta.user_ids || []
for (userId of Array.from(userIds)) {
if (UpdatesManager._validUserId(userId)) {
users[userId] = true
}
}
}
@ -766,12 +756,12 @@ module.exports = UpdatesManager = {
return callback(error)
}
for (update of Array.from(updates)) {
user_ids = update.meta.user_ids || []
userIds = update.meta.user_ids || []
update.meta.users = []
delete update.meta.user_ids
for (user_id of Array.from(user_ids)) {
if (UpdatesManager._validUserId(user_id)) {
update.meta.users.push(fetchedUserInfo[user_id])
for (userId of Array.from(userIds)) {
if (UpdatesManager._validUserId(userId)) {
update.meta.users.push(fetchedUserInfo[userId])
} else {
update.meta.users.push(null)
}
@ -782,11 +772,11 @@ module.exports = UpdatesManager = {
)
},
_validUserId(user_id) {
if (user_id == null) {
_validUserId(userId) {
if (userId == null) {
return false
} else {
return !!user_id.match(/^[a-f0-9]{24}$/)
return !!userId.match(/^[a-f0-9]{24}$/)
}
},
@ -799,7 +789,7 @@ module.exports = UpdatesManager = {
const summarizedUpdates = existingSummarizedUpdates.slice()
let previousUpdateWasBigDelete = false
for (const update of Array.from(updates)) {
let doc_id
let docId
const earliestUpdate = summarizedUpdates[summarizedUpdates.length - 1]
let shouldConcat = false
@ -837,13 +827,13 @@ module.exports = UpdatesManager = {
update.meta.user_id,
])
doc_id = update.doc_id.toString()
const doc = earliestUpdate.docs[doc_id]
docId = update.doc_id.toString()
const doc = earliestUpdate.docs[docId]
if (doc != null) {
doc.fromV = Math.min(doc.fromV, update.v)
doc.toV = Math.max(doc.toV, update.v)
} else {
earliestUpdate.docs[doc_id] = {
earliestUpdate.docs[docId] = {
fromV: update.v,
toV: update.v,
}

21
services/track-changes/app/js/WebApiManager.js
View file

@ -1,6 +1,3 @@
/* eslint-disable
camelcase,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
@ -58,21 +55,21 @@ module.exports = WebApiManager = {
)
},
getUserInfo(user_id, callback) {
getUserInfo(userId, callback) {
if (callback == null) {
callback = function () {}
}
const url = `/user/${user_id}/personal_info`
logger.debug({ user_id }, 'getting user info from web')
const url = `/user/${userId}/personal_info`
logger.debug({ userId }, 'getting user info from web')
return WebApiManager.sendRequest(url, function (error, body) {
let user
if (error != null) {
logger.error({ err: error, user_id, url }, 'error accessing web')
logger.error({ err: error, userId, url }, 'error accessing web')
return callback(error)
}
if (body === null) {
logger.error({ user_id, url }, 'no user found')
logger.error({ userId, url }, 'no user found')
return callback(null, null)
}
try {
@ -90,16 +87,16 @@ module.exports = WebApiManager = {
})
},
getProjectDetails(project_id, callback) {
getProjectDetails(projectId, callback) {
if (callback == null) {
callback = function () {}
}
const url = `/project/${project_id}/details`
logger.debug({ project_id }, 'getting project details from web')
const url = `/project/${projectId}/details`
logger.debug({ projectId }, 'getting project details from web')
return WebApiManager.sendRequest(url, function (error, body) {
let project
if (error != null) {
logger.error({ err: error, project_id, url }, 'error accessing web')
logger.error({ err: error, projectId, url }, 'error accessing web')
return callback(error)
}

5
services/track-changes/test/acceptance/js/ArchivingUpdatesTests.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
no-undef,
no-unused-vars,
*/
@ -268,11 +267,11 @@ describe('Archiving updates', function () {
if (error != null) {
throw error
}
const pack_id = index.packs[0]._id
const packId = index.packs[0]._id
return TrackChangesClient.getS3Doc(
this.project_id,
this.doc_id,
pack_id,
packId,
(error, doc) => {
if (error) return done(error)
doc.n.should.equal(1024)

5
services/track-changes/test/acceptance/js/helpers/MockDocStoreApi.js
View file

@ -1,6 +1,3 @@
/* eslint-disable
camelcase,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
@ -16,7 +13,7 @@ const app = express()
module.exports = MockDocUpdaterApi = {
docs: {},
getAllDoc(project_id, callback) {
getAllDoc(projectId, callback) {
if (callback == null) {
callback = function () {}
}

13
services/track-changes/test/acceptance/js/helpers/MockDocUpdaterApi.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
no-undef,
*/
// TODO: This file was created by bulk-decaffeinate.
@ -19,21 +18,21 @@ app.use(bodyParser.json())
module.exports = MockDocUpdaterApi = {
docs: {},
getDoc(project_id, doc_id, callback) {
getDoc(projectId, docId, callback) {
if (callback == null) {
callback = function () {}
}
return callback(null, this.docs[doc_id])
return callback(null, this.docs[docId])
},
setDoc(project_id, doc_id, lines, user_id, undoing, callback) {
setDoc(projectId, docId, lines, userId, undoing, callback) {
if (callback == null) {
callback = function () {}
}
if (!this.docs[doc_id]) {
this.docs[doc_id] = {}
if (!this.docs[docId]) {
this.docs[docId] = {}
}
this.docs[doc_id].lines = lines
this.docs[docId].lines = lines
return callback()
},

11
services/track-changes/test/acceptance/js/helpers/MockWebApi.js
View file

@ -1,6 +1,3 @@
/* eslint-disable
camelcase,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
@ -18,18 +15,18 @@ module.exports = MockWebApi = {
projects: {},
getUserInfo(user_id, callback) {
getUserInfo(userId, callback) {
if (callback == null) {
callback = function () {}
}
return callback(null, this.users[user_id] || null)
return callback(null, this.users[userId] || null)
},
getProjectDetails(project_id, callback) {
getProjectDetails(projectId, callback) {
if (callback == null) {
callback = function () {}
}
return callback(null, this.projects[project_id])
return callback(null, this.projects[projectId])
},
run() {

69
services/track-changes/test/acceptance/js/helpers/TrackChangesClient.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
@ -32,25 +31,25 @@ const s3 = new aws.S3({
const S3_BUCKET = Settings.trackchanges.stores.doc_history
module.exports = TrackChangesClient = {
flushAndGetCompressedUpdates(project_id, doc_id, callback) {
flushAndGetCompressedUpdates(projectId, docId, callback) {
if (callback == null) {
callback = function () {}
}
return TrackChangesClient.flushDoc(project_id, doc_id, error => {
return TrackChangesClient.flushDoc(projectId, docId, error => {
if (error != null) {
return callback(error)
}
return TrackChangesClient.getCompressedUpdates(doc_id, callback)
return TrackChangesClient.getCompressedUpdates(docId, callback)
})
},
flushDoc(project_id, doc_id, callback) {
flushDoc(projectId, docId, callback) {
if (callback == null) {
callback = function () {}
}
return request.post(
{
url: `http://localhost:3015/project/${project_id}/doc/${doc_id}/flush`,
url: `http://localhost:3015/project/${projectId}/doc/${docId}/flush`,
},
(error, response, body) => {
response.statusCode.should.equal(204)
@ -59,13 +58,13 @@ module.exports = TrackChangesClient = {
)
},
flushProject(project_id, callback) {
flushProject(projectId, callback) {
if (callback == null) {
callback = function () {}
}
return request.post(
{
url: `http://localhost:3015/project/${project_id}/flush`,
url: `http://localhost:3015/project/${projectId}/flush`,
},
(error, response, body) => {
response.statusCode.should.equal(204)
@ -74,35 +73,35 @@ module.exports = TrackChangesClient = {
)
},
getCompressedUpdates(doc_id, callback) {
getCompressedUpdates(docId, callback) {
if (callback == null) {
callback = function () {}
}
return db.docHistory
.find({ doc_id: ObjectId(doc_id) })
.find({ doc_id: ObjectId(docId) })
.sort({ 'meta.end_ts': 1 })
.toArray(callback)
},
getProjectMetaData(project_id, callback) {
getProjectMetaData(projectId, callback) {
if (callback == null) {
callback = function () {}
}
return db.projectHistoryMetaData.findOne(
{
project_id: ObjectId(project_id),
project_id: ObjectId(projectId),
},
callback
)
},
setPreserveHistoryForProject(project_id, callback) {
setPreserveHistoryForProject(projectId, callback) {
if (callback == null) {
callback = function () {}
}
return db.projectHistoryMetaData.updateOne(
{
project_id: ObjectId(project_id),
project_id: ObjectId(projectId),
},
{
$set: { preserveHistory: true },
@ -114,19 +113,19 @@ module.exports = TrackChangesClient = {
)
},
pushRawUpdates(project_id, doc_id, updates, callback) {
pushRawUpdates(projectId, docId, updates, callback) {
if (callback == null) {
callback = function () {}
}
return rclient.sadd(
Keys.docsWithHistoryOps({ project_id }),
doc_id,
Keys.docsWithHistoryOps({ project_id: projectId }),
docId,
error => {
if (error != null) {
return callback(error)
}
return rclient.rpush(
Keys.uncompressedHistoryOps({ doc_id }),
Keys.uncompressedHistoryOps({ doc_id: docId }),
...Array.from(Array.from(updates).map(u => JSON.stringify(u))),
callback
)
@ -134,13 +133,13 @@ module.exports = TrackChangesClient = {
)
},
getDiff(project_id, doc_id, from, to, callback) {
getDiff(projectId, docId, from, to, callback) {
if (callback == null) {
callback = function () {}
}
return request.get(
{
url: `http://localhost:3015/project/${project_id}/doc/${doc_id}/diff?from=${from}&to=${to}`,
url: `http://localhost:3015/project/${projectId}/doc/${docId}/diff?from=${from}&to=${to}`,
},
(error, response, body) => {
if (error) return callback(error)
@ -150,13 +149,13 @@ module.exports = TrackChangesClient = {
)
},
getUpdates(project_id, options, callback) {
getUpdates(projectId, options, callback) {
if (callback == null) {
callback = function () {}
}
return request.get(
{
url: `http://localhost:3015/project/${project_id}/updates?before=${options.before}&min_count=${options.min_count}`,
url: `http://localhost:3015/project/${projectId}/updates?before=${options.before}&min_count=${options.min_count}`,
},
(error, response, body) => {
if (error) return callback(error)
@ -166,9 +165,9 @@ module.exports = TrackChangesClient = {
)
},
exportProject(project_id, callback) {
exportProject(projectId, callback) {
request.get(
{ url: `http://localhost:3015/project/${project_id}/export`, json: true },
{ url: `http://localhost:3015/project/${projectId}/export`, json: true },
(error, response, updates) => {
if (error) return callback(error)
response.statusCode.should.equal(200)
@ -177,15 +176,15 @@ module.exports = TrackChangesClient = {
)
},
restoreDoc(project_id, doc_id, version, user_id, callback) {
restoreDoc(projectId, docId, version, userId, callback) {
if (callback == null) {
callback = function () {}
}
return request.post(
{
url: `http://localhost:3015/project/${project_id}/doc/${doc_id}/version/${version}/restore`,
url: `http://localhost:3015/project/${projectId}/doc/${docId}/version/${version}/restore`,
headers: {
'X-User-Id': user_id,
'X-User-Id': userId,
},
},
(error, response, body) => {
@ -196,13 +195,13 @@ module.exports = TrackChangesClient = {
)
},
pushDocHistory(project_id, doc_id, callback) {
pushDocHistory(projectId, docId, callback) {
if (callback == null) {
callback = function () {}
}
return request.post(
{
url: `http://localhost:3015/project/${project_id}/doc/${doc_id}/push`,
url: `http://localhost:3015/project/${projectId}/doc/${docId}/push`,
},
(error, response, body) => {
response.statusCode.should.equal(204)
@ -211,13 +210,13 @@ module.exports = TrackChangesClient = {
)
},
pullDocHistory(project_id, doc_id, callback) {
pullDocHistory(projectId, docId, callback) {
if (callback == null) {
callback = function () {}
}
return request.post(
{
url: `http://localhost:3015/project/${project_id}/doc/${doc_id}/pull`,
url: `http://localhost:3015/project/${projectId}/doc/${docId}/pull`,
},
(error, response, body) => {
response.statusCode.should.equal(204)
@ -250,13 +249,13 @@ module.exports = TrackChangesClient = {
})
},
getS3Doc(project_id, doc_id, pack_id, callback) {
getS3Doc(projectId, docId, packId, callback) {
if (callback == null) {
callback = function () {}
}
const params = {
Bucket: S3_BUCKET,
Key: `${project_id}/changes-${doc_id}/pack-${pack_id}`,
Key: `${projectId}/changes-${docId}/pack-${packId}`,
}
return s3.getObject(params, (error, data) => {
@ -276,13 +275,13 @@ module.exports = TrackChangesClient = {
})
},
removeS3Doc(project_id, doc_id, callback) {
removeS3Doc(projectId, docId, callback) {
if (callback == null) {
callback = function () {}
}
let params = {
Bucket: S3_BUCKET,
Prefix: `${project_id}/changes-${doc_id}`,
Prefix: `${projectId}/changes-${docId}`,
}
return s3.listObjects(params, (error, data) => {

5
services/track-changes/test/unit/js/DiffManager/DiffManagerTests.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
no-return-assign,
no-unused-vars,
*/
@ -239,8 +238,8 @@ describe('DiffManager', function () {
beforeEach(function () {
let retried = false
this.DiffManager._tryGetDocumentBeforeVersion = (
project_id,
doc_id,
projectId,
docId,
version,
callback
) => {

3
services/track-changes/test/unit/js/LockManager/LockManagerTests.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
mocha/no-nested-tests,
no-return-assign,
no-undef,
@ -152,7 +151,7 @@ describe('LockManager', function () {
beforeEach(function (done) {
const startTime = Date.now()
this.LockManager.LOCK_TEST_INTERVAL = 5
this.LockManager.tryLock = function (doc_id, callback) {
this.LockManager.tryLock = function (docId, callback) {
if (callback == null) {
callback = function () {}
}

9
services/track-changes/test/unit/js/RedisManager/RedisManagerTests.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
no-return-assign,
no-unused-vars,
*/
@ -32,11 +31,11 @@ describe('RedisManager', function () {
redis: {
history: {
key_schema: {
uncompressedHistoryOps({ doc_id }) {
return `UncompressedHistoryOps:${doc_id}`
uncompressedHistoryOps({ doc_id: docId }) {
return `UncompressedHistoryOps:${docId}`
},
docsWithHistoryOps({ project_id }) {
return `DocsWithHistoryOps:${project_id}`
docsWithHistoryOps({ project_id: projectId }) {
return `DocsWithHistoryOps:${projectId}`
},
},
},

19
services/track-changes/test/unit/js/UpdatesManager/UpdatesManagerTests.js
View file

@ -1,5 +1,4 @@
/* eslint-disable
camelcase,
no-return-assign,
no-unused-vars,
*/
@ -35,8 +34,8 @@ describe('UpdatesManager', function () {
redis: {
lock: {
key_schema: {
historyLock({ doc_id }) {
return `HistoryLock:${doc_id}`
historyLock({ doc_id: docId }) {
return `HistoryLock:${docId}`
},
},
},
@ -457,7 +456,7 @@ describe('UpdatesManager', function () {
]
this.redisArray = this.updates.slice()
this.RedisManager.getOldestDocUpdates = (
doc_id,
docId,
batchSize,
callback
) => {
@ -672,9 +671,9 @@ describe('UpdatesManager', function () {
})
it('should process the doc ops for the each doc_id', function () {
return Array.from(this.doc_ids).map(doc_id =>
return Array.from(this.doc_ids).map(docId =>
this.UpdatesManager._processUncompressedUpdatesForDocWithLock
.calledWith(this.project_id, doc_id, this.temporary)
.calledWith(this.project_id, docId, this.temporary)
.should.equal(true)
)
})
@ -887,11 +886,11 @@ describe('UpdatesManager', function () {
this.user_info[this.user_id_1] = { email: 'user1@sharelatex.com' }
this.user_info[this.user_id_2] = { email: 'user2@sharelatex.com' }
this.WebApiManager.getUserInfo = (user_id, callback) => {
this.WebApiManager.getUserInfo = (userId, callback) => {
if (callback == null) {
callback = function () {}
}
return callback(null, this.user_info[user_id])
return callback(null, this.user_info[userId])
}
sinon.spy(this.WebApiManager, 'getUserInfo')
@ -961,11 +960,11 @@ describe('UpdatesManager', function () {
op: 'mock-op-2',
},
]
this.WebApiManager.getUserInfo = (user_id, callback) => {
this.WebApiManager.getUserInfo = (userId, callback) => {
if (callback == null) {
callback = function () {}
}
return callback(null, this.user_info[user_id])
return callback(null, this.user_info[userId])
}
sinon.spy(this.WebApiManager, 'getUserInfo')