overleaf/services/real-time/app/js/DocumentUpdaterManager.js

157 lines
4.9 KiB
JavaScript
Raw Normal View History

const request = require('request')
const _ = require('lodash')
const OError = require('@overleaf/o-error')
const logger = require('@overleaf/logger')
const settings = require('@overleaf/settings')
const metrics = require('@overleaf/metrics')
const {
ClientRequestedMissingOpsError,
DocumentUpdaterRequestFailedError,
NullBytesInOpError,
2021-07-13 07:04:45 -04:00
UpdateTooLargeError,
} = require('./Errors')
2014-11-12 10:54:55 -05:00
const rclient = require('@overleaf/redis-wrapper').createClient(
settings.redis.documentupdater
)
const Keys = settings.redis.documentupdater.key_schema
const DocumentUpdaterManager = {
getDocument(projectId, docId, fromVersion, callback) {
const timer = new metrics.Timer('get-document')
const url = `${settings.apis.documentupdater.url}/project/${projectId}/doc/${docId}?fromVersion=${fromVersion}`
logger.debug(
{ projectId, docId, fromVersion },
'getting doc from document updater'
)
[misc] reland decaff cleanup (#166) * [misc] decaff cleanup: RoomManager * [misc] decaff cleanup: RedisClientManager * [misc] decaff cleanup: SafeJsonParse * [misc] decaff cleanup: WebApiManager * [misc] decaff cleanup: WebsocketController * [misc] decaff cleanup: WebsocketLoadBalancer * [misc] decaff cleanup: SessionSockets * [misc] decaff cleanup: HttpController * [misc] decaff cleanup: HttpApiController * [misc] decaff cleanup: HealthCheckManager * [misc] decaff cleanup: EventLogger * [misc] decaff cleanup: Errors o-error will eliminate most of it -- when we migrate over. * [misc] decaff cleanup: DrainManager * [misc] decaff cleanup: DocumentUpdaterManager * [misc] decaff cleanup: DocumentUpdaterController: no-unused-vars * [misc] decaff cleanup: DocumentUpdaterController: Array.from * [misc] decaff cleanup: DocumentUpdaterController: implicit return * [misc] decaff cleanup: DocumentUpdaterController: IIFE * [misc] decaff cleanup: DocumentUpdaterController: null checks * [misc] decaff cleanup: DocumentUpdaterController: simpler loops * [misc] decaff cleanup: DocumentUpdaterController: move module name def * [misc] decaff cleanup: ConnectedUsersManager: handle-callback-err * [misc] decaff cleanup: ConnectedUsersManager: implicit returns * [misc] decaff cleanup: ConnectedUsersManager: null checks * [misc] decaff cleanup: ChannelManager: no-unused-vars * [misc] decaff cleanup: ChannelManager: implicit returns * [misc] decaff cleanup: ChannelManager: other cleanup - var -> const - drop variable assignment before return * [misc] decaff cleanup: AuthorizationManager: handle-callback-err Note: This requires a change in WebsocketController to provide a dummy callback. * [misc] decaff cleanup: AuthorizationManager: Array.from * [misc] decaff cleanup: AuthorizationManager: implicit returns * [misc] decaff cleanup: AuthorizationManager: null checks * [misc] decaff cleanup: Router: handle-callback-err * [misc] decaff cleanup: Router: standard/no-callback-literal * [misc] decaff cleanup: Router: Array.from * [misc] decaff cleanup: Router: implicit returns * [misc] decaff cleanup: Router: refactor __guard__ wrapper * [misc] decaff cleanup: Router: null checks And a minor bug fix: user.id -> user._id * [misc] decaff cleanup: Router: move variable declarations to assignments * [misc] decaff cleanup: app: implicit returns * [misc] decaff cleanup: app: __guard__ * [misc] decaff cleanup: app: null checks * [misc] decaff cleanup: app: function definitions * [misc] decaff cleanup: app: drop unused next argument * [misc] decaff cleanup: app: var -> const
2020-07-07 06:06:02 -04:00
request.get(url, function (err, res, body) {
timer.done()
[misc] reland decaff cleanup (#166) * [misc] decaff cleanup: RoomManager * [misc] decaff cleanup: RedisClientManager * [misc] decaff cleanup: SafeJsonParse * [misc] decaff cleanup: WebApiManager * [misc] decaff cleanup: WebsocketController * [misc] decaff cleanup: WebsocketLoadBalancer * [misc] decaff cleanup: SessionSockets * [misc] decaff cleanup: HttpController * [misc] decaff cleanup: HttpApiController * [misc] decaff cleanup: HealthCheckManager * [misc] decaff cleanup: EventLogger * [misc] decaff cleanup: Errors o-error will eliminate most of it -- when we migrate over. * [misc] decaff cleanup: DrainManager * [misc] decaff cleanup: DocumentUpdaterManager * [misc] decaff cleanup: DocumentUpdaterController: no-unused-vars * [misc] decaff cleanup: DocumentUpdaterController: Array.from * [misc] decaff cleanup: DocumentUpdaterController: implicit return * [misc] decaff cleanup: DocumentUpdaterController: IIFE * [misc] decaff cleanup: DocumentUpdaterController: null checks * [misc] decaff cleanup: DocumentUpdaterController: simpler loops * [misc] decaff cleanup: DocumentUpdaterController: move module name def * [misc] decaff cleanup: ConnectedUsersManager: handle-callback-err * [misc] decaff cleanup: ConnectedUsersManager: implicit returns * [misc] decaff cleanup: ConnectedUsersManager: null checks * [misc] decaff cleanup: ChannelManager: no-unused-vars * [misc] decaff cleanup: ChannelManager: implicit returns * [misc] decaff cleanup: ChannelManager: other cleanup - var -> const - drop variable assignment before return * [misc] decaff cleanup: AuthorizationManager: handle-callback-err Note: This requires a change in WebsocketController to provide a dummy callback. * [misc] decaff cleanup: AuthorizationManager: Array.from * [misc] decaff cleanup: AuthorizationManager: implicit returns * [misc] decaff cleanup: AuthorizationManager: null checks * [misc] decaff cleanup: Router: handle-callback-err * [misc] decaff cleanup: Router: standard/no-callback-literal * [misc] decaff cleanup: Router: Array.from * [misc] decaff cleanup: Router: implicit returns * [misc] decaff cleanup: Router: refactor __guard__ wrapper * [misc] decaff cleanup: Router: null checks And a minor bug fix: user.id -> user._id * [misc] decaff cleanup: Router: move variable declarations to assignments * [misc] decaff cleanup: app: implicit returns * [misc] decaff cleanup: app: __guard__ * [misc] decaff cleanup: app: null checks * [misc] decaff cleanup: app: function definitions * [misc] decaff cleanup: app: drop unused next argument * [misc] decaff cleanup: app: var -> const
2020-07-07 06:06:02 -04:00
if (err) {
OError.tag(err, 'error getting doc from doc updater')
return callback(err)
}
if (res.statusCode >= 200 && res.statusCode < 300) {
logger.debug(
{ projectId, docId },
'got doc from document document updater'
)
try {
body = JSON.parse(body)
} catch (error) {
OError.tag(error, 'error parsing doc updater response')
return callback(error)
}
[misc] reland decaff cleanup (#166) * [misc] decaff cleanup: RoomManager * [misc] decaff cleanup: RedisClientManager * [misc] decaff cleanup: SafeJsonParse * [misc] decaff cleanup: WebApiManager * [misc] decaff cleanup: WebsocketController * [misc] decaff cleanup: WebsocketLoadBalancer * [misc] decaff cleanup: SessionSockets * [misc] decaff cleanup: HttpController * [misc] decaff cleanup: HttpApiController * [misc] decaff cleanup: HealthCheckManager * [misc] decaff cleanup: EventLogger * [misc] decaff cleanup: Errors o-error will eliminate most of it -- when we migrate over. * [misc] decaff cleanup: DrainManager * [misc] decaff cleanup: DocumentUpdaterManager * [misc] decaff cleanup: DocumentUpdaterController: no-unused-vars * [misc] decaff cleanup: DocumentUpdaterController: Array.from * [misc] decaff cleanup: DocumentUpdaterController: implicit return * [misc] decaff cleanup: DocumentUpdaterController: IIFE * [misc] decaff cleanup: DocumentUpdaterController: null checks * [misc] decaff cleanup: DocumentUpdaterController: simpler loops * [misc] decaff cleanup: DocumentUpdaterController: move module name def * [misc] decaff cleanup: ConnectedUsersManager: handle-callback-err * [misc] decaff cleanup: ConnectedUsersManager: implicit returns * [misc] decaff cleanup: ConnectedUsersManager: null checks * [misc] decaff cleanup: ChannelManager: no-unused-vars * [misc] decaff cleanup: ChannelManager: implicit returns * [misc] decaff cleanup: ChannelManager: other cleanup - var -> const - drop variable assignment before return * [misc] decaff cleanup: AuthorizationManager: handle-callback-err Note: This requires a change in WebsocketController to provide a dummy callback. * [misc] decaff cleanup: AuthorizationManager: Array.from * [misc] decaff cleanup: AuthorizationManager: implicit returns * [misc] decaff cleanup: AuthorizationManager: null checks * [misc] decaff cleanup: Router: handle-callback-err * [misc] decaff cleanup: Router: standard/no-callback-literal * [misc] decaff cleanup: Router: Array.from * [misc] decaff cleanup: Router: implicit returns * [misc] decaff cleanup: Router: refactor __guard__ wrapper * [misc] decaff cleanup: Router: null checks And a minor bug fix: user.id -> user._id * [misc] decaff cleanup: Router: move variable declarations to assignments * [misc] decaff cleanup: app: implicit returns * [misc] decaff cleanup: app: __guard__ * [misc] decaff cleanup: app: null checks * [misc] decaff cleanup: app: function definitions * [misc] decaff cleanup: app: drop unused next argument * [misc] decaff cleanup: app: var -> const
2020-07-07 06:06:02 -04:00
body = body || {}
callback(
null,
body.lines,
body.version,
body.ranges,
body.ops,
body.ttlInS
)
} else if (res.statusCode === 422 && body?.firstVersionInRedis) {
callback(new ClientRequestedMissingOpsError(422, body))
} else if ([404, 422].includes(res.statusCode)) {
callback(new ClientRequestedMissingOpsError(res.statusCode))
} else {
callback(
new DocumentUpdaterRequestFailedError('getDocument', res.statusCode)
)
}
})
},
checkDocument(projectId, docId, callback) {
// in this call fromVersion = -1 means get document without docOps
DocumentUpdaterManager.getDocument(projectId, docId, -1, callback)
},
flushProjectToMongoAndDelete(projectId, callback) {
// this method is called when the last connected user leaves the project
logger.debug({ projectId }, 'deleting project from document updater')
const timer = new metrics.Timer('delete.mongo.project')
// flush the project in the background when all users have left
const url =
`${settings.apis.documentupdater.url}/project/${projectId}?background=true` +
(settings.shutDownInProgress ? '&shutdown=true' : '')
[misc] reland decaff cleanup (#166) * [misc] decaff cleanup: RoomManager * [misc] decaff cleanup: RedisClientManager * [misc] decaff cleanup: SafeJsonParse * [misc] decaff cleanup: WebApiManager * [misc] decaff cleanup: WebsocketController * [misc] decaff cleanup: WebsocketLoadBalancer * [misc] decaff cleanup: SessionSockets * [misc] decaff cleanup: HttpController * [misc] decaff cleanup: HttpApiController * [misc] decaff cleanup: HealthCheckManager * [misc] decaff cleanup: EventLogger * [misc] decaff cleanup: Errors o-error will eliminate most of it -- when we migrate over. * [misc] decaff cleanup: DrainManager * [misc] decaff cleanup: DocumentUpdaterManager * [misc] decaff cleanup: DocumentUpdaterController: no-unused-vars * [misc] decaff cleanup: DocumentUpdaterController: Array.from * [misc] decaff cleanup: DocumentUpdaterController: implicit return * [misc] decaff cleanup: DocumentUpdaterController: IIFE * [misc] decaff cleanup: DocumentUpdaterController: null checks * [misc] decaff cleanup: DocumentUpdaterController: simpler loops * [misc] decaff cleanup: DocumentUpdaterController: move module name def * [misc] decaff cleanup: ConnectedUsersManager: handle-callback-err * [misc] decaff cleanup: ConnectedUsersManager: implicit returns * [misc] decaff cleanup: ConnectedUsersManager: null checks * [misc] decaff cleanup: ChannelManager: no-unused-vars * [misc] decaff cleanup: ChannelManager: implicit returns * [misc] decaff cleanup: ChannelManager: other cleanup - var -> const - drop variable assignment before return * [misc] decaff cleanup: AuthorizationManager: handle-callback-err Note: This requires a change in WebsocketController to provide a dummy callback. * [misc] decaff cleanup: AuthorizationManager: Array.from * [misc] decaff cleanup: AuthorizationManager: implicit returns * [misc] decaff cleanup: AuthorizationManager: null checks * [misc] decaff cleanup: Router: handle-callback-err * [misc] decaff cleanup: Router: standard/no-callback-literal * [misc] decaff cleanup: Router: Array.from * [misc] decaff cleanup: Router: implicit returns * [misc] decaff cleanup: Router: refactor __guard__ wrapper * [misc] decaff cleanup: Router: null checks And a minor bug fix: user.id -> user._id * [misc] decaff cleanup: Router: move variable declarations to assignments * [misc] decaff cleanup: app: implicit returns * [misc] decaff cleanup: app: __guard__ * [misc] decaff cleanup: app: null checks * [misc] decaff cleanup: app: function definitions * [misc] decaff cleanup: app: drop unused next argument * [misc] decaff cleanup: app: var -> const
2020-07-07 06:06:02 -04:00
request.del(url, function (err, res) {
timer.done()
[misc] reland decaff cleanup (#166) * [misc] decaff cleanup: RoomManager * [misc] decaff cleanup: RedisClientManager * [misc] decaff cleanup: SafeJsonParse * [misc] decaff cleanup: WebApiManager * [misc] decaff cleanup: WebsocketController * [misc] decaff cleanup: WebsocketLoadBalancer * [misc] decaff cleanup: SessionSockets * [misc] decaff cleanup: HttpController * [misc] decaff cleanup: HttpApiController * [misc] decaff cleanup: HealthCheckManager * [misc] decaff cleanup: EventLogger * [misc] decaff cleanup: Errors o-error will eliminate most of it -- when we migrate over. * [misc] decaff cleanup: DrainManager * [misc] decaff cleanup: DocumentUpdaterManager * [misc] decaff cleanup: DocumentUpdaterController: no-unused-vars * [misc] decaff cleanup: DocumentUpdaterController: Array.from * [misc] decaff cleanup: DocumentUpdaterController: implicit return * [misc] decaff cleanup: DocumentUpdaterController: IIFE * [misc] decaff cleanup: DocumentUpdaterController: null checks * [misc] decaff cleanup: DocumentUpdaterController: simpler loops * [misc] decaff cleanup: DocumentUpdaterController: move module name def * [misc] decaff cleanup: ConnectedUsersManager: handle-callback-err * [misc] decaff cleanup: ConnectedUsersManager: implicit returns * [misc] decaff cleanup: ConnectedUsersManager: null checks * [misc] decaff cleanup: ChannelManager: no-unused-vars * [misc] decaff cleanup: ChannelManager: implicit returns * [misc] decaff cleanup: ChannelManager: other cleanup - var -> const - drop variable assignment before return * [misc] decaff cleanup: AuthorizationManager: handle-callback-err Note: This requires a change in WebsocketController to provide a dummy callback. * [misc] decaff cleanup: AuthorizationManager: Array.from * [misc] decaff cleanup: AuthorizationManager: implicit returns * [misc] decaff cleanup: AuthorizationManager: null checks * [misc] decaff cleanup: Router: handle-callback-err * [misc] decaff cleanup: Router: standard/no-callback-literal * [misc] decaff cleanup: Router: Array.from * [misc] decaff cleanup: Router: implicit returns * [misc] decaff cleanup: Router: refactor __guard__ wrapper * [misc] decaff cleanup: Router: null checks And a minor bug fix: user.id -> user._id * [misc] decaff cleanup: Router: move variable declarations to assignments * [misc] decaff cleanup: app: implicit returns * [misc] decaff cleanup: app: __guard__ * [misc] decaff cleanup: app: null checks * [misc] decaff cleanup: app: function definitions * [misc] decaff cleanup: app: drop unused next argument * [misc] decaff cleanup: app: var -> const
2020-07-07 06:06:02 -04:00
if (err) {
OError.tag(err, 'error deleting project from document updater')
[misc] reland decaff cleanup (#166) * [misc] decaff cleanup: RoomManager * [misc] decaff cleanup: RedisClientManager * [misc] decaff cleanup: SafeJsonParse * [misc] decaff cleanup: WebApiManager * [misc] decaff cleanup: WebsocketController * [misc] decaff cleanup: WebsocketLoadBalancer * [misc] decaff cleanup: SessionSockets * [misc] decaff cleanup: HttpController * [misc] decaff cleanup: HttpApiController * [misc] decaff cleanup: HealthCheckManager * [misc] decaff cleanup: EventLogger * [misc] decaff cleanup: Errors o-error will eliminate most of it -- when we migrate over. * [misc] decaff cleanup: DrainManager * [misc] decaff cleanup: DocumentUpdaterManager * [misc] decaff cleanup: DocumentUpdaterController: no-unused-vars * [misc] decaff cleanup: DocumentUpdaterController: Array.from * [misc] decaff cleanup: DocumentUpdaterController: implicit return * [misc] decaff cleanup: DocumentUpdaterController: IIFE * [misc] decaff cleanup: DocumentUpdaterController: null checks * [misc] decaff cleanup: DocumentUpdaterController: simpler loops * [misc] decaff cleanup: DocumentUpdaterController: move module name def * [misc] decaff cleanup: ConnectedUsersManager: handle-callback-err * [misc] decaff cleanup: ConnectedUsersManager: implicit returns * [misc] decaff cleanup: ConnectedUsersManager: null checks * [misc] decaff cleanup: ChannelManager: no-unused-vars * [misc] decaff cleanup: ChannelManager: implicit returns * [misc] decaff cleanup: ChannelManager: other cleanup - var -> const - drop variable assignment before return * [misc] decaff cleanup: AuthorizationManager: handle-callback-err Note: This requires a change in WebsocketController to provide a dummy callback. * [misc] decaff cleanup: AuthorizationManager: Array.from * [misc] decaff cleanup: AuthorizationManager: implicit returns * [misc] decaff cleanup: AuthorizationManager: null checks * [misc] decaff cleanup: Router: handle-callback-err * [misc] decaff cleanup: Router: standard/no-callback-literal * [misc] decaff cleanup: Router: Array.from * [misc] decaff cleanup: Router: implicit returns * [misc] decaff cleanup: Router: refactor __guard__ wrapper * [misc] decaff cleanup: Router: null checks And a minor bug fix: user.id -> user._id * [misc] decaff cleanup: Router: move variable declarations to assignments * [misc] decaff cleanup: app: implicit returns * [misc] decaff cleanup: app: __guard__ * [misc] decaff cleanup: app: null checks * [misc] decaff cleanup: app: function definitions * [misc] decaff cleanup: app: drop unused next argument * [misc] decaff cleanup: app: var -> const
2020-07-07 06:06:02 -04:00
callback(err)
} else if (res.statusCode >= 200 && res.statusCode < 300) {
logger.debug({ projectId }, 'deleted project from document updater')
[misc] reland decaff cleanup (#166) * [misc] decaff cleanup: RoomManager * [misc] decaff cleanup: RedisClientManager * [misc] decaff cleanup: SafeJsonParse * [misc] decaff cleanup: WebApiManager * [misc] decaff cleanup: WebsocketController * [misc] decaff cleanup: WebsocketLoadBalancer * [misc] decaff cleanup: SessionSockets * [misc] decaff cleanup: HttpController * [misc] decaff cleanup: HttpApiController * [misc] decaff cleanup: HealthCheckManager * [misc] decaff cleanup: EventLogger * [misc] decaff cleanup: Errors o-error will eliminate most of it -- when we migrate over. * [misc] decaff cleanup: DrainManager * [misc] decaff cleanup: DocumentUpdaterManager * [misc] decaff cleanup: DocumentUpdaterController: no-unused-vars * [misc] decaff cleanup: DocumentUpdaterController: Array.from * [misc] decaff cleanup: DocumentUpdaterController: implicit return * [misc] decaff cleanup: DocumentUpdaterController: IIFE * [misc] decaff cleanup: DocumentUpdaterController: null checks * [misc] decaff cleanup: DocumentUpdaterController: simpler loops * [misc] decaff cleanup: DocumentUpdaterController: move module name def * [misc] decaff cleanup: ConnectedUsersManager: handle-callback-err * [misc] decaff cleanup: ConnectedUsersManager: implicit returns * [misc] decaff cleanup: ConnectedUsersManager: null checks * [misc] decaff cleanup: ChannelManager: no-unused-vars * [misc] decaff cleanup: ChannelManager: implicit returns * [misc] decaff cleanup: ChannelManager: other cleanup - var -> const - drop variable assignment before return * [misc] decaff cleanup: AuthorizationManager: handle-callback-err Note: This requires a change in WebsocketController to provide a dummy callback. * [misc] decaff cleanup: AuthorizationManager: Array.from * [misc] decaff cleanup: AuthorizationManager: implicit returns * [misc] decaff cleanup: AuthorizationManager: null checks * [misc] decaff cleanup: Router: handle-callback-err * [misc] decaff cleanup: Router: standard/no-callback-literal * [misc] decaff cleanup: Router: Array.from * [misc] decaff cleanup: Router: implicit returns * [misc] decaff cleanup: Router: refactor __guard__ wrapper * [misc] decaff cleanup: Router: null checks And a minor bug fix: user.id -> user._id * [misc] decaff cleanup: Router: move variable declarations to assignments * [misc] decaff cleanup: app: implicit returns * [misc] decaff cleanup: app: __guard__ * [misc] decaff cleanup: app: null checks * [misc] decaff cleanup: app: function definitions * [misc] decaff cleanup: app: drop unused next argument * [misc] decaff cleanup: app: var -> const
2020-07-07 06:06:02 -04:00
callback(null)
} else {
callback(
new DocumentUpdaterRequestFailedError(
'flushProjectToMongoAndDelete',
res.statusCode
)
)
}
})
},
2021-02-08 07:19:25 -05:00
_getPendingUpdateListKey() {
2021-02-09 05:48:40 -05:00
const shard = _.random(0, settings.pendingUpdateListShardCount - 1)
2021-02-08 07:19:25 -05:00
if (shard === 0) {
return 'pending-updates-list'
} else {
return `pending-updates-list-${shard}`
}
},
queueChange(projectId, docId, change, callback) {
const allowedKeys = [
'doc',
'op',
'v',
'dupIfSource',
'meta',
'lastV',
2021-07-13 07:04:45 -04:00
'hash',
]
change = _.pick(change, allowedKeys)
const jsonChange = JSON.stringify(change)
if (jsonChange.indexOf('\u0000') !== -1) {
// memory corruption check
return callback(new NullBytesInOpError(jsonChange))
}
const updateSize = jsonChange.length
if (updateSize > settings.maxUpdateSize) {
return callback(new UpdateTooLargeError(updateSize))
}
// record metric for each update added to queue
metrics.summary('redis.pendingUpdates', updateSize, { status: 'push' })
2020-04-06 11:24:10 -04:00
const docKey = `${projectId}:${docId}`
// Push onto pendingUpdates for doc_id first, because once the doc updater
// gets an entry on pending-updates-list, it starts processing.
2021-07-13 07:04:45 -04:00
rclient.rpush(
Keys.pendingUpdates({ doc_id: docId }),
2021-07-13 07:04:45 -04:00
jsonChange,
function (error) {
if (error) {
2021-07-13 07:04:45 -04:00
error = new OError('error pushing update into redis').withCause(error)
return callback(error)
}
2021-07-13 07:04:45 -04:00
const queueKey = DocumentUpdaterManager._getPendingUpdateListKey()
rclient.rpush(queueKey, docKey, function (error) {
2021-07-13 07:04:45 -04:00
if (error) {
error = new OError('error pushing doc_id into redis')
.withInfo({ queueKey })
.withCause(error)
}
callback(error)
})
}
)
},
}
module.exports = DocumentUpdaterManager