github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #17690 from overleaf/em-promisify-project-history-redis-manager

Browse source 
Promisify ProjectHistoryRedisManager

GitOrigin-RevId: 11908c6cc102653b34e2014af230e3912cd8b432
This commit is contained in:
Eric Mc Sween 2024-04-03 10:08:16 -04:00 committed by Copybot
parent caf715f539
commit 9d78fd0945
2 changed files with 67 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
services/document-updater/app/js/ProjectHistoryRedisManager.js
View file

@ -1,5 +1,5 @@
const Settings = require('@overleaf/settings') const Settings = require('@overleaf/settings')
const { promisifyAll } = require('@overleaf/promise-utils') const { callbackifyAll } = require('@overleaf/promise-utils')
const projectHistoryKeys = Settings.redis?.project_history?.key_schema const projectHistoryKeys = Settings.redis?.project_history?.key_schema
const rclient = require('@overleaf/redis-wrapper').createClient( const rclient = require('@overleaf/redis-wrapper').createClient(
Settings.redis.project_history Settings.redis.project_history
@ -7,12 +7,11 @@ const rclient = require('@overleaf/redis-wrapper').createClient(
const logger = require('@overleaf/logger') const logger = require('@overleaf/logger')
const metrics = require('./Metrics') const metrics = require('./Metrics')
const { docIsTooLarge } = require('./Limits') const { docIsTooLarge } = require('./Limits')
const OError = require('@overleaf/o-error')
const ProjectHistoryRedisManager = { const ProjectHistoryRedisManager = {
queueOps(projectId, ...rest) { async queueOps(projectId, ...ops) {
// Record metric for ops pushed onto queue // Record metric for ops pushed onto queue
const callback = rest.pop()
const ops = rest
for (const op of ops) { for (const op of ops) {
metrics.summary('redis.projectHistoryOps', op.length, { status: 'push' }) metrics.summary('redis.projectHistoryOps', op.length, { status: 'push' })
} }
@ -30,24 +29,18 @@ const ProjectHistoryRedisManager = {
}), }),
Date.now() Date.now()
) )
multi.exec(function (error, result) { const result = await multi.exec()
if (error) { return result[0]
return callback(error)
}
// return the number of entries pushed onto the project history queue
callback(null, result[0])
})
}, },
queueRenameEntity( async queueRenameEntity(
projectId, projectId,
projectHistoryId, projectHistoryId,
entityType, entityType,
entityId, entityId,
userId, userId,
projectUpdate, projectUpdate,
source, source
callback
) { ) {
projectUpdate = { projectUpdate = {
pathname: projectUpdate.pathname, pathname: projectUpdate.pathname,
@ -73,18 +66,17 @@ const ProjectHistoryRedisManager = {
) )
const jsonUpdate = JSON.stringify(projectUpdate) const jsonUpdate = JSON.stringify(projectUpdate)
ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate, callback) return await ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate)
}, },
queueAddEntity( async queueAddEntity(
projectId, projectId,
projectHistoryId, projectHistoryId,
entityType, entityType,
entityId, entityId,
userId, userId,
projectUpdate, projectUpdate,
source, source
callback
) { ) {
projectUpdate = { projectUpdate = {
pathname: projectUpdate.pathname, pathname: projectUpdate.pathname,
@ -111,16 +103,10 @@ const ProjectHistoryRedisManager = {
) )
const jsonUpdate = JSON.stringify(projectUpdate) const jsonUpdate = JSON.stringify(projectUpdate)
ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate, callback) return await ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate)
}, },
queueResyncProjectStructure( async queueResyncProjectStructure(projectId, projectHistoryId, docs, files) {
projectId,
projectHistoryId,
docs,
files,
callback
) {
logger.debug({ projectId, docs, files }, 'queue project structure resync') logger.debug({ projectId, docs, files }, 'queue project structure resync')
const projectUpdate = { const projectUpdate = {
resyncProjectStructure: { docs, files }, resyncProjectStructure: { docs, files },
@ -130,17 +116,16 @@ const ProjectHistoryRedisManager = {
}, },
} }
const jsonUpdate = JSON.stringify(projectUpdate) const jsonUpdate = JSON.stringify(projectUpdate)
ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate, callback) return await ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate)
}, },
queueResyncDocContent( async queueResyncDocContent(
projectId, projectId,
projectHistoryId, projectHistoryId,
docId, docId,
lines, lines,
version, version,
pathname, pathname
callback
) { ) {
logger.debug( logger.debug(
{ projectId, docId, lines, version, pathname }, { projectId, docId, lines, version, pathname },
@ -163,15 +148,16 @@ const ProjectHistoryRedisManager = {
// project update length as an upper bound // project update length as an upper bound
const sizeBound = jsonUpdate.length const sizeBound = jsonUpdate.length
if (docIsTooLarge(sizeBound, lines, Settings.max_doc_length)) { if (docIsTooLarge(sizeBound, lines, Settings.max_doc_length)) {
const err = new Error( throw new OError(
'blocking resync doc content insert into project history queue: doc is too large' 'blocking resync doc content insert into project history queue: doc is too large',
{ projectId, docId, docSize: sizeBound }
) )
logger.error({ projectId, docId, err, docSize: sizeBound }, err.message)
return callback(err)
} }
ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate, callback) return await ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate)
}, },
} }
module.exports = ProjectHistoryRedisManager module.exports = {
module.exports.promises = promisifyAll(ProjectHistoryRedisManager) ...callbackifyAll(ProjectHistoryRedisManager),
promises: ProjectHistoryRedisManager,
}

99
services/document-updater/test/unit/js/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.js
View file

@ -1,4 +1,5 @@
const sinon = require('sinon') const sinon = require('sinon')
const { expect } = require('chai')
const modulePath = '../../../../app/js/ProjectHistoryRedisManager.js' const modulePath = '../../../../app/js/ProjectHistoryRedisManager.js'
const SandboxedModule = require('sandboxed-module') const SandboxedModule = require('sandboxed-module')
const tk = require('timekeeper') const tk = require('timekeeper')
@ -8,7 +9,6 @@ describe('ProjectHistoryRedisManager', function () {
this.project_id = 'project-id-123' this.project_id = 'project-id-123'
this.projectHistoryId = 'history-id-123' this.projectHistoryId = 'history-id-123'
this.user_id = 'user-id-123' this.user_id = 'user-id-123'
this.callback = sinon.stub()
this.rclient = {} this.rclient = {}
this.source = 'editor' this.source = 'editor'
tk.freeze(new Date()) tk.freeze(new Date())
@ -48,17 +48,15 @@ describe('ProjectHistoryRedisManager', function () {
}) })
describe('queueOps', function () { describe('queueOps', function () {
beforeEach(function () { beforeEach(async function () {
this.ops = ['mock-op-1', 'mock-op-2'] this.ops = ['mock-op-1', 'mock-op-2']
this.multi = { exec: sinon.stub() } this.multi = { exec: sinon.stub().resolves([1]) }
this.multi.rpush = sinon.stub() this.multi.rpush = sinon.stub()
this.multi.setnx = sinon.stub() this.multi.setnx = sinon.stub()
this.rclient.multi = () => this.multi this.rclient.multi = () => this.multi
// @rclient = multi: () => @multi await this.ProjectHistoryRedisManager.promises.queueOps(
this.ProjectHistoryRedisManager.queueOps(
this.project_id, this.project_id,
...this.ops, ...this.ops
this.callback
) )
}) })
@ -83,7 +81,7 @@ describe('ProjectHistoryRedisManager', function () {
}) })
describe('queueRenameEntity', function () { describe('queueRenameEntity', function () {
beforeEach(function () { beforeEach(async function () {
this.file_id = 1234 this.file_id = 1234
this.rawUpdate = { this.rawUpdate = {
@ -92,16 +90,17 @@ describe('ProjectHistoryRedisManager', function () {
version: (this.version = 2), version: (this.version = 2),
} }
this.ProjectHistoryRedisManager.queueOps = sinon.stub() this.ProjectHistoryRedisManager.promises.queueOps = sinon
this.ProjectHistoryRedisManager.queueRenameEntity( .stub()
.resolves()
await this.ProjectHistoryRedisManager.promises.queueRenameEntity(
this.project_id, this.project_id,
this.projectHistoryId, this.projectHistoryId,
'file', 'file',
this.file_id, this.file_id,
this.user_id, this.user_id,
this.rawUpdate, this.rawUpdate,
this.source, this.source
this.callback
) )
}) })
@ -119,19 +118,14 @@ describe('ProjectHistoryRedisManager', function () {
file: this.file_id, file: this.file_id,
} }
this.ProjectHistoryRedisManager.queueOps this.ProjectHistoryRedisManager.promises.queueOps
.calledWithExactly( .calledWithExactly(this.project_id, JSON.stringify(update))
this.project_id,
JSON.stringify(update),
this.callback
)
.should.equal(true) .should.equal(true)
}) })
}) })
describe('queueAddEntity', function () { describe('queueAddEntity', function () {
beforeEach(function () { beforeEach(async function () {
this.rclient.rpush = sinon.stub().yields()
this.doc_id = 1234 this.doc_id = 1234
this.rawUpdate = { this.rawUpdate = {
@ -141,16 +135,17 @@ describe('ProjectHistoryRedisManager', function () {
url: (this.url = 'filestore.example.com'), url: (this.url = 'filestore.example.com'),
} }
this.ProjectHistoryRedisManager.queueOps = sinon.stub() this.ProjectHistoryRedisManager.promises.queueOps = sinon
this.ProjectHistoryRedisManager.queueAddEntity( .stub()
.resolves()
await this.ProjectHistoryRedisManager.promises.queueAddEntity(
this.project_id, this.project_id,
this.projectHistoryId, this.projectHistoryId,
'doc', 'doc',
this.doc_id, this.doc_id,
this.user_id, this.user_id,
this.rawUpdate, this.rawUpdate,
this.source, this.source
this.callback
) )
}) })
@ -169,12 +164,8 @@ describe('ProjectHistoryRedisManager', function () {
doc: this.doc_id, doc: this.doc_id,
} }
this.ProjectHistoryRedisManager.queueOps this.ProjectHistoryRedisManager.promises.queueOps
.calledWithExactly( .calledWithExactly(this.project_id, JSON.stringify(update))
this.project_id,
JSON.stringify(update),
this.callback
)
.should.equal(true) .should.equal(true)
}) })
@ -202,19 +193,20 @@ describe('ProjectHistoryRedisManager', function () {
}, },
} }
this.ProjectHistoryRedisManager.queueOps = sinon.stub() this.ProjectHistoryRedisManager.promises.queueOps = sinon
.stub()
.resolves()
}) })
describe('with a good doc', function () { describe('with a good doc', function () {
beforeEach(function () { beforeEach(async function () {
this.ProjectHistoryRedisManager.queueResyncDocContent( await this.ProjectHistoryRedisManager.promises.queueResyncDocContent(
this.project_id, this.project_id,
this.projectHistoryId, this.projectHistoryId,
this.doc_id, this.doc_id,
this.lines, this.lines,
this.version, this.version,
this.pathname, this.pathname
this.callback
) )
}) })
it('should check if the doc is too large', function () { it('should check if the doc is too large', function () {
@ -228,34 +220,31 @@ describe('ProjectHistoryRedisManager', function () {
}) })
it('should queue an update', function () { it('should queue an update', function () {
this.ProjectHistoryRedisManager.queueOps this.ProjectHistoryRedisManager.promises.queueOps
.calledWithExactly( .calledWithExactly(this.project_id, JSON.stringify(this.update))
this.project_id,
JSON.stringify(this.update),
this.callback
)
.should.equal(true) .should.equal(true)
}) })
}) })
describe('with a doc that is too large', function () { describe('with a doc that is too large', function () {
beforeEach(function () { beforeEach(async function () {
this.Limits.docIsTooLarge.returns(true) this.Limits.docIsTooLarge.returns(true)
this.ProjectHistoryRedisManager.queueResyncDocContent( await expect(
this.project_id, this.ProjectHistoryRedisManager.promises.queueResyncDocContent(
this.projectHistoryId, this.project_id,
this.doc_id, this.projectHistoryId,
this.lines, this.doc_id,
this.version, this.lines,
this.pathname, this.version,
this.callback this.pathname
) )
).to.be.rejected
}) })
it('should not queue an update if the doc is too large', function () { it('should not queue an update if the doc is too large', function () {
this.ProjectHistoryRedisManager.queueOps.called.should.equal(false) this.ProjectHistoryRedisManager.promises.queueOps.called.should.equal(
this.callback false
.calledWith(sinon.match.instanceOf(Error)) )
.should.equal(true)
}) })
}) })
}) })