[misc] exportProject: collect and send user ids of updates in trailer

This commit is contained in:
Jakob Ackermann 2021-02-25 09:52:54 +00:00
parent f411049b82
commit ed70b99d8a
5 changed files with 49 additions and 18 deletions

View file

@ -209,7 +209,7 @@ module.exports = HttpController = {
logger.log({ project_id }, 'exporting project history')
UpdatesManager.exportProject(project_id, function (
err,
updates,
{ updates, userIds },
confirmWrite
) {
const abortStreaming = req.aborted || res.finished || res.destroyed
@ -236,6 +236,7 @@ module.exports = HttpController = {
// The first write will emit the 200 status, headers and start of the
// response payload (open array)
res.setHeader('Content-Type', 'application/json')
res.setHeader('Trailer', 'X-User-Ids')
res.writeHead(200)
res.write('[')
}
@ -255,7 +256,8 @@ module.exports = HttpController = {
if (isLastWrite) {
// The last write will have no updates and will finish the response
// payload (close array).
// payload (close array) and emit the userIds as trailer.
res.addTrailers({ 'X-User-Ids': JSON.stringify(userIds) })
res.end(']')
}
})

View file

@ -641,6 +641,8 @@ module.exports = UpdatesManager = {
PackManager.makeProjectIterator(projectId, before, (err, iterator) => {
if (err) return consumer(err)
const accumulatedUserIds = new Set()
async.whilst(
() => !iterator.done(),
@ -654,13 +656,26 @@ module.exports = UpdatesManager = {
// call.
return cb()
}
updatesFromASinglePack.forEach((update) => {
accumulatedUserIds.add(
// Super defensive access on update details.
String(update && update.meta && update.meta.user_id)
)
})
// Emit updates and wait for the consumer.
consumer(null, updatesFromASinglePack, cb)
consumer(null, { updates: updatesFromASinglePack }, cb)
}),
(err) => {
if (err) return consumer(err)
consumer(null, [])
// Adding undefined can happen for broken updates.
accumulatedUserIds.delete('undefined')
consumer(null, {
updates: [],
userIds: Array.from(accumulatedUserIds).sort()
})
}
)
})

View file

@ -50,6 +50,7 @@ describe('Archiving updates', function () {
this.now = Date.now()
this.to = this.now
this.user_id = ObjectId().toString()
this.user_id_2 = ObjectId().toString()
this.doc_id = ObjectId().toString()
this.project_id = ObjectId().toString()
@ -92,7 +93,7 @@ describe('Archiving updates', function () {
op: [{ i: 'b', p: 0 }],
meta: {
ts: this.now + (i - 2048) * this.hours + 10 * this.minutes,
user_id: this.user_id
user_id: this.user_id_2
},
v: 2 * i + 2
})
@ -144,13 +145,17 @@ describe('Archiving updates', function () {
function testExportFeature() {
describe('exporting the project', function () {
before('fetch export', function (done) {
TrackChangesClient.exportProject(this.project_id, (error, updates) => {
if (error) {
return done(error)
TrackChangesClient.exportProject(
this.project_id,
(error, updates, userIds) => {
if (error) {
return done(error)
}
this.exportedUpdates = updates
this.exportedUserIds = userIds
done()
}
this.exportedUpdates = updates
done()
})
)
})
it('should include all the imported updates, with ids, sorted by timestamp', function () {
@ -175,6 +180,10 @@ describe('Archiving updates', function () {
return exportedUpdate
})
expect(this.exportedUpdates).to.deep.equal(expectedExportedUpdates)
expect(this.exportedUserIds).to.deep.equal([
this.user_id,
this.user_id_2
])
})
})
}

View file

@ -11,17 +11,22 @@ describe('ExportProject', function () {
describe('when there are no updates', function () {
before('fetch export', function (done) {
TrackChangesClient.exportProject(ObjectId(), (error, updates) => {
if (error) {
return done(error)
TrackChangesClient.exportProject(
ObjectId(),
(error, updates, userIds) => {
if (error) {
return done(error)
}
this.exportedUpdates = updates
this.exportedUserIds = userIds
done()
}
this.exportedUpdates = updates
done()
})
)
})
it('should export an empty array', function () {
expect(this.exportedUpdates).to.deep.equal([])
expect(this.exportedUserIds).to.deep.equal([])
})
})

View file

@ -171,7 +171,7 @@ module.exports = TrackChangesClient = {
(error, response, updates) => {
if (error) return callback(error)
response.statusCode.should.equal(200)
callback(null, updates)
callback(null, updates, JSON.parse(response.trailers['x-user-ids']))
}
)
},