2020-02-17 12:35:50 -05:00
|
|
|
/* eslint-disable
|
|
|
|
no-unused-vars,
|
|
|
|
*/
|
|
|
|
// TODO: This file was created by bulk-decaffeinate.
|
|
|
|
// Fix any style issues and re-enable lint.
|
2020-02-17 12:35:39 -05:00
|
|
|
/*
|
|
|
|
* decaffeinate suggestions:
|
|
|
|
* DS101: Remove unnecessary use of Array.from
|
|
|
|
* DS102: Remove unnecessary code created because of implicit returns
|
|
|
|
* DS207: Consider shorter variations of null checks
|
|
|
|
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
|
|
|
*/
|
2020-02-17 12:35:59 -05:00
|
|
|
let TrackChangesClient
|
|
|
|
const async = require('async')
|
|
|
|
const zlib = require('zlib')
|
|
|
|
const request = require('request')
|
2021-07-12 12:47:16 -04:00
|
|
|
const Settings = require('@overleaf/settings')
|
2020-11-10 06:32:05 -05:00
|
|
|
const rclient = require('@overleaf/redis-wrapper').createClient(
|
|
|
|
Settings.redis.history
|
|
|
|
) // Only works locally for now
|
2020-02-17 12:35:59 -05:00
|
|
|
const Keys = Settings.redis.history.key_schema
|
2020-09-10 11:48:09 -04:00
|
|
|
const { db, ObjectId } = require('../../../../app/js/mongodb')
|
2020-02-17 12:35:59 -05:00
|
|
|
|
|
|
|
const aws = require('aws-sdk')
|
2020-02-17 12:35:39 -05:00
|
|
|
const s3 = new aws.S3({
|
2020-02-17 12:35:59 -05:00
|
|
|
accessKeyId: Settings.trackchanges.s3.key,
|
|
|
|
secretAccessKey: Settings.trackchanges.s3.secret,
|
|
|
|
endpoint: Settings.trackchanges.s3.endpoint,
|
2021-07-13 07:04:43 -04:00
|
|
|
s3ForcePathStyle: Settings.trackchanges.s3.pathStyle,
|
2020-02-17 12:35:59 -05:00
|
|
|
})
|
|
|
|
const S3_BUCKET = Settings.trackchanges.stores.doc_history
|
|
|
|
|
|
|
|
module.exports = TrackChangesClient = {
|
2023-03-21 08:21:51 -04:00
|
|
|
flushAndGetCompressedUpdates(projectId, docId, callback) {
|
2020-02-17 12:35:59 -05:00
|
|
|
if (callback == null) {
|
2021-10-27 05:49:18 -04:00
|
|
|
callback = function () {}
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
2023-03-21 08:21:51 -04:00
|
|
|
return TrackChangesClient.flushDoc(projectId, docId, error => {
|
2020-02-17 12:35:59 -05:00
|
|
|
if (error != null) {
|
|
|
|
return callback(error)
|
|
|
|
}
|
2023-03-21 08:21:51 -04:00
|
|
|
return TrackChangesClient.getCompressedUpdates(docId, callback)
|
2020-02-17 12:35:59 -05:00
|
|
|
})
|
|
|
|
},
|
|
|
|
|
2023-03-21 08:21:51 -04:00
|
|
|
flushDoc(projectId, docId, callback) {
|
2020-02-17 12:35:59 -05:00
|
|
|
if (callback == null) {
|
2021-10-27 05:49:18 -04:00
|
|
|
callback = function () {}
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
|
|
|
return request.post(
|
|
|
|
{
|
2023-03-21 08:21:51 -04:00
|
|
|
url: `http://localhost:3015/project/${projectId}/doc/${docId}/flush`,
|
2020-02-17 12:35:59 -05:00
|
|
|
},
|
|
|
|
(error, response, body) => {
|
|
|
|
response.statusCode.should.equal(204)
|
|
|
|
return callback(error)
|
|
|
|
}
|
|
|
|
)
|
|
|
|
},
|
|
|
|
|
2023-03-21 08:21:51 -04:00
|
|
|
flushProject(projectId, callback) {
|
2020-02-17 12:35:59 -05:00
|
|
|
if (callback == null) {
|
2021-10-27 05:49:18 -04:00
|
|
|
callback = function () {}
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
|
|
|
return request.post(
|
|
|
|
{
|
2023-03-21 08:21:51 -04:00
|
|
|
url: `http://localhost:3015/project/${projectId}/flush`,
|
2020-02-17 12:35:59 -05:00
|
|
|
},
|
|
|
|
(error, response, body) => {
|
|
|
|
response.statusCode.should.equal(204)
|
|
|
|
return callback(error)
|
|
|
|
}
|
|
|
|
)
|
|
|
|
},
|
|
|
|
|
2023-03-21 08:21:51 -04:00
|
|
|
getCompressedUpdates(docId, callback) {
|
2020-02-17 12:35:59 -05:00
|
|
|
if (callback == null) {
|
2021-10-27 05:49:18 -04:00
|
|
|
callback = function () {}
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
|
|
|
return db.docHistory
|
2023-03-21 08:21:51 -04:00
|
|
|
.find({ doc_id: ObjectId(docId) })
|
2020-02-17 12:35:59 -05:00
|
|
|
.sort({ 'meta.end_ts': 1 })
|
|
|
|
.toArray(callback)
|
|
|
|
},
|
|
|
|
|
2023-03-21 08:21:51 -04:00
|
|
|
getProjectMetaData(projectId, callback) {
|
2020-02-17 12:35:59 -05:00
|
|
|
if (callback == null) {
|
2021-10-27 05:49:18 -04:00
|
|
|
callback = function () {}
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
2020-09-10 11:48:09 -04:00
|
|
|
return db.projectHistoryMetaData.findOne(
|
2020-02-17 12:35:59 -05:00
|
|
|
{
|
2023-03-21 08:21:51 -04:00
|
|
|
project_id: ObjectId(projectId),
|
2020-02-17 12:35:59 -05:00
|
|
|
},
|
2020-09-10 11:48:09 -04:00
|
|
|
callback
|
2020-02-17 12:35:59 -05:00
|
|
|
)
|
|
|
|
},
|
|
|
|
|
2023-03-21 08:21:51 -04:00
|
|
|
setPreserveHistoryForProject(projectId, callback) {
|
2020-02-17 12:35:59 -05:00
|
|
|
if (callback == null) {
|
2021-10-27 05:49:18 -04:00
|
|
|
callback = function () {}
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
2020-09-10 11:48:09 -04:00
|
|
|
return db.projectHistoryMetaData.updateOne(
|
2020-02-17 12:35:59 -05:00
|
|
|
{
|
2023-03-21 08:21:51 -04:00
|
|
|
project_id: ObjectId(projectId),
|
2020-02-17 12:35:59 -05:00
|
|
|
},
|
|
|
|
{
|
2021-07-13 07:04:43 -04:00
|
|
|
$set: { preserveHistory: true },
|
2020-02-17 12:35:59 -05:00
|
|
|
},
|
|
|
|
{
|
2021-07-13 07:04:43 -04:00
|
|
|
upsert: true,
|
2020-02-17 12:35:59 -05:00
|
|
|
},
|
|
|
|
callback
|
|
|
|
)
|
|
|
|
},
|
|
|
|
|
2023-03-21 08:21:51 -04:00
|
|
|
pushRawUpdates(projectId, docId, updates, callback) {
|
2020-02-17 12:35:59 -05:00
|
|
|
if (callback == null) {
|
2021-10-27 05:49:18 -04:00
|
|
|
callback = function () {}
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
|
|
|
return rclient.sadd(
|
2023-03-21 08:21:51 -04:00
|
|
|
Keys.docsWithHistoryOps({ project_id: projectId }),
|
|
|
|
docId,
|
2021-07-13 07:04:43 -04:00
|
|
|
error => {
|
2020-02-17 12:35:59 -05:00
|
|
|
if (error != null) {
|
|
|
|
return callback(error)
|
|
|
|
}
|
|
|
|
return rclient.rpush(
|
2023-03-21 08:21:51 -04:00
|
|
|
Keys.uncompressedHistoryOps({ doc_id: docId }),
|
2021-07-13 07:04:43 -04:00
|
|
|
...Array.from(Array.from(updates).map(u => JSON.stringify(u))),
|
2020-02-17 12:35:59 -05:00
|
|
|
callback
|
|
|
|
)
|
|
|
|
}
|
|
|
|
)
|
|
|
|
},
|
|
|
|
|
2023-03-21 08:21:51 -04:00
|
|
|
getDiff(projectId, docId, from, to, callback) {
|
2020-02-17 12:35:59 -05:00
|
|
|
if (callback == null) {
|
2021-10-27 05:49:18 -04:00
|
|
|
callback = function () {}
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
|
|
|
return request.get(
|
|
|
|
{
|
2023-03-21 08:21:51 -04:00
|
|
|
url: `http://localhost:3015/project/${projectId}/doc/${docId}/diff?from=${from}&to=${to}`,
|
2020-02-17 12:35:59 -05:00
|
|
|
},
|
|
|
|
(error, response, body) => {
|
2021-10-27 05:49:18 -04:00
|
|
|
if (error) return callback(error)
|
2020-02-17 12:35:59 -05:00
|
|
|
response.statusCode.should.equal(200)
|
|
|
|
return callback(null, JSON.parse(body))
|
|
|
|
}
|
|
|
|
)
|
|
|
|
},
|
|
|
|
|
2023-03-21 08:21:51 -04:00
|
|
|
getUpdates(projectId, options, callback) {
|
2020-02-17 12:35:59 -05:00
|
|
|
if (callback == null) {
|
2021-10-27 05:49:18 -04:00
|
|
|
callback = function () {}
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
|
|
|
return request.get(
|
|
|
|
{
|
2023-03-21 08:21:51 -04:00
|
|
|
url: `http://localhost:3015/project/${projectId}/updates?before=${options.before}&min_count=${options.min_count}`,
|
2020-02-17 12:35:59 -05:00
|
|
|
},
|
|
|
|
(error, response, body) => {
|
2021-10-27 05:49:18 -04:00
|
|
|
if (error) return callback(error)
|
2020-02-17 12:35:59 -05:00
|
|
|
response.statusCode.should.equal(200)
|
|
|
|
return callback(null, JSON.parse(body))
|
|
|
|
}
|
|
|
|
)
|
|
|
|
},
|
|
|
|
|
2023-03-21 08:21:51 -04:00
|
|
|
exportProject(projectId, callback) {
|
2021-02-23 08:57:27 -05:00
|
|
|
request.get(
|
2023-03-21 08:21:51 -04:00
|
|
|
{ url: `http://localhost:3015/project/${projectId}/export`, json: true },
|
2021-02-23 08:57:27 -05:00
|
|
|
(error, response, updates) => {
|
|
|
|
if (error) return callback(error)
|
|
|
|
response.statusCode.should.equal(200)
|
2021-02-25 04:52:54 -05:00
|
|
|
callback(null, updates, JSON.parse(response.trailers['x-user-ids']))
|
2021-02-23 08:57:27 -05:00
|
|
|
}
|
|
|
|
)
|
|
|
|
},
|
|
|
|
|
2023-03-21 08:21:51 -04:00
|
|
|
restoreDoc(projectId, docId, version, userId, callback) {
|
2020-02-17 12:35:59 -05:00
|
|
|
if (callback == null) {
|
2021-10-27 05:49:18 -04:00
|
|
|
callback = function () {}
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
|
|
|
return request.post(
|
|
|
|
{
|
2023-03-21 08:21:51 -04:00
|
|
|
url: `http://localhost:3015/project/${projectId}/doc/${docId}/version/${version}/restore`,
|
2020-02-17 12:35:59 -05:00
|
|
|
headers: {
|
2023-03-21 08:21:51 -04:00
|
|
|
'X-User-Id': userId,
|
2021-07-13 07:04:43 -04:00
|
|
|
},
|
2020-02-17 12:35:59 -05:00
|
|
|
},
|
|
|
|
(error, response, body) => {
|
2021-10-27 05:49:18 -04:00
|
|
|
if (error) return callback(error)
|
2020-02-17 12:35:59 -05:00
|
|
|
response.statusCode.should.equal(204)
|
|
|
|
return callback(null)
|
|
|
|
}
|
|
|
|
)
|
|
|
|
},
|
|
|
|
|
2023-03-21 08:21:51 -04:00
|
|
|
pushDocHistory(projectId, docId, callback) {
|
2020-02-17 12:35:59 -05:00
|
|
|
if (callback == null) {
|
2021-10-27 05:49:18 -04:00
|
|
|
callback = function () {}
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
|
|
|
return request.post(
|
|
|
|
{
|
2023-03-21 08:21:51 -04:00
|
|
|
url: `http://localhost:3015/project/${projectId}/doc/${docId}/push`,
|
2020-02-17 12:35:59 -05:00
|
|
|
},
|
|
|
|
(error, response, body) => {
|
|
|
|
response.statusCode.should.equal(204)
|
|
|
|
return callback(error)
|
|
|
|
}
|
|
|
|
)
|
|
|
|
},
|
|
|
|
|
2023-03-21 08:21:51 -04:00
|
|
|
pullDocHistory(projectId, docId, callback) {
|
2020-02-17 12:35:59 -05:00
|
|
|
if (callback == null) {
|
2021-10-27 05:49:18 -04:00
|
|
|
callback = function () {}
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
|
|
|
return request.post(
|
|
|
|
{
|
2023-03-21 08:21:51 -04:00
|
|
|
url: `http://localhost:3015/project/${projectId}/doc/${docId}/pull`,
|
2020-02-17 12:35:59 -05:00
|
|
|
},
|
|
|
|
(error, response, body) => {
|
|
|
|
response.statusCode.should.equal(204)
|
|
|
|
return callback(error)
|
|
|
|
}
|
|
|
|
)
|
|
|
|
},
|
|
|
|
|
|
|
|
waitForS3(done, retries) {
|
|
|
|
if (retries == null) {
|
|
|
|
retries = 42
|
|
|
|
}
|
|
|
|
if (!Settings.trackchanges.s3.endpoint) {
|
|
|
|
return done()
|
|
|
|
}
|
|
|
|
|
|
|
|
return request.get(`${Settings.trackchanges.s3.endpoint}/`, (err, res) => {
|
|
|
|
if (res && res.statusCode < 500) {
|
|
|
|
return done()
|
|
|
|
}
|
|
|
|
|
|
|
|
if (retries === 0) {
|
|
|
|
return done(err || new Error(`s3 returned ${res.statusCode}`))
|
|
|
|
}
|
|
|
|
|
|
|
|
return setTimeout(
|
|
|
|
() => TrackChangesClient.waitForS3(done, --retries),
|
|
|
|
1000
|
|
|
|
)
|
|
|
|
})
|
|
|
|
},
|
|
|
|
|
2023-03-21 08:21:51 -04:00
|
|
|
getS3Doc(projectId, docId, packId, callback) {
|
2020-02-17 12:35:59 -05:00
|
|
|
if (callback == null) {
|
2021-10-27 05:49:18 -04:00
|
|
|
callback = function () {}
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
|
|
|
const params = {
|
|
|
|
Bucket: S3_BUCKET,
|
2023-03-21 08:21:51 -04:00
|
|
|
Key: `${projectId}/changes-${docId}/pack-${packId}`,
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
return s3.getObject(params, (error, data) => {
|
|
|
|
if (error != null) {
|
|
|
|
return callback(error)
|
|
|
|
}
|
|
|
|
const body = data.Body
|
|
|
|
if (body == null) {
|
|
|
|
return callback(new Error('empty response from s3'))
|
|
|
|
}
|
|
|
|
return zlib.gunzip(body, (err, result) => {
|
|
|
|
if (err != null) {
|
|
|
|
return callback(err)
|
|
|
|
}
|
|
|
|
return callback(null, JSON.parse(result.toString()))
|
|
|
|
})
|
|
|
|
})
|
|
|
|
},
|
|
|
|
|
2023-03-21 08:21:51 -04:00
|
|
|
removeS3Doc(projectId, docId, callback) {
|
2020-02-17 12:35:59 -05:00
|
|
|
if (callback == null) {
|
2021-10-27 05:49:18 -04:00
|
|
|
callback = function () {}
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
|
|
|
let params = {
|
|
|
|
Bucket: S3_BUCKET,
|
2023-03-21 08:21:51 -04:00
|
|
|
Prefix: `${projectId}/changes-${docId}`,
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
return s3.listObjects(params, (error, data) => {
|
|
|
|
if (error != null) {
|
|
|
|
return callback(error)
|
|
|
|
}
|
|
|
|
|
|
|
|
params = {
|
|
|
|
Bucket: S3_BUCKET,
|
|
|
|
Delete: {
|
2021-07-13 07:04:43 -04:00
|
|
|
Objects: data.Contents.map(s3object => ({ Key: s3object.Key })),
|
|
|
|
},
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
return s3.deleteObjects(params, callback)
|
|
|
|
})
|
2021-07-13 07:04:43 -04:00
|
|
|
},
|
2020-02-17 12:35:59 -05:00
|
|
|
}
|