overleaf/services/clsi/app/js/UrlCache.js

274 lines
7.7 KiB
JavaScript
Raw Normal View History

/* eslint-disable
camelcase,
handle-callback-err,
no-return-assign,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let UrlCache
const db = require('./db')
const dbQueue = require('./DbQueue')
const UrlFetcher = require('./UrlFetcher')
const Settings = require('settings-sharelatex')
const crypto = require('crypto')
const fs = require('fs')
const logger = require('logger-sharelatex')
const async = require('async')
2014-02-12 12:27:43 -05:00
module.exports = UrlCache = {
downloadUrlToFile(project_id, url, destPath, lastModified, callback) {
if (callback == null) {
2020-08-10 12:01:11 -04:00
callback = function (error) {}
}
return UrlCache._ensureUrlIsInCache(
project_id,
url,
lastModified,
(error, pathToCachedUrl) => {
if (error != null) {
return callback(error)
}
2020-08-10 12:01:11 -04:00
return UrlCache._copyFile(pathToCachedUrl, destPath, function (error) {
if (error != null) {
return UrlCache._clearUrlDetails(project_id, url, () =>
callback(error)
)
} else {
return callback(error)
}
})
}
)
},
2014-02-12 12:27:43 -05:00
clearProject(project_id, callback) {
if (callback == null) {
2020-08-10 12:01:11 -04:00
callback = function (error) {}
}
2020-08-10 12:01:11 -04:00
return UrlCache._findAllUrlsInProject(project_id, function (error, urls) {
logger.log(
{ project_id, url_count: urls.length },
'clearing project URLs'
)
if (error != null) {
return callback(error)
}
2020-08-10 12:01:11 -04:00
const jobs = Array.from(urls || []).map((url) =>
((url) => (callback) =>
UrlCache._clearUrlFromCache(project_id, url, function (error) {
if (error != null) {
logger.error(
{ err: error, project_id, url },
'error clearing project URL'
)
}
return callback()
}))(url)
)
return async.series(jobs, callback)
})
},
2014-02-12 12:27:43 -05:00
_ensureUrlIsInCache(project_id, url, lastModified, callback) {
if (callback == null) {
2020-08-10 12:01:11 -04:00
callback = function (error, pathOnDisk) {}
}
if (lastModified != null) {
// MYSQL only stores dates to an accuracy of a second but the incoming lastModified might have milliseconds.
// So round down to seconds
lastModified = new Date(Math.floor(lastModified.getTime() / 1000) * 1000)
}
return UrlCache._doesUrlNeedDownloading(
project_id,
url,
lastModified,
(error, needsDownloading) => {
if (error != null) {
return callback(error)
}
if (needsDownloading) {
logger.log({ url, lastModified }, 'downloading URL')
return UrlFetcher.pipeUrlToFileWithRetry(
url,
UrlCache._cacheFilePathForUrl(project_id, url),
2020-08-10 12:01:11 -04:00
(error) => {
if (error != null) {
return callback(error)
}
return UrlCache._updateOrCreateUrlDetails(
project_id,
url,
lastModified,
2020-08-10 12:01:11 -04:00
(error) => {
if (error != null) {
return callback(error)
}
return callback(
null,
UrlCache._cacheFilePathForUrl(project_id, url)
)
}
)
}
)
} else {
logger.log({ url, lastModified }, 'URL is up to date in cache')
return callback(null, UrlCache._cacheFilePathForUrl(project_id, url))
}
}
)
},
2014-02-12 12:27:43 -05:00
_doesUrlNeedDownloading(project_id, url, lastModified, callback) {
if (callback == null) {
2020-08-10 12:01:11 -04:00
callback = function (error, needsDownloading) {}
}
if (lastModified == null) {
return callback(null, true)
}
2020-08-10 12:01:11 -04:00
return UrlCache._findUrlDetails(project_id, url, function (
error,
urlDetails
) {
if (error != null) {
return callback(error)
}
if (
urlDetails == null ||
urlDetails.lastModified == null ||
urlDetails.lastModified.getTime() < lastModified.getTime()
) {
return callback(null, true)
} else {
return callback(null, false)
}
})
},
2014-02-12 12:27:43 -05:00
_cacheFileNameForUrl(project_id, url) {
2020-08-10 12:01:11 -04:00
return project_id + ':' + crypto.createHash('md5').update(url).digest('hex')
},
2014-02-12 12:27:43 -05:00
_cacheFilePathForUrl(project_id, url) {
return `${Settings.path.clsiCacheDir}/${UrlCache._cacheFileNameForUrl(
project_id,
url
)}`
},
2014-02-12 12:27:43 -05:00
_copyFile(from, to, _callback) {
if (_callback == null) {
2020-08-10 12:01:11 -04:00
_callback = function (error) {}
}
2020-08-10 12:01:11 -04:00
const callbackOnce = function (error) {
if (error != null) {
logger.error({ err: error, from, to }, 'error copying file from cache')
}
_callback(error)
2020-08-10 12:01:11 -04:00
return (_callback = function () {})
}
const writeStream = fs.createWriteStream(to)
const readStream = fs.createReadStream(from)
writeStream.on('error', callbackOnce)
readStream.on('error', callbackOnce)
writeStream.on('close', callbackOnce)
return writeStream.on('open', () => readStream.pipe(writeStream))
},
2014-02-12 12:27:43 -05:00
_clearUrlFromCache(project_id, url, callback) {
if (callback == null) {
2020-08-10 12:01:11 -04:00
callback = function (error) {}
}
2020-08-10 12:01:11 -04:00
return UrlCache._clearUrlDetails(project_id, url, function (error) {
if (error != null) {
return callback(error)
}
2020-08-10 12:01:11 -04:00
return UrlCache._deleteUrlCacheFromDisk(project_id, url, function (
error
) {
if (error != null) {
return callback(error)
}
return callback(null)
})
})
},
2014-02-12 12:27:43 -05:00
_deleteUrlCacheFromDisk(project_id, url, callback) {
if (callback == null) {
2020-08-10 12:01:11 -04:00
callback = function (error) {}
}
2020-08-10 12:01:11 -04:00
return fs.unlink(UrlCache._cacheFilePathForUrl(project_id, url), function (
error
) {
if (error != null && error.code !== 'ENOENT') {
// no error if the file isn't present
return callback(error)
} else {
return callback()
}
})
},
2014-02-12 12:27:43 -05:00
_findUrlDetails(project_id, url, callback) {
if (callback == null) {
2020-08-10 12:01:11 -04:00
callback = function (error, urlDetails) {}
}
2020-08-10 12:01:11 -04:00
const job = (cb) =>
2020-03-16 11:31:02 -04:00
db.UrlCache.findOne({ where: { url, project_id } })
2020-08-10 12:01:11 -04:00
.then((urlDetails) => cb(null, urlDetails))
.error(cb)
return dbQueue.queue.push(job, callback)
},
2014-02-12 12:27:43 -05:00
_updateOrCreateUrlDetails(project_id, url, lastModified, callback) {
if (callback == null) {
2020-08-10 12:01:11 -04:00
callback = function (error) {}
}
2020-08-10 12:01:11 -04:00
const job = (cb) =>
db.UrlCache.findOrCreate({ where: { url, project_id } })
.spread((urlDetails, created) =>
urlDetails
2020-03-16 11:31:02 -04:00
.update({ lastModified })
.then(() => cb())
.error(cb)
)
.error(cb)
return dbQueue.queue.push(job, callback)
},
_clearUrlDetails(project_id, url, callback) {
if (callback == null) {
2020-08-10 12:01:11 -04:00
callback = function (error) {}
}
2020-08-10 12:01:11 -04:00
const job = (cb) =>
db.UrlCache.destroy({ where: { url, project_id } })
.then(() => cb(null))
.error(cb)
return dbQueue.queue.push(job, callback)
},
2014-02-12 12:27:43 -05:00
_findAllUrlsInProject(project_id, callback) {
if (callback == null) {
2020-08-10 12:01:11 -04:00
callback = function (error, urls) {}
}
2020-08-10 12:01:11 -04:00
const job = (cb) =>
db.UrlCache.findAll({ where: { project_id } })
2020-08-10 12:01:11 -04:00
.then((urlEntries) =>
cb(
null,
2020-08-10 12:01:11 -04:00
urlEntries.map((entry) => entry.url)
)
)
.error(cb)
return dbQueue.queue.push(job, callback)
}
}