github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-29 16:13:42 -05:00
Code Issues Projects Releases Packages Wiki Activity

added a queue with 1 concurency to db queries

Browse source
This commit is contained in:
Henry Oswald 2018-07-30 16:22:04 +01:00
parent daf40fc757
commit 1080c2c428
2 changed files with 46 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
services/clsi/app/coffee/ProjectPersistenceManager.coffee
View file

@ -11,6 +11,7 @@ module.exports = ProjectPersistenceManager =
EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5 EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5
markProjectAsJustAccessed: (project_id, callback = (error) ->) -> markProjectAsJustAccessed: (project_id, callback = (error) ->) ->
console.log("markProjectAsJustAccessed")
db.Project.findOrCreate(where: {project_id: project_id}) db.Project.findOrCreate(where: {project_id: project_id})
.spread( .spread(
(project, created) -> (project, created) ->
@ -53,11 +54,13 @@ module.exports = ProjectPersistenceManager =
callback() callback()
_clearProjectFromDatabase: (project_id, callback = (error) ->) -> _clearProjectFromDatabase: (project_id, callback = (error) ->) ->
console.log("_clearProjectFromDatabase")
db.Project.destroy(where: {project_id: project_id}) db.Project.destroy(where: {project_id: project_id})
.then(() -> callback()) .then(() -> callback())
.error callback .error callback
_findExpiredProjectIds: (callback = (error, project_ids) ->) -> _findExpiredProjectIds: (callback = (error, project_ids) ->) ->
console.log("_findExpiredProjectIds")
db.Project.findAll(where: ["lastAccessed < ?", new Date(Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT)]) db.Project.findAll(where: ["lastAccessed < ?", new Date(Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT)])
.then((projects) -> .then((projects) ->
callback null, projects.map((project) -> project.project_id) callback null, projects.map((project) -> project.project_id)

68
services/clsi/app/coffee/UrlCache.coffee
View file

@ -6,6 +6,15 @@ fs = require("fs")
logger = require "logger-sharelatex" logger = require "logger-sharelatex"
async = require "async" async = require "async"
queue = async.queue((task, cb)->
console.log("running task")
task(cb)
, 1)
console.log("hi there queue")
queue.drain = ()->
console.log('HI all items have been processed')
module.exports = UrlCache = module.exports = UrlCache =
downloadUrlToFile: (project_id, url, destPath, lastModified, callback = (error) ->) -> downloadUrlToFile: (project_id, url, destPath, lastModified, callback = (error) ->) ->
UrlCache._ensureUrlIsInCache project_id, url, lastModified, (error, pathToCachedUrl) => UrlCache._ensureUrlIsInCache project_id, url, lastModified, (error, pathToCachedUrl) =>
@ -51,8 +60,9 @@ module.exports = UrlCache =
_doesUrlNeedDownloading: (project_id, url, lastModified, callback = (error, needsDownloading) ->) -> _doesUrlNeedDownloading: (project_id, url, lastModified, callback = (error, needsDownloading) ->) ->
if !lastModified? if !lastModified?
return callback null, true return callback null, true
console.log "about to get _findUrlDetails"
UrlCache._findUrlDetails project_id, url, (error, urlDetails) -> UrlCache._findUrlDetails project_id, url, (error, urlDetails) ->
console.log error, urlDetails, "_findUrlDetails result"
return callback(error) if error? return callback(error) if error?
if !urlDetails? or !urlDetails.lastModified? or urlDetails.lastModified.getTime() < lastModified.getTime() if !urlDetails? or !urlDetails.lastModified? or urlDetails.lastModified.getTime() < lastModified.getTime()
return callback null, true return callback null, true
@ -94,36 +104,44 @@ module.exports = UrlCache =
return callback() return callback()
_findUrlDetails: (project_id, url, callback = (error, urlDetails) ->) -> _findUrlDetails: (project_id, url, callback = (error, urlDetails) ->) ->
console.log("_findUrlDetails") job = (cb)->
db.UrlCache.find(where: { url: url, project_id: project_id }) db.UrlCache.find(where: { url: url, project_id: project_id })
.then((urlDetails) -> callback null, urlDetails) .then((urlDetails) -> cb null, urlDetails)
.error callback .error cb
queue.push job, callback
_updateOrCreateUrlDetails: (project_id, url, lastModified, callback = (error) ->) -> _updateOrCreateUrlDetails: (project_id, url, lastModified, callback = (error) ->) ->
console.log("_updateOrCreateUrlDetails") job = (cb)->
db.UrlCache.findOrCreate(where: {url: url, project_id: project_id}) console.log("_updateOrCreateUrlDetails")
.spread( db.UrlCache.findOrCreate(where: {url: url, project_id: project_id})
(urlDetails, created) -> .spread(
urlDetails.updateAttributes(lastModified: lastModified) (urlDetails, created) ->
.then(() -> callback()) urlDetails.updateAttributes(lastModified: lastModified)
.error(callback) .then(() -> cb())
) .error(cb)
.error callback )
.error cb
queue.push(job, callback)
_clearUrlDetails: (project_id, url, callback = (error) ->) -> _clearUrlDetails: (project_id, url, callback = (error) ->) ->
console.log("_clearUrlDetails") job = (cb)->
db.UrlCache.destroy(where: {url: url, project_id: project_id}) console.log("_clearUrlDetails")
.then(() -> callback null) db.UrlCache.destroy(where: {url: url, project_id: project_id})
.error callback .then(() -> cb null)
.error cb
queue.push(job, callback)
_findAllUrlsInProject: (project_id, callback = (error, urls) ->) -> _findAllUrlsInProject: (project_id, callback = (error, urls) ->) ->
console.log("_findAllUrlsInProject") job = (cb)->
db.UrlCache.findAll(where: { project_id: project_id }) console.log("_findAllUrlsInProject")
.then( db.UrlCache.findAll(where: { project_id: project_id })
(urlEntries) -> .then(
callback null, urlEntries.map((entry) -> entry.url) (urlEntries) ->
) cb null, urlEntries.map((entry) -> entry.url)
.error callback )
.error cb
queue.push(job, callback)