github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-07 20:31:06 -05:00
Code Issues Projects Releases Packages Wiki Activity

added a queue with 1 concurency to db queries

Browse source
This commit is contained in:
Henry Oswald 2018-07-30 16:22:04 +01:00
parent daf40fc757
commit 1080c2c428
2 changed files with 46 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
services/clsi/app/coffee/ProjectPersistenceManager.coffee
View file

@ -11,6 +11,7 @@ module.exports = ProjectPersistenceManager =
EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5
markProjectAsJustAccessed: (project_id, callback = (error) ->) ->
console.log("markProjectAsJustAccessed")
db.Project.findOrCreate(where: {project_id: project_id})
.spread(
(project, created) ->
@ -53,11 +54,13 @@ module.exports = ProjectPersistenceManager =
callback()
_clearProjectFromDatabase: (project_id, callback = (error) ->) ->
console.log("_clearProjectFromDatabase")
db.Project.destroy(where: {project_id: project_id})
.then(() -> callback())
.error callback
_findExpiredProjectIds: (callback = (error, project_ids) ->) ->
console.log("_findExpiredProjectIds")
db.Project.findAll(where: ["lastAccessed < ?", new Date(Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT)])
.then((projects) ->
callback null, projects.map((project) -> project.project_id)

68
services/clsi/app/coffee/UrlCache.coffee
View file

@ -6,6 +6,15 @@ fs = require("fs")
logger = require "logger-sharelatex"
async = require "async"
queue = async.queue((task, cb)->
console.log("running task")
task(cb)
, 1)
console.log("hi there queue")
queue.drain = ()->
console.log('HI all items have been processed')
module.exports = UrlCache =
downloadUrlToFile: (project_id, url, destPath, lastModified, callback = (error) ->) ->
UrlCache._ensureUrlIsInCache project_id, url, lastModified, (error, pathToCachedUrl) =>
@ -51,8 +60,9 @@ module.exports = UrlCache =
_doesUrlNeedDownloading: (project_id, url, lastModified, callback = (error, needsDownloading) ->) ->
if !lastModified?
return callback null, true
console.log "about to get _findUrlDetails"
UrlCache._findUrlDetails project_id, url, (error, urlDetails) ->
console.log error, urlDetails, "_findUrlDetails result"
return callback(error) if error?
if !urlDetails? or !urlDetails.lastModified? or urlDetails.lastModified.getTime() < lastModified.getTime()
return callback null, true
@ -94,36 +104,44 @@ module.exports = UrlCache =
return callback()
_findUrlDetails: (project_id, url, callback = (error, urlDetails) ->) ->
console.log("_findUrlDetails")
db.UrlCache.find(where: { url: url, project_id: project_id })
.then((urlDetails) -> callback null, urlDetails)
.error callback
job = (cb)->
db.UrlCache.find(where: { url: url, project_id: project_id })
.then((urlDetails) -> cb null, urlDetails)
.error cb
queue.push job, callback
_updateOrCreateUrlDetails: (project_id, url, lastModified, callback = (error) ->) ->
console.log("_updateOrCreateUrlDetails")
db.UrlCache.findOrCreate(where: {url: url, project_id: project_id})
.spread(
(urlDetails, created) ->
urlDetails.updateAttributes(lastModified: lastModified)
.then(() -> callback())
.error(callback)
)
.error callback
job = (cb)->
console.log("_updateOrCreateUrlDetails")
db.UrlCache.findOrCreate(where: {url: url, project_id: project_id})
.spread(
(urlDetails, created) ->
urlDetails.updateAttributes(lastModified: lastModified)
.then(() -> cb())
.error(cb)
)
.error cb
queue.push(job, callback)
_clearUrlDetails: (project_id, url, callback = (error) ->) ->
console.log("_clearUrlDetails")
db.UrlCache.destroy(where: {url: url, project_id: project_id})
.then(() -> callback null)
.error callback
job = (cb)->
console.log("_clearUrlDetails")
db.UrlCache.destroy(where: {url: url, project_id: project_id})
.then(() -> cb null)
.error cb
queue.push(job, callback)
_findAllUrlsInProject: (project_id, callback = (error, urls) ->) ->
console.log("_findAllUrlsInProject")
db.UrlCache.findAll(where: { project_id: project_id })
.then(
(urlEntries) ->
callback null, urlEntries.map((entry) -> entry.url)
)
.error callback
job = (cb)->
console.log("_findAllUrlsInProject")
db.UrlCache.findAll(where: { project_id: project_id })
.then(
(urlEntries) ->
cb null, urlEntries.map((entry) -> entry.url)
)
.error cb
queue.push(job, callback)