From da86a094a8be4d4894ec31d2286e3c090b51f7ed Mon Sep 17 00:00:00 2001
From: Brian Gough <bjg@network-theory.co.uk>
Date: Tue, 1 Aug 2017 14:35:55 +0100
Subject: [PATCH] write files incrementally
---
.../clsi/app/coffee/CompileController.coffee | 5 ++-
.../clsi/app/coffee/CompileManager.coffee | 2 +-
services/clsi/app/coffee/RequestParser.coffee | 6 ++++
.../clsi/app/coffee/ResourceWriter.coffee | 34 ++++++++++++++++++-
4 files changed, 44 insertions(+), 3 deletions(-)
diff --git a/services/clsi/app/coffee/CompileController.coffee b/services/clsi/app/coffee/CompileController.coffee
index 250f9b8e7f..aefa70dd1b 100644
--- a/services/clsi/app/coffee/CompileController.coffee
+++ b/services/clsi/app/coffee/CompileController.coffee
@@ -15,7 +15,10 @@ module.exports = CompileController =
ProjectPersistenceManager.markProjectAsJustAccessed request.project_id, (error) ->
return next(error) if error?
CompileManager.doCompile request, (error, outputFiles = []) ->
- if error?.terminated
+ if error?.message is "invalid state"
+ code = 409 # Http 409 Conflict
+ status = "retry"
+ else if error?.terminated
status = "terminated"
else if error?.validate
status = "validation-#{error.validate}"
diff --git a/services/clsi/app/coffee/CompileManager.coffee b/services/clsi/app/coffee/CompileManager.coffee
index ae40bf7929..8673b52de5 100644
--- a/services/clsi/app/coffee/CompileManager.coffee
+++ b/services/clsi/app/coffee/CompileManager.coffee
@@ -31,7 +31,7 @@ module.exports = CompileManager =
timer = new Metrics.Timer("write-to-disk")
logger.log project_id: request.project_id, user_id: request.user_id, "syncing resources to disk"
- ResourceWriter.syncResourcesToDisk request.project_id, request.resources, compileDir, (error) ->
+ ResourceWriter.syncResourcesToDisk request, compileDir, (error) ->
if error?
logger.err err:error, project_id: request.project_id, user_id: request.user_id, "error writing resources to disk"
return callback(error)
diff --git a/services/clsi/app/coffee/RequestParser.coffee b/services/clsi/app/coffee/RequestParser.coffee
index 8fc4ecf370..fe22982fda 100644
--- a/services/clsi/app/coffee/RequestParser.coffee
+++ b/services/clsi/app/coffee/RequestParser.coffee
@@ -31,6 +31,12 @@ module.exports = RequestParser =
response.check = @_parseAttribute "check",
compile.options.check,
type: "string"
+ response.incremental = @_parseAttribute "incremental",
+ compile.options.incremental,
+ type: "string"
+ response.state = @_parseAttribute "state",
+ compile.options.state,
+ type: "string"
if response.timeout > RequestParser.MAX_TIMEOUT
response.timeout = RequestParser.MAX_TIMEOUT
diff --git a/services/clsi/app/coffee/ResourceWriter.coffee b/services/clsi/app/coffee/ResourceWriter.coffee
index e2a0e1f842..a4ae42517e 100644
--- a/services/clsi/app/coffee/ResourceWriter.coffee
+++ b/services/clsi/app/coffee/ResourceWriter.coffee
@@ -11,7 +11,39 @@ settings = require("settings-sharelatex")
parallelFileDownloads = settings.parallelFileDownloads or 1
module.exports = ResourceWriter =
- syncResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) ->
+
+ syncResourcesToDisk: (request, basePath, callback = (error) ->) ->
+ if request.incremental?
+ ResourceWriter.checkState request.incremental, basePath, (error, ok) ->
+ logger.log state: request.state, result:ok, "checked state on incremental request"
+ return callback new Error("invalid state") if not ok
+ ResourceWriter.saveIncrementalResourcesToDisk request.project_id, request.resources, basePath, callback
+ else
+ @saveAllResourcesToDisk request.project_id, request.resources, basePath, (error) ->
+ return callback(error) if error?
+ ResourceWriter.storeState request.state, basePath, callback
+
+ storeState: (state, basePath, callback) ->
+ logger.log state:state, basePath:basePath, "writing state"
+ fs.writeFile Path.join(basePath, ".resource-state"), state, {encoding: 'ascii'}, callback
+
+ checkState: (state, basePath, callback) ->
+ fs.readFile Path.join(basePath, ".resource-state"), {encoding:'ascii'}, (err, oldState) ->
+ logger.log state:state, oldState: oldState, basePath:basePath, err:err, "checking state"
+ if state is oldState
+ return callback(null, true)
+ else
+ return callback(null, false)
+
+ saveIncrementalResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) ->
+ @_createDirectory basePath, (error) =>
+ return callback(error) if error?
+ jobs = for resource in resources
+ do (resource) =>
+ (callback) => @_writeResourceToDisk(project_id, resource, basePath, callback)
+ async.parallelLimit jobs, parallelFileDownloads, callback
+
+ saveAllResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) ->
@_createDirectory basePath, (error) =>
return callback(error) if error?
@_removeExtraneousFiles resources, basePath, (error) =>