From 8415ea2f717e6f3d2247eeb724bd027ff1fcbcc5 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 17 Aug 2017 14:53:35 +0100 Subject: [PATCH] store the resource list in a file --- .../clsi/app/coffee/CompileManager.coffee | 13 ++++++---- .../app/coffee/ResourceListManager.coffee | 24 +++++++++++++++++++ .../clsi/app/coffee/ResourceWriter.coffee | 13 +++++++--- .../unit/coffee/CompileManagerTests.coffee | 2 +- .../unit/coffee/ResourceWriterTests.coffee | 3 +++ 5 files changed, 46 insertions(+), 9 deletions(-) create mode 100644 services/clsi/app/coffee/ResourceListManager.coffee diff --git a/services/clsi/app/coffee/CompileManager.coffee b/services/clsi/app/coffee/CompileManager.coffee index 2b563301b5..d6c79da0e3 100644 --- a/services/clsi/app/coffee/CompileManager.coffee +++ b/services/clsi/app/coffee/CompileManager.coffee @@ -31,7 +31,8 @@ module.exports = CompileManager = timer = new Metrics.Timer("write-to-disk") logger.log project_id: request.project_id, user_id: request.user_id, "syncing resources to disk" - ResourceWriter.syncResourcesToDisk request, compileDir, (error) -> + ResourceWriter.syncResourcesToDisk request, compileDir, (error, resourceList) -> + # NOTE: resourceList is insecure, it should only be used to exclude files from the output list if error? and error instanceof Errors.FilesOutOfSyncError logger.warn project_id: request.project_id, user_id: request.user_id, "files out of sync, please retry" return callback(error) @@ -40,15 +41,17 @@ module.exports = CompileManager = return callback(error) logger.log project_id: request.project_id, user_id: request.user_id, time_taken: Date.now() - timer.start, "written files to disk" timer.done() - + + # FIXME - for incremental compiles we don't want to inject this multiple times injectDraftModeIfRequired = (callback) -> if request.draft DraftModeManager.injectDraftMode Path.join(compileDir, request.rootResourcePath), callback else callback() + # FIXME - for incremental compiles we may need to update output.tex every time createTikzFileIfRequired = (callback) -> - if TikzManager.needsOutputFile(request.rootResourcePath, request.resources) + if TikzManager.needsOutputFile(request.rootResourcePath, resourceList) TikzManager.injectOutputFile compileDir, request.rootResourcePath, callback else callback() @@ -94,7 +97,7 @@ module.exports = CompileManager = error.validate = "fail" # compile was killed by user, was a validation, or a compile which failed validation if error?.terminated or error?.validate - OutputFileFinder.findOutputFiles request.resources, compileDir, (err, outputFiles) -> + OutputFileFinder.findOutputFiles resourceList, compileDir, (err, outputFiles) -> return callback(err) if err? callback(error, outputFiles) # return output files so user can check logs return @@ -114,7 +117,7 @@ module.exports = CompileManager = if stats?["latex-runs"] > 0 and timings?["cpu-time"] > 0 Metrics.timing("run-compile-cpu-time-per-pass", timings["cpu-time"] / stats["latex-runs"]) - OutputFileFinder.findOutputFiles request.resources, compileDir, (error, outputFiles) -> + OutputFileFinder.findOutputFiles resourceList, compileDir, (error, outputFiles) -> return callback(error) if error? OutputCacheManager.saveOutputFiles outputFiles, compileDir, (error, newOutputFiles) -> callback null, newOutputFiles diff --git a/services/clsi/app/coffee/ResourceListManager.coffee b/services/clsi/app/coffee/ResourceListManager.coffee new file mode 100644 index 0000000000..d68f5d1ad6 --- /dev/null +++ b/services/clsi/app/coffee/ResourceListManager.coffee @@ -0,0 +1,24 @@ +Path = require "path" +fs = require "fs" +mkdirp = require "mkdirp" +logger = require "logger-sharelatex" +settings = require("settings-sharelatex") + +module.exports = ResourceListManager = + + # This file is a list of the input files for the project, one per + # line, used to identify output files (i.e. files not on this list) + # when the incoming request is incremental. + RESOURCE_LIST_FILE: ".project-resource-list" + + saveResourceList: (resources, basePath, callback = (error) ->) -> + resourceListFile = Path.join(basePath, @RESOURCE_LIST_FILE) + resourceList = (resource.path for resource in resources) + fs.writeFile resourceListFile, resourceList.join("\n"), callback + + loadResourceList: (basePath, callback = (error) ->) -> + resourceListFile = Path.join(basePath, @RESOURCE_LIST_FILE) + fs.readFile resourceListFile, (err, resourceList) -> + return callback(err) if err? + resources = ({path: path} for path in resourceList?.toString()?.split("\n") or []) + callback(null, resources) diff --git a/services/clsi/app/coffee/ResourceWriter.coffee b/services/clsi/app/coffee/ResourceWriter.coffee index 9a78671ee6..18b8122e1e 100644 --- a/services/clsi/app/coffee/ResourceWriter.coffee +++ b/services/clsi/app/coffee/ResourceWriter.coffee @@ -4,6 +4,7 @@ fs = require "fs" async = require "async" mkdirp = require "mkdirp" OutputFileFinder = require "./OutputFileFinder" +ResourceListManager = require "./ResourceListManager" Metrics = require "./Metrics" Errors = require "./Errors" logger = require "logger-sharelatex" @@ -13,16 +14,22 @@ parallelFileDownloads = settings.parallelFileDownloads or 1 module.exports = ResourceWriter = - syncResourcesToDisk: (request, basePath, callback = (error) ->) -> + syncResourcesToDisk: (request, basePath, callback = (error, resourceList) ->) -> if request.syncType is "incremental" ResourceWriter.checkSyncState request.syncState, basePath, (error, syncStateOk) -> logger.log syncState: request.syncState, result:syncStateOk, "checked state on incremental request" return callback new Errors.FilesOutOfSyncError("invalid state for incremental update") if not syncStateOk - ResourceWriter.saveIncrementalResourcesToDisk request.project_id, request.resources, basePath, callback + ResourceWriter.saveIncrementalResourcesToDisk request.project_id, request.resources, basePath, (error) -> + return callback(error) if error? + ResourceListManager.loadResourceList basePath, callback else @saveAllResourcesToDisk request.project_id, request.resources, basePath, (error) -> return callback(error) if error? - ResourceWriter.storeSyncState request.syncState, basePath, callback + ResourceWriter.storeSyncState request.syncState, basePath, (error) -> + return callback(error) if error? + ResourceListManager.saveResourceList request.resources, basePath, (error) => + return callback(error) if error? + callback(null, request.resources) # The sync state is an identifier which must match for an # incremental update to be allowed. diff --git a/services/clsi/test/unit/coffee/CompileManagerTests.coffee b/services/clsi/test/unit/coffee/CompileManagerTests.coffee index 25109b621f..ff671b27c3 100644 --- a/services/clsi/test/unit/coffee/CompileManagerTests.coffee +++ b/services/clsi/test/unit/coffee/CompileManagerTests.coffee @@ -51,7 +51,7 @@ describe "CompileManager", -> @env = {} @Settings.compileDir = "compiles" @compileDir = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}" - @ResourceWriter.syncResourcesToDisk = sinon.stub().callsArg(2) + @ResourceWriter.syncResourcesToDisk = sinon.stub().callsArgWith(2, null, @resources) @LatexRunner.runLatex = sinon.stub().callsArg(2) @OutputFileFinder.findOutputFiles = sinon.stub().callsArgWith(2, null, @output_files) @OutputCacheManager.saveOutputFiles = sinon.stub().callsArgWith(2, null, @build_files) diff --git a/services/clsi/test/unit/coffee/ResourceWriterTests.coffee b/services/clsi/test/unit/coffee/ResourceWriterTests.coffee index d4dd9c16ee..0e602502f1 100644 --- a/services/clsi/test/unit/coffee/ResourceWriterTests.coffee +++ b/services/clsi/test/unit/coffee/ResourceWriterTests.coffee @@ -10,6 +10,7 @@ describe "ResourceWriter", -> "fs": @fs = mkdir: sinon.stub().callsArg(1) unlink: sinon.stub().callsArg(1) + "./ResourceListManager": @ResourceListManager = {} "wrench": @wrench = {} "./UrlCache" : @UrlCache = {} "mkdirp" : @mkdirp = sinon.stub().callsArg(1) @@ -33,6 +34,8 @@ describe "ResourceWriter", -> @ResourceWriter._removeExtraneousFiles = sinon.stub().callsArg(2) @ResourceWriter.checkSyncState = sinon.stub().callsArg(2) @ResourceWriter.storeSyncState = sinon.stub().callsArg(2) + @ResourceListManager.saveResourceList = sinon.stub().callsArg(2) + @ResourceListManager.loadResourceList = sinon.stub().callsArg(1) @ResourceWriter.syncResourcesToDisk({project_id: @project_id, resources: @resources}, @basePath, @callback) it "should remove old files", ->