store the resource list in a file

services/clsi/app/coffee/CompileManager.coffee

@@ -31,7 +31,8 @@ module.exports = CompileManager =
 
 		timer = new Metrics.Timer("write-to-disk")
 		logger.log project_id: request.project_id, user_id: request.user_id, "syncing resources to disk"
-		ResourceWriter.syncResourcesToDisk request, compileDir, (error) ->
+		ResourceWriter.syncResourcesToDisk request, compileDir, (error, resourceList) ->
+			# NOTE: resourceList is insecure, it should only be used to exclude files from the output list
 			if error? and error instanceof Errors.FilesOutOfSyncError
 				logger.warn project_id: request.project_id, user_id: request.user_id, "files out of sync, please retry"
 				return callback(error)
@@ -41,14 +42,16 @@ module.exports = CompileManager =
 			logger.log project_id: request.project_id, user_id: request.user_id, time_taken: Date.now() - timer.start, "written files to disk"
 			timer.done()
 
+			# FIXME - for incremental compiles we don't want to inject this multiple times
 			injectDraftModeIfRequired = (callback) ->
 				if request.draft
 					DraftModeManager.injectDraftMode Path.join(compileDir, request.rootResourcePath), callback
 				else
 					callback()
 
+			# FIXME - for incremental compiles we may need to update output.tex every time
 			createTikzFileIfRequired = (callback) ->
-				if TikzManager.needsOutputFile(request.rootResourcePath, request.resources)
+				if TikzManager.needsOutputFile(request.rootResourcePath, resourceList)
 					TikzManager.injectOutputFile compileDir, request.rootResourcePath, callback
 				else
 					callback()
@@ -94,7 +97,7 @@ module.exports = CompileManager =
 						error.validate = "fail"
 					# compile was killed by user, was a validation, or a compile which failed validation
 					if error?.terminated or error?.validate
-						OutputFileFinder.findOutputFiles request.resources, compileDir, (err, outputFiles) ->
+						OutputFileFinder.findOutputFiles resourceList, compileDir, (err, outputFiles) ->
 							return callback(err) if err?
 							callback(error, outputFiles) # return output files so user can check logs
 						return
@@ -114,7 +117,7 @@ module.exports = CompileManager =
 					if stats?["latex-runs"] > 0 and timings?["cpu-time"] > 0
 						Metrics.timing("run-compile-cpu-time-per-pass", timings["cpu-time"] / stats["latex-runs"])
 
-					OutputFileFinder.findOutputFiles request.resources, compileDir, (error, outputFiles) ->
+					OutputFileFinder.findOutputFiles resourceList, compileDir, (error, outputFiles) ->
 						return callback(error) if error?
 						OutputCacheManager.saveOutputFiles outputFiles, compileDir,  (error, newOutputFiles) ->
 							callback null, newOutputFiles

services/clsi/app/coffee/ResourceListManager.coffee

@@ -0,0 +1,24 @@
+Path = require "path"
+fs = require "fs"
+mkdirp = require "mkdirp"
+logger = require "logger-sharelatex"
+settings = require("settings-sharelatex")
+
+module.exports = ResourceListManager =
+
+	# This file is a list of the input files for the project, one per
+	# line, used to identify output files (i.e. files not on this list)
+	# when the incoming request is incremental.
+	RESOURCE_LIST_FILE: ".project-resource-list"
+
+	saveResourceList: (resources, basePath, callback = (error) ->) ->
+		resourceListFile = Path.join(basePath, @RESOURCE_LIST_FILE)
+		resourceList = (resource.path for resource in resources)
+		fs.writeFile resourceListFile, resourceList.join("\n"), callback
+
+	loadResourceList: (basePath, callback = (error) ->) ->
+		resourceListFile = Path.join(basePath, @RESOURCE_LIST_FILE)
+		fs.readFile resourceListFile, (err, resourceList) ->
+			return callback(err) if err?
+			resources = ({path: path} for path in resourceList?.toString()?.split("\n") or [])
+			callback(null, resources)

services/clsi/app/coffee/ResourceWriter.coffee

@@ -4,6 +4,7 @@ fs = require "fs"
 async = require "async"
 mkdirp = require "mkdirp"
 OutputFileFinder = require "./OutputFileFinder"
+ResourceListManager = require "./ResourceListManager"
 Metrics = require "./Metrics"
 Errors = require "./Errors"
 logger = require "logger-sharelatex"
@@ -13,16 +14,22 @@ parallelFileDownloads = settings.parallelFileDownloads or 1
 
 module.exports = ResourceWriter =
 
-	syncResourcesToDisk: (request, basePath, callback = (error) ->) ->
+	syncResourcesToDisk: (request, basePath, callback = (error, resourceList) ->) ->
 		if request.syncType is "incremental"
 			ResourceWriter.checkSyncState request.syncState, basePath, (error, syncStateOk) ->
 				logger.log syncState: request.syncState, result:syncStateOk, "checked state on incremental request"
 				return callback new Errors.FilesOutOfSyncError("invalid state for incremental update") if not syncStateOk
-				ResourceWriter.saveIncrementalResourcesToDisk request.project_id, request.resources, basePath, callback
+				ResourceWriter.saveIncrementalResourcesToDisk request.project_id, request.resources, basePath, (error) ->
+					return callback(error) if error?
+					ResourceListManager.loadResourceList basePath, callback
 		else
 			@saveAllResourcesToDisk request.project_id, request.resources, basePath, (error) ->
 				return callback(error) if error?
-				ResourceWriter.storeSyncState request.syncState, basePath, callback
+				ResourceWriter.storeSyncState request.syncState, basePath, (error) ->
+					return callback(error) if error?
+					ResourceListManager.saveResourceList request.resources, basePath, (error) =>
+						return callback(error) if error?
+						callback(null, request.resources)
 
 	# The sync state is an identifier which must match for an
 	# incremental update to be allowed.

services/clsi/test/unit/coffee/CompileManagerTests.coffee

@@ -51,7 +51,7 @@ describe "CompileManager", ->
 			@env = {}
 			@Settings.compileDir = "compiles"
 			@compileDir = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"
-			@ResourceWriter.syncResourcesToDisk = sinon.stub().callsArg(2)
+			@ResourceWriter.syncResourcesToDisk = sinon.stub().callsArgWith(2, null, @resources)
 			@LatexRunner.runLatex = sinon.stub().callsArg(2)
 			@OutputFileFinder.findOutputFiles = sinon.stub().callsArgWith(2, null, @output_files)
 			@OutputCacheManager.saveOutputFiles = sinon.stub().callsArgWith(2, null, @build_files)

services/clsi/test/unit/coffee/ResourceWriterTests.coffee

@@ -10,6 +10,7 @@ describe "ResourceWriter", ->
 			"fs": @fs =
 				mkdir: sinon.stub().callsArg(1)
 				unlink: sinon.stub().callsArg(1)
+			"./ResourceListManager": @ResourceListManager = {}
 			"wrench": @wrench = {}
 			"./UrlCache" : @UrlCache = {}
 			"mkdirp" : @mkdirp = sinon.stub().callsArg(1)
@@ -33,6 +34,8 @@ describe "ResourceWriter", ->
 			@ResourceWriter._removeExtraneousFiles = sinon.stub().callsArg(2)
 			@ResourceWriter.checkSyncState = sinon.stub().callsArg(2)
 			@ResourceWriter.storeSyncState = sinon.stub().callsArg(2)
+			@ResourceListManager.saveResourceList = sinon.stub().callsArg(2)
+			@ResourceListManager.loadResourceList = sinon.stub().callsArg(1)
 			@ResourceWriter.syncResourcesToDisk({project_id: @project_id, resources: @resources}, @basePath, @callback)
 
 		it "should remove old files", ->