combine the resource state and resource list

to prevent them getting out of sync
This commit is contained in:
Brian Gough 2017-09-07 11:54:38 +01:00
parent 5b33129138
commit 3d1c3a1d27
4 changed files with 26 additions and 61 deletions

View file

@ -1,25 +0,0 @@
Path = require "path"
fs = require "fs"
logger = require "logger-sharelatex"
settings = require("settings-sharelatex")
SafeReader = require "./SafeReader"
module.exports = ResourceListManager =
# This file is a list of the input files for the project, one per
# line, used to identify output files (i.e. files not on this list)
# when the incoming request is incremental.
RESOURCE_LIST_FILE: ".project-resource-list"
saveResourceList: (resources, basePath, callback = (error) ->) ->
resourceListFile = Path.join(basePath, @RESOURCE_LIST_FILE)
resourceList = (resource.path for resource in resources)
fs.writeFile resourceListFile, resourceList.join("\n"), callback
loadResourceList: (basePath, callback = (error) ->) ->
resourceListFile = Path.join(basePath, @RESOURCE_LIST_FILE)
# limit file to 128K, compile directory is user accessible
SafeReader.readFile resourceListFile, 128*1024, 'utf8', (err, resourceList) ->
return callback(err) if err?
resources = ({path: path} for path in resourceList?.toString()?.split("\n") or [])
callback(null, resources)

View file

@ -11,10 +11,11 @@ module.exports = ResourceStateManager =
# incremental update to be allowed.
#
# The initial value is passed in and stored on a full
# compile.
# compile, along with the list of resources..
#
# Subsequent incremental compiles must come with the same value - if
# not they will be rejected with a 409 Conflict response.
# not they will be rejected with a 409 Conflict response. The
# previous list of resources is returned.
#
# An incremental compile can only update existing files with new
# content. The sync state identifier must change if any docs or
@ -22,7 +23,7 @@ module.exports = ResourceStateManager =
SYNC_STATE_FILE: ".project-sync-state"
saveProjectStateHash: (state, basePath, callback) ->
saveProjectStateHash: (state, resources, basePath, callback = (error) ->) ->
stateFile = Path.join(basePath, @SYNC_STATE_FILE)
if not state? # remove the file if no state passed in
logger.log state:state, basePath:basePath, "clearing sync state"
@ -33,14 +34,18 @@ module.exports = ResourceStateManager =
return callback()
else
logger.log state:state, basePath:basePath, "writing sync state"
fs.writeFile stateFile, state, {encoding: 'ascii'}, callback
resourceList = (resource.path for resource in resources)
fs.writeFile stateFile, [resourceList..., "stateHash:#{state}"].join("\n"), callback
checkProjectStateHashMatches: (state, basePath, callback) ->
checkProjectStateHashMatches: (state, basePath, callback = (error, resources) ->) ->
stateFile = Path.join(basePath, @SYNC_STATE_FILE)
SafeReader.readFile stateFile, 64, 'ascii', (err, oldState) ->
SafeReader.readFile stateFile, 128*1024, 'utf8', (err, result) ->
return callback(err) if err?
logger.log state:state, oldState: oldState, basePath:basePath, stateMatches: !(state isnt oldState), "checking sync state"
if state isnt oldState
[resourceList..., oldState] = result?.toString()?.split("\n") or []
newState = "stateHash:#{state}"
logger.log state:state, oldState: oldState, basePath:basePath, stateMatches: (newState is oldState), "checking sync state"
if newState isnt oldState
return callback new Errors.FilesOutOfSyncError("invalid state for incremental update")
else
callback(null)
resources = ({path: path} for path in resourceList)
callback(null, resources)

View file

@ -5,7 +5,6 @@ async = require "async"
mkdirp = require "mkdirp"
OutputFileFinder = require "./OutputFileFinder"
ResourceStateManager = require "./ResourceStateManager"
ResourceListManager = require "./ResourceListManager"
Metrics = require "./Metrics"
logger = require "logger-sharelatex"
settings = require("settings-sharelatex")
@ -17,24 +16,20 @@ module.exports = ResourceWriter =
syncResourcesToDisk: (request, basePath, callback = (error, resourceList) ->) ->
if request.syncType is "incremental"
logger.log project_id: request.project_id, user_id: request.user_id, "incremental sync"
ResourceStateManager.checkProjectStateHashMatches request.syncState, basePath, (error) ->
ResourceStateManager.checkProjectStateHashMatches request.syncState, basePath, (error, resourceList) ->
return callback(error) if error?
ResourceListManager.loadResourceList basePath, (error, resourceList) ->
ResourceWriter._removeExtraneousFiles resourceList, basePath, (error) =>
return callback(error) if error?
ResourceWriter._removeExtraneousFiles resourceList, basePath, (error) =>
ResourceWriter.saveIncrementalResourcesToDisk request.project_id, request.resources, basePath, (error) ->
return callback(error) if error?
ResourceWriter.saveIncrementalResourcesToDisk request.project_id, request.resources, basePath, (error) ->
return callback(error) if error?
callback(null, resourceList)
callback(null, resourceList)
else
logger.log project_id: request.project_id, user_id: request.user_id, "full sync"
@saveAllResourcesToDisk request.project_id, request.resources, basePath, (error) ->
return callback(error) if error?
ResourceStateManager.saveProjectStateHash request.syncState, basePath, (error) ->
ResourceStateManager.saveProjectStateHash request.syncState, request.resources, basePath, (error) ->
return callback(error) if error?
ResourceListManager.saveResourceList request.resources, basePath, (error) =>
return callback(error) if error?
callback(null, request.resources)
callback(null, request.resources)
saveIncrementalResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) ->
@_createDirectory basePath, (error) =>
@ -81,7 +76,7 @@ module.exports = ResourceWriter =
should_delete = true
if path.match(/^output\./) or path.match(/\.aux$/) or path.match(/^cache\//) # knitr cache
should_delete = false
if path in ['.project-resource-list', '.project-sync-state']
if path == '.project-sync-state'
should_delete = false
if path == "output.pdf" or path == "output.dvi" or path == "output.log" or path == "output.xdv"
should_delete = true

View file

@ -10,7 +10,6 @@ describe "ResourceWriter", ->
"fs": @fs =
mkdir: sinon.stub().callsArg(1)
unlink: sinon.stub().callsArg(1)
"./ResourceListManager": @ResourceListManager = {}
"./ResourceStateManager": @ResourceStateManager = {}
"wrench": @wrench = {}
"./UrlCache" : @UrlCache = {}
@ -34,9 +33,7 @@ describe "ResourceWriter", ->
@ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3)
@ResourceWriter._removeExtraneousFiles = sinon.stub().callsArg(2)
@ResourceStateManager.checkProjectStateHashMatches = sinon.stub().callsArg(2)
@ResourceStateManager.saveProjectStateHash = sinon.stub().callsArg(2)
@ResourceListManager.saveResourceList = sinon.stub().callsArg(2)
@ResourceListManager.loadResourceList = sinon.stub().callsArg(1)
@ResourceStateManager.saveProjectStateHash = sinon.stub().callsArg(3)
@ResourceWriter.syncResourcesToDisk({
project_id: @project_id
syncState: @syncState = "0123456789abcdef"
@ -54,14 +51,9 @@ describe "ResourceWriter", ->
.calledWith(@project_id, resource, @basePath)
.should.equal true
it "should store the sync state", ->
it "should store the sync state and resource list", ->
@ResourceStateManager.saveProjectStateHash
.calledWith(@syncState, @basePath)
.should.equal true
it "should save the resource list", ->
@ResourceListManager.saveResourceList
.calledWith(@resources, @basePath)
.calledWith(@syncState, @resources, @basePath)
.should.equal true
it "should call the callback", ->
@ -74,10 +66,8 @@ describe "ResourceWriter", ->
]
@ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3)
@ResourceWriter._removeExtraneousFiles = sinon.stub().callsArg(2)
@ResourceStateManager.checkProjectStateHashMatches = sinon.stub().callsArg(2)
@ResourceStateManager.saveProjectStateHash = sinon.stub().callsArg(2)
@ResourceListManager.saveResourceList = sinon.stub().callsArg(2)
@ResourceListManager.loadResourceList = sinon.stub().callsArgWith(1, null, @resources)
@ResourceStateManager.checkProjectStateHashMatches = sinon.stub().callsArgWith(2, null, @resources)
@ResourceStateManager.saveProjectStateHash = sinon.stub().callsArg(3)
@ResourceWriter.syncResourcesToDisk({
project_id: @project_id,
syncType: "incremental",