github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-12-26 20:04:25 +00:00
Code Issues Projects Releases Packages Wiki Activity

add sha1 hash support on writes

Browse source
This commit is contained in:
Brian Gough 2017-02-14 16:11:43 +00:00
parent a33d4f505b
commit bd70aaa76c
5 changed files with 32 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
services/document-updater/app/coffee/RedisKeyBuilder.coffee
View file

@ -28,6 +28,8 @@ module.exports = RedisKeyBuilder =
return (key_schema) -> key_schema.docOps({doc_id})
docVersion: ({doc_id}) ->
return (key_schema) -> key_schema.docVersion({doc_id})
docHash: ({doc_id}) ->
return (key_schema) -> key_schema.docHash({doc_id})
projectKey: ({doc_id}) ->
return (key_schema) -> key_schema.projectKey({doc_id})
uncompressedHistoryOp: ({doc_id}) ->

28
services/document-updater/app/coffee/RedisManager.coffee
View file

@ -6,6 +6,7 @@ keys = require('./RedisKeyBuilder')
logger = require('logger-sharelatex')
metrics = require('./Metrics')
Errors = require "./Errors"
crypto = require "crypto"
# Make times easy to read
minutes = 60 # seconds for Redis expire
@ -18,12 +19,15 @@ module.exports = RedisManager =
callback = (error) ->
timer.done()
_callback(error)
logger.log project_id:project_id, doc_id:doc_id, version: version, "putting doc in redis"
docLines = JSON.stringify(docLines)
docHash = RedisManager._computeHash(docLines)
logger.log project_id:project_id, doc_id:doc_id, version: version, hash:docHash, "putting doc in redis"
ranges = RedisManager._serializeRanges(ranges)
multi = rclient.multi()
multi.set keys.docLines(doc_id:doc_id), JSON.stringify(docLines)
multi.set keys.docLines(doc_id:doc_id), docLines
multi.set keys.projectKey({doc_id:doc_id}), project_id
multi.set keys.docVersion(doc_id:doc_id), version
multi.set keys.docHash(doc_id:doc_id), docHash
if ranges?
multi.set keys.ranges(doc_id:doc_id), ranges
else
@ -46,6 +50,7 @@ module.exports = RedisManager =
multi.del keys.docLines(doc_id:doc_id)
multi.del keys.projectKey(doc_id:doc_id)
multi.del keys.docVersion(doc_id:doc_id)
multi.del keys.docHash(doc_id:doc_id)
multi.del keys.ranges(doc_id:doc_id)
multi.exec (error) ->
return callback(error) if error?
@ -56,11 +61,17 @@ module.exports = RedisManager =
multi = rclient.multi()
multi.get keys.docLines(doc_id:doc_id)
multi.get keys.docVersion(doc_id:doc_id)
multi.get keys.docHash(doc_id:doc_id)
multi.get keys.projectKey(doc_id:doc_id)
multi.get keys.ranges(doc_id:doc_id)
multi.exec (error, [docLines, version, doc_project_id, ranges])->
multi.exec (error, [docLines, version, storedHash, doc_project_id, ranges])->
timer.done()
return callback(error) if error?
if docLines?
computedHash = RedisManager._computeHash(docLines)
if computedHash isnt storedHash
logger.error project_id: project_id, doc_id: doc_id, doc_project_id: doc_project_id, computedHash: computedHash, storedHash: storedHash, "hash mismatch on retrieved document"
try
docLines = JSON.parse docLines
ranges = RedisManager._deserializeRanges(ranges)
@ -121,8 +132,11 @@ module.exports = RedisManager =
return callback(error)
jsonOps = appliedOps.map (op) -> JSON.stringify op
multi = rclient.multi()
multi.set keys.docLines(doc_id:doc_id), JSON.stringify(docLines)
newDocLines = JSON.stringify(docLines)
newHash = RedisManager._computeHash(newDocLines)
multi.set keys.docLines(doc_id:doc_id), newDocLines
multi.set keys.docVersion(doc_id:doc_id), newVersion
multi.set keys.docHash(doc_id:doc_id), newHash
if jsonOps.length > 0
multi.rpush keys.docOps(doc_id: doc_id), jsonOps...
multi.expire keys.docOps(doc_id: doc_id), RedisManager.DOC_OPS_TTL
@ -150,4 +164,8 @@ module.exports = RedisManager =
if !ranges? or ranges == ""
return {}
else
return JSON.parse(ranges)
return JSON.parse(ranges)
_computeHash: (docLines) ->
# use sha1 checksum of doclines to detect data corruption
return crypto.createHash('sha1').update(docLines).digest('hex')

1
services/document-updater/config/settings.defaults.coffee
View file

@ -30,6 +30,7 @@ module.exports =
docLines: ({doc_id}) -> "doclines:#{doc_id}"
docOps: ({doc_id}) -> "DocOps:#{doc_id}"
docVersion: ({doc_id}) -> "DocVersion:#{doc_id}"
docHash: ({doc_id}) -> "DocHash:#{doc_id}"
projectKey: ({doc_id}) -> "ProjectId:#{doc_id}"
docsInProject: ({project_id}) -> "DocsIn:#{project_id}"
ranges: ({doc_id}) -> "Ranges:#{doc_id}"

2
services/document-updater/test/unit/coffee/RedisBackend/RedisBackendTests.coffee
View file

@ -19,6 +19,7 @@ describe "RedisBackend", ->
docLines: ({doc_id}) -> "doclines:#{doc_id}"
docOps: ({doc_id}) -> "DocOps:#{doc_id}"
docVersion: ({doc_id}) -> "DocVersion:#{doc_id}"
docHash: ({doc_id}) -> "DocHash:#{doc_id}"
projectKey: ({doc_id}) -> "ProjectId:#{doc_id}"
pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}"
docsInProject: ({project_id}) -> "DocsIn:#{project_id}"
@ -33,6 +34,7 @@ describe "RedisBackend", ->
docLines: ({doc_id}) -> "doclines:{#{doc_id}}"
docOps: ({doc_id}) -> "DocOps:{#{doc_id}}"
docVersion: ({doc_id}) -> "DocVersion:{#{doc_id}}"
docHash: ({doc_id}) -> "DocHash:{#{doc_id}}"
projectKey: ({doc_id}) -> "ProjectId:{#{doc_id}}"
pendingUpdates: ({doc_id}) -> "PendingUpdates:{#{doc_id}}"
docsInProject: ({project_id}) -> "DocsIn:{#{project_id}}"

5
services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee
View file

@ -4,6 +4,7 @@ should = chai.should()
modulePath = "../../../../app/js/RedisManager.js"
SandboxedModule = require('sandboxed-module')
Errors = require "../../../../app/js/Errors"
crypto = require "crypto"
describe "RedisManager", ->
beforeEach ->
@ -19,6 +20,7 @@ describe "RedisManager", ->
docLines: ({doc_id}) -> "doclines:#{doc_id}"
docOps: ({doc_id}) -> "DocOps:#{doc_id}"
docVersion: ({doc_id}) -> "DocVersion:#{doc_id}"
docHash: ({doc_id}) -> "DocHash:#{doc_id}"
projectKey: ({doc_id}) -> "ProjectId:#{doc_id}"
pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}"
docsInProject: ({project_id}) -> "DocsIn:#{project_id}"
@ -38,10 +40,11 @@ describe "RedisManager", ->
@lines = ["one", "two", "three"]
@jsonlines = JSON.stringify @lines
@version = 42
@hash = crypto.createHash('sha1').update(@jsonlines).digest('hex')
@ranges = { comments: "mock", entries: "mock" }
@json_ranges = JSON.stringify @ranges
@rclient.get = sinon.stub()
@rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @project_id, @json_ranges])
@rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @hash, @project_id, @json_ranges])
describe "successfully", ->
beforeEach ->