github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Revert "change send doc lines using tpds to work with stream and doc store"

Browse source 
This reverts commit ba37710b087b4919c01eed3d8cfeb9d61af43a4d.
This commit is contained in:
James Allen 2014-05-20 11:14:06 +01:00
parent 22c577f9b5
commit 93d418b716
5 changed files with 30 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee
View file

@ -86,7 +86,7 @@ module.exports = ProjectEntityHandler =
for docPath, doc of docs for docPath, doc of docs
do (docPath, doc) -> do (docPath, doc) ->
requests.push (callback) -> requests.push (callback) ->
tpdsUpdateSender.addDoc {project_id:project_id, doc_id:doc._id, path:docPath, project_name:project.name, rev:doc.rev||0}, tpdsUpdateSender.addDoc {project_id:project_id, docLines:doc.lines, path:docPath, project_name:project.name, rev:doc.rev||0},
sl_req_id, sl_req_id,
callback callback
self.getAllFiles project_id, (error, files) -> self.getAllFiles project_id, (error, files) ->
@ -127,7 +127,7 @@ module.exports = ProjectEntityHandler =
return callback(err) if err? return callback(err) if err?
tpdsUpdateSender.addDoc { tpdsUpdateSender.addDoc {
project_id: project._id, project_id: project._id,
doc_id: doc._id docLines: docLines,
path: result.path.fileSystem, path: result.path.fileSystem,
project_name: project.name, project_name: project.name,
rev: 0 rev: 0
@ -256,7 +256,7 @@ module.exports = ProjectEntityHandler =
if modified if modified
# Don't need to block for marking as updated # Don't need to block for marking as updated
projectUpdateHandler.markAsUpdated project_id projectUpdateHandler.markAsUpdated project_id
tpdsUpdateSender.addDoc {project_id:project_id, path:path.fileSystem, doc_id:doc_id, project_name:project.name, rev:rev}, callback tpdsUpdateSender.addDoc {project_id:project_id, path:path.fileSystem, docLines:lines, project_name:project.name, rev:rev}, callback
else else
callback() callback()

32
services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateSender.coffee
View file

@ -15,8 +15,10 @@ buildPath = (user_id, project_name, filePath)->
queue = require('fairy').connect(settings.redis.fairy).queue(keys.queue.web_to_tpds_http_requests) queue = require('fairy').connect(settings.redis.fairy).queue(keys.queue.web_to_tpds_http_requests)
module.exports = module.exports =
 
_addEntity: (options, sl_req_id, callback = (err)->)-> addFile : (options, sl_req_id, callback = (err)->)->
metrics.inc("tpds.add-file")
{callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id)
getProjectsUsersIds options.project_id, (err, user_id, allUserIds)-> getProjectsUsersIds options.project_id, (err, user_id, allUserIds)->
logger.log project_id: options.project_id, user_id:user_id, path: options.path, uri:options.uri, sl_req_id:sl_req_id, rev:options.rev, "sending file to third party data store" logger.log project_id: options.project_id, user_id:user_id, path: options.path, uri:options.uri, sl_req_id:sl_req_id, rev:options.rev, "sending file to third party data store"
postOptions = postOptions =
@ -27,23 +29,29 @@ module.exports =
sl_project_id:options.project_id sl_project_id:options.project_id
sl_all_user_ids:JSON.stringify(allUserIds) sl_all_user_ids:JSON.stringify(allUserIds)
uri : "#{settings.apis.thirdPartyDataStore.url}#{buildPath(user_id, options.project_name, options.path)}" uri : "#{settings.apis.thirdPartyDataStore.url}#{buildPath(user_id, options.project_name, options.path)}"
title: "addFile" title:"addFile"
streamOrigin : options.streamOrigin streamOrigin : settings.apis.filestore.url + path.join("/project/#{options.project_id}/file/","#{options.file_id}")
queue.enqueue options.project_id, "pipeStreamFrom", postOptions, -> queue.enqueue options.project_id, "pipeStreamFrom", postOptions, ->
logger.log project_id: options.project_id, user_id:user_id, path: options.path, uri:options.uri, sl_req_id:sl_req_id, rev:options.rev, "sending file to third party data store queued up for processing" logger.log project_id: options.project_id, user_id:user_id, path: options.path, uri:options.uri, sl_req_id:sl_req_id, rev:options.rev, "sending file to third party data store queued up for processing"
callback() callback()
addFile : (options, sl_req_id, callback = (err)->)->
metrics.inc("tpds.add-file")
{callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id)
options.streamOrigin = settings.apis.filestore.url + path.join("/project/#{options.project_id}/file/","#{options.file_id}")
@_addEntity(options, sl_req_id, callback)
addDoc : (options, sl_req_id, callback = (err)->)-> addDoc : (options, sl_req_id, callback = (err)->)->
metrics.inc("tpds.add-doc") metrics.inc("tpds.add-doc")
{callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id)
options.streamOrigin = settings.apis.docstore.url + path.join("/project/#{options.project_id}/doc/","#{options.doc_id}") getProjectsUsersIds options.project_id, (err, user_id, allUserIds)->
@_addEntity(options, sl_req_id, callback) return callback(err) if err?
logger.log project_id: options.project_id, user_id:user_id, path: options.path, rev:options.rev, uri:options.uri, project_name:options.project_name, docLines:options.docLines, sl_req_id:sl_req_id, "sending doc to third party data store"
postOptions =
method : "post"
headers:
"sl_req_id":sl_req_id,
sl_entity_rev:options.rev,
sl_project_id:options.project_id
sl_all_user_ids:JSON.stringify(allUserIds)
uri : "#{settings.apis.thirdPartyDataStore.url}#{buildPath(user_id, options.project_name, options.path)}"
title: "addDoc"
docLines: options.docLines
queue.enqueue options.project_id, "sendDoc", postOptions, callback
     
moveEntity : (options, sl_req_id, callback = (err)->)-> moveEntity : (options, sl_req_id, callback = (err)->)->

9
services/web/test/UnitTests/coffee/Project/ProjectEntityHandlerTests.coffee
View file

@ -35,11 +35,10 @@ describe 'ProjectEntityHandler', ->
@rev = 0 @rev = 0
save:(callback)->callback() save:(callback)->callback()
rootFolder:[@rootFolder] rootFolder:[@rootFolder]
@DocModel = class Doc @DocModel = class Doc
constructor:(options)-> constructor:(options)->
{@name, @lines} = options {@name, @lines} = options
@_id = doc_id @_id = "mock-id"
@rev = 0 @rev = 0
@FileModel = class File @FileModel = class File
constructor:(options)-> constructor:(options)->
@ -327,7 +326,7 @@ describe 'ProjectEntityHandler', ->
@tpdsUpdateSender.addDoc @tpdsUpdateSender.addDoc
.calledWith({ .calledWith({
project_id: project_id project_id: project_id
doc_id: doc_id docLines: @lines
path: @path path: @path
project_name: @project.name project_name: @project.name
rev: 0 rev: 0
@ -505,7 +504,7 @@ describe 'ProjectEntityHandler', ->
.calledWith({ .calledWith({
project_id: project_id project_id: project_id
project_name: @project.name project_name: @project.name
doc_id: doc_id docLines: @lines
rev: @rev rev: @rev
path: @path path: @path
}) })
@ -700,7 +699,7 @@ describe 'ProjectEntityHandler', ->
@tpdsUpdateSender.addDoc @tpdsUpdateSender.addDoc
.calledWith({ .calledWith({
project_id: project_id, project_id: project_id,
doc_id: doc._id docLines: doc.lines
project_name: @project.name project_name: @project.name
rev: doc.rev rev: doc.rev
path: path path: path

7
services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsUpdateSenderTests.coffee
View file

@ -22,7 +22,6 @@ describe 'TpdsUpdateSender', ->
@requestQueuer = regist:(queue, meth, opts, callback)-> @requestQueuer = regist:(queue, meth, opts, callback)->
project = {owner_ref:user_id,readOnly_refs:[read_only_ref_1], collaberator_refs:[collaberator_ref_1]} project = {owner_ref:user_id,readOnly_refs:[read_only_ref_1], collaberator_refs:[collaberator_ref_1]}
@Project = findById:sinon.stub().callsArgWith(2, null, project) @Project = findById:sinon.stub().callsArgWith(2, null, project)
@docstoreUrl = "docstore.sharelatex.env"
@updateSender = SandboxedModule.require modulePath, requires: @updateSender = SandboxedModule.require modulePath, requires:
'fairy':{connect:=>{queue:=>@requestQueuer}} 'fairy':{connect:=>{queue:=>@requestQueuer}}
"settings-sharelatex": "settings-sharelatex":
@ -32,8 +31,6 @@ describe 'TpdsUpdateSender', ->
thirdPartyDataStore: {url: thirdPartyDataStoreApiUrl} thirdPartyDataStore: {url: thirdPartyDataStoreApiUrl}
filestore: filestore:
url: filestoreUrl url: filestoreUrl
docstore:
url: @docstoreUrl
redis:fairy:{} redis:fairy:{}
"logger-sharelatex":{log:->} "logger-sharelatex":{log:->}
'../../models/Project': Project:@Project '../../models/Project': Project:@Project
@ -59,12 +56,12 @@ describe 'TpdsUpdateSender', ->
path = "/some/path/here.tex" path = "/some/path/here.tex"
lines = ["line1", "line2", "line3"] lines = ["line1", "line2", "line3"]
@requestQueuer.enqueue = (uid, method, job, callback)=> @requestQueuer.enqueue = (uid, method, job, callback)->
uid.should.equal project_id uid.should.equal project_id
job.method.should.equal "post" job.method.should.equal "post"
expectedUrl = "#{thirdPartyDataStoreApiUrl}/user/#{user_id}/entity/#{encodeURIComponent(project_name)}#{encodeURIComponent(path)}" expectedUrl = "#{thirdPartyDataStoreApiUrl}/user/#{user_id}/entity/#{encodeURIComponent(project_name)}#{encodeURIComponent(path)}"
job.uri.should.equal expectedUrl job.uri.should.equal expectedUrl
job.streamOrigin.should.equal "#{@docstoreUrl}/project/#{project_id}/doc/#{doc_id}" assert.deepEqual job.docLines, lines
job.headers.sl_all_user_ids.should.eql(JSON.stringify([collaberator_ref_1, read_only_ref_1, user_id])) job.headers.sl_all_user_ids.should.eql(JSON.stringify([collaberator_ref_1, read_only_ref_1, user_id]))
done() done()
@updateSender.addDoc {project_id:project_id, doc_id:doc_id, path:path, docLines:lines,project_name:project_name} @updateSender.addDoc {project_id:project_id, doc_id:doc_id, path:path, docLines:lines,project_name:project_name}