Index '.bib' file objects.

This commit is contained in:
Shane Kilkelly 2016-05-19 14:58:12 +01:00
parent b649c13c17
commit 0ea16f0bcc
3 changed files with 67 additions and 9 deletions

View file

@ -16,19 +16,31 @@ module.exports = ReferencesHandler =
_buildDocUrl: (projectId, docId) ->
"#{settings.apis.docstore.url}/project/#{projectId}/doc/#{docId}/raw"
_buildFileUrl: (projectId, fileId) ->
"#{settings.apis.filestore.url}/project/#{projectId}/file/#{fileId}"
_findBibFileIds: (project) ->
ids = []
_process = (folder) ->
(folder.fileRefs or []).forEach (file) ->
if file?.name?.match(/^.*\.bib$/)
ids.push(file._id)
(folder.folders or []).forEach (folder) ->
_process(folder)
(project.rootFolder or []).forEach (rootFolder) ->
_process(rootFolder)
return ids
_findBibDocIds: (project) ->
ids = []
_process = (folder) ->
(folder.docs or []).forEach (doc) ->
if doc?.name?.match(/^.*\.bib$/)
ids.push(doc._id)
(folder.folders or []).forEach (folder) ->
_process(folder)
(project.rootFolder or []).forEach (rootFolder) ->
_process(rootFolder)
return ids
_isFullIndex: (project, callback = (err, result) ->) ->
@ -43,16 +55,17 @@ module.exports = ReferencesHandler =
return callback(err)
logger.log {projectId}, "indexing all bib files in project"
docIds = ReferencesHandler._findBibDocIds(project)
ReferencesHandler._doIndexOperation(projectId, project, docIds, callback)
fileIds = ReferencesHandler._findBibFileIds(project)
ReferencesHandler._doIndexOperation(projectId, project, docIds, fileIds, callback)
index: (projectId, docIds, callback=(err, data)->) ->
ProjectGetter.getProject projectId, {rootFolder: true, owner_ref: 1}, (err, project) ->
if err
logger.err {err, projectId}, "error finding project"
return callback(err)
ReferencesHandler._doIndexOperation(projectId, project, docIds, callback)
ReferencesHandler._doIndexOperation(projectId, project, docIds, [], callback)
_doIndexOperation: (projectId, project, docIds, callback) ->
_doIndexOperation: (projectId, project, docIds, fileIds, callback) ->
ReferencesHandler._isFullIndex project, (err, isFullIndex) ->
if err
logger.err {err, projectId}, "error checking whether to do full index"
@ -67,11 +80,14 @@ module.exports = ReferencesHandler =
return callback(err)
bibDocUrls = docIds.map (docId) ->
ReferencesHandler._buildDocUrl projectId, docId
bibFileUrls = fileIds.map (fileId) ->
ReferencesHandler._buildFileUrl projectId, fileId
allUrls = bibDocUrls.concat(bibFileUrls)
logger.log {projectId, isFullIndex, docIds, bibDocUrls}, "sending request to references service"
request.post {
url: "#{settings.apis.references.url}/project/#{projectId}/index"
json:
docUrls: bibDocUrls
docUrls: allUrls
fullIndex: isFullIndex
}, (err, res, data) ->
if err

View file

@ -11,6 +11,9 @@ define [
if entity?.name?.match /.*\.bib$/
@indexReferences([doc.doc_id], true)
@$scope.$on 'references:should-reindex', (e, data) =>
@indexAllReferences(true)
# When we join the project:
# index all references files
# and don't broadcast to all clients

View file

@ -21,7 +21,11 @@ describe 'ReferencesHandler', ->
{name: 'two.txt', _id: 'bbb'},
]
folders: [
{docs: [{name: 'three.bib', _id: 'ccc'}], folders: []}
{
docs: [{name: 'three.bib', _id: 'ccc'}],
fileRefs: [{name: 'four.bib', _id: 'ghg'}],
folders: []
}
]
]
@docIds = ['aaa', 'ccc']
@ -34,6 +38,7 @@ describe 'ReferencesHandler', ->
apis:
references: {url: 'http://some.url/references'}
docstore: {url: 'http://some.url/docstore'}
filestore: {url: 'http://some.url/filestore'}
}
'request': @request = {
get: sinon.stub()
@ -56,6 +61,7 @@ describe 'ReferencesHandler', ->
beforeEach ->
sinon.stub(@handler, '_findBibDocIds')
sinon.stub(@handler, '_findBibFileIds')
sinon.stub(@handler, '_isFullIndex').callsArgWith(1, null, true)
@request.post.callsArgWith(1, null, {statusCode: 200}, @fakeResponseData)
@call = (callback) =>
@ -198,6 +204,7 @@ describe 'ReferencesHandler', ->
beforeEach ->
sinon.stub(@handler, '_findBibDocIds').returns(['aaa', 'ccc'])
sinon.stub(@handler, '_findBibFileIds').returns(['fff', 'ggg'])
sinon.stub(@handler, '_isFullIndex').callsArgWith(1, null, true)
@request.post.callsArgWith(1, null, {statusCode: 200}, @fakeResponseData)
@call = (callback) =>
@ -209,6 +216,12 @@ describe 'ReferencesHandler', ->
@handler._findBibDocIds.calledWith(@fakeProject).should.equal true
done()
it 'should call _findBibFileIds', (done) ->
@call (err, data) =>
@handler._findBibDocIds.callCount.should.equal 1
@handler._findBibDocIds.calledWith(@fakeProject).should.equal true
done()
it 'should call DocumentUpdaterHandler.flushDocToMongo', (done) ->
@call (err, data) =>
@DocumentUpdaterHandler.flushDocToMongo.callCount.should.equal 2
@ -219,7 +232,7 @@ describe 'ReferencesHandler', ->
@request.post.callCount.should.equal 1
arg = @request.post.firstCall.args[0]
expect(arg.json).to.have.all.keys 'docUrls', 'fullIndex'
expect(arg.json.docUrls.length).to.equal 2
expect(arg.json.docUrls.length).to.equal 4
expect(arg.json.fullIndex).to.equal true
done()
@ -309,6 +322,32 @@ describe 'ReferencesHandler', ->
result = @handler._findBibDocIds(@fakeProject)
expect(result).to.deep.equal @expectedIds
describe '_findBibFileIds', ->
beforeEach ->
@fakeProject =
rootFolder: [
docs: [
{name: 'one.bib', _id: 'aaa'},
{name: 'two.txt', _id: 'bbb'},
]
fileRefs: [
{name: 'other.bib', _id: 'ddd'}
],
folders: [
{
docs: [{name: 'three.bib', _id: 'ccc'}],
fileRefs: [{name: 'four.bib', _id: 'ghg'}],
folders: []
}
]
]
@expectedIds = ['ddd', 'ghg']
it 'should select the correct docIds', ->
result = @handler._findBibFileIds(@fakeProject)
expect(result).to.deep.equal @expectedIds
describe '_isFullIndex', ->
beforeEach ->