mirror of
https://github.com/overleaf/overleaf.git
synced 2025-04-22 02:18:27 +00:00
Merge pull request #22231 from overleaf/jpa-references-hash
[misc] references: read files from history-v1 w/ fallback to filestore GitOrigin-RevId: e8e05fbe5e5ea9fe83c6b3b00e0ee85d038ff5f0
This commit is contained in:
Jakob Ackermann
Copybot
parent
aae03bdf6e
commit
8db1f2c9bc
2 changed files with 72 additions and 22 deletions
|
|
@ -33,15 +33,25 @@ if (!Features.hasFeature('references')) {
|
|||
|
||||
export default ReferencesHandler = {
|
||||
_buildDocUrl(projectId, docId) {
|
||||
return `${settings.apis.docstore.url}/project/${projectId}/doc/${docId}/raw`
|
||||
return {
|
||||
url: `${settings.apis.docstore.url}/project/${projectId}/doc/${docId}/raw`,
|
||||
}
|
||||
},
|
||||
|
||||
_buildFileUrl(projectId, fileId) {
|
||||
return `${settings.apis.filestore.url}/project/${projectId}/file/${fileId}`
|
||||
_buildFileUrl(projectId, historyId, fileRef) {
|
||||
const filestoreURL = `${settings.apis.filestore.url}/project/${projectId}/file/${fileRef._id}`
|
||||
if (fileRef.hash) {
|
||||
return {
|
||||
url: `${settings.apis.project_history.url}/project/${historyId}/blob/${fileRef.hash}`,
|
||||
fallbackURL: filestoreURL,
|
||||
}
|
||||
} else {
|
||||
return { url: filestoreURL }
|
||||
}
|
||||
},
|
||||
|
||||
_findBibFileIds(project) {
|
||||
const ids = []
|
||||
_findBibFileRefs(project) {
|
||||
const fileRefs = []
|
||||
function _process(folder) {
|
||||
_.forEach(folder.fileRefs || [], function (file) {
|
||||
if (
|
||||
|
|
@ -49,13 +59,13 @@ export default ReferencesHandler = {
|
|||
x1.match(/^.*\.bib$/)
|
||||
)
|
||||
) {
|
||||
return ids.push(file._id)
|
||||
return fileRefs.push(file)
|
||||
}
|
||||
})
|
||||
return _.forEach(folder.folders || [], folder => _process(folder))
|
||||
}
|
||||
_.forEach(project.rootFolder || [], rootFolder => _process(rootFolder))
|
||||
return ids
|
||||
return fileRefs
|
||||
},
|
||||
|
||||
_findBibDocIds(project) {
|
||||
|
|
@ -103,7 +113,7 @@ export default ReferencesHandler = {
|
|||
}
|
||||
return ProjectGetter.getProject(
|
||||
projectId,
|
||||
{ rootFolder: true, owner_ref: 1 },
|
||||
{ rootFolder: true, owner_ref: 1, 'overleaf.history.id': 1 },
|
||||
function (err, project) {
|
||||
if (err) {
|
||||
OError.tag(err, 'error finding project', {
|
||||
|
|
@ -118,22 +128,28 @@ export default ReferencesHandler = {
|
|||
}
|
||||
logger.debug({ projectId }, 'indexing all bib files in project')
|
||||
const docIds = ReferencesHandler._findBibDocIds(project)
|
||||
const fileIds = ReferencesHandler._findBibFileIds(project)
|
||||
const fileRefs = ReferencesHandler._findBibFileRefs(project)
|
||||
return ReferencesHandler._doIndexOperation(
|
||||
projectId,
|
||||
project,
|
||||
docIds,
|
||||
fileIds,
|
||||
fileRefs,
|
||||
callback
|
||||
)
|
||||
}
|
||||
)
|
||||
},
|
||||
|
||||
_doIndexOperation(projectId, project, docIds, fileIds, callback) {
|
||||
_doIndexOperation(projectId, project, docIds, fileRefs, callback) {
|
||||
if (!Features.hasFeature('references')) {
|
||||
return callback()
|
||||
}
|
||||
const historyId = project?.overleaf?.history?.id
|
||||
if (!historyId) {
|
||||
return callback(
|
||||
new OError('project does not have a history id', { projectId })
|
||||
)
|
||||
}
|
||||
return ReferencesHandler._isFullIndex(project, function (err, isFullIndex) {
|
||||
if (err) {
|
||||
OError.tag(err, 'error checking whether to do full index', {
|
||||
|
|
@ -162,15 +178,16 @@ export default ReferencesHandler = {
|
|||
const bibDocUrls = docIds.map(docId =>
|
||||
ReferencesHandler._buildDocUrl(projectId, docId)
|
||||
)
|
||||
const bibFileUrls = fileIds.map(fileId =>
|
||||
ReferencesHandler._buildFileUrl(projectId, fileId)
|
||||
const bibFileUrls = fileRefs.map(fileRef =>
|
||||
ReferencesHandler._buildFileUrl(projectId, historyId, fileRef)
|
||||
)
|
||||
const allUrls = bibDocUrls.concat(bibFileUrls)
|
||||
const sourceURLs = bibDocUrls.concat(bibFileUrls)
|
||||
return request.post(
|
||||
{
|
||||
url: `${settings.apis.references.url}/project/${projectId}/index`,
|
||||
json: {
|
||||
docUrls: allUrls,
|
||||
docUrls: sourceURLs.map(item => item.fallbackURL || item.url),
|
||||
sourceURLs,
|
||||
fullIndex: isFullIndex,
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ const modulePath =
|
|||
describe('ReferencesHandler', function () {
|
||||
beforeEach(async function () {
|
||||
this.projectId = '222'
|
||||
this.historyId = 42
|
||||
this.fakeProject = {
|
||||
_id: this.projectId,
|
||||
owner_ref: (this.fakeOwner = {
|
||||
|
|
@ -33,12 +34,16 @@ describe('ReferencesHandler', function () {
|
|||
folders: [
|
||||
{
|
||||
docs: [{ name: 'three.bib', _id: 'ccc' }],
|
||||
fileRefs: [{ name: 'four.bib', _id: 'ghg' }],
|
||||
fileRefs: [
|
||||
{ name: 'four.bib', _id: 'fff' },
|
||||
{ name: 'five.bib', _id: 'ggg', hash: 'hash' },
|
||||
],
|
||||
folders: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
overleaf: { history: { id: this.historyId } },
|
||||
}
|
||||
this.docIds = ['aaa', 'ccc']
|
||||
this.handler = await esmock.strict(modulePath, {
|
||||
|
|
@ -47,6 +52,7 @@ describe('ReferencesHandler', function () {
|
|||
references: { url: 'http://some.url/references' },
|
||||
docstore: { url: 'http://some.url/docstore' },
|
||||
filestore: { url: 'http://some.url/filestore' },
|
||||
project_history: { url: 'http://project-history.local' },
|
||||
},
|
||||
}),
|
||||
request: (this.request = {
|
||||
|
|
@ -77,7 +83,9 @@ describe('ReferencesHandler', function () {
|
|||
describe('indexAll', function () {
|
||||
beforeEach(function () {
|
||||
sinon.stub(this.handler, '_findBibDocIds').returns(['aaa', 'ccc'])
|
||||
sinon.stub(this.handler, '_findBibFileIds').returns(['fff', 'ggg'])
|
||||
sinon
|
||||
.stub(this.handler, '_findBibFileRefs')
|
||||
.returns([{ _id: 'fff' }, { _id: 'ggg', hash: 'hash' }])
|
||||
sinon.stub(this.handler, '_isFullIndex').callsArgWith(1, null, true)
|
||||
this.request.post.callsArgWith(
|
||||
1,
|
||||
|
|
@ -101,7 +109,7 @@ describe('ReferencesHandler', function () {
|
|||
})
|
||||
})
|
||||
|
||||
it('should call _findBibFileIds', function (done) {
|
||||
it('should call _findBibFileRefs', function (done) {
|
||||
return this.call((err, data) => {
|
||||
expect(err).to.be.null
|
||||
this.handler._findBibDocIds.callCount.should.equal(1)
|
||||
|
|
@ -125,8 +133,30 @@ describe('ReferencesHandler', function () {
|
|||
expect(err).to.be.null
|
||||
this.request.post.callCount.should.equal(1)
|
||||
const arg = this.request.post.firstCall.args[0]
|
||||
expect(arg.json).to.have.all.keys('docUrls', 'fullIndex')
|
||||
expect(arg.json).to.have.all.keys('docUrls', 'sourceURLs', 'fullIndex')
|
||||
expect(arg.json.docUrls.length).to.equal(4)
|
||||
expect(arg.json.docUrls).to.deep.equal([
|
||||
`${this.settings.apis.docstore.url}/project/${this.projectId}/doc/aaa/raw`,
|
||||
`${this.settings.apis.docstore.url}/project/${this.projectId}/doc/ccc/raw`,
|
||||
`${this.settings.apis.filestore.url}/project/${this.projectId}/file/fff`,
|
||||
`${this.settings.apis.filestore.url}/project/${this.projectId}/file/ggg`,
|
||||
])
|
||||
expect(arg.json.sourceURLs.length).to.equal(4)
|
||||
expect(arg.json.sourceURLs).to.deep.equal([
|
||||
{
|
||||
url: `${this.settings.apis.docstore.url}/project/${this.projectId}/doc/aaa/raw`,
|
||||
},
|
||||
{
|
||||
url: `${this.settings.apis.docstore.url}/project/${this.projectId}/doc/ccc/raw`,
|
||||
},
|
||||
{
|
||||
url: `${this.settings.apis.filestore.url}/project/${this.projectId}/file/fff`,
|
||||
},
|
||||
{
|
||||
url: `${this.settings.apis.project_history.url}/project/${this.historyId}/blob/hash`,
|
||||
fallbackURL: `${this.settings.apis.filestore.url}/project/${this.projectId}/file/ggg`,
|
||||
},
|
||||
])
|
||||
expect(arg.json.fullIndex).to.equal(true)
|
||||
return done()
|
||||
})
|
||||
|
|
@ -274,7 +304,7 @@ describe('ReferencesHandler', function () {
|
|||
})
|
||||
})
|
||||
|
||||
describe('_findBibFileIds', function () {
|
||||
describe('_findBibFileRefs', function () {
|
||||
beforeEach(function () {
|
||||
this.fakeProject = {
|
||||
rootFolder: [
|
||||
|
|
@ -294,11 +324,14 @@ describe('ReferencesHandler', function () {
|
|||
},
|
||||
],
|
||||
}
|
||||
return (this.expectedIds = ['ddd', 'ghg'])
|
||||
this.expectedIds = [
|
||||
this.fakeProject.rootFolder[0].fileRefs[0],
|
||||
this.fakeProject.rootFolder[0].folders[0].fileRefs[0],
|
||||
]
|
||||
})
|
||||
|
||||
it('should select the correct docIds', function () {
|
||||
const result = this.handler._findBibFileIds(this.fakeProject)
|
||||
const result = this.handler._findBibFileRefs(this.fakeProject)
|
||||
return expect(result).to.deep.equal(this.expectedIds)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue