mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-21 20:47:08 -05:00
Merge pull request #165 from overleaf/ho-retry-url-downloads
add pipeUrlToFileWithRetry for file downloads
This commit is contained in:
commit
457b7d6657
5 changed files with 153 additions and 109 deletions
|
@ -95,7 +95,7 @@ module.exports = UrlCache = {
|
|||
}
|
||||
if (needsDownloading) {
|
||||
logger.log({ url, lastModified }, 'downloading URL')
|
||||
return UrlFetcher.pipeUrlToFile(
|
||||
return UrlFetcher.pipeUrlToFileWithRetry(
|
||||
url,
|
||||
UrlCache._cacheFilePathForUrl(project_id, url),
|
||||
error => {
|
||||
|
|
|
@ -12,16 +12,23 @@
|
|||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let UrlFetcher
|
||||
const request = require('request').defaults({ jar: false })
|
||||
const fs = require('fs')
|
||||
const logger = require('logger-sharelatex')
|
||||
const settings = require('settings-sharelatex')
|
||||
const URL = require('url')
|
||||
const async = require('async')
|
||||
|
||||
const oneMinute = 60 * 1000
|
||||
|
||||
module.exports = UrlFetcher = {
|
||||
pipeUrlToFileWithRetry(url, filePath, callback) {
|
||||
const doDownload = function(cb) {
|
||||
UrlFetcher.pipeUrlToFile(url, filePath, cb)
|
||||
}
|
||||
async.retry(3, doDownload, callback)
|
||||
},
|
||||
|
||||
pipeUrlToFile(url, filePath, _callback) {
|
||||
if (_callback == null) {
|
||||
_callback = function(error) {}
|
||||
|
|
|
@ -11,7 +11,6 @@
|
|||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const Client = require('./helpers/Client')
|
||||
const request = require('request')
|
||||
require('chai').should()
|
||||
const sinon = require('sinon')
|
||||
const ClsiApp = require('./helpers/ClsiApp')
|
||||
|
|
|
@ -160,7 +160,7 @@ describe('UrlCache', function() {
|
|||
|
||||
describe('_ensureUrlIsInCache', function() {
|
||||
beforeEach(function() {
|
||||
this.UrlFetcher.pipeUrlToFile = sinon.stub().callsArg(2)
|
||||
this.UrlFetcher.pipeUrlToFileWithRetry = sinon.stub().callsArg(2)
|
||||
return (this.UrlCache._updateOrCreateUrlDetails = sinon
|
||||
.stub()
|
||||
.callsArg(3))
|
||||
|
@ -190,7 +190,7 @@ describe('UrlCache', function() {
|
|||
})
|
||||
|
||||
it('should download the URL to the cache file', function() {
|
||||
return this.UrlFetcher.pipeUrlToFile
|
||||
return this.UrlFetcher.pipeUrlToFileWithRetry
|
||||
.calledWith(
|
||||
this.url,
|
||||
this.UrlCache._cacheFilePathForUrl(this.project_id, this.url)
|
||||
|
@ -232,7 +232,7 @@ describe('UrlCache', function() {
|
|||
})
|
||||
|
||||
it('should not download the URL to the cache file', function() {
|
||||
return this.UrlFetcher.pipeUrlToFile.called.should.equal(false)
|
||||
return this.UrlFetcher.pipeUrlToFileWithRetry.called.should.equal(false)
|
||||
})
|
||||
|
||||
return it('should return the callback with the cache file path', function() {
|
||||
|
|
|
@ -33,7 +33,42 @@ describe('UrlFetcher', function() {
|
|||
}
|
||||
}))
|
||||
})
|
||||
describe('pipeUrlToFileWithRetry', function() {
|
||||
this.beforeEach(function() {
|
||||
this.UrlFetcher.pipeUrlToFile = sinon.stub()
|
||||
})
|
||||
|
||||
it('should call pipeUrlToFile', function(done) {
|
||||
this.UrlFetcher.pipeUrlToFile.callsArgWith(2)
|
||||
this.UrlFetcher.pipeUrlToFileWithRetry(this.url, this.path, err => {
|
||||
expect(err).to.equal(undefined)
|
||||
this.UrlFetcher.pipeUrlToFile.called.should.equal(true)
|
||||
done()
|
||||
})
|
||||
})
|
||||
|
||||
it('should call pipeUrlToFile multiple times on error', function(done) {
|
||||
error = new Error("couldn't download file")
|
||||
this.UrlFetcher.pipeUrlToFile.callsArgWith(2, error)
|
||||
this.UrlFetcher.pipeUrlToFileWithRetry(this.url, this.path, err => {
|
||||
expect(err).to.equal(error)
|
||||
this.UrlFetcher.pipeUrlToFile.callCount.should.equal(3)
|
||||
done()
|
||||
})
|
||||
})
|
||||
|
||||
it('should call pipeUrlToFile twice if only 1 error', function(done) {
|
||||
this.UrlFetcher.pipeUrlToFile.onCall(0).callsArgWith(2, 'error')
|
||||
this.UrlFetcher.pipeUrlToFile.onCall(1).callsArgWith(2)
|
||||
this.UrlFetcher.pipeUrlToFileWithRetry(this.url, this.path, err => {
|
||||
expect(err).to.equal(undefined)
|
||||
this.UrlFetcher.pipeUrlToFile.callCount.should.equal(2)
|
||||
done()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('pipeUrlToFile', function() {
|
||||
it('should turn off the cookie jar in request', function() {
|
||||
return this.defaults.calledWith({ jar: false }).should.equal(true)
|
||||
})
|
||||
|
@ -145,7 +180,9 @@ describe('UrlFetcher', function() {
|
|||
this.callback.calledWith(sinon.match(Error)).should.equal(true)
|
||||
|
||||
const message = this.callback.args[0][0].message
|
||||
expect(message).to.include('URL returned non-success status code: 404')
|
||||
expect(message).to.include(
|
||||
'URL returned non-success status code: 404'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
|
@ -172,3 +209,4 @@ describe('UrlFetcher', function() {
|
|||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
Loading…
Reference in a new issue