Add option --retry-sleep (#3059)

Closes #2852
This commit is contained in:
pukkandan 2022-05-19 20:00:31 +05:30
parent 9e49146352
commit 23326151c4
No known key found for this signature in database
GPG key ID: 7EEE9E1E817D0A39
6 changed files with 56 additions and 6 deletions

View file

@ -453,6 +453,9 @@ class YoutubeDL:
Allowed keys are 'download', 'postprocess',
'download-title' (console title) and 'postprocess-title'.
The template is mapped on a dictionary with keys 'progress' and 'info'
retry_sleep_functions: Dictionary of functions that takes the number of attempts
as argument and returns the time to sleep in seconds.
Allowed keys are 'http', 'fragment', 'file_access'
The following parameters are not used by YoutubeDL itself, they are used by
the downloader (see yt_dlp/downloader/common.py):

View file

@ -247,6 +247,28 @@ def parse_retries(name, value):
opts.extractor_retries = parse_retries('extractor', opts.extractor_retries)
opts.file_access_retries = parse_retries('file access', opts.file_access_retries)
# Retry sleep function
def parse_sleep_func(expr):
NUMBER_RE = r'\d+(?:\.\d+)?'
op, start, limit, step, *_ = tuple(re.fullmatch(
rf'(?:(linear|exp)=)?({NUMBER_RE})(?::({NUMBER_RE}))?(?::({NUMBER_RE}))?',
expr.strip()).groups()) + (None, None)
if op == 'exp':
return lambda n: min(float(start) * (float(step or 2) ** n), float(limit or 'inf'))
else:
default_step = start if op or limit else 0
return lambda n: min(float(start) + float(step or default_step) * n, float(limit or 'inf'))
for key, expr in opts.retry_sleep.items():
if not expr:
del opts.retry_sleep[key]
continue
try:
opts.retry_sleep[key] = parse_sleep_func(expr)
except AttributeError as e:
raise ValueError(f'invalid {key} retry sleep expression {expr!r}: {e}')
# Bytes
def parse_bytes(name, value):
if value is None:
@ -694,6 +716,7 @@ def parse_options(argv=None):
'file_access_retries': opts.file_access_retries,
'fragment_retries': opts.fragment_retries,
'extractor_retries': opts.extractor_retries,
'retry_sleep_functions': opts.retry_sleep,
'skip_unavailable_fragments': opts.skip_unavailable_fragments,
'keep_fragments': opts.keep_fragments,
'concurrent_fragment_downloads': opts.concurrent_fragment_downloads,

View file

@ -19,6 +19,7 @@
encodeFilename,
error_to_compat_str,
format_bytes,
int_or_none,
sanitize_open,
shell_quote,
timeconvert,
@ -64,6 +65,7 @@ class FileDownloader:
useful for bypassing bandwidth throttling imposed by
a webserver (experimental)
progress_template: See YoutubeDL.py
retry_sleep_functions: See YoutubeDL.py
Subclasses of this one must re-define the real_download method.
"""
@ -98,6 +100,8 @@ def _set_ydl(self, ydl):
def to_screen(self, *args, **kargs):
self.ydl.to_screen(*args, quiet=self.params.get('quiet'), **kargs)
__to_screen = to_screen
@property
def FD_NAME(self):
return re.sub(r'(?<!^)(?=[A-Z])', '_', type(self).__name__[:-2]).lower()
@ -232,7 +236,8 @@ def inner(self, *args, **kwargs):
self.to_screen(
f'[download] Unable to {action} file due to file access error. '
f'Retrying (attempt {retry} of {self.format_retries(file_access_retries)}) ...')
time.sleep(0.01)
if not self.sleep_retry('file_access', retry):
time.sleep(0.01)
return inner
return outer
@ -390,14 +395,23 @@ def report_resuming_byte(self, resume_len):
def report_retry(self, err, count, retries):
"""Report retry in case of HTTP error 5xx"""
self.to_screen(
self.__to_screen(
'[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...'
% (error_to_compat_str(err), count, self.format_retries(retries)))
self.sleep_retry('http', count)
def report_unable_to_resume(self):
"""Report it was impossible to resume download."""
self.to_screen('[download] Unable to resume')
def sleep_retry(self, retry_type, count):
sleep_func = self.params.get('retry_sleep_functions', {}).get(retry_type)
delay = int_or_none(sleep_func(n=count - 1)) if sleep_func else None
if delay:
self.__to_screen(f'Sleeping {delay} seconds ...')
time.sleep(delay)
return sleep_func is not None
@staticmethod
def supports_manifest(manifest):
""" Whether the downloader can download the fragments from the manifest.

View file

@ -142,6 +142,7 @@ def _call_downloader(self, tmpfilename, info_dict):
self.to_screen(
'[%s] Got error. Retrying fragments (attempt %d of %s)...'
% (self.get_basename(), count, self.format_retries(fragment_retries)))
self.sleep_retry('fragment', count)
if count > fragment_retries:
if not skip_unavailable_fragments:
self.report_error('Giving up after %s fragment retries' % fragment_retries)

View file

@ -25,10 +25,6 @@ def to_screen(self, *args, **kargs):
console_title = to_screen
def report_retry(self, err, count, retries):
super().to_screen(
f'[download] Got server HTTP error: {err}. Retrying (attempt {count} of {self.format_retries(retries)}) ...')
class FragmentFD(FileDownloader):
"""
@ -70,6 +66,7 @@ def report_retry_fragment(self, err, frag_index, count, retries):
self.to_screen(
'\r[download] Got server HTTP error: %s. Retrying fragment %d (attempt %d of %s) ...'
% (error_to_compat_str(err), frag_index, count, self.format_retries(retries)))
self.sleep_retry('fragment', count)
def report_skip_fragment(self, frag_index, err=None):
err = f' {err};' if err else ''

View file

@ -828,6 +828,18 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
'--fragment-retries',
dest='fragment_retries', metavar='RETRIES', default=10,
help='Number of retries for a fragment (default is %default), or "infinite" (DASH, hlsnative and ISM)')
downloader.add_option(
'--retry-sleep',
dest='retry_sleep', metavar='[TYPE:]EXPR', default={}, type='str',
action='callback', callback=_dict_from_options_callback,
callback_kwargs={
'allowed_keys': 'http|fragment|file_access',
'default_key': 'http',
}, help=(
'An expression for the time to sleep between retries in seconds (optionally) prefixed '
'by the type of retry (http (default), fragment, file_access) to apply the sleep to. '
'EXPR can be a number, or of the forms linear=START[:END[:STEP=1]] or exp=START[:END[:BASE=2]]. '
'Eg: --retry-sleep linear=1::2 --retry-sleep fragment:exp=1:20'))
downloader.add_option(
'--skip-unavailable-fragments', '--no-abort-on-unavailable-fragment',
action='store_true', dest='skip_unavailable_fragments', default=True,