From f2ebc5c7be57ef41d3fd5508e75c5ca26d36a0d1 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 29 Nov 2021 00:27:44 +0530 Subject: [PATCH] Option `--wait-for-video` to wait for scheduled streams --- yt_dlp/YoutubeDL.py | 59 ++++++++++++++++++++++++++++++++++---- yt_dlp/__init__.py | 9 ++++++ yt_dlp/extractor/common.py | 9 ++++-- yt_dlp/options.py | 10 +++++++ yt_dlp/utils.py | 13 ++++++++- 5 files changed, 91 insertions(+), 9 deletions(-) diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 29c9ecd16a..5e2b633b7a 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -93,6 +93,7 @@ PostProcessingError, preferredencoding, prepend_extension, + ReExtractInfo, register_socks_protocols, RejectedVideoReached, render_table, @@ -109,7 +110,7 @@ strftime_or_none, subtitles_filename, supports_terminal_sequences, - ThrottledDownload, + timetuple_from_msec, to_high_limit_path, traverse_obj, try_get, @@ -333,6 +334,9 @@ class YoutubeDL(object): extract_flat: Do not resolve URLs, return the immediate result. Pass in 'in_playlist' to only show this behavior for playlist items. + wait_for_video: If given, wait for scheduled streams to become available. + The value should be a tuple containing the range + (min_secs, max_secs) to wait between retries postprocessors: A list of dictionaries, each with an entry * key: The name of the postprocessor. See yt_dlp/postprocessor/__init__.py for a list. @@ -1328,9 +1332,12 @@ def wrapper(self, *args, **kwargs): self.report_error(msg) except ExtractorError as e: # An error we somewhat expected self.report_error(compat_str(e), e.format_traceback()) - except ThrottledDownload as e: - self.to_stderr('\r') - self.report_warning(f'{e}; Re-extracting data') + except ReExtractInfo as e: + if e.expected: + self.to_screen(f'{e}; Re-extracting data') + else: + self.to_stderr('\r') + self.report_warning(f'{e}; Re-extracting data') return wrapper(self, *args, **kwargs) except (DownloadCancelled, LazyList.IndexError, PagedList.IndexError): raise @@ -1341,6 +1348,47 @@ def wrapper(self, *args, **kwargs): raise return wrapper + def _wait_for_video(self, ie_result): + if (not self.params.get('wait_for_video') + or ie_result.get('_type', 'video') != 'video' + or ie_result.get('formats') or ie_result.get('url')): + return + + format_dur = lambda dur: '%02d:%02d:%02d' % timetuple_from_msec(dur * 1000)[:-1] + last_msg = '' + + def progress(msg): + nonlocal last_msg + self.to_screen(msg + ' ' * (len(last_msg) - len(msg)) + '\r', skip_eol=True) + last_msg = msg + + min_wait, max_wait = self.params.get('wait_for_video') + diff = try_get(ie_result, lambda x: x['release_timestamp'] - time.time()) + if diff is None and ie_result.get('live_status') == 'is_upcoming': + diff = random.randrange(min_wait or 0, max_wait) if max_wait else min_wait + self.report_warning('Release time of video is not known') + elif (diff or 0) <= 0: + self.report_warning('Video should already be available according to extracted info') + diff = min(max(diff, min_wait or 0), max_wait or float('inf')) + self.to_screen(f'[wait] Waiting for {format_dur(diff)} - Press Ctrl+C to try now') + + wait_till = time.time() + diff + try: + while True: + diff = wait_till - time.time() + if diff <= 0: + progress('') + raise ReExtractInfo('[wait] Wait period ended', expected=True) + progress(f'[wait] Remaining time until next attempt: {self._format_screen(format_dur(diff), self.Styles.EMPHASIS)}') + time.sleep(1) + except KeyboardInterrupt: + progress('') + raise ReExtractInfo('[wait] Interrupted by user', expected=True) + except BaseException as e: + if not isinstance(e, ReExtractInfo): + self.to_screen('') + raise + @__handle_extraction_exceptions def __extract_info(self, url, ie, download, extra_info, process): ie_result = ie.extract(url) @@ -1356,6 +1404,7 @@ def __extract_info(self, url, ie, download, extra_info, process): ie_result.setdefault('original_url', extra_info['original_url']) self.add_default_extra_info(ie_result, ie, url) if process: + self._wait_for_video(ie_result) return self.process_ie_result(ie_result, download, extra_info) else: return ie_result @@ -3007,7 +3056,7 @@ def download_with_info_file(self, info_filename): info = self.sanitize_info(json.loads('\n'.join(f)), self.params.get('clean_infojson', True)) try: self.__download_wrapper(self.process_ie_result)(info, download=True) - except (DownloadError, EntryNotInPlaylist, ThrottledDownload) as e: + except (DownloadError, EntryNotInPlaylist, ReExtractInfo) as e: if not isinstance(e, EntryNotInPlaylist): self.to_stderr('\r') webpage_url = info.get('webpage_url') diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index d56c55b569..2a1b83b26a 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -196,6 +196,14 @@ def _real_main(argv=None): opts.continue_dl = False if opts.concurrent_fragment_downloads <= 0: raise ValueError('Concurrent fragments must be positive') + if opts.wait_for_video is not None: + mobj = re.match(r'(?P\d+)(?:-(?P\d+))?$', opts.wait_for_video) + if not mobj: + parser.error('Invalid time range to wait') + min_wait, max_wait = map(int_or_none, mobj.group('min', 'max')) + if max_wait is not None and max_wait < min_wait: + parser.error('Invalid time range to wait') + opts.wait_for_video = (min_wait, max_wait) def parse_retries(retries, name=''): if retries in ('inf', 'infinite'): @@ -720,6 +728,7 @@ def report_args_compat(arg, name): 'youtube_include_hls_manifest': opts.youtube_include_hls_manifest, 'encoding': opts.encoding, 'extract_flat': opts.extract_flat, + 'wait_for_video': opts.wait_for_video, 'mark_watched': opts.mark_watched, 'merge_output_format': opts.merge_output_format, 'final_ext': final_ext, diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index fc28bca2e1..49c454d393 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -1079,7 +1079,8 @@ def report_login(self): def raise_login_required( self, msg='This video is only available for registered users', metadata_available=False, method='any'): - if metadata_available and self.get_param('ignore_no_formats_error'): + if metadata_available and ( + self.get_param('ignore_no_formats_error') or self.get_param('wait_for_video')): self.report_warning(msg) if method is not None: msg = '%s. %s' % (msg, self._LOGIN_HINTS[method]) @@ -1088,13 +1089,15 @@ def raise_login_required( def raise_geo_restricted( self, msg='This video is not available from your location due to geo restriction', countries=None, metadata_available=False): - if metadata_available and self.get_param('ignore_no_formats_error'): + if metadata_available and ( + self.get_param('ignore_no_formats_error') or self.get_param('wait_for_video')): self.report_warning(msg) else: raise GeoRestrictedError(msg, countries=countries) def raise_no_formats(self, msg, expected=False, video_id=None): - if expected and self.get_param('ignore_no_formats_error'): + if expected and ( + self.get_param('ignore_no_formats_error') or self.get_param('wait_for_video')): self.report_warning(msg, video_id) elif isinstance(msg, ExtractorError): raise msg diff --git a/yt_dlp/options.py b/yt_dlp/options.py index 4c19204040..b3cb7746f3 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -258,6 +258,16 @@ def _dict_from_options_callback( '--no-flat-playlist', action='store_false', dest='extract_flat', help='Extract the videos of a playlist') + general.add_option( + '--wait-for-video', + dest='wait_for_video', metavar='MIN[-MAX]', default=None, + help=( + 'Wait for scheduled streams to become available. ' + 'Pass the minimum number of seconds (or range) to wait between retries')) + general.add_option( + '--no-wait-for-video', + dest='wait_for_video', action='store_const', const=None, + help='Do not wait for scheduled streams (default)') general.add_option( '--mark-watched', action='store_true', dest='mark_watched', default=False, diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index ade2bbff16..582cc99fb2 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -2600,10 +2600,21 @@ class MaxDownloadsReached(DownloadCancelled): msg = 'Maximum number of downloads reached, stopping due to --max-downloads' -class ThrottledDownload(YoutubeDLError): +class ReExtractInfo(YoutubeDLError): + """ Video info needs to be re-extracted. """ + + def __init__(self, msg, expected=False): + super().__init__(msg) + self.expected = expected + + +class ThrottledDownload(ReExtractInfo): """ Download speed below --throttled-rate. """ msg = 'The download speed is below throttle limit' + def __init__(self, msg): + super().__init__(msg, expected=False) + class UnavailableVideoError(YoutubeDLError): """Unavailable Format exception.