Improved progress reporting (See desc) (#1125)

* Separate `--console-title` and `--no-progress`
* Add option `--progress` to show progress-bar even in quiet mode
* Fix and refactor `minicurses`
* Use `minicurses` for all progress reporting
* Standardize use of terminal sequences and enable color support for windows 10
* Add option `--progress-template` to customize progress-bar and console-title
* Add postprocessor hooks and progress reporting

Closes: #906, #901, #1085, #1170
This commit is contained in:
pukkandan 2021-10-09 00:41:59 +05:30 committed by GitHub
parent fee3f44f5f
commit 819e05319b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 301 additions and 206 deletions

View file

@ -604,7 +604,18 @@ ## Verbosity and Simulation Options:
(Alias: --force-download-archive) (Alias: --force-download-archive)
--newline Output progress bar as new lines --newline Output progress bar as new lines
--no-progress Do not print progress bar --no-progress Do not print progress bar
--progress Show progress bar, even if in quiet mode
--console-title Display progress in console titlebar --console-title Display progress in console titlebar
--progress-template [TYPES:]TEMPLATE
Template for progress outputs, optionally
prefixed with one of "download:" (default),
"download-title:" (the console title),
"postprocess:", or "postprocess-title:".
The video's fields are accessible under the
"info" key and the progress attributes are
accessible under "progress" key. Eg:
--console-title --progress-template
"download-title:%(info.id)s-%(progress.eta)s"
-v, --verbose Print various debugging information -v, --verbose Print various debugging information
--dump-pages Print downloaded pages encoded using base64 --dump-pages Print downloaded pages encoded using base64
to debug problems (very verbose) to debug problems (very verbose)

View file

@ -666,8 +666,7 @@ def test(tmpl, expected, *, info=None, **params):
ydl._num_downloads = 1 ydl._num_downloads = 1
self.assertEqual(ydl.validate_outtmpl(tmpl), None) self.assertEqual(ydl.validate_outtmpl(tmpl), None)
outtmpl, tmpl_dict = ydl.prepare_outtmpl(tmpl, info or self.outtmpl_info) out = ydl.evaluate_outtmpl(tmpl, info or self.outtmpl_info)
out = ydl.escape_outtmpl(outtmpl) % tmpl_dict
fname = ydl.prepare_filename(info or self.outtmpl_info) fname = ydl.prepare_filename(info or self.outtmpl_info)
if not isinstance(expected, (list, tuple)): if not isinstance(expected, (list, tuple)):

View file

@ -42,6 +42,7 @@
compat_urllib_error, compat_urllib_error,
compat_urllib_request, compat_urllib_request,
compat_urllib_request_DataHandler, compat_urllib_request_DataHandler,
windows_enable_vt_mode,
) )
from .cookies import load_cookies from .cookies import load_cookies
from .utils import ( from .utils import (
@ -67,8 +68,6 @@
float_or_none, float_or_none,
format_bytes, format_bytes,
format_field, format_field,
STR_FORMAT_RE_TMPL,
STR_FORMAT_TYPES,
formatSeconds, formatSeconds,
GeoRestrictedError, GeoRestrictedError,
HEADRequest, HEADRequest,
@ -101,9 +100,13 @@
sanitize_url, sanitize_url,
sanitized_Request, sanitized_Request,
std_headers, std_headers,
STR_FORMAT_RE_TMPL,
STR_FORMAT_TYPES,
str_or_none, str_or_none,
strftime_or_none, strftime_or_none,
subtitles_filename, subtitles_filename,
supports_terminal_sequences,
TERMINAL_SEQUENCES,
ThrottledDownload, ThrottledDownload,
to_high_limit_path, to_high_limit_path,
traverse_obj, traverse_obj,
@ -248,6 +251,7 @@ class YoutubeDL(object):
rejecttitle: Reject downloads for matching titles. rejecttitle: Reject downloads for matching titles.
logger: Log messages to a logging.Logger instance. logger: Log messages to a logging.Logger instance.
logtostderr: Log messages to stderr instead of stdout. logtostderr: Log messages to stderr instead of stdout.
consoletitle: Display progress in console window's titlebar.
writedescription: Write the video description to a .description file writedescription: Write the video description to a .description file
writeinfojson: Write the video description to a .info.json file writeinfojson: Write the video description to a .info.json file
clean_infojson: Remove private fields from the infojson clean_infojson: Remove private fields from the infojson
@ -353,6 +357,15 @@ class YoutubeDL(object):
Progress hooks are guaranteed to be called at least once Progress hooks are guaranteed to be called at least once
(with status "finished") if the download is successful. (with status "finished") if the download is successful.
postprocessor_hooks: A list of functions that get called on postprocessing
progress, with a dictionary with the entries
* status: One of "started", "processing", or "finished".
Check this first and ignore unknown values.
* postprocessor: Name of the postprocessor
* info_dict: The extracted info_dict
Progress hooks are guaranteed to be called at least twice
(with status "started" and "finished") if the processing is successful.
merge_output_format: Extension to use when merging formats. merge_output_format: Extension to use when merging formats.
final_ext: Expected final extension; used to detect when the file was final_ext: Expected final extension; used to detect when the file was
already downloaded and converted. "merge_output_format" is already downloaded and converted. "merge_output_format" is
@ -412,11 +425,15 @@ class YoutubeDL(object):
filename, abort-on-error, multistreams, no-live-chat, filename, abort-on-error, multistreams, no-live-chat,
no-clean-infojson, no-playlist-metafiles, no-keep-subs. no-clean-infojson, no-playlist-metafiles, no-keep-subs.
Refer __init__.py for their implementation Refer __init__.py for their implementation
progress_template: Dictionary of templates for progress outputs.
Allowed keys are 'download', 'postprocess',
'download-title' (console title) and 'postprocess-title'.
The template is mapped on a dictionary with keys 'progress' and 'info'
The following parameters are not used by YoutubeDL itself, they are used by The following parameters are not used by YoutubeDL itself, they are used by
the downloader (see yt_dlp/downloader/common.py): the downloader (see yt_dlp/downloader/common.py):
nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize, nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize,
max_filesize, test, noresizebuffer, retries, continuedl, noprogress, consoletitle, max_filesize, test, noresizebuffer, retries, continuedl, noprogress,
xattr_set_filesize, external_downloader_args, hls_use_mpegts, http_chunk_size. xattr_set_filesize, external_downloader_args, hls_use_mpegts, http_chunk_size.
The following options are used by the post processors: The following options are used by the post processors:
@ -484,26 +501,27 @@ def __init__(self, params=None, auto_init=True):
self._first_webpage_request = True self._first_webpage_request = True
self._post_hooks = [] self._post_hooks = []
self._progress_hooks = [] self._progress_hooks = []
self._postprocessor_hooks = []
self._download_retcode = 0 self._download_retcode = 0
self._num_downloads = 0 self._num_downloads = 0
self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
self._err_file = sys.stderr self._err_file = sys.stderr
self.params = { self.params = params
# Default parameters
'nocheckcertificate': False,
}
self.params.update(params)
self.cache = Cache(self) self.cache = Cache(self)
windows_enable_vt_mode()
self.params['no_color'] = self.params.get('no_color') or not supports_terminal_sequences(self._err_file)
if sys.version_info < (3, 6): if sys.version_info < (3, 6):
self.report_warning( self.report_warning(
'Python version %d.%d is not supported! Please update to Python 3.6 or above' % sys.version_info[:2]) 'Python version %d.%d is not supported! Please update to Python 3.6 or above' % sys.version_info[:2])
if self.params.get('allow_unplayable_formats'): if self.params.get('allow_unplayable_formats'):
self.report_warning( self.report_warning(
'You have asked for unplayable formats to be listed/downloaded. ' f'You have asked for {self._color_text("unplayable formats", "blue")} to be listed/downloaded. '
'This is a developer option intended for debugging. ' 'This is a developer option intended for debugging. \n'
'If you experience any issues while using this option, DO NOT open a bug report') ' If you experience any issues while using this option, '
f'{self._color_text("DO NOT", "red")} open a bug report')
def check_deprecated(param, option, suggestion): def check_deprecated(param, option, suggestion):
if self.params.get(param) is not None: if self.params.get(param) is not None:
@ -675,9 +693,13 @@ def add_post_hook(self, ph):
self._post_hooks.append(ph) self._post_hooks.append(ph)
def add_progress_hook(self, ph): def add_progress_hook(self, ph):
"""Add the progress hook (currently only for the file downloader)""" """Add the download progress hook"""
self._progress_hooks.append(ph) self._progress_hooks.append(ph)
def add_postprocessor_hook(self, ph):
"""Add the postprocessing progress hook"""
self._postprocessor_hooks.append(ph)
def _bidi_workaround(self, message): def _bidi_workaround(self, message):
if not hasattr(self, '_output_channel'): if not hasattr(self, '_output_channel'):
return message return message
@ -790,6 +812,11 @@ def to_screen(self, message, skip_eol=False):
self.to_stdout( self.to_stdout(
message, skip_eol, quiet=self.params.get('quiet', False)) message, skip_eol, quiet=self.params.get('quiet', False))
def _color_text(self, text, color):
if self.params.get('no_color'):
return text
return f'{TERMINAL_SEQUENCES[color.upper()]}{text}{TERMINAL_SEQUENCES["RESET_STYLE"]}'
def report_warning(self, message, only_once=False): def report_warning(self, message, only_once=False):
''' '''
Print the message to stderr, it will be prefixed with 'WARNING:' Print the message to stderr, it will be prefixed with 'WARNING:'
@ -800,24 +827,14 @@ def report_warning(self, message, only_once=False):
else: else:
if self.params.get('no_warnings'): if self.params.get('no_warnings'):
return return
if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt': self.to_stderr(f'{self._color_text("WARNING:", "yellow")} {message}', only_once)
_msg_header = '\033[0;33mWARNING:\033[0m'
else:
_msg_header = 'WARNING:'
warning_message = '%s %s' % (_msg_header, message)
self.to_stderr(warning_message, only_once)
def report_error(self, message, tb=None): def report_error(self, message, tb=None):
''' '''
Do the same as trouble, but prefixes the message with 'ERROR:', colored Do the same as trouble, but prefixes the message with 'ERROR:', colored
in red if stderr is a tty file. in red if stderr is a tty file.
''' '''
if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt': self.trouble(f'{self._color_text("ERROR:", "red")} {message}', tb)
_msg_header = '\033[0;31mERROR:\033[0m'
else:
_msg_header = 'ERROR:'
error_message = '%s %s' % (_msg_header, message)
self.trouble(error_message, tb)
def write_debug(self, message, only_once=False): def write_debug(self, message, only_once=False):
'''Log debug message or Print message to stderr''' '''Log debug message or Print message to stderr'''
@ -919,7 +936,7 @@ def validate_outtmpl(cls, outtmpl):
return err return err
def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None): def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None):
""" Make the template and info_dict suitable for substitution : ydl.outtmpl_escape(outtmpl) % info_dict """ """ Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict """
info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set
info_dict = dict(info_dict) # Do not sanitize so as not to consume LazyList info_dict = dict(info_dict) # Do not sanitize so as not to consume LazyList
@ -1073,6 +1090,10 @@ def create_key(outer_mobj):
return EXTERNAL_FORMAT_RE.sub(create_key, outtmpl), TMPL_DICT return EXTERNAL_FORMAT_RE.sub(create_key, outtmpl), TMPL_DICT
def evaluate_outtmpl(self, outtmpl, info_dict, *args, **kwargs):
outtmpl, info_dict = self.prepare_outtmpl(outtmpl, info_dict, *args, **kwargs)
return self.escape_outtmpl(outtmpl) % info_dict
def _prepare_filename(self, info_dict, tmpl_type='default'): def _prepare_filename(self, info_dict, tmpl_type='default'):
try: try:
sanitize = lambda k, v: sanitize_filename( sanitize = lambda k, v: sanitize_filename(
@ -2431,10 +2452,8 @@ def print_optional(field):
if self.params.get('forceprint') or self.params.get('forcejson'): if self.params.get('forceprint') or self.params.get('forcejson'):
self.post_extract(info_dict) self.post_extract(info_dict)
for tmpl in self.params.get('forceprint', []): for tmpl in self.params.get('forceprint', []):
if re.match(r'\w+$', tmpl): self.to_stdout(self.evaluate_outtmpl(
tmpl = '%({})s'.format(tmpl) f'%({tmpl})s' if re.match(r'\w+$', tmpl) else tmpl, info_dict))
tmpl, info_copy = self.prepare_outtmpl(tmpl, info_dict)
self.to_stdout(self.escape_outtmpl(tmpl) % info_copy)
print_mandatory('title') print_mandatory('title')
print_mandatory('id') print_mandatory('id')

View file

@ -302,11 +302,14 @@ def validate_outtmpl(tmpl, msg):
parser.error('invalid %s %r: %s' % (msg, tmpl, error_to_compat_str(err))) parser.error('invalid %s %r: %s' % (msg, tmpl, error_to_compat_str(err)))
for k, tmpl in opts.outtmpl.items(): for k, tmpl in opts.outtmpl.items():
validate_outtmpl(tmpl, '%s output template' % k) validate_outtmpl(tmpl, f'{k} output template')
opts.forceprint = opts.forceprint or [] opts.forceprint = opts.forceprint or []
for tmpl in opts.forceprint or []: for tmpl in opts.forceprint or []:
validate_outtmpl(tmpl, 'print template') validate_outtmpl(tmpl, 'print template')
validate_outtmpl(opts.sponsorblock_chapter_title, 'SponsorBlock chapter title') validate_outtmpl(opts.sponsorblock_chapter_title, 'SponsorBlock chapter title')
for k, tmpl in opts.progress_template.items():
k = f'{k[:-6]} console title' if '-title' in k else f'{k} progress'
validate_outtmpl(tmpl, f'{k} template')
if opts.extractaudio and not opts.keepvideo and opts.format is None: if opts.extractaudio and not opts.keepvideo and opts.format is None:
opts.format = 'bestaudio/best' opts.format = 'bestaudio/best'
@ -633,8 +636,9 @@ def report_args_compat(arg, name):
'noresizebuffer': opts.noresizebuffer, 'noresizebuffer': opts.noresizebuffer,
'http_chunk_size': opts.http_chunk_size, 'http_chunk_size': opts.http_chunk_size,
'continuedl': opts.continue_dl, 'continuedl': opts.continue_dl,
'noprogress': opts.noprogress, 'noprogress': opts.quiet if opts.noprogress is None else opts.noprogress,
'progress_with_newline': opts.progress_with_newline, 'progress_with_newline': opts.progress_with_newline,
'progress_template': opts.progress_template,
'playliststart': opts.playliststart, 'playliststart': opts.playliststart,
'playlistend': opts.playlistend, 'playlistend': opts.playlistend,
'playlistreverse': opts.playlist_reverse, 'playlistreverse': opts.playlist_reverse,

View file

@ -159,6 +159,12 @@ def compat_expanduser(path):
compat_pycrypto_AES = None compat_pycrypto_AES = None
def windows_enable_vt_mode(): # TODO: Do this the proper way https://bugs.python.org/issue30075
if compat_os_name != 'nt':
return
os.system('')
# Deprecated # Deprecated
compat_basestring = str compat_basestring = str
@ -281,5 +287,6 @@ def compat_expanduser(path):
'compat_xml_parse_error', 'compat_xml_parse_error',
'compat_xpath', 'compat_xpath',
'compat_zip', 'compat_zip',
'windows_enable_vt_mode',
'workaround_optparse_bug9161', 'workaround_optparse_bug9161',
] ]

View file

@ -7,7 +7,6 @@
import time import time
import random import random
from ..compat import compat_os_name
from ..utils import ( from ..utils import (
decodeArgument, decodeArgument,
encodeFilename, encodeFilename,
@ -17,6 +16,7 @@
timeconvert, timeconvert,
) )
from ..minicurses import ( from ..minicurses import (
MultilineLogger,
MultilinePrinter, MultilinePrinter,
QuietMultilinePrinter, QuietMultilinePrinter,
BreaklineStatusPrinter BreaklineStatusPrinter
@ -44,8 +44,6 @@ class FileDownloader(object):
noresizebuffer: Do not automatically resize the download buffer. noresizebuffer: Do not automatically resize the download buffer.
continuedl: Try to continue downloads if possible. continuedl: Try to continue downloads if possible.
noprogress: Do not print the progress bar. noprogress: Do not print the progress bar.
logtostderr: Log messages to stderr instead of stdout.
consoletitle: Display progress in console window's titlebar.
nopart: Do not use temporary .part files. nopart: Do not use temporary .part files.
updatetime: Use the Last-modified header to set output file timestamps. updatetime: Use the Last-modified header to set output file timestamps.
test: Download only first bytes to test the downloader. test: Download only first bytes to test the downloader.
@ -61,6 +59,7 @@ class FileDownloader(object):
http_chunk_size: Size of a chunk for chunk-based HTTP downloading. May be http_chunk_size: Size of a chunk for chunk-based HTTP downloading. May be
useful for bypassing bandwidth throttling imposed by useful for bypassing bandwidth throttling imposed by
a webserver (experimental) a webserver (experimental)
progress_template: See YoutubeDL.py
Subclasses of this one must re-define the real_download method. Subclasses of this one must re-define the real_download method.
""" """
@ -73,7 +72,7 @@ def __init__(self, ydl, params):
self.ydl = ydl self.ydl = ydl
self._progress_hooks = [] self._progress_hooks = []
self.params = params self.params = params
self._multiline = None self._prepare_multiline_status()
self.add_progress_hook(self.report_progress) self.add_progress_hook(self.report_progress)
@staticmethod @staticmethod
@ -242,55 +241,46 @@ def report_destination(self, filename):
"""Report destination filename.""" """Report destination filename."""
self.to_screen('[download] Destination: ' + filename) self.to_screen('[download] Destination: ' + filename)
def _prepare_multiline_status(self, lines): def _prepare_multiline_status(self, lines=1):
if self.params.get('quiet'): if self.params.get('noprogress'):
self._multiline = QuietMultilinePrinter() self._multiline = QuietMultilinePrinter()
elif self.params.get('progress_with_newline', False): elif self.ydl.params.get('logger'):
self._multiline = MultilineLogger(self.ydl.params['logger'], lines)
elif self.params.get('progress_with_newline'):
self._multiline = BreaklineStatusPrinter(sys.stderr, lines) self._multiline = BreaklineStatusPrinter(sys.stderr, lines)
elif self.params.get('noprogress', False):
self._multiline = None
else: else:
self._multiline = MultilinePrinter(sys.stderr, lines) self._multiline = MultilinePrinter(sys.stderr, lines, not self.params.get('quiet'))
def _finish_multiline_status(self): def _finish_multiline_status(self):
if self._multiline is not None: self._multiline.end()
self._multiline.end()
def _report_progress_status(self, msg, is_last_line=False, progress_line=None): def _report_progress_status(self, s):
fullmsg = '[download] ' + msg progress_dict = s.copy()
if self.params.get('progress_with_newline', False): progress_dict.pop('info_dict')
self.to_screen(fullmsg) progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
elif progress_line is not None and self._multiline is not None:
self._multiline.print_at_line(fullmsg, progress_line) progress_template = self.params.get('progress_template', {})
else: self._multiline.print_at_line(self.ydl.evaluate_outtmpl(
if compat_os_name == 'nt' or not sys.stderr.isatty(): progress_template.get('download') or '[download] %(progress._default_template)s',
prev_len = getattr(self, '_report_progress_prev_line_length', 0) progress_dict), s.get('progress_idx') or 0)
if prev_len > len(fullmsg): self.to_console_title(self.ydl.evaluate_outtmpl(
fullmsg += ' ' * (prev_len - len(fullmsg)) progress_template.get('download-title') or 'yt-dlp %(progress._default_template)s',
self._report_progress_prev_line_length = len(fullmsg) progress_dict))
clear_line = '\r'
else:
clear_line = '\r\x1b[K'
self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
self.to_console_title('yt-dlp ' + msg)
def report_progress(self, s): def report_progress(self, s):
if s['status'] == 'finished': if s['status'] == 'finished':
if self.params.get('noprogress', False): if self.params.get('noprogress'):
self.to_screen('[download] Download completed') self.to_screen('[download] Download completed')
else: msg_template = '100%%'
msg_template = '100%%' if s.get('total_bytes') is not None:
if s.get('total_bytes') is not None: s['_total_bytes_str'] = format_bytes(s['total_bytes'])
s['_total_bytes_str'] = format_bytes(s['total_bytes']) msg_template += ' of %(_total_bytes_str)s'
msg_template += ' of %(_total_bytes_str)s' if s.get('elapsed') is not None:
if s.get('elapsed') is not None: s['_elapsed_str'] = self.format_seconds(s['elapsed'])
s['_elapsed_str'] = self.format_seconds(s['elapsed']) msg_template += ' in %(_elapsed_str)s'
msg_template += ' in %(_elapsed_str)s' s['_percent_str'] = self.format_percent(100)
self._report_progress_status( s['_default_template'] = msg_template % s
msg_template % s, is_last_line=True, progress_line=s.get('progress_idx')) self._report_progress_status(s)
return
if self.params.get('noprogress'):
return return
if s['status'] != 'downloading': if s['status'] != 'downloading':
@ -332,8 +322,8 @@ def report_progress(self, s):
msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s' msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
else: else:
msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s' msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
s['_default_template'] = msg_template % s
self._report_progress_status(msg_template % s, progress_line=s.get('progress_idx')) self._report_progress_status(s)
def report_resuming_byte(self, resume_len): def report_resuming_byte(self, resume_len):
"""Report attempt to resume at given byte.""" """Report attempt to resume at given byte."""
@ -405,7 +395,9 @@ def download(self, filename, info_dict, subtitle=False):
'[download] Sleeping %s seconds ...' % ( '[download] Sleeping %s seconds ...' % (
sleep_interval_sub)) sleep_interval_sub))
time.sleep(sleep_interval_sub) time.sleep(sleep_interval_sub)
return self.real_download(filename, info_dict), True ret = self.real_download(filename, info_dict)
self._finish_multiline_status()
return ret, True
def real_download(self, filename, info_dict): def real_download(self, filename, info_dict):
"""Real download process. Redefine in subclasses.""" """Real download process. Redefine in subclasses."""

View file

@ -393,9 +393,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
result = result and job.result() result = result and job.result()
finally: finally:
tpe.shutdown(wait=True) tpe.shutdown(wait=True)
return result
self._finish_multiline_status()
return True
def download_and_append_fragments(self, ctx, fragments, info_dict, *, pack_func=None, finish_func=None, tpe=None): def download_and_append_fragments(self, ctx, fragments, info_dict, *, pack_func=None, finish_func=None, tpe=None):
fragment_retries = self.params.get('fragment_retries', 0) fragment_retries = self.params.get('fragment_retries', 0)

View file

@ -1134,10 +1134,7 @@ def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, f
if mobj: if mobj:
break break
if not self.get_param('no_color') and compat_os_name != 'nt' and sys.stderr.isatty(): _name = self._downloader._color_text(name, 'blue')
_name = '\033[0;34m%s\033[0m' % name
else:
_name = name
if mobj: if mobj:
if group is None: if group is None:

View file

@ -1,10 +1,12 @@
import os
from threading import Lock from threading import Lock
from .utils import compat_os_name, get_windows_version from .utils import supports_terminal_sequences, TERMINAL_SEQUENCES
class MultilinePrinterBase(): class MultilinePrinterBase:
def __init__(self, stream=None, lines=1):
self.stream = stream
self.maximum = lines - 1
def __enter__(self): def __enter__(self):
return self return self
@ -17,119 +19,87 @@ def print_at_line(self, text, pos):
def end(self): def end(self):
pass pass
def _add_line_number(self, text, line):
class MultilinePrinter(MultilinePrinterBase): if self.maximum:
return f'{line + 1}: {text}'
def __init__(self, stream, lines): return text
"""
@param stream stream to write to
@lines number of lines to be written
"""
self.stream = stream
is_win10 = compat_os_name == 'nt' and get_windows_version() >= (10, )
self.CARRIAGE_RETURN = '\r'
if os.getenv('TERM') and self._isatty() or is_win10:
# reason not to use curses https://github.com/yt-dlp/yt-dlp/pull/1036#discussion_r713851492
# escape sequences for Win10 https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences
self.UP = '\x1b[A'
self.DOWN = '\n'
self.ERASE_LINE = '\x1b[K'
self._HAVE_FULLCAP = self._isatty() or is_win10
else:
self.UP = self.DOWN = self.ERASE_LINE = None
self._HAVE_FULLCAP = False
# lines are numbered from top to bottom, counting from 0 to self.maximum
self.maximum = lines - 1
self.lastline = 0
self.lastlength = 0
self.movelock = Lock()
@property
def have_fullcap(self):
"""
True if the TTY is allowing to control cursor,
so that multiline progress works
"""
return self._HAVE_FULLCAP
def _isatty(self):
try:
return self.stream.isatty()
except BaseException:
return False
def _move_cursor(self, dest):
current = min(self.lastline, self.maximum)
self.stream.write(self.CARRIAGE_RETURN)
if current == dest:
# current and dest are at same position, no need to move cursor
return
elif current > dest:
# when maximum == 2,
# 0. dest
# 1.
# 2. current
self.stream.write(self.UP * (current - dest))
elif current < dest:
# when maximum == 2,
# 0. current
# 1.
# 2. dest
self.stream.write(self.DOWN * (dest - current))
self.lastline = dest
def print_at_line(self, text, pos):
with self.movelock:
if self.have_fullcap:
self._move_cursor(pos)
self.stream.write(self.ERASE_LINE)
self.stream.write(text)
else:
if self.maximum != 0:
# let user know about which line is updating the status
text = f'{pos + 1}: {text}'
textlen = len(text)
if self.lastline == pos:
# move cursor at the start of progress when writing to same line
self.stream.write(self.CARRIAGE_RETURN)
if self.lastlength > textlen:
text += ' ' * (self.lastlength - textlen)
self.lastlength = textlen
else:
# otherwise, break the line
self.stream.write('\n')
self.lastlength = 0
self.stream.write(text)
self.lastline = pos
def end(self):
with self.movelock:
# move cursor to the end of the last line, and write line break
# so that other to_screen calls can precede
self._move_cursor(self.maximum)
self.stream.write('\n')
class QuietMultilinePrinter(MultilinePrinterBase): class QuietMultilinePrinter(MultilinePrinterBase):
def __init__(self): pass
self.have_fullcap = True
class MultilineLogger(MultilinePrinterBase):
def print_at_line(self, text, pos):
# stream is the logger object, not an actual stream
self.stream.debug(self._add_line_number(text, pos))
class BreaklineStatusPrinter(MultilinePrinterBase): class BreaklineStatusPrinter(MultilinePrinterBase):
def __init__(self, stream, lines):
"""
@param stream stream to write to
"""
self.stream = stream
self.maximum = lines
self.have_fullcap = True
def print_at_line(self, text, pos): def print_at_line(self, text, pos):
if self.maximum != 0: self.stream.write(self._add_line_number(text, pos) + '\n')
# let user know about which line is updating the status
text = f'{pos + 1}: {text}'
self.stream.write(text + '\n') class MultilinePrinter(MultilinePrinterBase):
def __init__(self, stream=None, lines=1, preserve_output=True):
super().__init__(stream, lines)
self.preserve_output = preserve_output
self._lastline = self._lastlength = 0
self._movelock = Lock()
self._HAVE_FULLCAP = supports_terminal_sequences(self.stream)
def lock(func):
def wrapper(self, *args, **kwargs):
with self._movelock:
return func(self, *args, **kwargs)
return wrapper
def _move_cursor(self, dest):
current = min(self._lastline, self.maximum)
self.stream.write('\r')
distance = dest - current
if distance < 0:
self.stream.write(TERMINAL_SEQUENCES['UP'] * -distance)
elif distance > 0:
self.stream.write(TERMINAL_SEQUENCES['DOWN'] * distance)
self._lastline = dest
@lock
def print_at_line(self, text, pos):
if self._HAVE_FULLCAP:
self._move_cursor(pos)
self.stream.write(TERMINAL_SEQUENCES['ERASE_LINE'])
self.stream.write(text)
return
text = self._add_line_number(text, pos)
textlen = len(text)
if self._lastline == pos:
# move cursor at the start of progress when writing to same line
self.stream.write('\r')
if self._lastlength > textlen:
text += ' ' * (self._lastlength - textlen)
self._lastlength = textlen
else:
# otherwise, break the line
self.stream.write('\n')
self._lastlength = textlen
self.stream.write(text)
self._lastline = pos
@lock
def end(self):
# move cursor to the end of the last line, and write line break
# so that other to_screen calls can precede
if self._HAVE_FULLCAP:
self._move_cursor(self.maximum)
if self.preserve_output:
self.stream.write('\n')
return
if self._HAVE_FULLCAP:
self.stream.write(
TERMINAL_SEQUENCES['ERASE_LINE']
+ f'{TERMINAL_SEQUENCES["UP"]}{TERMINAL_SEQUENCES["ERASE_LINE"]}' * self.maximum)
else:
self.stream.write(' ' * self._lastlength)

View file

@ -910,12 +910,30 @@ def _dict_from_options_callback(
help='Output progress bar as new lines') help='Output progress bar as new lines')
verbosity.add_option( verbosity.add_option(
'--no-progress', '--no-progress',
action='store_true', dest='noprogress', default=False, action='store_true', dest='noprogress', default=None,
help='Do not print progress bar') help='Do not print progress bar')
verbosity.add_option(
'--progress',
action='store_false', dest='noprogress',
help='Show progress bar, even if in quiet mode')
verbosity.add_option( verbosity.add_option(
'--console-title', '--console-title',
action='store_true', dest='consoletitle', default=False, action='store_true', dest='consoletitle', default=False,
help='Display progress in console titlebar') help='Display progress in console titlebar')
verbosity.add_option(
'--progress-template',
metavar='[TYPES:]TEMPLATE', dest='progress_template', default={}, type='str',
action='callback', callback=_dict_from_options_callback,
callback_kwargs={
'allowed_keys': '(download|postprocess)(-title)?',
'default_key': 'download'
}, help=(
'Template for progress outputs, optionally prefixed with one of "download:" (default), '
'"download-title:" (the console title), "postprocess:", or "postprocess-title:". '
'The video\'s fields are accessible under the "info" key and '
'the progress attributes are accessible under "progress" key. Eg: '
# TODO: Document the fields inside "progress"
'--console-title --progress-template "download-title:%(info.id)s-%(progress.eta)s"'))
verbosity.add_option( verbosity.add_option(
'-v', '--verbose', '-v', '--verbose',
action='store_true', dest='verbose', default=False, action='store_true', dest='verbose', default=False,

View file

@ -1,5 +1,6 @@
from __future__ import unicode_literals from __future__ import unicode_literals
import copy
import functools import functools
import os import os
@ -11,7 +12,26 @@
) )
class PostProcessor(object): class PostProcessorMetaClass(type):
@staticmethod
def run_wrapper(func):
@functools.wraps(func)
def run(self, info, *args, **kwargs):
self._hook_progress({'status': 'started'}, info)
ret = func(self, info, *args, **kwargs)
if ret is not None:
_, info = ret
self._hook_progress({'status': 'finished'}, info)
return ret
return run
def __new__(cls, name, bases, attrs):
if 'run' in attrs:
attrs['run'] = cls.run_wrapper(attrs['run'])
return type.__new__(cls, name, bases, attrs)
class PostProcessor(metaclass=PostProcessorMetaClass):
"""Post Processor class. """Post Processor class.
PostProcessor objects can be added to downloaders with their PostProcessor objects can be added to downloaders with their
@ -34,7 +54,9 @@ class PostProcessor(object):
_downloader = None _downloader = None
def __init__(self, downloader=None): def __init__(self, downloader=None):
self._downloader = downloader self._progress_hooks = []
self.add_progress_hook(self.report_progress)
self.set_downloader(downloader)
self.PP_NAME = self.pp_key() self.PP_NAME = self.pp_key()
@classmethod @classmethod
@ -68,6 +90,10 @@ def get_param(self, name, default=None, *args, **kwargs):
def set_downloader(self, downloader): def set_downloader(self, downloader):
"""Sets the downloader for this PP.""" """Sets the downloader for this PP."""
self._downloader = downloader self._downloader = downloader
if not downloader:
return
for ph in downloader._postprocessor_hooks:
self.add_progress_hook(ph)
@staticmethod @staticmethod
def _restrict_to(*, video=True, audio=True, images=True): def _restrict_to(*, video=True, audio=True, images=True):
@ -115,6 +141,39 @@ def _configuration_args(self, exe, *args, **kwargs):
return _configuration_args( return _configuration_args(
self.pp_key(), self.get_param('postprocessor_args'), exe, *args, **kwargs) self.pp_key(), self.get_param('postprocessor_args'), exe, *args, **kwargs)
def _hook_progress(self, status, info_dict):
if not self._progress_hooks:
return
info_dict = dict(info_dict)
for key in ('__original_infodict', '__postprocessors'):
info_dict.pop(key, None)
status.update({
'info_dict': copy.deepcopy(info_dict),
'postprocessor': self.pp_key(),
})
for ph in self._progress_hooks:
ph(status)
def add_progress_hook(self, ph):
# See YoutubeDl.py (search for postprocessor_hooks) for a description of this interface
self._progress_hooks.append(ph)
def report_progress(self, s):
s['_default_template'] = '%(postprocessor)s %(status)s' % s
progress_dict = s.copy()
progress_dict.pop('info_dict')
progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
progress_template = self.get_param('progress_template', {})
tmpl = progress_template.get('postprocess')
if tmpl:
self._downloader.to_stdout(self._downloader.evaluate_outtmpl(tmpl, progress_dict))
self._downloader.to_console_title(self._downloader.evaluate_outtmpl(
progress_template.get('postprocess-title') or 'yt-dlp %(progress._default_template)s',
progress_dict))
class AudioConversionError(PostProcessingError): class AudioConversionError(PostProcessingError):
pass pass

View file

@ -62,8 +62,7 @@ def run(self, info):
def interpretter(self, inp, out): def interpretter(self, inp, out):
def f(info): def f(info):
outtmpl, tmpl_dict = self._downloader.prepare_outtmpl(template, info) data_to_parse = self._downloader.evaluate_outtmpl(template, info)
data_to_parse = self._downloader.escape_outtmpl(outtmpl) % tmpl_dict
self.write_debug(f'Searching for {out_re.pattern!r} in {template!r}') self.write_debug(f'Searching for {out_re.pattern!r} in {template!r}')
match = out_re.search(data_to_parse) match = out_re.search(data_to_parse)
if match is None: if match is None:

View file

@ -292,8 +292,7 @@ def _remove_tiny_rename_sponsors(self, chapters):
'name': SponsorBlockPP.CATEGORIES[category], 'name': SponsorBlockPP.CATEGORIES[category],
'category_names': [SponsorBlockPP.CATEGORIES[c] for c in cats] 'category_names': [SponsorBlockPP.CATEGORIES[c] for c in cats]
}) })
outtmpl, tmpl_dict = self._downloader.prepare_outtmpl(self._sponsorblock_chapter_title, c) c['title'] = self._downloader.evaluate_outtmpl(self._sponsorblock_chapter_title, c)
c['title'] = self._downloader.escape_outtmpl(outtmpl) % tmpl_dict
# Merge identically named sponsors. # Merge identically named sponsors.
if (new_chapters and 'categories' in new_chapters[-1] if (new_chapters and 'categories' in new_chapters[-1]
and new_chapters[-1]['title'] == c['title']): and new_chapters[-1]['title'] == c['title']):

View file

@ -6440,3 +6440,26 @@ def jwt_encode_hs256(payload_data, key, headers={}):
signature_b64 = base64.b64encode(h.digest()) signature_b64 = base64.b64encode(h.digest())
token = header_b64 + b'.' + payload_b64 + b'.' + signature_b64 token = header_b64 + b'.' + payload_b64 + b'.' + signature_b64
return token return token
def supports_terminal_sequences(stream):
if compat_os_name == 'nt':
if get_windows_version() < (10, ):
return False
elif not os.getenv('TERM'):
return False
try:
return stream.isatty()
except BaseException:
return False
TERMINAL_SEQUENCES = {
'DOWN': '\n',
'UP': '\x1b[A',
'ERASE_LINE': '\x1b[K',
'RED': '\033[0;31m',
'YELLOW': '\033[0;33m',
'BLUE': '\033[0;34m',
'RESET_STYLE': '\033[0m',
}