From 6f7563beb7509aba2f8b1f03fd37e52427dcfecb Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 1 May 2022 04:46:05 +0530 Subject: [PATCH] [XAttrMetadata] Refactor and document dependencies --- README.md | 1 + yt_dlp/dependencies.py | 9 +++ yt_dlp/options.py | 2 +- yt_dlp/postprocessor/xattrpp.py | 76 +++++++++------------- yt_dlp/utils.py | 111 ++++++++++++-------------------- 5 files changed, 81 insertions(+), 118 deletions(-) diff --git a/README.md b/README.md index 556977dfa..dc1fad5b3 100644 --- a/README.md +++ b/README.md @@ -273,6 +273,7 @@ ## DEPENDENCIES * [**secretstorage**](https://github.com/mitya57/secretstorage) - For accessing the Gnome keyring while decrypting cookies of Chromium-based browsers on Linux. Licensed under [BSD-3-Clause](https://github.com/mitya57/secretstorage/blob/master/LICENSE) * [**brotli**](https://github.com/google/brotli)\* or [**brotlicffi**](https://github.com/python-hyper/brotlicffi) - [Brotli](https://en.wikipedia.org/wiki/Brotli) content encoding support. Both licensed under MIT [1](https://github.com/google/brotli/blob/master/LICENSE) [2](https://github.com/python-hyper/brotlicffi/blob/master/LICENSE) * [**certifi**](https://github.com/certifi/python-certifi)\* - Provides Mozilla's root certificate bundle. Licensed under [MPLv2](https://github.com/certifi/python-certifi/blob/master/LICENSE) +* [**xattr**](https://github.com/xattr/xattr), [**pyxattr**](https://github.com/iustin/pyxattr) or [**setfattr**](http://savannah.nongnu.org/projects/attr) - For writing xattr metadata on Linux. Licensed under [MIT](https://github.com/xattr/xattr/blob/master/LICENSE.txt), [LGPL2.1](https://github.com/iustin/pyxattr/blob/master/COPYING) and [GPLv2+](http://git.savannah.nongnu.org/cgit/attr.git/tree/doc/COPYING) respectively * [**AtomicParsley**](https://github.com/wez/atomicparsley) - For embedding thumbnail in mp4/m4a if mutagen/ffmpeg cannot. Licensed under [GPLv2+](https://github.com/wez/atomicparsley/blob/master/COPYING) * [**rtmpdump**](http://rtmpdump.mplayerhq.hu) - For downloading `rtmp` streams. ffmpeg will be used as a fallback. Licensed under [GPLv2+](http://rtmpdump.mplayerhq.hu) * [**mplayer**](http://mplayerhq.hu/design7/info.html) or [**mpv**](https://mpv.io) - For downloading `rstp` streams. ffmpeg will be used as a fallback. Licensed under [GPLv2+](https://github.com/mpv-player/mpv/blob/master/Copyright) diff --git a/yt_dlp/dependencies.py b/yt_dlp/dependencies.py index a4c2e5f06..772cfb576 100644 --- a/yt_dlp/dependencies.py +++ b/yt_dlp/dependencies.py @@ -75,6 +75,15 @@ websockets = None +try: + import xattr # xattr or pyxattr +except ImportError: + xattr = None +else: + if hasattr(xattr, 'set'): # pyxattr + xattr._yt_dlp__identifier = 'pyxattr' + + all_dependencies = {k: v for k, v in globals().items() if not k.startswith('_')} diff --git a/yt_dlp/options.py b/yt_dlp/options.py index a62681cbc..c03f69319 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -1422,7 +1422,7 @@ def _dict_from_options_callback( dest='parse_metadata', metavar='FIELDS REGEX REPLACE', action='append', nargs=3, help='Replace text in a metadata field using the given regex. This option can be used multiple times') postproc.add_option( - '--xattrs', + '--xattrs', '--xattr', action='store_true', dest='xattrs', default=False, help='Write metadata to the video file\'s xattrs (using dublin core and xdg standards)') postproc.add_option( diff --git a/yt_dlp/postprocessor/xattrpp.py b/yt_dlp/postprocessor/xattrpp.py index d6ac9b876..065ddf963 100644 --- a/yt_dlp/postprocessor/xattrpp.py +++ b/yt_dlp/postprocessor/xattrpp.py @@ -12,68 +12,52 @@ class XAttrMetadataPP(PostProcessor): - # - # More info about extended attributes for media: - # http://freedesktop.org/wiki/CommonExtendedAttributes/ - # http://www.freedesktop.org/wiki/PhreedomDraft/ - # http://dublincore.org/documents/usageguide/elements.shtml - # - # TODO: - # * capture youtube keywords and put them in 'user.dublincore.subject' (comma-separated) - # * figure out which xattrs can be used for 'duration', 'thumbnail', 'resolution' - # + """Set extended attributes on downloaded file (if xattr support is found) + + More info about extended attributes for media: + http://freedesktop.org/wiki/CommonExtendedAttributes/ + http://www.freedesktop.org/wiki/PhreedomDraft/ + http://dublincore.org/documents/usageguide/elements.shtml + + TODO: + * capture youtube keywords and put them in 'user.dublincore.subject' (comma-separated) + * figure out which xattrs can be used for 'duration', 'thumbnail', 'resolution' + """ + + XATTR_MAPPING = { + 'user.xdg.referrer.url': 'webpage_url', + # 'user.xdg.comment': 'description', + 'user.dublincore.title': 'title', + 'user.dublincore.date': 'upload_date', + 'user.dublincore.description': 'description', + 'user.dublincore.contributor': 'uploader', + 'user.dublincore.format': 'format', + } def run(self, info): - """ Set extended attributes on downloaded file (if xattr support is found). """ - - # Write the metadata to the file's xattrs + mtime = os.stat(info['filepath']).st_mtime self.to_screen('Writing metadata to file\'s xattrs') - - filename = info['filepath'] - mtime = os.stat(filename).st_mtime - try: - xattr_mapping = { - 'user.xdg.referrer.url': 'webpage_url', - # 'user.xdg.comment': 'description', - 'user.dublincore.title': 'title', - 'user.dublincore.date': 'upload_date', - 'user.dublincore.description': 'description', - 'user.dublincore.contributor': 'uploader', - 'user.dublincore.format': 'format', - } - - num_written = 0 - for xattrname, infoname in xattr_mapping.items(): - + for xattrname, infoname in self.XATTR_MAPPING.items(): value = info.get(infoname) - if value: if infoname == 'upload_date': value = hyphenate_date(value) - - byte_value = value.encode('utf-8') - write_xattr(filename, xattrname, byte_value) - num_written += 1 + write_xattr(info['filepath'], xattrname, value.encode('utf-8')) except XAttrUnavailableError as e: raise PostProcessingError(str(e)) - except XAttrMetadataError as e: if e.reason == 'NO_SPACE': self.report_warning( 'There\'s no disk space left, disk quota exceeded or filesystem xattr limit exceeded. ' - + (('Some ' if num_written else '') + 'extended attributes are not written.').capitalize()) + 'Some extended attributes are not written') elif e.reason == 'VALUE_TOO_LONG': - self.report_warning( - 'Unable to write extended attributes due to too long values.') + self.report_warning('Unable to write extended attributes due to too long values.') else: - msg = 'This filesystem doesn\'t support extended attributes. ' - if compat_os_name == 'nt': - msg += 'You need to use NTFS.' - else: - msg += '(You may have to enable them in your /etc/fstab)' - raise PostProcessingError(str(e)) + tip = ('You need to use NTFS' if compat_os_name == 'nt' + else 'You may have to enable them in your "/etc/fstab"') + raise PostProcessingError(f'This filesystem doesn\'t support extended attributes. {tip}') - self.try_utime(filename, mtime, mtime) + self.try_utime(info['filepath'], mtime, mtime) return [], info diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 3b75ab6b3..fc9eb253b 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -4673,87 +4673,56 @@ def _get_pixel(idx): def write_xattr(path, key, value): - # This mess below finds the best xattr tool for the job - try: - # try the pyxattr module... - import xattr + # Windows: Write xattrs to NTFS Alternate Data Streams: + # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29 + if compat_os_name == 'nt': + assert ':' not in key + assert os.path.exists(path) - if hasattr(xattr, 'set'): # pyxattr - # Unicode arguments are not supported in python-pyxattr until - # version 0.5.0 - # See https://github.com/ytdl-org/youtube-dl/issues/5498 - pyxattr_required_version = '0.5.0' - if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version): - # TODO: fallback to CLI tools - raise XAttrUnavailableError( - 'python-pyxattr is detected but is too old. ' - 'yt-dlp requires %s or above while your version is %s. ' - 'Falling back to other xattr implementations' % ( - pyxattr_required_version, xattr.__version__)) + try: + with open(f'{path}:{key}', 'wb') as f: + f.write(value) + except OSError as e: + raise XAttrMetadataError(e.errno, e.strerror) + return + # UNIX Method 1. Use xattrs/pyxattrs modules + from .dependencies import xattr + + setxattr = None + if getattr(xattr, '_yt_dlp__identifier', None) == 'pyxattr': + # Unicode arguments are not supported in pyxattr until version 0.5.0 + # See https://github.com/ytdl-org/youtube-dl/issues/5498 + if version_tuple(xattr.__version__) >= (0, 5, 0): setxattr = xattr.set - else: # xattr - setxattr = xattr.setxattr + elif xattr: + setxattr = xattr.setxattr + if setxattr: try: setxattr(path, key, value) except OSError as e: raise XAttrMetadataError(e.errno, e.strerror) + return - except ImportError: - if compat_os_name == 'nt': - # Write xattrs to NTFS Alternate Data Streams: - # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29 - assert ':' not in key - assert os.path.exists(path) + # UNIX Method 2. Use setfattr/xattr executables + exe = ('setfattr' if check_executable('setfattr', ['--version']) + else 'xattr' if check_executable('xattr', ['-h']) else None) + if not exe: + raise XAttrUnavailableError( + 'Couldn\'t find a tool to set the xattrs. Install either the python "xattr" or "pyxattr" modules or the ' + + ('"xattr" binary' if sys.platform != 'linux' else 'GNU "attr" package (which contains the "setfattr" tool)')) - ads_fn = path + ':' + key - try: - with open(ads_fn, 'wb') as f: - f.write(value) - except OSError as e: - raise XAttrMetadataError(e.errno, e.strerror) - else: - user_has_setfattr = check_executable('setfattr', ['--version']) - user_has_xattr = check_executable('xattr', ['-h']) - - if user_has_setfattr or user_has_xattr: - - value = value.decode('utf-8') - if user_has_setfattr: - executable = 'setfattr' - opts = ['-n', key, '-v', value] - elif user_has_xattr: - executable = 'xattr' - opts = ['-w', key, value] - - cmd = ([encodeFilename(executable, True)] - + [encodeArgument(o) for o in opts] - + [encodeFilename(path, True)]) - - try: - p = Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - except OSError as e: - raise XAttrMetadataError(e.errno, e.strerror) - stdout, stderr = p.communicate_or_kill() - stderr = stderr.decode('utf-8', 'replace') - if p.returncode != 0: - raise XAttrMetadataError(p.returncode, stderr) - - else: - # On Unix, and can't find pyxattr, setfattr, or xattr. - if sys.platform.startswith('linux'): - raise XAttrUnavailableError( - "Couldn't find a tool to set the xattrs. " - "Install either the python 'pyxattr' or 'xattr' " - "modules, or the GNU 'attr' package " - "(which contains the 'setfattr' tool).") - else: - raise XAttrUnavailableError( - "Couldn't find a tool to set the xattrs. " - "Install either the python 'xattr' module, " - "or the 'xattr' binary.") + value = value.decode('utf-8') + try: + p = Popen( + [exe, '-w', key, value, path] if exe == 'xattr' else [exe, '-n', key, '-v', value, path], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + except OSError as e: + raise XAttrMetadataError(e.errno, e.strerror) + stderr = p.communicate_or_kill()[1].decode('utf-8', 'replace') + if p.returncode: + raise XAttrMetadataError(p.returncode, stderr) def random_birthday(year_field, month_field, day_field):