From cb23bcba294563857561914a19e7d06990c71829 Mon Sep 17 00:00:00 2001 From: fnord Date: Mon, 27 Jul 2015 04:24:04 -0500 Subject: [PATCH 1/2] BBCIE: Fix missing .mp4 formats on news sites Prior to merge BBCNewsIE from pr #6026 was rewritten into BBCIE in 9afa177. Support was added for non-news sites (/travel, /sports), however support for the news mediaselector was removed to support these sites. This removed support for .mp4. pr #6026 ( news site, news mediaseletor : http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/ ) format code extension resolution note journalism_nonuk_stream_h264_flv_200_akamai flv 400x224 h264@ 176k, 1001.77KiB journalism_nonuk_stream_h264_flv_400_akamai flv 400x224 h264@ 512k, 2.85MiB journalism_nonuk_stream_h264_flv_med_akamai flv 640x360 h264@ 800k, 4.43MiB journalism_nonuk_stream_h264_flv_hi_akamai flv 688x384 h264@1500k, 8.34MiB journalism_world_stream_h264_http_200_sis_news_http mp4 400x224 h264@ 176k, 1001.77KiB journalism_world_stream_h264_http_400_sis_news_http mp4 400x224 h264@ 512k, 2.85MiB journalism_world_stream_h264_http_hi_sis_news_http mp4 688x384 h264@1500k, 8.34MiB (best) # 9afa177 ( same url, non-news mediaselector: http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/ ) format code extension resolution note journalism_nonuk_stream_h264_flv_lo_akamai flv 384x216 h264@ 496k, 2.76MiB journalism_nonuk_stream_h264_flv_med_akamai flv 640x360 h264@ 800k, 4.43MiB journalism_nonuk_stream_h264_flv_hi_akamai flv 688x384 h264@1500k, 8.34MiB (best) This change corrects the above, by trying /mediaselector/5 if /mediaselector/4 fails. --- youtube_dl/extractor/bbc.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py index 66e52641b..0f0ea7cfd 100644 --- a/youtube_dl/extractor/bbc.py +++ b/youtube_dl/extractor/bbc.py @@ -14,7 +14,6 @@ ) from ..compat import compat_HTTPError - class BBCCoUkIE(InfoExtractor): IE_NAME = 'bbc.co.uk' IE_DESC = 'BBC iPlayer' @@ -271,8 +270,16 @@ def _get_subtitles(self, media, programme_id): return subtitles def _download_media_selector(self, programme_id): - return self._download_media_selector_url( - self._MEDIASELECTOR_URL % programme_id, programme_id) + try: + return self._download_media_selector_url( + self._MEDIASELECTOR_URL % programme_id, programme_id) + except ExtractorError as e: + if hasattr(self, '_MEDIASELECTOR_ALT_URL') and str(e) == 'bbc returned error: notukerror': + # notukerror on bbc.com/travel using bbc news mediaselector: fallback to /mediaselector/5/ + return self._download_media_selector_url( + self._MEDIASELECTOR_ALT_URL % programme_id, programme_id) + else: + raise def _download_media_selector_url(self, url, programme_id=None): try: @@ -297,7 +304,6 @@ def _process_media_selector(self, media_selection, programme_id): formats.extend(self._extract_video(media, programme_id)) elif kind == 'captions': subtitles = self.extract_subtitles(media, programme_id) - return formats, subtitles def _download_playlist(self, playlist_id): @@ -426,9 +432,10 @@ class BBCIE(BBCCoUkIE): IE_DESC = 'BBC' _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P[^/#?]+)' - # fails with notukerror for some videos - # _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s' - _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s' + # fails with notukerror for some videos ( non news sites such as bbc.com/travel ) + _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s' + # limited selection of formats but may work where the above does not + _MEDIASELECTOR_ALT_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s' _TESTS = [{ # article with multiple videos embedded with data-media-meta containing @@ -455,7 +462,7 @@ class BBCIE(BBCCoUkIE): 'url': 'http://www.bbc.com/news/world-europe-32041533', 'info_dict': { 'id': 'p02mprgb', - 'ext': 'flv', + 'ext': 'mp4', 'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV', 'duration': 47, 'timestamp': 1427219242, @@ -515,7 +522,7 @@ class BBCIE(BBCCoUkIE): 'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star', 'info_dict': { 'id': 'p018zqqg', - 'ext': 'flv', + 'ext': 'mp4', 'title': 'Hyundai Santa Fe Sport: Rock star', 'description': 'md5:b042a26142c4154a6e472933cf20793d', 'timestamp': 1368473503, @@ -530,7 +537,7 @@ class BBCIE(BBCCoUkIE): 'url': 'http://www.bbc.com/sport/0/football/33653409', 'info_dict': { 'id': 'p02xycnp', - 'ext': 'flv', + 'ext': 'mp4', 'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?', 'description': 'md5:398fca0e2e701c609d726e034fa1fc89', 'duration': 140, From d12a1a47d5ff0833a1ecd7fe47f4ffa67a9b4a50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 30 Jul 2015 00:55:06 +0600 Subject: [PATCH 2/2] [bbc] Improve work with mediaselection URLs --- youtube_dl/extractor/bbc.py | 50 ++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py index 3d9366644..9a1b6e3dc 100644 --- a/youtube_dl/extractor/bbc.py +++ b/youtube_dl/extractor/bbc.py @@ -14,12 +14,15 @@ ) from ..compat import compat_HTTPError + class BBCCoUkIE(InfoExtractor): IE_NAME = 'bbc.co.uk' IE_DESC = 'BBC iPlayer' _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P[\da-z]{8})' - _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s' + _MEDIASELECTOR_URLS = [ + 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s', + ] _TESTS = [ { @@ -161,6 +164,10 @@ class BBCCoUkIE(InfoExtractor): } ] + class MediaSelectionError(Exception): + def __init__(self, id): + self.id = id + def _extract_asx_playlist(self, connection, programme_id): asx = self._download_xml(connection.get('href'), programme_id, 'Downloading ASX playlist') return [ref.get('href') for ref in asx.findall('./Entry/ref')] @@ -211,8 +218,7 @@ def _extract_items(self, playlist): def _extract_medias(self, media_selection): error = media_selection.find('./{http://bbc.co.uk/2008/mp/mediaselection}error') if error is not None: - raise ExtractorError( - '%s returned error: %s' % (self.IE_NAME, error.get('id')), expected=True) + raise BBCCoUkIE.MediaSelectionError(error.get('id')) return media_selection.findall('./{http://bbc.co.uk/2008/mp/mediaselection}media') def _extract_connections(self, media): @@ -269,17 +275,23 @@ def _get_subtitles(self, media, programme_id): ] return subtitles + def _raise_extractor_error(self, media_selection_error): + raise ExtractorError( + '%s returned error: %s' % (self.IE_NAME, media_selection_error.id), + expected=True) + def _download_media_selector(self, programme_id): - try: - return self._download_media_selector_url( - self._MEDIASELECTOR_URL % programme_id, programme_id) - except ExtractorError as e: - if hasattr(self, '_MEDIASELECTOR_ALT_URL') and str(e) == 'bbc returned error: notukerror': - # notukerror on bbc.com/travel using bbc news mediaselector: fallback to /mediaselector/5/ - return self._download_media_selector_url( - self._MEDIASELECTOR_ALT_URL % programme_id, programme_id) - else: - raise + last_exception = None + for mediaselector_url in self._MEDIASELECTOR_URLS: + try: + return self._download_media_selector_url( + mediaselector_url % programme_id, programme_id) + except BBCCoUkIE.MediaSelectionError as e: + if e.id == 'notukerror': + last_exception = e + continue + self._raise_extractor_error(e) + self._raise_extractor_error(last_exception) def _download_media_selector_url(self, url, programme_id=None): try: @@ -432,10 +444,14 @@ class BBCIE(BBCCoUkIE): IE_DESC = 'BBC' _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P[^/#?]+)' - # fails with notukerror for some videos ( non news sites such as bbc.com/travel ) - _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s' - # limited selection of formats but may work where the above does not - _MEDIASELECTOR_ALT_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s' + _MEDIASELECTOR_URLS = [ + # Provides more formats, namely direct mp4 links, but fails on some videos with + # notukerror for non UK (?) users (e.g. + # http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret) + 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s', + # Provides fewer formats, but works everywhere for everybody (hopefully) + 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s', + ] _TESTS = [{ # article with multiple videos embedded with data-media-meta containing