From 2632941f327c8b013e5fbc736317fc897876ab73 Mon Sep 17 00:00:00 2001 From: ping Date: Wed, 20 May 2015 15:53:45 +0800 Subject: [PATCH 1/5] [soompi] Add new extractor for tv.soompi.com --- youtube_dl/extractor/__init__.py | 4 + youtube_dl/extractor/soompi.py | 130 +++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 youtube_dl/extractor/soompi.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 6f8c261d5..2a5cf9547 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -482,6 +482,10 @@ from .snotr import SnotrIE from .sockshare import SockshareIE from .sohu import SohuIE +from .soompi import ( + SoompiIE, + SoompiShowIE, +) from .soundcloud import ( SoundcloudIE, SoundcloudSetIE, diff --git a/youtube_dl/extractor/soompi.py b/youtube_dl/extractor/soompi.py new file mode 100644 index 000000000..5ecf40b7f --- /dev/null +++ b/youtube_dl/extractor/soompi.py @@ -0,0 +1,130 @@ +# encoding: utf-8 +from __future__ import unicode_literals + +import re +import json +import base64 +import xml.etree.ElementTree + +# Soompi uses the same subtitle encryption as crunchyroll +from .crunchyroll import CrunchyrollIE + + +class SoompiIE(CrunchyrollIE): + IE_NAME = 'soompi' + _VALID_URL = r'^https?://tv\.soompi\.com/en/watch/(?P[0-9]+)' + _TESTS = [{ + 'url': 'http://tv.soompi.com/en/watch/23363', + 'info_dict': { + 'id': '23363', + 'ext': 'mp4', + 'title': 'Liar Game CM1', + 'description': '15sec' + }, + 'params': { + 'skip_download': True, + }, + }] + + def _get_episodes(self, webpage, episode_filter=None): + episodes = json.loads( + self._search_regex(r'\s+VIDEOS\s+= (\[.+?\]);', webpage, "episodes meta")) + return [ep for ep in episodes if episode_filter is None or episode_filter(ep)] + + def _get_subtitles(self, video_id, show_format_xml): + subtitles = {} + subtitle_info_nodes = show_format_xml.findall('./{default}preload/subtitles/subtitle') + subtitle_nodes = show_format_xml.findall('./{default}preload/subtitle') + + sub_langs = {} + for i in subtitle_info_nodes: + sub_langs[i.attrib["id"]] = i.attrib["title"] + + for s in subtitle_nodes: + lang_code = sub_langs.get(s.attrib["id"], None) + if lang_code is None: + continue + + sub_id = int(s.attrib["id"]) + iv = base64.b64decode(s.find("iv").text) + data = base64.b64decode(s.find("data").text) + subtitle = self._decrypt_subtitles(data, iv, sub_id).decode('utf-8') + sub_root = xml.etree.ElementTree.fromstring(subtitle) + + subtitles[lang_code] = [{ + 'ext': 'srt', 'data': self._convert_subtitles_to_srt(sub_root) + }, { + 'ext': 'ass', 'data': self._convert_subtitles_to_ass(sub_root) + }] + return subtitles + + def _real_extract(self, url): + video_id = self._match_id(url) + + webpage = self._download_webpage( + url, video_id, note="Downloading episode page", + errnote="Video may not be available for your location") + vid_formats = re.findall(r"\?quality=q([0-9]+)", webpage) + + show_meta = json.loads( + self._search_regex(r'\s+var show = (\{.+?\});', webpage, "show meta")) + episodes = self._get_episodes( + webpage, episode_filter=lambda x: x['id'] == video_id) + + title = episodes[0]["name"] + description = episodes[0]["description"] + duration = int(episodes[0]["duration"]) + slug = show_meta["slug"] + + formats = [] + show_format_xml = None + for vf in vid_formats: + show_format_url = "http://tv.soompi.com/en/show/%s/%s-config.xml?mode=hls&quality=q%s" \ + % (slug, video_id, vf) + show_format_xml = self._download_xml( + show_format_url, video_id, note="Downloading q%s show xml" % vf) + avail_formats = self._extract_m3u8_formats( + show_format_xml.find('./{default}preload/stream_info/file').text, + video_id, ext="mp4", m3u8_id=vf, preference=int(vf)) + formats.extend(avail_formats) + self._sort_formats(formats) + + subtitles = self.extract_subtitles(video_id, show_format_xml) + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'duration': duration, + 'formats': formats, + 'subtitles': subtitles + } + + +class SoompiShowIE(SoompiIE): + IE_NAME = 'soompi:show' + _VALID_URL = r'^https?://tv\.soompi\.com/en/shows/(?P[0-9a-zA-Z\-_]+)' + _TESTS = [{ + 'url': 'http://tv.soompi.com/en/shows/liar-game', + 'info_dict': { + 'id': 'liar-game', + 'title': 'Liar Game', + 'description': 'md5:52c02bce0c1a622a95823591d0589b66', + }, + 'playlist_count': 14, + }] + + def _real_extract(self, url): + show_id = self._match_id(url) + + webpage = self._download_webpage(url, show_id, note="Downloading show page") + title = self._og_search_title(webpage).replace("SoompiTV | ", "") + description = self._og_search_description(webpage) + + episodes = self._get_episodes(webpage) + entries = [] + for ep in episodes: + entries.append(self.url_result( + 'http://tv.soompi.com/en/watch/%s' % ep['id'], 'Soompi', ep['id'])) + + return self.playlist_result(entries, show_id, title, description) From 5137adb94dcce98a3c14fb3892c5c72f70ff34ea Mon Sep 17 00:00:00 2001 From: ping Date: Wed, 20 May 2015 16:16:10 +0800 Subject: [PATCH 2/5] [soompi] Switch to non-geoblocked test video --- youtube_dl/extractor/soompi.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/soompi.py b/youtube_dl/extractor/soompi.py index 5ecf40b7f..4726872dc 100644 --- a/youtube_dl/extractor/soompi.py +++ b/youtube_dl/extractor/soompi.py @@ -14,12 +14,12 @@ class SoompiIE(CrunchyrollIE): IE_NAME = 'soompi' _VALID_URL = r'^https?://tv\.soompi\.com/en/watch/(?P[0-9]+)' _TESTS = [{ - 'url': 'http://tv.soompi.com/en/watch/23363', + 'url': 'http://tv.soompi.com/en/watch/29235', 'info_dict': { - 'id': '23363', + 'id': '29235', 'ext': 'mp4', - 'title': 'Liar Game CM1', - 'description': '15sec' + 'title': 'Episode 1096', + 'description': '2015-05-20' }, 'params': { 'skip_download': True, From 0385d642232ba4e8b455d0c4eb95c7985f22f276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 30 May 2015 14:12:58 +0600 Subject: [PATCH 3/5] [crunchyroll] Extract subtitles extraction routine --- youtube_dl/extractor/crunchyroll.py | 30 +++++++++++++---------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py index 1c77df47e..4ac537a6d 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dl/extractor/crunchyroll.py @@ -76,8 +76,8 @@ def _real_initialize(self): self._login() def _decrypt_subtitles(self, data, iv, id): - data = bytes_to_intlist(data) - iv = bytes_to_intlist(iv) + data = bytes_to_intlist(base64.b64decode(data)) + iv = bytes_to_intlist(base64.b64decode(iv)) id = int(id) def obfuscate_key_aux(count, modulo, start): @@ -179,6 +179,16 @@ def ass_bool(strvalue): return output + def _extract_subtitles(self, subtitle): + sub_root = xml.etree.ElementTree.fromstring(subtitle) + return [{ + 'ext': 'srt', + 'data': self._convert_subtitles_to_srt(sub_root), + }, { + 'ext': 'ass', + 'data': self._convert_subtitles_to_ass(sub_root), + }] + def _get_subtitles(self, video_id, webpage): subtitles = {} for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage): @@ -190,25 +200,11 @@ def _get_subtitles(self, video_id, webpage): data = self._search_regex(r'([^<]+)', sub_page, 'subtitle_data', fatal=False) if not id or not iv or not data: continue - id = int(id) - iv = base64.b64decode(iv) - data = base64.b64decode(data) - subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8') lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False) if not lang_code: continue - sub_root = xml.etree.ElementTree.fromstring(subtitle) - subtitles[lang_code] = [ - { - 'ext': 'srt', - 'data': self._convert_subtitles_to_srt(sub_root), - }, - { - 'ext': 'ass', - 'data': self._convert_subtitles_to_ass(sub_root), - }, - ] + subtitles[lang_code] = self._extract_subtitles(subtitle) return subtitles def _real_extract(self, url): From b2cf6543b21bbe0954c45b35b1402eaca5187c0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 30 May 2015 14:30:04 +0600 Subject: [PATCH 4/5] [soompi] Improve and simplify --- youtube_dl/extractor/soompi.py | 146 ++++++++++++++++++--------------- 1 file changed, 81 insertions(+), 65 deletions(-) diff --git a/youtube_dl/extractor/soompi.py b/youtube_dl/extractor/soompi.py index 4726872dc..5da66ca9e 100644 --- a/youtube_dl/extractor/soompi.py +++ b/youtube_dl/extractor/soompi.py @@ -2,17 +2,31 @@ from __future__ import unicode_literals import re -import json -import base64 -import xml.etree.ElementTree -# Soompi uses the same subtitle encryption as crunchyroll from .crunchyroll import CrunchyrollIE +from .common import InfoExtractor +from ..compat import compat_HTTPError +from ..utils import ( + ExtractorError, + int_or_none, + remove_start, + xpath_text, +) -class SoompiIE(CrunchyrollIE): + +class SoompiBaseIE(InfoExtractor): + def _get_episodes(self, webpage, episode_filter=None): + episodes = self._parse_json( + self._search_regex( + r'VIDEOS\s*=\s*(\[.+?\]);', webpage, 'episodes JSON'), + None) + return list(filter(episode_filter, episodes)) + + +class SoompiIE(SoompiBaseIE, CrunchyrollIE): IE_NAME = 'soompi' - _VALID_URL = r'^https?://tv\.soompi\.com/en/watch/(?P[0-9]+)' + _VALID_URL = r'https?://tv\.soompi\.com/(?:en/)?watch/(?P[0-9]+)' _TESTS = [{ 'url': 'http://tv.soompi.com/en/watch/29235', 'info_dict': { @@ -26,84 +40,86 @@ class SoompiIE(CrunchyrollIE): }, }] - def _get_episodes(self, webpage, episode_filter=None): - episodes = json.loads( - self._search_regex(r'\s+VIDEOS\s+= (\[.+?\]);', webpage, "episodes meta")) - return [ep for ep in episodes if episode_filter is None or episode_filter(ep)] - - def _get_subtitles(self, video_id, show_format_xml): - subtitles = {} - subtitle_info_nodes = show_format_xml.findall('./{default}preload/subtitles/subtitle') - subtitle_nodes = show_format_xml.findall('./{default}preload/subtitle') + def _get_episode(self, webpage, video_id): + return self._get_episodes(webpage, lambda x: x['id'] == video_id)[0] + def _get_subtitles(self, config, video_id): sub_langs = {} - for i in subtitle_info_nodes: - sub_langs[i.attrib["id"]] = i.attrib["title"] + for subtitle in config.findall('./{default}preload/subtitles/subtitle'): + sub_langs[subtitle.attrib['id']] = subtitle.attrib['title'] - for s in subtitle_nodes: - lang_code = sub_langs.get(s.attrib["id"], None) - if lang_code is None: + subtitles = {} + for s in config.findall('./{default}preload/subtitle'): + lang_code = sub_langs.get(s.attrib['id']) + if not lang_code: + continue + sub_id = s.get('id') + data = xpath_text(s, './data', 'data') + iv = xpath_text(s, './iv', 'iv') + if not id or not iv or not data: continue - - sub_id = int(s.attrib["id"]) - iv = base64.b64decode(s.find("iv").text) - data = base64.b64decode(s.find("data").text) subtitle = self._decrypt_subtitles(data, iv, sub_id).decode('utf-8') - sub_root = xml.etree.ElementTree.fromstring(subtitle) - - subtitles[lang_code] = [{ - 'ext': 'srt', 'data': self._convert_subtitles_to_srt(sub_root) - }, { - 'ext': 'ass', 'data': self._convert_subtitles_to_ass(sub_root) - }] + subtitles[lang_code] = self._extract_subtitles(subtitle) return subtitles def _real_extract(self, url): video_id = self._match_id(url) - webpage = self._download_webpage( - url, video_id, note="Downloading episode page", - errnote="Video may not be available for your location") - vid_formats = re.findall(r"\?quality=q([0-9]+)", webpage) - - show_meta = json.loads( - self._search_regex(r'\s+var show = (\{.+?\});', webpage, "show meta")) - episodes = self._get_episodes( - webpage, episode_filter=lambda x: x['id'] == video_id) - - title = episodes[0]["name"] - description = episodes[0]["description"] - duration = int(episodes[0]["duration"]) - slug = show_meta["slug"] + try: + webpage = self._download_webpage( + url, video_id, 'Downloading episode page') + except ExtractorError as ee: + if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403: + webpage = ee.cause.read() + block_message = self._html_search_regex( + r'(?s)
(.+?)
', webpage, + 'block message', default=None) + if block_message: + raise ExtractorError(block_message, expected=True) + raise formats = [] - show_format_xml = None - for vf in vid_formats: - show_format_url = "http://tv.soompi.com/en/show/%s/%s-config.xml?mode=hls&quality=q%s" \ - % (slug, video_id, vf) - show_format_xml = self._download_xml( - show_format_url, video_id, note="Downloading q%s show xml" % vf) - avail_formats = self._extract_m3u8_formats( - show_format_xml.find('./{default}preload/stream_info/file').text, - video_id, ext="mp4", m3u8_id=vf, preference=int(vf)) - formats.extend(avail_formats) + config = None + for format_id in re.findall(r'\?quality=([0-9a-zA-Z]+)', webpage): + config = self._download_xml( + 'http://tv.soompi.com/en/show/_/%s-config.xml?mode=hls&quality=%s' % (video_id, format_id), + video_id, 'Downloading %s XML' % format_id) + m3u8_url = xpath_text( + config, './{default}preload/stream_info/file', + '%s m3u8 URL' % format_id) + if not m3u8_url: + continue + formats.extend(self._extract_m3u8_formats( + m3u8_url, video_id, 'mp4', m3u8_id=format_id)) self._sort_formats(formats) - subtitles = self.extract_subtitles(video_id, show_format_xml) + episode = self._get_episode(webpage, video_id) + + title = episode['name'] + description = episode.get('description') + duration = int_or_none(episode.get('duration')) + + thumbnails = [{ + 'id': thumbnail_id, + 'url': thumbnail_url, + } for thumbnail_id, thumbnail_url in episode.get('img_url', {}).items()] + + subtitles = self.extract_subtitles(config, video_id) return { 'id': video_id, 'title': title, 'description': description, + 'thumbnails': thumbnails, 'duration': duration, 'formats': formats, 'subtitles': subtitles } -class SoompiShowIE(SoompiIE): +class SoompiShowIE(SoompiBaseIE): IE_NAME = 'soompi:show' - _VALID_URL = r'^https?://tv\.soompi\.com/en/shows/(?P[0-9a-zA-Z\-_]+)' + _VALID_URL = r'https?://tv\.soompi\.com/en/shows/(?P[0-9a-zA-Z\-_]+)' _TESTS = [{ 'url': 'http://tv.soompi.com/en/shows/liar-game', 'info_dict': { @@ -117,14 +133,14 @@ class SoompiShowIE(SoompiIE): def _real_extract(self, url): show_id = self._match_id(url) - webpage = self._download_webpage(url, show_id, note="Downloading show page") - title = self._og_search_title(webpage).replace("SoompiTV | ", "") + webpage = self._download_webpage( + url, show_id, 'Downloading show page') + + title = remove_start(self._og_search_title(webpage), 'SoompiTV | ') description = self._og_search_description(webpage) - episodes = self._get_episodes(webpage) - entries = [] - for ep in episodes: - entries.append(self.url_result( - 'http://tv.soompi.com/en/watch/%s' % ep['id'], 'Soompi', ep['id'])) + entries = [ + self.url_result('http://tv.soompi.com/en/watch/%s' % episode['id'], 'Soompi') + for episode in self._get_episodes(webpage)] return self.playlist_result(entries, show_id, title, description) From 1a5b77dc21384c462e0be86a1638cafd15a6e236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 30 May 2015 14:36:45 +0600 Subject: [PATCH 5/5] [crunchyroll] Fix python 3.2 --- youtube_dl/extractor/crunchyroll.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py index 4ac537a6d..41f0c736d 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dl/extractor/crunchyroll.py @@ -76,8 +76,8 @@ def _real_initialize(self): self._login() def _decrypt_subtitles(self, data, iv, id): - data = bytes_to_intlist(base64.b64decode(data)) - iv = bytes_to_intlist(base64.b64decode(iv)) + data = bytes_to_intlist(base64.b64decode(data.encode('utf-8'))) + iv = bytes_to_intlist(base64.b64decode(iv.encode('utf-8'))) id = int(id) def obfuscate_key_aux(count, modulo, start):