From b7f59a3bf69b5c935be085551d30ce4d0b8a97d4 Mon Sep 17 00:00:00 2001 From: Philip Xu Date: Thu, 13 Oct 2016 21:51:26 -0400 Subject: [PATCH 1/4] [huajiao] Add new extractor --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/huajiao.py | 50 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 youtube_dl/extractor/huajiao.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 08bed8b0c..75e16af4e 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -372,6 +372,7 @@ HRTiIE, HRTiPlaylistIE, ) +from .huajiao import HuajiaoIE from .huffpost import HuffPostIE from .hypem import HypemIE from .iconosquare import IconosquareIE diff --git a/youtube_dl/extractor/huajiao.py b/youtube_dl/extractor/huajiao.py new file mode 100644 index 000000000..352b48120 --- /dev/null +++ b/youtube_dl/extractor/huajiao.py @@ -0,0 +1,50 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from ..utils import parse_duration, parse_iso8601 +from .common import InfoExtractor + + +class HuajiaoIE(InfoExtractor): + IE_DESC = '花椒直播' + _VALID_URL = r'https?://(?:www\.)?huajiao\.com/l/(?P[0-9]+)' + _TEST = { + 'url': 'http://www.huajiao.com/l/38941232', + 'md5': 'd08bf9ac98787d24d1e4c0283f2d372d', + 'info_dict': { + 'id': '38941232', + 'ext': 'mp4', + 'title': '#新人求关注#', + 'description': 're:.*', + 'duration': 2424.0, + 'thumbnail': 're:^https?://.*\.jpg$', + 'timestamp': 1475866459, + 'upload_date': '20161007', + 'uploader': 'Penny_余姿昀', + 'uploader_id': '75206005', + } + } + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + feed_json = self._search_regex( + r'var\s*feed\s*=\s*({.*})', webpage, 'feed json str') + feed = self._parse_json(feed_json, video_id) + + description = self._html_search_meta( + 'description', webpage, 'description', fatal=False) + + return { + 'id': video_id, + 'title': feed['feed']['formated_title'], + 'description': description, + 'duration': parse_duration(feed['feed']['duration']), + 'thumbnail': feed['feed']['image'], + 'timestamp': parse_iso8601(feed['creatime'], ' '), + 'uploader': feed['author']['nickname'], + 'uploader_id': feed['author']['uid'], + 'formats': self._extract_m3u8_formats( + feed['feed']['m3u8'], video_id, 'mp4', 'm3u8_native'), + } From 2e638d7bcae26ef3f8a9d707d49c897e19571a0e Mon Sep 17 00:00:00 2001 From: Philip Xu Date: Fri, 14 Oct 2016 14:12:06 -0400 Subject: [PATCH 2/4] Made optional fields optional --- youtube_dl/extractor/huajiao.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/huajiao.py b/youtube_dl/extractor/huajiao.py index 352b48120..005b8068f 100644 --- a/youtube_dl/extractor/huajiao.py +++ b/youtube_dl/extractor/huajiao.py @@ -36,15 +36,18 @@ def _real_extract(self, url): description = self._html_search_meta( 'description', webpage, 'description', fatal=False) + def get(section, field): + return feed.get(section, {}).get(field) + return { 'id': video_id, 'title': feed['feed']['formated_title'], 'description': description, - 'duration': parse_duration(feed['feed']['duration']), - 'thumbnail': feed['feed']['image'], - 'timestamp': parse_iso8601(feed['creatime'], ' '), - 'uploader': feed['author']['nickname'], - 'uploader_id': feed['author']['uid'], + 'duration': parse_duration(get('feed', 'duration')), + 'thumbnail': get('feed', 'image'), + 'timestamp': parse_iso8601(feed.get('creatime'), ' '), + 'uploader': get('author', 'nickname'), + 'uploader_id': get('author', 'uid'), 'formats': self._extract_m3u8_formats( feed['feed']['m3u8'], video_id, 'mp4', 'm3u8_native'), } From d32fa0f12c43342baf8f19678bad38a1b1b52483 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 15 Oct 2016 14:53:53 +0800 Subject: [PATCH 3/4] [huajiao] Coding style --- youtube_dl/extractor/huajiao.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/huajiao.py b/youtube_dl/extractor/huajiao.py index 005b8068f..c17cbb624 100644 --- a/youtube_dl/extractor/huajiao.py +++ b/youtube_dl/extractor/huajiao.py @@ -1,8 +1,11 @@ # coding: utf-8 from __future__ import unicode_literals -from ..utils import parse_duration, parse_iso8601 from .common import InfoExtractor +from ..utils import ( + parse_duration, + parse_iso8601, +) class HuajiaoIE(InfoExtractor): From f605242bfcfe53403019f8bfcb3e10466992df7c Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 15 Oct 2016 14:55:36 +0800 Subject: [PATCH 4/4] [ChangeLog] Update for #10917 --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index c1173150d..3f2299a37 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,7 @@ version Extractors ++ [huajiao] New extractor (#10917) * [cmt] Fix mgid extraction (#10813) * [chirbit] Fix extraction of user profile pages * [charambatv] Fix extraction