[grooveshark,http] Make HTTP POST downloads work

This commit is contained in:
Philipp Hagemeister 2014-08-24 01:31:35 +02:00
parent 00558d9414
commit d769be6c96
3 changed files with 42 additions and 41 deletions

View file

@ -27,8 +27,16 @@ def real_download(self, filename, info_dict):
headers['Youtubedl-user-agent'] = info_dict['user_agent']
if 'http_referer' in info_dict:
headers['Referer'] = info_dict['http_referer']
basic_request = compat_urllib_request.Request(url, None, headers)
request = compat_urllib_request.Request(url, None, headers)
add_headers = info_dict.get('http_headers')
if add_headers:
headers.update(add_headers)
data = info_dict.get('http_post_data')
http_method = info_dict.get('http_method')
basic_request = compat_urllib_request.Request(url, data, headers)
request = compat_urllib_request.Request(url, data, headers)
if http_method is not None:
basic_request.get_method = lambda: http_method
request.get_method = lambda: http_method
is_test = self.params.get('test', False)

View file

@ -84,6 +84,12 @@ class InfoExtractor(object):
format, irrespective of the file format.
-1 for default (order by other properties),
-2 or smaller for less than default.
* http_referer HTTP Referer header value to set.
* http_method HTTP method to use for the download.
* http_headers A dictionary of additional HTTP headers
to add to the request.
* http_post_data Additional data to send with a POST
request.
url: Final video URL.
ext: Video filename extension.
format: The video format, defaults to ext (used for --get-format)

View file

@ -12,7 +12,7 @@
from ..utils import ExtractorError, compat_urllib_request, compat_html_parser
from ..utils import compat_urlparse
urlparse = compat_urlparse.urlparse
urlunparse = compat_urlparse.urlunparse
urldefrag = compat_urlparse.urldefrag
@ -52,7 +52,7 @@ class GroovesharkIE(InfoExtractor):
'id': '6SS1DW',
'title': 'Jolene (Tenth Key Remix ft. Will Sessions)',
'ext': 'mp3',
'duration': 227
'duration': 227,
}
}
@ -60,7 +60,7 @@ class GroovesharkIE(InfoExtractor):
do_bootstrap_request = True
def _parse_target(self, target):
uri = urlparse(target)
uri = compat_urlparse.urlparse(target)
hash = uri.fragment[1:].split('?')[0]
token = basename(hash.rstrip('/'))
return (uri, hash, token)
@ -123,22 +123,36 @@ def _get_bootstrap(self, target):
def _get_playerpage(self, target):
(_, _, token) = self._parse_target(target)
res = self._download_webpage(
webpage = self._download_webpage(
target, token,
note='Downloading player page',
errnote='Unable to download player page',
fatal=False)
if res is not None:
o = GroovesharkHtmlParser.extract_object_tags(res)
return (res, [x for x in o if x['attrs']['id'] == 'jsPlayerEmbed'])
if webpage is not None:
# Search (for example German) error message
error_msg = self._html_search_regex(
r'<div id="content">\s*<h2>(.*?)</h2>', webpage,
'error message', default=None)
if error_msg is not None:
error_msg = error_msg.replace('\n', ' ')
raise ExtractorError('Grooveshark said: %s' % error_msg)
return (res, None)
if webpage is not None:
o = GroovesharkHtmlParser.extract_object_tags(webpage)
return (webpage, [x for x in o if x['attrs']['id'] == 'jsPlayerEmbed'])
return (webpage, None)
def _real_initialize(self):
self.ts = int(time.time() * 1000) # timestamp in millis
def _real_extract(self, url):
(target_uri, _, token) = self._parse_target(url)
# 1. Fill cookiejar by making a request to the player page
swf_referer = None
if self.do_playerpage_request:
(_, player_objs) = self._get_playerpage(url)
if player_objs is not None:
@ -162,11 +176,10 @@ def _real_extract(self, url):
'Content-Length': len(post_data),
'Content-Type': 'application/x-www-form-urlencoded'
}
if 'swf_referer' in locals():
if swf_referer is not None:
headers['Referer'] = swf_referer
info_dict = {
return {
'id': token,
'title': meta['song']['Name'],
'http_method': 'POST',
@ -174,32 +187,6 @@ def _real_extract(self, url):
'ext': 'mp3',
'format': 'mp3 audio',
'duration': duration,
# various ways of supporting the download request.
# remove keys unnecessary to the eventual post implementation
'post_data': post_data,
'post_dict': post_dict,
'headers': headers
'http_post_data': post_data,
'http_headers': headers,
}
if 'swf_referer' in locals():
info_dict['http_referer'] = swf_referer
return info_dict
def _real_initialize(self):
self.ts = int(time.time() * 1000) # timestamp in millis
def _download_json(self, url_or_request, video_id,
note=u'Downloading JSON metadata',
errnote=u'Unable to download JSON metadata',
fatal=True,
transform_source=None):
try:
out = super(GroovesharkIE, self)._download_json(
url_or_request, video_id, note, errnote, transform_source)
return out
except ExtractorError as ee:
if fatal:
raise ee
return None