github/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2024-11-21 20:46:36 -05:00
Code Issues Projects Releases Packages Wiki Activity

[soundcloud] Retry playlist pages on 502 error

Browse source 
Closes #872
This commit is contained in:
pukkandan 2021-09-05 10:27:49 +05:30
parent aa6c25309a
commit e04a1ff92e
No known key found for this signature in database
GPG key ID: 0F00D95A001F4698
1 changed files with 21 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
yt_dlp/extractor/soundcloud.py
View file

@ -23,6 +23,8 @@
int_or_none, int_or_none,
KNOWN_EXTENSIONS, KNOWN_EXTENSIONS,
mimetype2ext, mimetype2ext,
network_exceptions,
remove_end,
parse_qs, parse_qs,
str_or_none, str_or_none,
try_get, try_get,
@ -662,7 +664,7 @@ def _extract_playlist(self, base_url, playlist_id, playlist_title):
'entries': self._entries(base_url, playlist_id), 'entries': self._entries(base_url, playlist_id),
} }
def _entries(self, base_url, playlist_id): def _entries(self, url, playlist_id):
# Per the SoundCloud documentation, the maximum limit for a linked partitioning query is 200. # Per the SoundCloud documentation, the maximum limit for a linked partitioning query is 200.
# https://developers.soundcloud.com/blog/offset-pagination-deprecated # https://developers.soundcloud.com/blog/offset-pagination-deprecated
query = { query = {
@ -671,12 +673,25 @@ def _entries(self, base_url, playlist_id):
'offset': 0, 'offset': 0,
} }
retries = self.get_param('extractor_retries', 3)
next_href = base_url
for i in itertools.count(): for i in itertools.count():
response = self._download_json( attempt, last_error = -1, None
next_href, playlist_id, while attempt < retries:
'Downloading track page %s' % (i + 1), query=query, headers=self._HEADERS) attempt += 1
if last_error:
self.report_warning('%s. Retrying ...' % remove_end(last_error, '.'), playlist_id)
try:
response = self._download_json(
url, playlist_id, query=query, headers=self._HEADERS,
note='Downloading track page %s%s' % (i + 1, f' (retry #{attempt})' if attempt else ''))
break
except ExtractorError as e:
# Downloading page may result in intermittent 502 HTTP error
# See https://github.com/yt-dlp/yt-dlp/issues/872
if attempt >= retries or not isinstance(e.cause, compat_HTTPError) or e.cause.code != 502:
raise
last_error = str(e.cause or e.msg)
def resolve_entry(*candidates): def resolve_entry(*candidates):
for cand in candidates: for cand in candidates:
@ -692,7 +707,7 @@ def resolve_entry(*candidates):
for e in response['collection'] or []: for e in response['collection'] or []:
yield resolve_entry(e, e.get('track'), e.get('playlist')) yield resolve_entry(e, e.get('track'), e.get('playlist'))
next_href = response.get('next_href') url = response.get('next_href')
query.pop('offset', None) query.pop('offset', None)