github/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2024-11-21 20:46:36 -05:00
Code Issues Projects Releases Packages Wiki Activity

[utils] OnDemandPagedList: Do not download pages after error

Browse source
This commit is contained in:
pukkandan 2022-03-03 02:42:52 +05:30
parent c2ae48dbd5
commit f1d130902b
No known key found for this signature in database
GPG key ID: 7EEE9E1E817D0A39
1 changed files with 9 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
yt_dlp/utils.py
View file

@ -2799,13 +2799,14 @@ def __len__(self):
def __init__(self, pagefunc, pagesize, use_cache=True):
self._pagefunc = pagefunc
self._pagesize = pagesize
self._pagecount = float('inf')
self._use_cache = use_cache
self._cache = {}
def getpage(self, pagenum):
page_results = self._cache.get(pagenum)
if page_results is None:
page_results = list(self._pagefunc(pagenum))
page_results = [] if pagenum > self._pagecount else list(self._pagefunc(pagenum))
if self._use_cache:
self._cache[pagenum] = page_results
return page_results
@ -2817,7 +2818,7 @@ def _getslice(self, start, end):
raise NotImplementedError('This method must be implemented by subclasses')
def __getitem__(self, idx):
# NOTE: cache must be enabled if this is used
assert self._use_cache, 'Indexing PagedList requires cache'
if not isinstance(idx, int) or idx < 0:
raise TypeError('indices must be non-negative integers')
entries = self.getslice(idx, idx + 1)
@ -2843,7 +2844,11 @@ def _getslice(self, start, end):
if (end is not None and firstid <= end <= nextfirstid)
else None)
page_results = self.getpage(pagenum)
try:
page_results = self.getpage(pagenum)
except Exception:
self._pagecount = pagenum - 1
raise
if startv != 0 or endv is not None:
page_results = page_results[startv:endv]
yield from page_results
@ -2863,8 +2868,8 @@ def _getslice(self, start, end):
class InAdvancePagedList(PagedList):
def __init__(self, pagefunc, pagecount, pagesize):
self._pagecount = pagecount
PagedList.__init__(self, pagefunc, pagesize, True)
self._pagecount = pagecount
def _getslice(self, start, end):
start_page = start // self._pagesize