w/youtube-dl
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

[soundcloud] Remove limit on search results

Browse source
This commit is contained in:
reiv 2015-10-30 23:56:07 +01:00 committed by Sergey M․
parent 417b453699
commit 328a22e175
1 changed files with 13 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
youtube_dl/extractor/soundcloud.py
View file

@ -477,7 +477,7 @@ class SoundcloudPlaylistIE(SoundcloudIE):
class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE): class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
IE_NAME = 'soundcloud:search' IE_NAME = 'soundcloud:search'
IE_DESC = 'Soundcloud search' IE_DESC = 'Soundcloud search'
_MAX_RESULTS = 200 _MAX_RESULTS = float('inf')
_TESTS = [{ _TESTS = [{
'url': 'scsearch15:post-avant jazzcore', 'url': 'scsearch15:post-avant jazzcore',
'info_dict': { 'info_dict': {
@ -487,24 +487,28 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
}] }]
_SEARCH_KEY = 'scsearch' _SEARCH_KEY = 'scsearch'
_RESULTS_PER_PAGE = 50 _MAX_RESULTS_PER_PAGE = 200
_DEFAULT_RESULTS_PER_PAGE = 50
_API_V2_BASE = 'https://api-v2.soundcloud.com' _API_V2_BASE = 'https://api-v2.soundcloud.com'
def _get_collection(self, endpoint, collection_id, **query): def _get_collection(self, endpoint, collection_id, **query):
query['limit'] = self._RESULTS_PER_PAGE query['limit'] = results_per_page = min(
query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
self._MAX_RESULTS_PER_PAGE)
query['client_id'] = self._CLIENT_ID query['client_id'] = self._CLIENT_ID
query['linked_partitioning'] = '1' query['linked_partitioning'] = '1'
total_results = self._MAX_RESULTS total_results = None
collected_results = 0 collected_results = 0
next_url = None next_url = None
for i in itertools.count(): for i in itertools.count():
if not next_url: if not next_url:
query['offset'] = i * self._RESULTS_PER_PAGE query['offset'] = i * results_per_page
data = compat_urllib_parse.urlencode(query) data = compat_urllib_parse.urlencode(query)
next_url = '{0}{1}?{2}'.format(self._API_V2_BASE, endpoint, data) next_url = '{0}{1}?{2}'.format(
self._API_V2_BASE, endpoint, data)
response = self._download_json(next_url, response = self._download_json(next_url,
video_id=collection_id, video_id=collection_id,
@ -520,7 +524,8 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
for item in filter(bool, collection): for item in filter(bool, collection):
yield item yield item
if collected_results >= total_results or not collection: if (total_results is not None and
collected_results >= total_results) or not collection:
break break
next_url = response.get('next_href', None) next_url = response.get('next_href', None)
@ -528,7 +533,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
def _get_n_results(self, query, n): def _get_n_results(self, query, n):
tracks = self._get_collection('/search/tracks', tracks = self._get_collection('/search/tracks',
collection_id='Query "{0}"'.format(query), collection_id='Query "{0}"'.format(query),
q=query.encode('utf-8')) limit=n, q=query)
results = [self.url_result(url=track['uri']) results = [self.url_result(url=track['uri'])
for track in itertools.islice(tracks, n)] for track in itertools.islice(tracks, n)]