From 9f4b9118ccaef5cd7c414a78c5622968e8c3343f Mon Sep 17 00:00:00 2001 From: Slava Shklyaev Date: Tue, 2 Jun 2015 12:47:52 +0300 Subject: [PATCH 01/10] [nova] Fix display_id extraction bug Make id group non-greedy so that .html is not included in it. --- youtube_dl/extractor/nova.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py index 4e999b237..1dd18511e 100644 --- a/youtube_dl/extractor/nova.py +++ b/youtube_dl/extractor/nova.py @@ -8,7 +8,7 @@ from .common import InfoExtractor class NovaIE(InfoExtractor): IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz' - _VALID_URL = 'http://(?:[^.]+\.)?(?Ptv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P[^/]+)(?:\.html|/?)' + _VALID_URL = 'http://(?:[^.]+\.)?(?Ptv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P[^/]+?)(?:\.html|/?)$' _TESTS = [{ 'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html', 'info_dict': { From 9464a194dbf48989c486fa2de9e1aebc59e28ed4 Mon Sep 17 00:00:00 2001 From: Slava Shklyaev Date: Tue, 2 Jun 2015 12:52:39 +0300 Subject: [PATCH 02/10] [nova] Fix extension extraction bug Replace the hardcoded flv with determine_ext. Let rtmpdump parse the url. --- youtube_dl/extractor/nova.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py index 1dd18511e..fd5f9cb0e 100644 --- a/youtube_dl/extractor/nova.py +++ b/youtube_dl/extractor/nova.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import re from .common import InfoExtractor +from ..utils import determine_ext class NovaIE(InfoExtractor): @@ -39,7 +40,7 @@ class NovaIE(InfoExtractor): 'info_dict': { 'id': '1756825', 'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove', - 'ext': 'flv', + 'ext': 'mp4', 'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově', 'description': 'md5:d804ba6b30bc7da2705b1fea961bddfe', 'thumbnail': 're:^https?://.*\.(?:jpg)', @@ -108,21 +109,8 @@ class NovaIE(InfoExtractor): mediafile = config['mediafile'] video_url = mediafile['src'] - - m = re.search(r'^(?Prtmpe?://[^/]+/(?P[^/]+?))/&*(?P.+)$', video_url) - if m: - formats = [{ - 'url': m.group('url'), - 'app': m.group('app'), - 'play_path': m.group('playpath'), - 'player_path': 'http://tvnoviny.nova.cz/static/shared/app/videojs/video-js.swf', - 'ext': 'flv', - }] - else: - formats = [{ - 'url': video_url, - }] - self._sort_formats(formats) + ext = determine_ext(video_url) + video_url = video_url.replace('&{}:'.format(ext), '') title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage) description = self._og_search_description(webpage) @@ -134,5 +122,6 @@ class NovaIE(InfoExtractor): 'title': title, 'description': description, 'thumbnail': thumbnail, - 'formats': formats, + 'url': video_url, + 'ext': ext, } From fcb04bcaca1b83cd3f13f494d7d775e35e0b6182 Mon Sep 17 00:00:00 2001 From: Slava Shklyaev Date: Tue, 2 Jun 2015 12:55:41 +0300 Subject: [PATCH 03/10] [nova] Extract upload_date in some cases --- youtube_dl/extractor/nova.py | 39 ++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py index fd5f9cb0e..30c64aaf8 100644 --- a/youtube_dl/extractor/nova.py +++ b/youtube_dl/extractor/nova.py @@ -49,6 +49,33 @@ class NovaIE(InfoExtractor): # rtmp download 'skip_download': True, } + }, { + 'url': 'http://novaplus.nova.cz/porad/televizni-noviny/video/5585-televizni-noviny-30-5-2015/', + 'info_dict': { + 'id': '1756858', + 'ext': 'mp4', + 'title': 'Televizní noviny - 30. 5. 2015', + 'thumbnail': 're:^https?://.*\.(?:jpg)', + 'upload_date': '20150530', + }, + 'params': { + # rtmp download + 'skip_download': True, + } + }, { + 'url': 'http://fanda.nova.cz/clanek/fun-and-games/krvavy-epos-zaklinac-3-divoky-hon-vychazi-vyhrajte-ho-pro-sebe.html', + 'info_dict': { + 'id': '1753621', + 'ext': 'mp4', + 'title': 'Zaklínač 3: Divoký hon', + 'description': 're:.*Pokud se stejně jako my nemůžete.*', + 'thumbnail': 're:https?://.*\.jpg(\?.*)?', + 'upload_date': '20150521', + }, + 'params': { + # rtmp download + 'skip_download': True, + } }, { 'url': 'http://sport.tn.nova.cz/clanek/sport/hokej/nhl/zivot-jde-dal-hodnotil-po-vyrazeni-z-playoff-jiri-sekac.html', 'only_matching': True, @@ -116,11 +143,23 @@ class NovaIE(InfoExtractor): description = self._og_search_description(webpage) thumbnail = config.get('poster') + mobj = None + if site == 'novaplus': + mobj = re.search(r'(?P\d{1,2})-(?P\d{1,2})-(?P\d{4})$', display_id) + if site == 'fanda': + mobj = re.search( + r'(?P\d{1,2})\.(?P\d{1,2})\.(?P\d{4})\b', webpage) + if mobj: + upload_date = '{}{:02d}{:02d}'.format(mobj.group('year'), int(mobj.group('month')), int(mobj.group('day'))) + else: + upload_date = None + return { 'id': video_id, 'display_id': display_id, 'title': title, 'description': description, + 'upload_date': upload_date, 'thumbnail': thumbnail, 'url': video_url, 'ext': ext, From 34c0f95db273ac5e7a7f8a6d23a3f90ceadf4695 Mon Sep 17 00:00:00 2001 From: Slava Shklyaev Date: Tue, 2 Jun 2015 12:56:36 +0300 Subject: [PATCH 04/10] [nova] Remove html tags from description --- youtube_dl/extractor/nova.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py index 30c64aaf8..140312f9c 100644 --- a/youtube_dl/extractor/nova.py +++ b/youtube_dl/extractor/nova.py @@ -4,7 +4,7 @@ from __future__ import unicode_literals import re from .common import InfoExtractor -from ..utils import determine_ext +from ..utils import clean_html, determine_ext class NovaIE(InfoExtractor): @@ -42,7 +42,7 @@ class NovaIE(InfoExtractor): 'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove', 'ext': 'mp4', 'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově', - 'description': 'md5:d804ba6b30bc7da2705b1fea961bddfe', + 'description': 'md5:dc24e50be5908df83348e50d1431295e', 'thumbnail': 're:^https?://.*\.(?:jpg)', }, 'params': { @@ -140,7 +140,7 @@ class NovaIE(InfoExtractor): video_url = video_url.replace('&{}:'.format(ext), '') title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage) - description = self._og_search_description(webpage) + description = clean_html(self._og_search_description(webpage, default=None)) thumbnail = config.get('poster') mobj = None From a00234f1c517d077a237da576be638fef980d79e Mon Sep 17 00:00:00 2001 From: Slava Shklyaev Date: Tue, 2 Jun 2015 12:57:03 +0300 Subject: [PATCH 05/10] [nova] Minor style improvement --- youtube_dl/extractor/nova.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py index 140312f9c..10957e5fa 100644 --- a/youtube_dl/extractor/nova.py +++ b/youtube_dl/extractor/nova.py @@ -132,7 +132,7 @@ class NovaIE(InfoExtractor): config = self._download_json( config_url, display_id, 'Downloading config JSON', - transform_source=lambda s: re.sub(r'var\s+[\da-zA-Z_]+\s*=\s*({.+?});', r'\1', s)) + transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1]) mediafile = config['mediafile'] video_url = mediafile['src'] From 4b5fe1349f5568f3b9b939520db0a1ddc598b4b3 Mon Sep 17 00:00:00 2001 From: Slava Shklyaev Date: Tue, 2 Jun 2015 18:15:05 +0300 Subject: [PATCH 06/10] [nova] Comply with review --- youtube_dl/extractor/nova.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py index 10957e5fa..4a2d76506 100644 --- a/youtube_dl/extractor/nova.py +++ b/youtube_dl/extractor/nova.py @@ -4,7 +4,11 @@ from __future__ import unicode_literals import re from .common import InfoExtractor -from ..utils import clean_html, determine_ext +from ..utils import ( + clean_html, + determine_ext, + unified_strdate, +) class NovaIE(InfoExtractor): @@ -143,14 +147,12 @@ class NovaIE(InfoExtractor): description = clean_html(self._og_search_description(webpage, default=None)) thumbnail = config.get('poster') - mobj = None if site == 'novaplus': - mobj = re.search(r'(?P\d{1,2})-(?P\d{1,2})-(?P\d{4})$', display_id) - if site == 'fanda': - mobj = re.search( - r'(?P\d{1,2})\.(?P\d{1,2})\.(?P\d{4})\b', webpage) - if mobj: - upload_date = '{}{:02d}{:02d}'.format(mobj.group('year'), int(mobj.group('month')), int(mobj.group('day'))) + upload_date = unified_strdate(self._search_regex( + r'(\d{1,2}-\d{1,2}-\d{4})$', display_id, 'upload date', default=None)) + elif site == 'fanda': + upload_date = unified_strdate(self._search_regex( + r'(\d{1,2}\.\d{1,2}\.\d{4})', webpage, 'upload date', default=None)) else: upload_date = None From 08b7968e2873b45dafe465ec04541db8fcd4967d Mon Sep 17 00:00:00 2001 From: Slava Shklyaev Date: Tue, 2 Jun 2015 17:49:15 +0300 Subject: [PATCH 07/10] [nova] Fix display_id extraction bug --- youtube_dl/extractor/nova.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py index 4a2d76506..8360a65d9 100644 --- a/youtube_dl/extractor/nova.py +++ b/youtube_dl/extractor/nova.py @@ -13,9 +13,9 @@ from ..utils import ( class NovaIE(InfoExtractor): IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz' - _VALID_URL = 'http://(?:[^.]+\.)?(?Ptv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P[^/]+?)(?:\.html|/?)$' + _VALID_URL = 'http://(?:[^.]+\.)?(?Ptv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P[^/]+?)(?:\.html|/|$)' _TESTS = [{ - 'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html', + 'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html?utm_source=tvnoviny&utm_medium=cpfooter&utm_campaign=novaplus', 'info_dict': { 'id': '1608920', 'display_id': 'co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou', @@ -29,7 +29,7 @@ class NovaIE(InfoExtractor): 'skip_download': True, } }, { - 'url': 'http://tn.nova.cz/clanek/tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci.html', + 'url': 'http://tn.nova.cz/clanek/tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci.html#player_13260', 'md5': '1dd7b9d5ea27bc361f110cd855a19bd3', 'info_dict': { 'id': '1757139', @@ -40,7 +40,7 @@ class NovaIE(InfoExtractor): 'thumbnail': 're:^https?://.*\.(?:jpg)', } }, { - 'url': 'http://novaplus.nova.cz/porad/policie-modrava/video/5591-policie-modrava-15-dil-blondynka-na-hrbitove/', + 'url': 'http://novaplus.nova.cz/porad/policie-modrava/video/5591-policie-modrava-15-dil-blondynka-na-hrbitove', 'info_dict': { 'id': '1756825', 'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove', From b0cda32f726443d464a68a34b22a2e02ef8b29b0 Mon Sep 17 00:00:00 2001 From: Slava Shklyaev Date: Tue, 2 Jun 2015 18:17:33 +0300 Subject: [PATCH 08/10] [nova] Fix Python 2.6 compatability issue --- youtube_dl/extractor/nova.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py index 8360a65d9..7e3498eea 100644 --- a/youtube_dl/extractor/nova.py +++ b/youtube_dl/extractor/nova.py @@ -141,7 +141,7 @@ class NovaIE(InfoExtractor): mediafile = config['mediafile'] video_url = mediafile['src'] ext = determine_ext(video_url) - video_url = video_url.replace('&{}:'.format(ext), '') + video_url = video_url.replace('&%s:' % ext, '') title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage) description = clean_html(self._og_search_description(webpage, default=None)) From fa971259e69a8031c384754b6238cfff71bea773 Mon Sep 17 00:00:00 2001 From: Slava Shklyaev Date: Tue, 2 Jun 2015 19:09:47 +0300 Subject: [PATCH 09/10] [nova] Add a comment about html in description --- youtube_dl/extractor/nova.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py index 7e3498eea..85253b6ed 100644 --- a/youtube_dl/extractor/nova.py +++ b/youtube_dl/extractor/nova.py @@ -46,7 +46,7 @@ class NovaIE(InfoExtractor): 'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove', 'ext': 'mp4', 'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově', - 'description': 'md5:dc24e50be5908df83348e50d1431295e', + 'description': 'md5:dc24e50be5908df83348e50d1431295e', # Make sure this description is clean of html tags 'thumbnail': 're:^https?://.*\.(?:jpg)', }, 'params': { From e4ac7bb1e598b0317742737ea06c162fa7f22cd4 Mon Sep 17 00:00:00 2001 From: Slava Shklyaev Date: Wed, 3 Jun 2015 19:18:41 +0300 Subject: [PATCH 10/10] [nova] Revert "Fix extension extraction bug" This reverts commit 9464a194dbf48989c486fa2de9e1aebc59e28ed4. --- youtube_dl/extractor/nova.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py index 85253b6ed..3f9c776ef 100644 --- a/youtube_dl/extractor/nova.py +++ b/youtube_dl/extractor/nova.py @@ -6,7 +6,6 @@ import re from .common import InfoExtractor from ..utils import ( clean_html, - determine_ext, unified_strdate, ) @@ -44,7 +43,7 @@ class NovaIE(InfoExtractor): 'info_dict': { 'id': '1756825', 'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove', - 'ext': 'mp4', + 'ext': 'flv', 'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově', 'description': 'md5:dc24e50be5908df83348e50d1431295e', # Make sure this description is clean of html tags 'thumbnail': 're:^https?://.*\.(?:jpg)', @@ -57,7 +56,7 @@ class NovaIE(InfoExtractor): 'url': 'http://novaplus.nova.cz/porad/televizni-noviny/video/5585-televizni-noviny-30-5-2015/', 'info_dict': { 'id': '1756858', - 'ext': 'mp4', + 'ext': 'flv', 'title': 'Televizní noviny - 30. 5. 2015', 'thumbnail': 're:^https?://.*\.(?:jpg)', 'upload_date': '20150530', @@ -140,8 +139,21 @@ class NovaIE(InfoExtractor): mediafile = config['mediafile'] video_url = mediafile['src'] - ext = determine_ext(video_url) - video_url = video_url.replace('&%s:' % ext, '') + + m = re.search(r'^(?Prtmpe?://[^/]+/(?P[^/]+?))/&*(?P.+)$', video_url) + if m: + formats = [{ + 'url': m.group('url'), + 'app': m.group('app'), + 'play_path': m.group('playpath'), + 'player_path': 'http://tvnoviny.nova.cz/static/shared/app/videojs/video-js.swf', + 'ext': 'flv', + }] + else: + formats = [{ + 'url': video_url, + }] + self._sort_formats(formats) title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage) description = clean_html(self._og_search_description(webpage, default=None)) @@ -163,6 +175,5 @@ class NovaIE(InfoExtractor): 'description': description, 'upload_date': upload_date, 'thumbnail': thumbnail, - 'url': video_url, - 'ext': ext, + 'formats': formats, }