[funimation] Use mobile webpage for workaround hulu error

youtube_dl/extractor/funimation.py

@@ -18,7 +18,7 @@ from ..utils import (
 class FunimationIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/official/(?P<id>[^/?#&]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.funimation.com/shows/air/videos/official/breeze',
         'info_dict': {
             'id': '658',
@@ -28,19 +28,17 @@ class FunimationIE(InfoExtractor):
             'description': 'md5:1769f43cd5fc130ace8fd87232207892',
             'thumbnail': 're:https?://.*\.jpg',
         },
-    }
+    }, {
-
+        'url': 'http://www.funimation.com/shows/hacksign/videos/official/role-play',
-    def _download_webpage(self, url_or_request, video_id, note='Downloading webpage'):
+        'info_dict': {
-        HEADERS = {
+            'id': '31128',
-            'User-Agent': 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0',
+            'display_id': 'role-play',
-        }
+            'ext': 'mp4',
-        if isinstance(url_or_request, compat_urllib_request.Request):
+            'title': '.hack//SIGN - 1 - Role Play',
-            for header, value in HEADERS.items():
+            'description': 'md5:b602bdc15eef4c9bbb201bb6e6a4a2dd',
-                url_or_request.add_header(header, value)
+            'thumbnail': 're:https?://.*\.jpg',
-        else:
+        },
-            url_or_request = sanitized_Request(url_or_request, headers=HEADERS)
+    }]
-        response = super(FunimationIE, self)._download_webpage(url_or_request, video_id, note)
-        return response
 
     def _login(self):
         (username, password) = self._get_login_info()
@@ -51,6 +49,7 @@ class FunimationIE(InfoExtractor):
             'password_field': password,
         }))
         login_request = sanitized_Request('http://www.funimation.com/login', data, headers={
+            'User-Agent': 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0',
             'Content-Type': 'application/x-www-form-urlencoded'
         })
         login = self._download_webpage(
@@ -64,15 +63,8 @@ class FunimationIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, display_id)
+        errors = []
-
+        formats = []
-        items = self._parse_json(
-            self._search_regex(
-                r'var\s+playersData\s*=\s*(\[.+?\]);\n',
-                webpage, 'players data'),
-            display_id)[0]['playlist'][0]['items']
-
-        item = next(item for item in items if item.get('itemAK') == display_id)
 
         ERRORS_MAP = {
             'ERROR_MATURE_CONTENT_LOGGED_IN': 'matureContentLoggedIn',
@@ -87,6 +79,29 @@ class FunimationIE(InfoExtractor):
             'ERROR_STREAM_NOT_FOUND': 'streamNotFound',
         }
 
+        USER_AGENTS = (
+            # PC UA is served with m3u8 that provides some bonus lower quality formats
+            ('pc', 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0'),
+            # Mobile UA allows to extract direct links and also does not fail when
+            # PC UA fails with hulu error (e.g.
+            # http://www.funimation.com/shows/hacksign/videos/official/role-play)
+            ('mobile', 'Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36'),
+        )
+
+        for kind, user_agent in USER_AGENTS:
+            request = sanitized_Request(url)
+            request.add_header('User-Agent', user_agent)
+            webpage = self._download_webpage(
+                request, display_id, 'Downloading %s webpage' % kind)
+
+            items = self._parse_json(
+                self._search_regex(
+                    r'var\s+playersData\s*=\s*(\[.+?\]);\n',
+                    webpage, 'players data'),
+                display_id)[0]['playlist'][0]['items']
+
+            item = next(item for item in items if item.get('itemAK') == display_id)
+
             error_messages = {}
             video_error_messages = self._search_regex(
                 r'var\s+videoErrorMessages\s*=\s*({.+?});\n',
@@ -103,13 +118,12 @@ class FunimationIE(InfoExtractor):
                             if error_message:
                                 error_messages[error_message] = content
 
-        errors = []
+            for video in item.get('videoSet', []):
-        formats = []
-        for video in item['videoSet']:
                 auth_token = video.get('authToken')
                 if not auth_token:
                     continue
                 funimation_id = video.get('FUNImationID') or video.get('videoId')
+                preference = 1 if video.get('languageMode') == 'dub' else 0
                 if not auth_token.startswith('?'):
                     auth_token = '?%s' % auth_token
                 for quality in ('sd', 'hd', 'hd1080'):
@@ -122,13 +136,22 @@ class FunimationIE(InfoExtractor):
                     if determine_ext(format_url) == 'm3u8':
                         m3u8_formats = self._extract_m3u8_formats(
                             format_url + auth_token, display_id, 'mp4', entry_protocol='m3u8_native',
-                        m3u8_id=funimation_id or 'hls', fatal=False)
+                            preference=preference, m3u8_id=funimation_id or 'hls', fatal=False)
                         if m3u8_formats:
                             formats.extend(m3u8_formats)
                     else:
-                    formats.append({
+                        f = {
                             'url': format_url + auth_token,
+                            'format_id': funimation_id,
+                            'preference': preference,
+                        }
+                        mobj = re.search(r'(?P<height>\d+)-(?P<tbr>\d+)[Kk]', format_url)
+                        if mobj:
+                            f.update({
+                                'height': int(mobj.group('height')),
+                                'tbr': int(mobj.group('tbr')),
                             })
+                        formats.append(f)
 
         if not formats and errors:
             raise ExtractorError(
@@ -136,6 +159,8 @@ class FunimationIE(InfoExtractor):
                 % (self.IE_NAME, clean_html(error_messages.get(errors[0], errors[0]))),
                 expected=True)
 
+        self._sort_formats(formats)
+
         title = item['title']
         artist = item.get('artist')
         if artist: