w/youtube-dl
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

[packtpub] fix extraction(closes #21268)

Browse source
This commit is contained in:
Remita Amine 2019-07-09 08:28:39 +01:00
parent a6389abfd7
commit 25d71fb058
1 changed files with 51 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

107
youtube_dl/extractor/packtpub.py
View file

@ -5,26 +5,27 @@ import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_str, # compat_str,
compat_HTTPError, compat_HTTPError,
) )
from ..utils import ( from ..utils import (
clean_html, clean_html,
ExtractorError, ExtractorError,
remove_end, # remove_end,
str_or_none,
strip_or_none, strip_or_none,
unified_timestamp, unified_timestamp,
urljoin, # urljoin,
) )
class PacktPubBaseIE(InfoExtractor): class PacktPubBaseIE(InfoExtractor):
_PACKT_BASE = 'https://www.packtpub.com' # _PACKT_BASE = 'https://www.packtpub.com'
_MAPT_REST = '%s/mapt-rest' % _PACKT_BASE _STATIC_PRODUCTS_BASE = 'https://static.packt-cdn.com/products/'
class PacktPubIE(PacktPubBaseIE): class PacktPubIE(PacktPubBaseIE):
_VALID_URL = r'https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<course_id>\d+)/(?P<chapter_id>\d+)/(?P<id>\d+)' _VALID_URL = r'https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<course_id>\d+)/(?P<chapter_id>\d+)/(?P<id>\d+)(?:/(?P<display_id>[^/?&#]+))?'
_TESTS = [{ _TESTS = [{
'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215/20528/20530/Project+Intro', 'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215/20528/20530/Project+Intro',
@ -50,9 +51,9 @@ class PacktPubIE(PacktPubBaseIE):
return return
try: try:
self._TOKEN = self._download_json( self._TOKEN = self._download_json(
self._MAPT_REST + '/users/tokens', None, 'https://services.packtpub.com/auth-v1/users/tokens', None,
'Downloading Authorization Token', data=json.dumps({ 'Downloading Authorization Token', data=json.dumps({
'email': username, 'username': username,
'password': password, 'password': password,
}).encode())['data']['access'] }).encode())['data']['access']
except ExtractorError as e: except ExtractorError as e:
@ -61,54 +62,40 @@ class PacktPubIE(PacktPubBaseIE):
raise ExtractorError(message, expected=True) raise ExtractorError(message, expected=True)
raise raise
def _handle_error(self, response):
if response.get('status') != 'success':
raise ExtractorError(
'% said: %s' % (self.IE_NAME, response['message']),
expected=True)
def _download_json(self, *args, **kwargs):
response = super(PacktPubIE, self)._download_json(*args, **kwargs)
self._handle_error(response)
return response
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) course_id, chapter_id, video_id, display_id = re.match(self._VALID_URL, url).groups()
course_id, chapter_id, video_id = mobj.group(
'course_id', 'chapter_id', 'id')
headers = {} headers = {}
if self._TOKEN: if self._TOKEN:
headers['Authorization'] = 'Bearer ' + self._TOKEN headers['Authorization'] = 'Bearer ' + self._TOKEN
video = self._download_json( try:
'%s/users/me/products/%s/chapters/%s/sections/%s' video_url = self._download_json(
% (self._MAPT_REST, course_id, chapter_id, video_id), video_id, 'https://services.packtpub.com/products-v1/products/%s/%s/%s' % (course_id, chapter_id, video_id), video_id,
'Downloading JSON video', headers=headers)['data'] 'Downloading JSON video', headers=headers)['data']
except ExtractorError as e:
content = video.get('content') if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
if not content:
self.raise_login_required('This video is locked') self.raise_login_required('This video is locked')
raise
video_url = content['file'] # TODO: find a better way to avoid duplicating course requests
# metadata = self._download_json(
# '%s/products/%s/chapters/%s/sections/%s/metadata'
# % (self._MAPT_REST, course_id, chapter_id, video_id),
# video_id)['data']
metadata = self._download_json( # title = metadata['pageTitle']
'%s/products/%s/chapters/%s/sections/%s/metadata' # course_title = metadata.get('title')
% (self._MAPT_REST, course_id, chapter_id, video_id), # if course_title:
video_id)['data'] # title = remove_end(title, ' - %s' % course_title)
# timestamp = unified_timestamp(metadata.get('publicationDate'))
title = metadata['pageTitle'] # thumbnail = urljoin(self._PACKT_BASE, metadata.get('filepath'))
course_title = metadata.get('title')
if course_title:
title = remove_end(title, ' - %s' % course_title)
timestamp = unified_timestamp(metadata.get('publicationDate'))
thumbnail = urljoin(self._PACKT_BASE, metadata.get('filepath'))
return { return {
'id': video_id, 'id': video_id,
'url': video_url, 'url': video_url,
'title': title, 'title': display_id or video_id, # title,
'thumbnail': thumbnail, # 'thumbnail': thumbnail,
'timestamp': timestamp, # 'timestamp': timestamp,
} }
@ -119,6 +106,7 @@ class PacktPubCourseIE(PacktPubBaseIE):
'info_dict': { 'info_dict': {
'id': '9781787122215', 'id': '9781787122215',
'title': 'Learn Nodejs by building 12 projects [Video]', 'title': 'Learn Nodejs by building 12 projects [Video]',
'description': 'md5:489da8d953f416e51927b60a1c7db0aa',
}, },
'playlist_count': 90, 'playlist_count': 90,
}, { }, {
@ -136,35 +124,38 @@ class PacktPubCourseIE(PacktPubBaseIE):
url, course_id = mobj.group('url', 'id') url, course_id = mobj.group('url', 'id')
course = self._download_json( course = self._download_json(
'%s/products/%s/metadata' % (self._MAPT_REST, course_id), self._STATIC_PRODUCTS_BASE + '%s/toc' % course_id, course_id)
course_id)['data'] metadata = self._download_json(
self._STATIC_PRODUCTS_BASE + '%s/summary' % course_id,
course_id, fatal=False) or {}
entries = [] entries = []
for chapter_num, chapter in enumerate(course['tableOfContents'], 1): for chapter_num, chapter in enumerate(course['chapters'], 1):
if chapter.get('type') != 'chapter': chapter_id = str_or_none(chapter.get('id'))
continue sections = chapter.get('sections')
children = chapter.get('children') if not chapter_id or not isinstance(sections, list):
if not isinstance(children, list):
continue continue
chapter_info = { chapter_info = {
'chapter': chapter.get('title'), 'chapter': chapter.get('title'),
'chapter_number': chapter_num, 'chapter_number': chapter_num,
'chapter_id': chapter.get('id'), 'chapter_id': chapter_id,
} }
for section in children: for section in sections:
if section.get('type') != 'section': section_id = str_or_none(section.get('id'))
continue if not section_id or section.get('contentType') != 'video':
section_url = section.get('seoUrl')
if not isinstance(section_url, compat_str):
continue continue
entry = { entry = {
'_type': 'url_transparent', '_type': 'url_transparent',
'url': urljoin(url + '/', section_url), 'url': '/'.join([url, chapter_id, section_id]),
'title': strip_or_none(section.get('title')), 'title': strip_or_none(section.get('title')),
'description': clean_html(section.get('summary')), 'description': clean_html(section.get('summary')),
'thumbnail': metadata.get('coverImage'),
'timestamp': unified_timestamp(metadata.get('publicationDate')),
'ie_key': PacktPubIE.ie_key(), 'ie_key': PacktPubIE.ie_key(),
} }
entry.update(chapter_info) entry.update(chapter_info)
entries.append(entry) entries.append(entry)
return self.playlist_result(entries, course_id, course.get('title')) return self.playlist_result(
entries, course_id, metadata.get('title'),
clean_html(metadata.get('about')))