[packtpub] fix extraction(closes #21268)
This commit is contained in:
Remita Amine
parent
a6389abfd7
commit
25d71fb058
1 changed files with 51 additions and 60 deletions
|
|
@ -5,26 +5,27 @@ import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_str,
|
# compat_str,
|
||||||
compat_HTTPError,
|
compat_HTTPError,
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
clean_html,
|
clean_html,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
remove_end,
|
# remove_end,
|
||||||
|
str_or_none,
|
||||||
strip_or_none,
|
strip_or_none,
|
||||||
unified_timestamp,
|
unified_timestamp,
|
||||||
urljoin,
|
# urljoin,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class PacktPubBaseIE(InfoExtractor):
|
class PacktPubBaseIE(InfoExtractor):
|
||||||
_PACKT_BASE = 'https://www.packtpub.com'
|
# _PACKT_BASE = 'https://www.packtpub.com'
|
||||||
_MAPT_REST = '%s/mapt-rest' % _PACKT_BASE
|
_STATIC_PRODUCTS_BASE = 'https://static.packt-cdn.com/products/'
|
||||||
|
|
||||||
|
|
||||||
class PacktPubIE(PacktPubBaseIE):
|
class PacktPubIE(PacktPubBaseIE):
|
||||||
_VALID_URL = r'https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<course_id>\d+)/(?P<chapter_id>\d+)/(?P<id>\d+)'
|
_VALID_URL = r'https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<course_id>\d+)/(?P<chapter_id>\d+)/(?P<id>\d+)(?:/(?P<display_id>[^/?&#]+))?'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215/20528/20530/Project+Intro',
|
'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215/20528/20530/Project+Intro',
|
||||||
|
|
@ -50,9 +51,9 @@ class PacktPubIE(PacktPubBaseIE):
|
||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
self._TOKEN = self._download_json(
|
self._TOKEN = self._download_json(
|
||||||
self._MAPT_REST + '/users/tokens', None,
|
'https://services.packtpub.com/auth-v1/users/tokens', None,
|
||||||
'Downloading Authorization Token', data=json.dumps({
|
'Downloading Authorization Token', data=json.dumps({
|
||||||
'email': username,
|
'username': username,
|
||||||
'password': password,
|
'password': password,
|
||||||
}).encode())['data']['access']
|
}).encode())['data']['access']
|
||||||
except ExtractorError as e:
|
except ExtractorError as e:
|
||||||
|
|
@ -61,54 +62,40 @@ class PacktPubIE(PacktPubBaseIE):
|
||||||
raise ExtractorError(message, expected=True)
|
raise ExtractorError(message, expected=True)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
def _handle_error(self, response):
|
|
||||||
if response.get('status') != 'success':
|
|
||||||
raise ExtractorError(
|
|
||||||
'% said: %s' % (self.IE_NAME, response['message']),
|
|
||||||
expected=True)
|
|
||||||
|
|
||||||
def _download_json(self, *args, **kwargs):
|
|
||||||
response = super(PacktPubIE, self)._download_json(*args, **kwargs)
|
|
||||||
self._handle_error(response)
|
|
||||||
return response
|
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
course_id, chapter_id, video_id, display_id = re.match(self._VALID_URL, url).groups()
|
||||||
course_id, chapter_id, video_id = mobj.group(
|
|
||||||
'course_id', 'chapter_id', 'id')
|
|
||||||
|
|
||||||
headers = {}
|
headers = {}
|
||||||
if self._TOKEN:
|
if self._TOKEN:
|
||||||
headers['Authorization'] = 'Bearer ' + self._TOKEN
|
headers['Authorization'] = 'Bearer ' + self._TOKEN
|
||||||
video = self._download_json(
|
try:
|
||||||
'%s/users/me/products/%s/chapters/%s/sections/%s'
|
video_url = self._download_json(
|
||||||
% (self._MAPT_REST, course_id, chapter_id, video_id), video_id,
|
'https://services.packtpub.com/products-v1/products/%s/%s/%s' % (course_id, chapter_id, video_id), video_id,
|
||||||
'Downloading JSON video', headers=headers)['data']
|
'Downloading JSON video', headers=headers)['data']
|
||||||
|
except ExtractorError as e:
|
||||||
|
if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
|
||||||
|
self.raise_login_required('This video is locked')
|
||||||
|
raise
|
||||||
|
|
||||||
content = video.get('content')
|
# TODO: find a better way to avoid duplicating course requests
|
||||||
if not content:
|
# metadata = self._download_json(
|
||||||
self.raise_login_required('This video is locked')
|
# '%s/products/%s/chapters/%s/sections/%s/metadata'
|
||||||
|
# % (self._MAPT_REST, course_id, chapter_id, video_id),
|
||||||
|
# video_id)['data']
|
||||||
|
|
||||||
video_url = content['file']
|
# title = metadata['pageTitle']
|
||||||
|
# course_title = metadata.get('title')
|
||||||
metadata = self._download_json(
|
# if course_title:
|
||||||
'%s/products/%s/chapters/%s/sections/%s/metadata'
|
# title = remove_end(title, ' - %s' % course_title)
|
||||||
% (self._MAPT_REST, course_id, chapter_id, video_id),
|
# timestamp = unified_timestamp(metadata.get('publicationDate'))
|
||||||
video_id)['data']
|
# thumbnail = urljoin(self._PACKT_BASE, metadata.get('filepath'))
|
||||||
|
|
||||||
title = metadata['pageTitle']
|
|
||||||
course_title = metadata.get('title')
|
|
||||||
if course_title:
|
|
||||||
title = remove_end(title, ' - %s' % course_title)
|
|
||||||
timestamp = unified_timestamp(metadata.get('publicationDate'))
|
|
||||||
thumbnail = urljoin(self._PACKT_BASE, metadata.get('filepath'))
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'url': video_url,
|
'url': video_url,
|
||||||
'title': title,
|
'title': display_id or video_id, # title,
|
||||||
'thumbnail': thumbnail,
|
# 'thumbnail': thumbnail,
|
||||||
'timestamp': timestamp,
|
# 'timestamp': timestamp,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -119,6 +106,7 @@ class PacktPubCourseIE(PacktPubBaseIE):
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '9781787122215',
|
'id': '9781787122215',
|
||||||
'title': 'Learn Nodejs by building 12 projects [Video]',
|
'title': 'Learn Nodejs by building 12 projects [Video]',
|
||||||
|
'description': 'md5:489da8d953f416e51927b60a1c7db0aa',
|
||||||
},
|
},
|
||||||
'playlist_count': 90,
|
'playlist_count': 90,
|
||||||
}, {
|
}, {
|
||||||
|
|
@ -136,35 +124,38 @@ class PacktPubCourseIE(PacktPubBaseIE):
|
||||||
url, course_id = mobj.group('url', 'id')
|
url, course_id = mobj.group('url', 'id')
|
||||||
|
|
||||||
course = self._download_json(
|
course = self._download_json(
|
||||||
'%s/products/%s/metadata' % (self._MAPT_REST, course_id),
|
self._STATIC_PRODUCTS_BASE + '%s/toc' % course_id, course_id)
|
||||||
course_id)['data']
|
metadata = self._download_json(
|
||||||
|
self._STATIC_PRODUCTS_BASE + '%s/summary' % course_id,
|
||||||
|
course_id, fatal=False) or {}
|
||||||
|
|
||||||
entries = []
|
entries = []
|
||||||
for chapter_num, chapter in enumerate(course['tableOfContents'], 1):
|
for chapter_num, chapter in enumerate(course['chapters'], 1):
|
||||||
if chapter.get('type') != 'chapter':
|
chapter_id = str_or_none(chapter.get('id'))
|
||||||
continue
|
sections = chapter.get('sections')
|
||||||
children = chapter.get('children')
|
if not chapter_id or not isinstance(sections, list):
|
||||||
if not isinstance(children, list):
|
|
||||||
continue
|
continue
|
||||||
chapter_info = {
|
chapter_info = {
|
||||||
'chapter': chapter.get('title'),
|
'chapter': chapter.get('title'),
|
||||||
'chapter_number': chapter_num,
|
'chapter_number': chapter_num,
|
||||||
'chapter_id': chapter.get('id'),
|
'chapter_id': chapter_id,
|
||||||
}
|
}
|
||||||
for section in children:
|
for section in sections:
|
||||||
if section.get('type') != 'section':
|
section_id = str_or_none(section.get('id'))
|
||||||
continue
|
if not section_id or section.get('contentType') != 'video':
|
||||||
section_url = section.get('seoUrl')
|
|
||||||
if not isinstance(section_url, compat_str):
|
|
||||||
continue
|
continue
|
||||||
entry = {
|
entry = {
|
||||||
'_type': 'url_transparent',
|
'_type': 'url_transparent',
|
||||||
'url': urljoin(url + '/', section_url),
|
'url': '/'.join([url, chapter_id, section_id]),
|
||||||
'title': strip_or_none(section.get('title')),
|
'title': strip_or_none(section.get('title')),
|
||||||
'description': clean_html(section.get('summary')),
|
'description': clean_html(section.get('summary')),
|
||||||
|
'thumbnail': metadata.get('coverImage'),
|
||||||
|
'timestamp': unified_timestamp(metadata.get('publicationDate')),
|
||||||
'ie_key': PacktPubIE.ie_key(),
|
'ie_key': PacktPubIE.ie_key(),
|
||||||
}
|
}
|
||||||
entry.update(chapter_info)
|
entry.update(chapter_info)
|
||||||
entries.append(entry)
|
entries.append(entry)
|
||||||
|
|
||||||
return self.playlist_result(entries, course_id, course.get('title'))
|
return self.playlist_result(
|
||||||
|
entries, course_id, metadata.get('title'),
|
||||||
|
clean_html(metadata.get('about')))
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue