Convert all tabs to 4 spaces (PEP8)
This commit is contained in:
Philipp Hagemeister
parent
40b35b4aa6
commit
59ae15a507
9 changed files with 4841 additions and 4841 deletions
|
|
@ -13,186 +13,186 @@ from youtube_dl.InfoExtractors import CollegeHumorIE, XNXXIE
|
|||
|
||||
|
||||
class DownloadTest(unittest.TestCase):
|
||||
PARAMETERS_FILE = "test/parameters.json"
|
||||
#calculated with md5sum:
|
||||
#md5sum (GNU coreutils) 8.19
|
||||
PARAMETERS_FILE = "test/parameters.json"
|
||||
#calculated with md5sum:
|
||||
#md5sum (GNU coreutils) 8.19
|
||||
|
||||
YOUTUBE_SIZE = 1993883
|
||||
YOUTUBE_URL = "http://www.youtube.com/watch?v=BaW_jenozKc"
|
||||
YOUTUBE_FILE = "BaW_jenozKc.mp4"
|
||||
YOUTUBE_SIZE = 1993883
|
||||
YOUTUBE_URL = "http://www.youtube.com/watch?v=BaW_jenozKc"
|
||||
YOUTUBE_FILE = "BaW_jenozKc.mp4"
|
||||
|
||||
DAILYMOTION_MD5 = "d363a50e9eb4f22ce90d08d15695bb47"
|
||||
DAILYMOTION_URL = "http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech"
|
||||
DAILYMOTION_FILE = "x33vw9.mp4"
|
||||
DAILYMOTION_MD5 = "d363a50e9eb4f22ce90d08d15695bb47"
|
||||
DAILYMOTION_URL = "http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech"
|
||||
DAILYMOTION_FILE = "x33vw9.mp4"
|
||||
|
||||
METACAFE_SIZE = 5754305
|
||||
METACAFE_URL = "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/"
|
||||
METACAFE_FILE = "_aUehQsCQtM.flv"
|
||||
METACAFE_SIZE = 5754305
|
||||
METACAFE_URL = "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/"
|
||||
METACAFE_FILE = "_aUehQsCQtM.flv"
|
||||
|
||||
BLIP_MD5 = "93c24d2f4e0782af13b8a7606ea97ba7"
|
||||
BLIP_URL = "http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352"
|
||||
BLIP_FILE = "5779306.m4v"
|
||||
BLIP_MD5 = "93c24d2f4e0782af13b8a7606ea97ba7"
|
||||
BLIP_URL = "http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352"
|
||||
BLIP_FILE = "5779306.m4v"
|
||||
|
||||
XVIDEO_MD5 = "1ab4dedc01f771cb2a65e91caa801aaf"
|
||||
XVIDEO_URL = "http://www.xvideos.com/video939581/funny_porns_by_s_-1"
|
||||
XVIDEO_FILE = "939581.flv"
|
||||
XVIDEO_MD5 = "1ab4dedc01f771cb2a65e91caa801aaf"
|
||||
XVIDEO_URL = "http://www.xvideos.com/video939581/funny_porns_by_s_-1"
|
||||
XVIDEO_FILE = "939581.flv"
|
||||
|
||||
VIMEO_MD5 = "1ab4dedc01f771cb2a65e91caa801aaf"
|
||||
VIMEO_URL = "http://vimeo.com/14160053"
|
||||
VIMEO_FILE = ""
|
||||
VIMEO_MD5 = "1ab4dedc01f771cb2a65e91caa801aaf"
|
||||
VIMEO_URL = "http://vimeo.com/14160053"
|
||||
VIMEO_FILE = ""
|
||||
|
||||
VIMEO2_MD5 = ""
|
||||
VIMEO2_URL = "http://player.vimeo.com/video/47019590"
|
||||
VIMEO2_FILE = ""
|
||||
VIMEO2_MD5 = ""
|
||||
VIMEO2_URL = "http://player.vimeo.com/video/47019590"
|
||||
VIMEO2_FILE = ""
|
||||
|
||||
SOUNDCLOUD_MD5 = "ce3775768ebb6432fa8495d446a078ed"
|
||||
SOUNDCLOUD_URL = "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy"
|
||||
SOUNDCLOUD_FILE = "n6FLbx6ZzMiu.mp3"
|
||||
SOUNDCLOUD_MD5 = "ce3775768ebb6432fa8495d446a078ed"
|
||||
SOUNDCLOUD_URL = "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy"
|
||||
SOUNDCLOUD_FILE = "n6FLbx6ZzMiu.mp3"
|
||||
|
||||
STANDFORD_MD5 = "22c8206291368c4e2c9c1a307f0ea0f4"
|
||||
STANDFORD_URL = "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100"
|
||||
STANDFORD_FILE = "PracticalUnix_intro-environment.mp4"
|
||||
STANDFORD_MD5 = "22c8206291368c4e2c9c1a307f0ea0f4"
|
||||
STANDFORD_URL = "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100"
|
||||
STANDFORD_FILE = "PracticalUnix_intro-environment.mp4"
|
||||
|
||||
COLLEGEHUMOR_MD5 = ""
|
||||
COLLEGEHUMOR_URL = "http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody"
|
||||
COLLEGEHUMOR_FILE = ""
|
||||
COLLEGEHUMOR_MD5 = ""
|
||||
COLLEGEHUMOR_URL = "http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody"
|
||||
COLLEGEHUMOR_FILE = ""
|
||||
|
||||
XNXX_MD5 = "5f0469c8d1dfd1bc38c8e6deb5e0a21d"
|
||||
XNXX_URL = "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_"
|
||||
XNXX_FILE = "1135332.flv"
|
||||
XNXX_MD5 = "5f0469c8d1dfd1bc38c8e6deb5e0a21d"
|
||||
XNXX_URL = "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_"
|
||||
XNXX_FILE = "1135332.flv"
|
||||
|
||||
def test_youtube(self):
|
||||
#let's download a file from youtube
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(YoutubeIE())
|
||||
fd.download([DownloadTest.YOUTUBE_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.YOUTUBE_FILE))
|
||||
self.assertEqual(os.path.getsize(DownloadTest.YOUTUBE_FILE), DownloadTest.YOUTUBE_SIZE)
|
||||
def test_youtube(self):
|
||||
#let's download a file from youtube
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(YoutubeIE())
|
||||
fd.download([DownloadTest.YOUTUBE_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.YOUTUBE_FILE))
|
||||
self.assertEqual(os.path.getsize(DownloadTest.YOUTUBE_FILE), DownloadTest.YOUTUBE_SIZE)
|
||||
|
||||
def test_dailymotion(self):
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(DailymotionIE())
|
||||
fd.download([DownloadTest.DAILYMOTION_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.DAILYMOTION_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.DAILYMOTION_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.DAILYMOTION_MD5)
|
||||
def test_dailymotion(self):
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(DailymotionIE())
|
||||
fd.download([DownloadTest.DAILYMOTION_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.DAILYMOTION_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.DAILYMOTION_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.DAILYMOTION_MD5)
|
||||
|
||||
def test_metacafe(self):
|
||||
#this emulate a skip,to be 2.6 compatible
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(MetacafeIE())
|
||||
fd.add_info_extractor(YoutubeIE())
|
||||
fd.download([DownloadTest.METACAFE_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.METACAFE_FILE))
|
||||
self.assertEqual(os.path.getsize(DownloadTest.METACAFE_FILE), DownloadTest.METACAFE_SIZE)
|
||||
def test_metacafe(self):
|
||||
#this emulate a skip,to be 2.6 compatible
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(MetacafeIE())
|
||||
fd.add_info_extractor(YoutubeIE())
|
||||
fd.download([DownloadTest.METACAFE_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.METACAFE_FILE))
|
||||
self.assertEqual(os.path.getsize(DownloadTest.METACAFE_FILE), DownloadTest.METACAFE_SIZE)
|
||||
|
||||
def test_blip(self):
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(BlipTVIE())
|
||||
fd.download([DownloadTest.BLIP_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.BLIP_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.BLIP_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.BLIP_MD5)
|
||||
def test_blip(self):
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(BlipTVIE())
|
||||
fd.download([DownloadTest.BLIP_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.BLIP_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.BLIP_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.BLIP_MD5)
|
||||
|
||||
def test_xvideo(self):
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(XVideosIE())
|
||||
fd.download([DownloadTest.XVIDEO_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.XVIDEO_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.XVIDEO_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.XVIDEO_MD5)
|
||||
def test_xvideo(self):
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(XVideosIE())
|
||||
fd.download([DownloadTest.XVIDEO_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.XVIDEO_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.XVIDEO_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.XVIDEO_MD5)
|
||||
|
||||
def test_vimeo(self):
|
||||
#skipped for the moment produce an error
|
||||
return
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(VimeoIE())
|
||||
fd.download([DownloadTest.VIMEO_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.VIMEO_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.VIMEO_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.VIMEO_MD5)
|
||||
def test_vimeo(self):
|
||||
#skipped for the moment produce an error
|
||||
return
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(VimeoIE())
|
||||
fd.download([DownloadTest.VIMEO_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.VIMEO_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.VIMEO_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.VIMEO_MD5)
|
||||
|
||||
def test_vimeo2(self):
|
||||
#skipped for the moment produce an error
|
||||
return
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(VimeoIE())
|
||||
fd.download([DownloadTest.VIMEO2_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.VIMEO2_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.VIMEO2_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.VIMEO2_MD5)
|
||||
def test_vimeo2(self):
|
||||
#skipped for the moment produce an error
|
||||
return
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(VimeoIE())
|
||||
fd.download([DownloadTest.VIMEO2_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.VIMEO2_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.VIMEO2_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.VIMEO2_MD5)
|
||||
|
||||
def test_soundcloud(self):
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(SoundcloudIE())
|
||||
fd.download([DownloadTest.SOUNDCLOUD_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.SOUNDCLOUD_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.SOUNDCLOUD_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.SOUNDCLOUD_MD5)
|
||||
def test_soundcloud(self):
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(SoundcloudIE())
|
||||
fd.download([DownloadTest.SOUNDCLOUD_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.SOUNDCLOUD_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.SOUNDCLOUD_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.SOUNDCLOUD_MD5)
|
||||
|
||||
def test_standford(self):
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(StanfordOpenClassroomIE())
|
||||
fd.download([DownloadTest.STANDFORD_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.STANDFORD_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.STANDFORD_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.STANDFORD_MD5)
|
||||
def test_standford(self):
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(StanfordOpenClassroomIE())
|
||||
fd.download([DownloadTest.STANDFORD_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.STANDFORD_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.STANDFORD_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.STANDFORD_MD5)
|
||||
|
||||
def test_collegehumor(self):
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(CollegeHumorIE())
|
||||
fd.download([DownloadTest.COLLEGEHUMOR_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.COLLEGEHUMOR_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.COLLEGEHUMOR_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.COLLEGEHUMOR_MD5)
|
||||
def test_collegehumor(self):
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(CollegeHumorIE())
|
||||
fd.download([DownloadTest.COLLEGEHUMOR_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.COLLEGEHUMOR_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.COLLEGEHUMOR_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.COLLEGEHUMOR_MD5)
|
||||
|
||||
def test_xnxx(self):
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(XNXXIE())
|
||||
fd.download([DownloadTest.XNXX_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.XNXX_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.XNXX_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.XNXX_MD5)
|
||||
def test_xnxx(self):
|
||||
with open(DownloadTest.PARAMETERS_FILE) as f:
|
||||
fd = FileDownloader(json.load(f))
|
||||
fd.add_info_extractor(XNXXIE())
|
||||
fd.download([DownloadTest.XNXX_URL])
|
||||
self.assertTrue(os.path.exists(DownloadTest.XNXX_FILE))
|
||||
md5_down_file = md5_for_file(DownloadTest.XNXX_FILE)
|
||||
self.assertEqual(md5_down_file, DownloadTest.XNXX_MD5)
|
||||
|
||||
def tearDown(self):
|
||||
if os.path.exists(DownloadTest.YOUTUBE_FILE):
|
||||
os.remove(DownloadTest.YOUTUBE_FILE)
|
||||
if os.path.exists(DownloadTest.DAILYMOTION_FILE):
|
||||
os.remove(DownloadTest.DAILYMOTION_FILE)
|
||||
if os.path.exists(DownloadTest.METACAFE_FILE):
|
||||
os.remove(DownloadTest.METACAFE_FILE)
|
||||
if os.path.exists(DownloadTest.BLIP_FILE):
|
||||
os.remove(DownloadTest.BLIP_FILE)
|
||||
if os.path.exists(DownloadTest.XVIDEO_FILE):
|
||||
os.remove(DownloadTest.XVIDEO_FILE)
|
||||
if os.path.exists(DownloadTest.VIMEO_FILE):
|
||||
os.remove(DownloadTest.VIMEO_FILE)
|
||||
if os.path.exists(DownloadTest.SOUNDCLOUD_FILE):
|
||||
os.remove(DownloadTest.SOUNDCLOUD_FILE)
|
||||
if os.path.exists(DownloadTest.STANDFORD_FILE):
|
||||
os.remove(DownloadTest.STANDFORD_FILE)
|
||||
if os.path.exists(DownloadTest.COLLEGEHUMOR_FILE):
|
||||
os.remove(DownloadTest.COLLEGEHUMOR_FILE)
|
||||
if os.path.exists(DownloadTest.XNXX_FILE):
|
||||
os.remove(DownloadTest.XNXX_FILE)
|
||||
def tearDown(self):
|
||||
if os.path.exists(DownloadTest.YOUTUBE_FILE):
|
||||
os.remove(DownloadTest.YOUTUBE_FILE)
|
||||
if os.path.exists(DownloadTest.DAILYMOTION_FILE):
|
||||
os.remove(DownloadTest.DAILYMOTION_FILE)
|
||||
if os.path.exists(DownloadTest.METACAFE_FILE):
|
||||
os.remove(DownloadTest.METACAFE_FILE)
|
||||
if os.path.exists(DownloadTest.BLIP_FILE):
|
||||
os.remove(DownloadTest.BLIP_FILE)
|
||||
if os.path.exists(DownloadTest.XVIDEO_FILE):
|
||||
os.remove(DownloadTest.XVIDEO_FILE)
|
||||
if os.path.exists(DownloadTest.VIMEO_FILE):
|
||||
os.remove(DownloadTest.VIMEO_FILE)
|
||||
if os.path.exists(DownloadTest.SOUNDCLOUD_FILE):
|
||||
os.remove(DownloadTest.SOUNDCLOUD_FILE)
|
||||
if os.path.exists(DownloadTest.STANDFORD_FILE):
|
||||
os.remove(DownloadTest.STANDFORD_FILE)
|
||||
if os.path.exists(DownloadTest.COLLEGEHUMOR_FILE):
|
||||
os.remove(DownloadTest.COLLEGEHUMOR_FILE)
|
||||
if os.path.exists(DownloadTest.XNXX_FILE):
|
||||
os.remove(DownloadTest.XNXX_FILE)
|
||||
|
||||
def md5_for_file(filename, block_size=2**20):
|
||||
with open(filename) as f:
|
||||
md5 = hashlib.md5()
|
||||
while True:
|
||||
data = f.read(block_size)
|
||||
if not data:
|
||||
break
|
||||
md5.update(data)
|
||||
return md5.hexdigest()
|
||||
with open(filename) as f:
|
||||
md5 = hashlib.md5()
|
||||
while True:
|
||||
data = f.read(block_size)
|
||||
if not data:
|
||||
break
|
||||
md5.update(data)
|
||||
return md5.hexdigest()
|
||||
|
|
|
|||
|
|
@ -5,9 +5,9 @@ import os.path
|
|||
import subprocess
|
||||
|
||||
class TestImport(unittest.TestCase):
|
||||
def test_import(self):
|
||||
rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
|
||||
def test_import(self):
|
||||
rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -14,79 +14,79 @@ from youtube_dl.utils import unescapeHTML
|
|||
from youtube_dl.utils import orderedSet
|
||||
|
||||
if sys.version_info < (3,0):
|
||||
_compat_str = lambda b: b.decode('unicode-escape')
|
||||
_compat_str = lambda b: b.decode('unicode-escape')
|
||||
else:
|
||||
_compat_str = lambda s: s
|
||||
_compat_str = lambda s: s
|
||||
|
||||
class TestUtil(unittest.TestCase):
|
||||
def test_timeconvert(self):
|
||||
self.assertTrue(timeconvert('') is None)
|
||||
self.assertTrue(timeconvert('bougrg') is None)
|
||||
def test_timeconvert(self):
|
||||
self.assertTrue(timeconvert('') is None)
|
||||
self.assertTrue(timeconvert('bougrg') is None)
|
||||
|
||||
def test_sanitize_filename(self):
|
||||
self.assertEqual(sanitize_filename('abc'), 'abc')
|
||||
self.assertEqual(sanitize_filename('abc_d-e'), 'abc_d-e')
|
||||
def test_sanitize_filename(self):
|
||||
self.assertEqual(sanitize_filename('abc'), 'abc')
|
||||
self.assertEqual(sanitize_filename('abc_d-e'), 'abc_d-e')
|
||||
|
||||
self.assertEqual(sanitize_filename('123'), '123')
|
||||
self.assertEqual(sanitize_filename('123'), '123')
|
||||
|
||||
self.assertEqual('abc_de', sanitize_filename('abc/de'))
|
||||
self.assertFalse('/' in sanitize_filename('abc/de///'))
|
||||
self.assertEqual('abc_de', sanitize_filename('abc/de'))
|
||||
self.assertFalse('/' in sanitize_filename('abc/de///'))
|
||||
|
||||
self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de'))
|
||||
self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|'))
|
||||
self.assertEqual('yes no', sanitize_filename('yes? no'))
|
||||
self.assertEqual('this - that', sanitize_filename('this: that'))
|
||||
self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de'))
|
||||
self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|'))
|
||||
self.assertEqual('yes no', sanitize_filename('yes? no'))
|
||||
self.assertEqual('this - that', sanitize_filename('this: that'))
|
||||
|
||||
self.assertEqual(sanitize_filename('AT&T'), 'AT&T')
|
||||
aumlaut = _compat_str('\xe4')
|
||||
self.assertEqual(sanitize_filename(aumlaut), aumlaut)
|
||||
tests = _compat_str('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0446\u0430')
|
||||
self.assertEqual(sanitize_filename(tests), tests)
|
||||
self.assertEqual(sanitize_filename('AT&T'), 'AT&T')
|
||||
aumlaut = _compat_str('\xe4')
|
||||
self.assertEqual(sanitize_filename(aumlaut), aumlaut)
|
||||
tests = _compat_str('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0446\u0430')
|
||||
self.assertEqual(sanitize_filename(tests), tests)
|
||||
|
||||
forbidden = '"\0\\/'
|
||||
for fc in forbidden:
|
||||
for fbc in forbidden:
|
||||
self.assertTrue(fbc not in sanitize_filename(fc))
|
||||
forbidden = '"\0\\/'
|
||||
for fc in forbidden:
|
||||
for fbc in forbidden:
|
||||
self.assertTrue(fbc not in sanitize_filename(fc))
|
||||
|
||||
def test_sanitize_filename_restricted(self):
|
||||
self.assertEqual(sanitize_filename('abc', restricted=True), 'abc')
|
||||
self.assertEqual(sanitize_filename('abc_d-e', restricted=True), 'abc_d-e')
|
||||
def test_sanitize_filename_restricted(self):
|
||||
self.assertEqual(sanitize_filename('abc', restricted=True), 'abc')
|
||||
self.assertEqual(sanitize_filename('abc_d-e', restricted=True), 'abc_d-e')
|
||||
|
||||
self.assertEqual(sanitize_filename('123', restricted=True), '123')
|
||||
self.assertEqual(sanitize_filename('123', restricted=True), '123')
|
||||
|
||||
self.assertEqual('abc_de', sanitize_filename('abc/de', restricted=True))
|
||||
self.assertFalse('/' in sanitize_filename('abc/de///', restricted=True))
|
||||
self.assertEqual('abc_de', sanitize_filename('abc/de', restricted=True))
|
||||
self.assertFalse('/' in sanitize_filename('abc/de///', restricted=True))
|
||||
|
||||
self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de', restricted=True))
|
||||
self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|', restricted=True))
|
||||
self.assertEqual('yes_no', sanitize_filename('yes? no', restricted=True))
|
||||
self.assertEqual('this_-_that', sanitize_filename('this: that', restricted=True))
|
||||
self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de', restricted=True))
|
||||
self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|', restricted=True))
|
||||
self.assertEqual('yes_no', sanitize_filename('yes? no', restricted=True))
|
||||
self.assertEqual('this_-_that', sanitize_filename('this: that', restricted=True))
|
||||
|
||||
tests =_compat_str('a\xe4b\u4e2d\u56fd\u7684c')
|
||||
self.assertEqual(sanitize_filename(tests, restricted=True), 'a_b_c')
|
||||
self.assertTrue(sanitize_filename(_compat_str('\xf6'), restricted=True) != '') # No empty filename
|
||||
tests =_compat_str('a\xe4b\u4e2d\u56fd\u7684c')
|
||||
self.assertEqual(sanitize_filename(tests, restricted=True), 'a_b_c')
|
||||
self.assertTrue(sanitize_filename(_compat_str('\xf6'), restricted=True) != '') # No empty filename
|
||||
|
||||
forbidden = '"\0\\/&!: \'\t\n'
|
||||
for fc in forbidden:
|
||||
for fbc in forbidden:
|
||||
self.assertTrue(fbc not in sanitize_filename(fc, restricted=True))
|
||||
forbidden = '"\0\\/&!: \'\t\n'
|
||||
for fc in forbidden:
|
||||
for fbc in forbidden:
|
||||
self.assertTrue(fbc not in sanitize_filename(fc, restricted=True))
|
||||
|
||||
# Handle a common case more neatly
|
||||
self.assertEqual(sanitize_filename(_compat_str('\u5927\u58f0\u5e26 - Song'), restricted=True), 'Song')
|
||||
self.assertEqual(sanitize_filename(_compat_str('\u603b\u7edf: Speech'), restricted=True), 'Speech')
|
||||
# .. but make sure the file name is never empty
|
||||
self.assertTrue(sanitize_filename('-', restricted=True) != '')
|
||||
self.assertTrue(sanitize_filename(':', restricted=True) != '')
|
||||
# Handle a common case more neatly
|
||||
self.assertEqual(sanitize_filename(_compat_str('\u5927\u58f0\u5e26 - Song'), restricted=True), 'Song')
|
||||
self.assertEqual(sanitize_filename(_compat_str('\u603b\u7edf: Speech'), restricted=True), 'Speech')
|
||||
# .. but make sure the file name is never empty
|
||||
self.assertTrue(sanitize_filename('-', restricted=True) != '')
|
||||
self.assertTrue(sanitize_filename(':', restricted=True) != '')
|
||||
|
||||
def test_ordered_set(self):
|
||||
self.assertEqual(orderedSet([1,1,2,3,4,4,5,6,7,3,5]), [1,2,3,4,5,6,7])
|
||||
self.assertEqual(orderedSet([]), [])
|
||||
self.assertEqual(orderedSet([1]), [1])
|
||||
#keep the list ordered
|
||||
self.assertEqual(orderedSet([135,1,1,1]), [135,1])
|
||||
def test_ordered_set(self):
|
||||
self.assertEqual(orderedSet([1,1,2,3,4,4,5,6,7,3,5]), [1,2,3,4,5,6,7])
|
||||
self.assertEqual(orderedSet([]), [])
|
||||
self.assertEqual(orderedSet([1]), [1])
|
||||
#keep the list ordered
|
||||
self.assertEqual(orderedSet([135,1,1,1]), [135,1])
|
||||
|
||||
def test_unescape_html(self):
|
||||
self.assertEqual(unescapeHTML(_compat_str('%20;')), _compat_str('%20;'))
|
||||
def test_unescape_html(self):
|
||||
self.assertEqual(unescapeHTML(_compat_str('%20;')), _compat_str('%20;'))
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
unittest.main()
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -10,189 +10,189 @@ from utils import *
|
|||
|
||||
|
||||
class PostProcessor(object):
|
||||
"""Post Processor class.
|
||||
"""Post Processor class.
|
||||
|
||||
PostProcessor objects can be added to downloaders with their
|
||||
add_post_processor() method. When the downloader has finished a
|
||||
successful download, it will take its internal chain of PostProcessors
|
||||
and start calling the run() method on each one of them, first with
|
||||
an initial argument and then with the returned value of the previous
|
||||
PostProcessor.
|
||||
PostProcessor objects can be added to downloaders with their
|
||||
add_post_processor() method. When the downloader has finished a
|
||||
successful download, it will take its internal chain of PostProcessors
|
||||
and start calling the run() method on each one of them, first with
|
||||
an initial argument and then with the returned value of the previous
|
||||
PostProcessor.
|
||||
|
||||
The chain will be stopped if one of them ever returns None or the end
|
||||
of the chain is reached.
|
||||
The chain will be stopped if one of them ever returns None or the end
|
||||
of the chain is reached.
|
||||
|
||||
PostProcessor objects follow a "mutual registration" process similar
|
||||
to InfoExtractor objects.
|
||||
"""
|
||||
PostProcessor objects follow a "mutual registration" process similar
|
||||
to InfoExtractor objects.
|
||||
"""
|
||||
|
||||
_downloader = None
|
||||
_downloader = None
|
||||
|
||||
def __init__(self, downloader=None):
|
||||
self._downloader = downloader
|
||||
def __init__(self, downloader=None):
|
||||
self._downloader = downloader
|
||||
|
||||
def set_downloader(self, downloader):
|
||||
"""Sets the downloader for this PP."""
|
||||
self._downloader = downloader
|
||||
def set_downloader(self, downloader):
|
||||
"""Sets the downloader for this PP."""
|
||||
self._downloader = downloader
|
||||
|
||||
def run(self, information):
|
||||
"""Run the PostProcessor.
|
||||
def run(self, information):
|
||||
"""Run the PostProcessor.
|
||||
|
||||
The "information" argument is a dictionary like the ones
|
||||
composed by InfoExtractors. The only difference is that this
|
||||
one has an extra field called "filepath" that points to the
|
||||
downloaded file.
|
||||
The "information" argument is a dictionary like the ones
|
||||
composed by InfoExtractors. The only difference is that this
|
||||
one has an extra field called "filepath" that points to the
|
||||
downloaded file.
|
||||
|
||||
When this method returns None, the postprocessing chain is
|
||||
stopped. However, this method may return an information
|
||||
dictionary that will be passed to the next postprocessing
|
||||
object in the chain. It can be the one it received after
|
||||
changing some fields.
|
||||
When this method returns None, the postprocessing chain is
|
||||
stopped. However, this method may return an information
|
||||
dictionary that will be passed to the next postprocessing
|
||||
object in the chain. It can be the one it received after
|
||||
changing some fields.
|
||||
|
||||
In addition, this method may raise a PostProcessingError
|
||||
exception that will be taken into account by the downloader
|
||||
it was called from.
|
||||
"""
|
||||
return information # by default, do nothing
|
||||
In addition, this method may raise a PostProcessingError
|
||||
exception that will be taken into account by the downloader
|
||||
it was called from.
|
||||
"""
|
||||
return information # by default, do nothing
|
||||
|
||||
class AudioConversionError(BaseException):
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
|
||||
class FFmpegExtractAudioPP(PostProcessor):
|
||||
def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False):
|
||||
PostProcessor.__init__(self, downloader)
|
||||
if preferredcodec is None:
|
||||
preferredcodec = 'best'
|
||||
self._preferredcodec = preferredcodec
|
||||
self._preferredquality = preferredquality
|
||||
self._keepvideo = keepvideo
|
||||
self._exes = self.detect_executables()
|
||||
def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False):
|
||||
PostProcessor.__init__(self, downloader)
|
||||
if preferredcodec is None:
|
||||
preferredcodec = 'best'
|
||||
self._preferredcodec = preferredcodec
|
||||
self._preferredquality = preferredquality
|
||||
self._keepvideo = keepvideo
|
||||
self._exes = self.detect_executables()
|
||||
|
||||
@staticmethod
|
||||
def detect_executables():
|
||||
def executable(exe):
|
||||
try:
|
||||
subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
|
||||
except OSError:
|
||||
return False
|
||||
return exe
|
||||
programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
|
||||
return dict((program, executable(program)) for program in programs)
|
||||
@staticmethod
|
||||
def detect_executables():
|
||||
def executable(exe):
|
||||
try:
|
||||
subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
|
||||
except OSError:
|
||||
return False
|
||||
return exe
|
||||
programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
|
||||
return dict((program, executable(program)) for program in programs)
|
||||
|
||||
def get_audio_codec(self, path):
|
||||
if not self._exes['ffprobe'] and not self._exes['avprobe']: return None
|
||||
try:
|
||||
cmd = [self._exes['avprobe'] or self._exes['ffprobe'], '-show_streams', '--', encodeFilename(path)]
|
||||
handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
|
||||
output = handle.communicate()[0]
|
||||
if handle.wait() != 0:
|
||||
return None
|
||||
except (IOError, OSError):
|
||||
return None
|
||||
audio_codec = None
|
||||
for line in output.split('\n'):
|
||||
if line.startswith('codec_name='):
|
||||
audio_codec = line.split('=')[1].strip()
|
||||
elif line.strip() == 'codec_type=audio' and audio_codec is not None:
|
||||
return audio_codec
|
||||
return None
|
||||
def get_audio_codec(self, path):
|
||||
if not self._exes['ffprobe'] and not self._exes['avprobe']: return None
|
||||
try:
|
||||
cmd = [self._exes['avprobe'] or self._exes['ffprobe'], '-show_streams', '--', encodeFilename(path)]
|
||||
handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
|
||||
output = handle.communicate()[0]
|
||||
if handle.wait() != 0:
|
||||
return None
|
||||
except (IOError, OSError):
|
||||
return None
|
||||
audio_codec = None
|
||||
for line in output.split('\n'):
|
||||
if line.startswith('codec_name='):
|
||||
audio_codec = line.split('=')[1].strip()
|
||||
elif line.strip() == 'codec_type=audio' and audio_codec is not None:
|
||||
return audio_codec
|
||||
return None
|
||||
|
||||
def run_ffmpeg(self, path, out_path, codec, more_opts):
|
||||
if not self._exes['ffmpeg'] and not self._exes['avconv']:
|
||||
raise AudioConversionError('ffmpeg or avconv not found. Please install one.')
|
||||
if codec is None:
|
||||
acodec_opts = []
|
||||
else:
|
||||
acodec_opts = ['-acodec', codec]
|
||||
cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y', '-i', encodeFilename(path), '-vn']
|
||||
+ acodec_opts + more_opts +
|
||||
['--', encodeFilename(out_path)])
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
stdout,stderr = p.communicate()
|
||||
if p.returncode != 0:
|
||||
msg = stderr.strip().split('\n')[-1]
|
||||
raise AudioConversionError(msg)
|
||||
def run_ffmpeg(self, path, out_path, codec, more_opts):
|
||||
if not self._exes['ffmpeg'] and not self._exes['avconv']:
|
||||
raise AudioConversionError('ffmpeg or avconv not found. Please install one.')
|
||||
if codec is None:
|
||||
acodec_opts = []
|
||||
else:
|
||||
acodec_opts = ['-acodec', codec]
|
||||
cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y', '-i', encodeFilename(path), '-vn']
|
||||
+ acodec_opts + more_opts +
|
||||
['--', encodeFilename(out_path)])
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
stdout,stderr = p.communicate()
|
||||
if p.returncode != 0:
|
||||
msg = stderr.strip().split('\n')[-1]
|
||||
raise AudioConversionError(msg)
|
||||
|
||||
def run(self, information):
|
||||
path = information['filepath']
|
||||
def run(self, information):
|
||||
path = information['filepath']
|
||||
|
||||
filecodec = self.get_audio_codec(path)
|
||||
if filecodec is None:
|
||||
self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe')
|
||||
return None
|
||||
filecodec = self.get_audio_codec(path)
|
||||
if filecodec is None:
|
||||
self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe')
|
||||
return None
|
||||
|
||||
more_opts = []
|
||||
if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
|
||||
if self._preferredcodec == 'm4a' and filecodec == 'aac':
|
||||
# Lossless, but in another container
|
||||
acodec = 'copy'
|
||||
extension = self._preferredcodec
|
||||
more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
|
||||
elif filecodec in ['aac', 'mp3', 'vorbis']:
|
||||
# Lossless if possible
|
||||
acodec = 'copy'
|
||||
extension = filecodec
|
||||
if filecodec == 'aac':
|
||||
more_opts = ['-f', 'adts']
|
||||
if filecodec == 'vorbis':
|
||||
extension = 'ogg'
|
||||
else:
|
||||
# MP3 otherwise.
|
||||
acodec = 'libmp3lame'
|
||||
extension = 'mp3'
|
||||
more_opts = []
|
||||
if self._preferredquality is not None:
|
||||
if int(self._preferredquality) < 10:
|
||||
more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
|
||||
else:
|
||||
more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
|
||||
else:
|
||||
# We convert the audio (lossy)
|
||||
acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
|
||||
extension = self._preferredcodec
|
||||
more_opts = []
|
||||
if self._preferredquality is not None:
|
||||
if int(self._preferredquality) < 10:
|
||||
more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
|
||||
else:
|
||||
more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
|
||||
if self._preferredcodec == 'aac':
|
||||
more_opts += ['-f', 'adts']
|
||||
if self._preferredcodec == 'm4a':
|
||||
more_opts += [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
|
||||
if self._preferredcodec == 'vorbis':
|
||||
extension = 'ogg'
|
||||
if self._preferredcodec == 'wav':
|
||||
extension = 'wav'
|
||||
more_opts += ['-f', 'wav']
|
||||
more_opts = []
|
||||
if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
|
||||
if self._preferredcodec == 'm4a' and filecodec == 'aac':
|
||||
# Lossless, but in another container
|
||||
acodec = 'copy'
|
||||
extension = self._preferredcodec
|
||||
more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
|
||||
elif filecodec in ['aac', 'mp3', 'vorbis']:
|
||||
# Lossless if possible
|
||||
acodec = 'copy'
|
||||
extension = filecodec
|
||||
if filecodec == 'aac':
|
||||
more_opts = ['-f', 'adts']
|
||||
if filecodec == 'vorbis':
|
||||
extension = 'ogg'
|
||||
else:
|
||||
# MP3 otherwise.
|
||||
acodec = 'libmp3lame'
|
||||
extension = 'mp3'
|
||||
more_opts = []
|
||||
if self._preferredquality is not None:
|
||||
if int(self._preferredquality) < 10:
|
||||
more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
|
||||
else:
|
||||
more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
|
||||
else:
|
||||
# We convert the audio (lossy)
|
||||
acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
|
||||
extension = self._preferredcodec
|
||||
more_opts = []
|
||||
if self._preferredquality is not None:
|
||||
if int(self._preferredquality) < 10:
|
||||
more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
|
||||
else:
|
||||
more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
|
||||
if self._preferredcodec == 'aac':
|
||||
more_opts += ['-f', 'adts']
|
||||
if self._preferredcodec == 'm4a':
|
||||
more_opts += [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
|
||||
if self._preferredcodec == 'vorbis':
|
||||
extension = 'ogg'
|
||||
if self._preferredcodec == 'wav':
|
||||
extension = 'wav'
|
||||
more_opts += ['-f', 'wav']
|
||||
|
||||
prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups
|
||||
new_path = prefix + sep + extension
|
||||
self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path)
|
||||
try:
|
||||
self.run_ffmpeg(path, new_path, acodec, more_opts)
|
||||
except:
|
||||
etype,e,tb = sys.exc_info()
|
||||
if isinstance(e, AudioConversionError):
|
||||
self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message)
|
||||
else:
|
||||
self._downloader.to_stderr(u'ERROR: error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg'))
|
||||
return None
|
||||
prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups
|
||||
new_path = prefix + sep + extension
|
||||
self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path)
|
||||
try:
|
||||
self.run_ffmpeg(path, new_path, acodec, more_opts)
|
||||
except:
|
||||
etype,e,tb = sys.exc_info()
|
||||
if isinstance(e, AudioConversionError):
|
||||
self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message)
|
||||
else:
|
||||
self._downloader.to_stderr(u'ERROR: error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg'))
|
||||
return None
|
||||
|
||||
# Try to update the date time for extracted audio file.
|
||||
if information.get('filetime') is not None:
|
||||
try:
|
||||
os.utime(encodeFilename(new_path), (time.time(), information['filetime']))
|
||||
except:
|
||||
self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file')
|
||||
# Try to update the date time for extracted audio file.
|
||||
if information.get('filetime') is not None:
|
||||
try:
|
||||
os.utime(encodeFilename(new_path), (time.time(), information['filetime']))
|
||||
except:
|
||||
self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file')
|
||||
|
||||
if not self._keepvideo:
|
||||
try:
|
||||
os.remove(encodeFilename(path))
|
||||
except (IOError, OSError):
|
||||
self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
|
||||
return None
|
||||
if not self._keepvideo:
|
||||
try:
|
||||
os.remove(encodeFilename(path))
|
||||
except (IOError, OSError):
|
||||
self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
|
||||
return None
|
||||
|
||||
information['filepath'] = new_path
|
||||
return information
|
||||
information['filepath'] = new_path
|
||||
return information
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -4,4 +4,4 @@
|
|||
import __init__
|
||||
|
||||
if __name__ == '__main__':
|
||||
__init__.main()
|
||||
__init__.main()
|
||||
|
|
|
|||
|
|
@ -12,490 +12,490 @@ import email.utils
|
|||
import json
|
||||
|
||||
try:
|
||||
import urllib.request as compat_urllib_request
|
||||
import urllib.request as compat_urllib_request
|
||||
except ImportError: # Python 2
|
||||
import urllib2 as compat_urllib_request
|
||||
import urllib2 as compat_urllib_request
|
||||
|
||||
try:
|
||||
import urllib.error as compat_urllib_error
|
||||
import urllib.error as compat_urllib_error
|
||||
except ImportError: # Python 2
|
||||
import urllib2 as compat_urllib_error
|
||||
import urllib2 as compat_urllib_error
|
||||
|
||||
try:
|
||||
import urllib.parse as compat_urllib_parse
|
||||
import urllib.parse as compat_urllib_parse
|
||||
except ImportError: # Python 2
|
||||
import urllib as compat_urllib_parse
|
||||
import urllib as compat_urllib_parse
|
||||
|
||||
try:
|
||||
import http.cookiejar as compat_cookiejar
|
||||
import http.cookiejar as compat_cookiejar
|
||||
except ImportError: # Python 2
|
||||
import cookielib as compat_cookiejar
|
||||
import cookielib as compat_cookiejar
|
||||
|
||||
try:
|
||||
import html.entities as compat_html_entities
|
||||
import html.entities as compat_html_entities
|
||||
except ImportError: # Python 2
|
||||
import htmlentitydefs as compat_html_entities
|
||||
import htmlentitydefs as compat_html_entities
|
||||
|
||||
try:
|
||||
import html.parser as compat_html_parser
|
||||
import html.parser as compat_html_parser
|
||||
except ImportError: # Python 2
|
||||
import HTMLParser as compat_html_parser
|
||||
import HTMLParser as compat_html_parser
|
||||
|
||||
try:
|
||||
import http.client as compat_http_client
|
||||
import http.client as compat_http_client
|
||||
except ImportError: # Python 2
|
||||
import httplib as compat_http_client
|
||||
import httplib as compat_http_client
|
||||
|
||||
try:
|
||||
from urllib.parse import parse_qs as compat_parse_qs
|
||||
from urllib.parse import parse_qs as compat_parse_qs
|
||||
except ImportError: # Python 2
|
||||
# HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
|
||||
# Python 2's version is apparently totally broken
|
||||
def _unquote(string, encoding='utf-8', errors='replace'):
|
||||
if string == '':
|
||||
return string
|
||||
res = string.split('%')
|
||||
if len(res) == 1:
|
||||
return string
|
||||
if encoding is None:
|
||||
encoding = 'utf-8'
|
||||
if errors is None:
|
||||
errors = 'replace'
|
||||
# pct_sequence: contiguous sequence of percent-encoded bytes, decoded
|
||||
pct_sequence = b''
|
||||
string = res[0]
|
||||
for item in res[1:]:
|
||||
try:
|
||||
if not item:
|
||||
raise ValueError
|
||||
pct_sequence += item[:2].decode('hex')
|
||||
rest = item[2:]
|
||||
if not rest:
|
||||
# This segment was just a single percent-encoded character.
|
||||
# May be part of a sequence of code units, so delay decoding.
|
||||
# (Stored in pct_sequence).
|
||||
continue
|
||||
except ValueError:
|
||||
rest = '%' + item
|
||||
# Encountered non-percent-encoded characters. Flush the current
|
||||
# pct_sequence.
|
||||
string += pct_sequence.decode(encoding, errors) + rest
|
||||
pct_sequence = b''
|
||||
if pct_sequence:
|
||||
# Flush the final pct_sequence
|
||||
string += pct_sequence.decode(encoding, errors)
|
||||
return string
|
||||
# HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
|
||||
# Python 2's version is apparently totally broken
|
||||
def _unquote(string, encoding='utf-8', errors='replace'):
|
||||
if string == '':
|
||||
return string
|
||||
res = string.split('%')
|
||||
if len(res) == 1:
|
||||
return string
|
||||
if encoding is None:
|
||||
encoding = 'utf-8'
|
||||
if errors is None:
|
||||
errors = 'replace'
|
||||
# pct_sequence: contiguous sequence of percent-encoded bytes, decoded
|
||||
pct_sequence = b''
|
||||
string = res[0]
|
||||
for item in res[1:]:
|
||||
try:
|
||||
if not item:
|
||||
raise ValueError
|
||||
pct_sequence += item[:2].decode('hex')
|
||||
rest = item[2:]
|
||||
if not rest:
|
||||
# This segment was just a single percent-encoded character.
|
||||
# May be part of a sequence of code units, so delay decoding.
|
||||
# (Stored in pct_sequence).
|
||||
continue
|
||||
except ValueError:
|
||||
rest = '%' + item
|
||||
# Encountered non-percent-encoded characters. Flush the current
|
||||
# pct_sequence.
|
||||
string += pct_sequence.decode(encoding, errors) + rest
|
||||
pct_sequence = b''
|
||||
if pct_sequence:
|
||||
# Flush the final pct_sequence
|
||||
string += pct_sequence.decode(encoding, errors)
|
||||
return string
|
||||
|
||||
def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
|
||||
encoding='utf-8', errors='replace'):
|
||||
qs, _coerce_result = qs, unicode
|
||||
pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
|
||||
r = []
|
||||
for name_value in pairs:
|
||||
if not name_value and not strict_parsing:
|
||||
continue
|
||||
nv = name_value.split('=', 1)
|
||||
if len(nv) != 2:
|
||||
if strict_parsing:
|
||||
raise ValueError("bad query field: %r" % (name_value,))
|
||||
# Handle case of a control-name with no equal sign
|
||||
if keep_blank_values:
|
||||
nv.append('')
|
||||
else:
|
||||
continue
|
||||
if len(nv[1]) or keep_blank_values:
|
||||
name = nv[0].replace('+', ' ')
|
||||
name = _unquote(name, encoding=encoding, errors=errors)
|
||||
name = _coerce_result(name)
|
||||
value = nv[1].replace('+', ' ')
|
||||
value = _unquote(value, encoding=encoding, errors=errors)
|
||||
value = _coerce_result(value)
|
||||
r.append((name, value))
|
||||
return r
|
||||
def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
|
||||
encoding='utf-8', errors='replace'):
|
||||
qs, _coerce_result = qs, unicode
|
||||
pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
|
||||
r = []
|
||||
for name_value in pairs:
|
||||
if not name_value and not strict_parsing:
|
||||
continue
|
||||
nv = name_value.split('=', 1)
|
||||
if len(nv) != 2:
|
||||
if strict_parsing:
|
||||
raise ValueError("bad query field: %r" % (name_value,))
|
||||
# Handle case of a control-name with no equal sign
|
||||
if keep_blank_values:
|
||||
nv.append('')
|
||||
else:
|
||||
continue
|
||||
if len(nv[1]) or keep_blank_values:
|
||||
name = nv[0].replace('+', ' ')
|
||||
name = _unquote(name, encoding=encoding, errors=errors)
|
||||
name = _coerce_result(name)
|
||||
value = nv[1].replace('+', ' ')
|
||||
value = _unquote(value, encoding=encoding, errors=errors)
|
||||
value = _coerce_result(value)
|
||||
r.append((name, value))
|
||||
return r
|
||||
|
||||
def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
|
||||
encoding='utf-8', errors='replace'):
|
||||
parsed_result = {}
|
||||
pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
|
||||
encoding=encoding, errors=errors)
|
||||
for name, value in pairs:
|
||||
if name in parsed_result:
|
||||
parsed_result[name].append(value)
|
||||
else:
|
||||
parsed_result[name] = [value]
|
||||
return parsed_result
|
||||
def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
|
||||
encoding='utf-8', errors='replace'):
|
||||
parsed_result = {}
|
||||
pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
|
||||
encoding=encoding, errors=errors)
|
||||
for name, value in pairs:
|
||||
if name in parsed_result:
|
||||
parsed_result[name].append(value)
|
||||
else:
|
||||
parsed_result[name] = [value]
|
||||
return parsed_result
|
||||
|
||||
try:
|
||||
compat_str = unicode # Python 2
|
||||
compat_str = unicode # Python 2
|
||||
except NameError:
|
||||
compat_str = str
|
||||
compat_str = str
|
||||
|
||||
try:
|
||||
compat_chr = unichr # Python 2
|
||||
compat_chr = unichr # Python 2
|
||||
except NameError:
|
||||
compat_chr = chr
|
||||
compat_chr = chr
|
||||
|
||||
std_headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
|
||||
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'Accept-Language': 'en-us,en;q=0.5',
|
||||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
|
||||
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'Accept-Language': 'en-us,en;q=0.5',
|
||||
}
|
||||
def preferredencoding():
|
||||
"""Get preferred encoding.
|
||||
"""Get preferred encoding.
|
||||
|
||||
Returns the best encoding scheme for the system, based on
|
||||
locale.getpreferredencoding() and some further tweaks.
|
||||
"""
|
||||
try:
|
||||
pref = locale.getpreferredencoding()
|
||||
u'TEST'.encode(pref)
|
||||
except:
|
||||
pref = 'UTF-8'
|
||||
Returns the best encoding scheme for the system, based on
|
||||
locale.getpreferredencoding() and some further tweaks.
|
||||
"""
|
||||
try:
|
||||
pref = locale.getpreferredencoding()
|
||||
u'TEST'.encode(pref)
|
||||
except:
|
||||
pref = 'UTF-8'
|
||||
|
||||
return pref
|
||||
return pref
|
||||
|
||||
if sys.version_info < (3,0):
|
||||
def compat_print(s):
|
||||
print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
|
||||
def compat_print(s):
|
||||
print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
|
||||
else:
|
||||
def compat_print(s):
|
||||
assert type(s) == type(u'')
|
||||
print(s)
|
||||
def compat_print(s):
|
||||
assert type(s) == type(u'')
|
||||
print(s)
|
||||
|
||||
def htmlentity_transform(matchobj):
|
||||
"""Transforms an HTML entity to a character.
|
||||
"""Transforms an HTML entity to a character.
|
||||
|
||||
This function receives a match object and is intended to be used with
|
||||
the re.sub() function.
|
||||
"""
|
||||
entity = matchobj.group(1)
|
||||
This function receives a match object and is intended to be used with
|
||||
the re.sub() function.
|
||||
"""
|
||||
entity = matchobj.group(1)
|
||||
|
||||
# Known non-numeric HTML entity
|
||||
if entity in compat_html_entities.name2codepoint:
|
||||
return compat_chr(compat_html_entities.name2codepoint[entity])
|
||||
# Known non-numeric HTML entity
|
||||
if entity in compat_html_entities.name2codepoint:
|
||||
return compat_chr(compat_html_entities.name2codepoint[entity])
|
||||
|
||||
mobj = re.match(u'(?u)#(x?\\d+)', entity)
|
||||
if mobj is not None:
|
||||
numstr = mobj.group(1)
|
||||
if numstr.startswith(u'x'):
|
||||
base = 16
|
||||
numstr = u'0%s' % numstr
|
||||
else:
|
||||
base = 10
|
||||
return compat_chr(int(numstr, base))
|
||||
mobj = re.match(u'(?u)#(x?\\d+)', entity)
|
||||
if mobj is not None:
|
||||
numstr = mobj.group(1)
|
||||
if numstr.startswith(u'x'):
|
||||
base = 16
|
||||
numstr = u'0%s' % numstr
|
||||
else:
|
||||
base = 10
|
||||
return compat_chr(int(numstr, base))
|
||||
|
||||
# Unknown entity in name, return its literal representation
|
||||
return (u'&%s;' % entity)
|
||||
# Unknown entity in name, return its literal representation
|
||||
return (u'&%s;' % entity)
|
||||
|
||||
compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
|
||||
class IDParser(compat_html_parser.HTMLParser):
|
||||
"""Modified HTMLParser that isolates a tag with the specified id"""
|
||||
def __init__(self, id):
|
||||
self.id = id
|
||||
self.result = None
|
||||
self.started = False
|
||||
self.depth = {}
|
||||
self.html = None
|
||||
self.watch_startpos = False
|
||||
self.error_count = 0
|
||||
compat_html_parser.HTMLParser.__init__(self)
|
||||
"""Modified HTMLParser that isolates a tag with the specified id"""
|
||||
def __init__(self, id):
|
||||
self.id = id
|
||||
self.result = None
|
||||
self.started = False
|
||||
self.depth = {}
|
||||
self.html = None
|
||||
self.watch_startpos = False
|
||||
self.error_count = 0
|
||||
compat_html_parser.HTMLParser.__init__(self)
|
||||
|
||||
def error(self, message):
|
||||
if self.error_count > 10 or self.started:
|
||||
raise compat_html_parser.HTMLParseError(message, self.getpos())
|
||||
self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
|
||||
self.error_count += 1
|
||||
self.goahead(1)
|
||||
def error(self, message):
|
||||
if self.error_count > 10 or self.started:
|
||||
raise compat_html_parser.HTMLParseError(message, self.getpos())
|
||||
self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
|
||||
self.error_count += 1
|
||||
self.goahead(1)
|
||||
|
||||
def loads(self, html):
|
||||
self.html = html
|
||||
self.feed(html)
|
||||
self.close()
|
||||
def loads(self, html):
|
||||
self.html = html
|
||||
self.feed(html)
|
||||
self.close()
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attrs = dict(attrs)
|
||||
if self.started:
|
||||
self.find_startpos(None)
|
||||
if 'id' in attrs and attrs['id'] == self.id:
|
||||
self.result = [tag]
|
||||
self.started = True
|
||||
self.watch_startpos = True
|
||||
if self.started:
|
||||
if not tag in self.depth: self.depth[tag] = 0
|
||||
self.depth[tag] += 1
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attrs = dict(attrs)
|
||||
if self.started:
|
||||
self.find_startpos(None)
|
||||
if 'id' in attrs and attrs['id'] == self.id:
|
||||
self.result = [tag]
|
||||
self.started = True
|
||||
self.watch_startpos = True
|
||||
if self.started:
|
||||
if not tag in self.depth: self.depth[tag] = 0
|
||||
self.depth[tag] += 1
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if self.started:
|
||||
if tag in self.depth: self.depth[tag] -= 1
|
||||
if self.depth[self.result[0]] == 0:
|
||||
self.started = False
|
||||
self.result.append(self.getpos())
|
||||
def handle_endtag(self, tag):
|
||||
if self.started:
|
||||
if tag in self.depth: self.depth[tag] -= 1
|
||||
if self.depth[self.result[0]] == 0:
|
||||
self.started = False
|
||||
self.result.append(self.getpos())
|
||||
|
||||
def find_startpos(self, x):
|
||||
"""Needed to put the start position of the result (self.result[1])
|
||||
after the opening tag with the requested id"""
|
||||
if self.watch_startpos:
|
||||
self.watch_startpos = False
|
||||
self.result.append(self.getpos())
|
||||
handle_entityref = handle_charref = handle_data = handle_comment = \
|
||||
handle_decl = handle_pi = unknown_decl = find_startpos
|
||||
def find_startpos(self, x):
|
||||
"""Needed to put the start position of the result (self.result[1])
|
||||
after the opening tag with the requested id"""
|
||||
if self.watch_startpos:
|
||||
self.watch_startpos = False
|
||||
self.result.append(self.getpos())
|
||||
handle_entityref = handle_charref = handle_data = handle_comment = \
|
||||
handle_decl = handle_pi = unknown_decl = find_startpos
|
||||
|
||||
def get_result(self):
|
||||
if self.result is None:
|
||||
return None
|
||||
if len(self.result) != 3:
|
||||
return None
|
||||
lines = self.html.split('\n')
|
||||
lines = lines[self.result[1][0]-1:self.result[2][0]]
|
||||
lines[0] = lines[0][self.result[1][1]:]
|
||||
if len(lines) == 1:
|
||||
lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
|
||||
lines[-1] = lines[-1][:self.result[2][1]]
|
||||
return '\n'.join(lines).strip()
|
||||
def get_result(self):
|
||||
if self.result is None:
|
||||
return None
|
||||
if len(self.result) != 3:
|
||||
return None
|
||||
lines = self.html.split('\n')
|
||||
lines = lines[self.result[1][0]-1:self.result[2][0]]
|
||||
lines[0] = lines[0][self.result[1][1]:]
|
||||
if len(lines) == 1:
|
||||
lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
|
||||
lines[-1] = lines[-1][:self.result[2][1]]
|
||||
return '\n'.join(lines).strip()
|
||||
|
||||
def get_element_by_id(id, html):
|
||||
"""Return the content of the tag with the specified id in the passed HTML document"""
|
||||
parser = IDParser(id)
|
||||
try:
|
||||
parser.loads(html)
|
||||
except compat_html_parser.HTMLParseError:
|
||||
pass
|
||||
return parser.get_result()
|
||||
"""Return the content of the tag with the specified id in the passed HTML document"""
|
||||
parser = IDParser(id)
|
||||
try:
|
||||
parser.loads(html)
|
||||
except compat_html_parser.HTMLParseError:
|
||||
pass
|
||||
return parser.get_result()
|
||||
|
||||
|
||||
def clean_html(html):
|
||||
"""Clean an HTML snippet into a readable string"""
|
||||
# Newline vs <br />
|
||||
html = html.replace('\n', ' ')
|
||||
html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html)
|
||||
# Strip html tags
|
||||
html = re.sub('<.*?>', '', html)
|
||||
# Replace html entities
|
||||
html = unescapeHTML(html)
|
||||
return html
|
||||
"""Clean an HTML snippet into a readable string"""
|
||||
# Newline vs <br />
|
||||
html = html.replace('\n', ' ')
|
||||
html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html)
|
||||
# Strip html tags
|
||||
html = re.sub('<.*?>', '', html)
|
||||
# Replace html entities
|
||||
html = unescapeHTML(html)
|
||||
return html
|
||||
|
||||
|
||||
def sanitize_open(filename, open_mode):
|
||||
"""Try to open the given filename, and slightly tweak it if this fails.
|
||||
"""Try to open the given filename, and slightly tweak it if this fails.
|
||||
|
||||
Attempts to open the given filename. If this fails, it tries to change
|
||||
the filename slightly, step by step, until it's either able to open it
|
||||
or it fails and raises a final exception, like the standard open()
|
||||
function.
|
||||
Attempts to open the given filename. If this fails, it tries to change
|
||||
the filename slightly, step by step, until it's either able to open it
|
||||
or it fails and raises a final exception, like the standard open()
|
||||
function.
|
||||
|
||||
It returns the tuple (stream, definitive_file_name).
|
||||
"""
|
||||
try:
|
||||
if filename == u'-':
|
||||
if sys.platform == 'win32':
|
||||
import msvcrt
|
||||
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
|
||||
return (sys.stdout, filename)
|
||||
stream = open(encodeFilename(filename), open_mode)
|
||||
return (stream, filename)
|
||||
except (IOError, OSError) as err:
|
||||
# In case of error, try to remove win32 forbidden chars
|
||||
filename = re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', filename)
|
||||
It returns the tuple (stream, definitive_file_name).
|
||||
"""
|
||||
try:
|
||||
if filename == u'-':
|
||||
if sys.platform == 'win32':
|
||||
import msvcrt
|
||||
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
|
||||
return (sys.stdout, filename)
|
||||
stream = open(encodeFilename(filename), open_mode)
|
||||
return (stream, filename)
|
||||
except (IOError, OSError) as err:
|
||||
# In case of error, try to remove win32 forbidden chars
|
||||
filename = re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', filename)
|
||||
|
||||
# An exception here should be caught in the caller
|
||||
stream = open(encodeFilename(filename), open_mode)
|
||||
return (stream, filename)
|
||||
# An exception here should be caught in the caller
|
||||
stream = open(encodeFilename(filename), open_mode)
|
||||
return (stream, filename)
|
||||
|
||||
|
||||
def timeconvert(timestr):
|
||||
"""Convert RFC 2822 defined time string into system timestamp"""
|
||||
timestamp = None
|
||||
timetuple = email.utils.parsedate_tz(timestr)
|
||||
if timetuple is not None:
|
||||
timestamp = email.utils.mktime_tz(timetuple)
|
||||
return timestamp
|
||||
"""Convert RFC 2822 defined time string into system timestamp"""
|
||||
timestamp = None
|
||||
timetuple = email.utils.parsedate_tz(timestr)
|
||||
if timetuple is not None:
|
||||
timestamp = email.utils.mktime_tz(timetuple)
|
||||
return timestamp
|
||||
|
||||
def sanitize_filename(s, restricted=False):
|
||||
"""Sanitizes a string so it could be used as part of a filename.
|
||||
If restricted is set, use a stricter subset of allowed characters.
|
||||
"""
|
||||
def replace_insane(char):
|
||||
if char == '?' or ord(char) < 32 or ord(char) == 127:
|
||||
return ''
|
||||
elif char == '"':
|
||||
return '' if restricted else '\''
|
||||
elif char == ':':
|
||||
return '_-' if restricted else ' -'
|
||||
elif char in '\\/|*<>':
|
||||
return '_'
|
||||
if restricted and (char in '!&\'' or char.isspace()):
|
||||
return '_'
|
||||
if restricted and ord(char) > 127:
|
||||
return '_'
|
||||
return char
|
||||
"""Sanitizes a string so it could be used as part of a filename.
|
||||
If restricted is set, use a stricter subset of allowed characters.
|
||||
"""
|
||||
def replace_insane(char):
|
||||
if char == '?' or ord(char) < 32 or ord(char) == 127:
|
||||
return ''
|
||||
elif char == '"':
|
||||
return '' if restricted else '\''
|
||||
elif char == ':':
|
||||
return '_-' if restricted else ' -'
|
||||
elif char in '\\/|*<>':
|
||||
return '_'
|
||||
if restricted and (char in '!&\'' or char.isspace()):
|
||||
return '_'
|
||||
if restricted and ord(char) > 127:
|
||||
return '_'
|
||||
return char
|
||||
|
||||
result = u''.join(map(replace_insane, s))
|
||||
while '__' in result:
|
||||
result = result.replace('__', '_')
|
||||
result = result.strip('_')
|
||||
# Common case of "Foreign band name - English song title"
|
||||
if restricted and result.startswith('-_'):
|
||||
result = result[2:]
|
||||
if not result:
|
||||
result = '_'
|
||||
return result
|
||||
result = u''.join(map(replace_insane, s))
|
||||
while '__' in result:
|
||||
result = result.replace('__', '_')
|
||||
result = result.strip('_')
|
||||
# Common case of "Foreign band name - English song title"
|
||||
if restricted and result.startswith('-_'):
|
||||
result = result[2:]
|
||||
if not result:
|
||||
result = '_'
|
||||
return result
|
||||
|
||||
def orderedSet(iterable):
|
||||
""" Remove all duplicates from the input iterable """
|
||||
res = []
|
||||
for el in iterable:
|
||||
if el not in res:
|
||||
res.append(el)
|
||||
return res
|
||||
""" Remove all duplicates from the input iterable """
|
||||
res = []
|
||||
for el in iterable:
|
||||
if el not in res:
|
||||
res.append(el)
|
||||
return res
|
||||
|
||||
def unescapeHTML(s):
|
||||
"""
|
||||
@param s a string
|
||||
"""
|
||||
assert type(s) == type(u'')
|
||||
"""
|
||||
@param s a string
|
||||
"""
|
||||
assert type(s) == type(u'')
|
||||
|
||||
result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s)
|
||||
return result
|
||||
result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s)
|
||||
return result
|
||||
|
||||
def encodeFilename(s):
|
||||
"""
|
||||
@param s The name of the file
|
||||
"""
|
||||
"""
|
||||
@param s The name of the file
|
||||
"""
|
||||
|
||||
assert type(s) == type(u'')
|
||||
assert type(s) == type(u'')
|
||||
|
||||
# Python 3 has a Unicode API
|
||||
if sys.version_info >= (3, 0):
|
||||
return s
|
||||
# Python 3 has a Unicode API
|
||||
if sys.version_info >= (3, 0):
|
||||
return s
|
||||
|
||||
if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
|
||||
# Pass u'' directly to use Unicode APIs on Windows 2000 and up
|
||||
# (Detecting Windows NT 4 is tricky because 'major >= 4' would
|
||||
# match Windows 9x series as well. Besides, NT 4 is obsolete.)
|
||||
return s
|
||||
else:
|
||||
return s.encode(sys.getfilesystemencoding(), 'ignore')
|
||||
if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
|
||||
# Pass u'' directly to use Unicode APIs on Windows 2000 and up
|
||||
# (Detecting Windows NT 4 is tricky because 'major >= 4' would
|
||||
# match Windows 9x series as well. Besides, NT 4 is obsolete.)
|
||||
return s
|
||||
else:
|
||||
return s.encode(sys.getfilesystemencoding(), 'ignore')
|
||||
|
||||
class DownloadError(Exception):
|
||||
"""Download Error exception.
|
||||
"""Download Error exception.
|
||||
|
||||
This exception may be thrown by FileDownloader objects if they are not
|
||||
configured to continue on errors. They will contain the appropriate
|
||||
error message.
|
||||
"""
|
||||
pass
|
||||
This exception may be thrown by FileDownloader objects if they are not
|
||||
configured to continue on errors. They will contain the appropriate
|
||||
error message.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class SameFileError(Exception):
|
||||
"""Same File exception.
|
||||
"""Same File exception.
|
||||
|
||||
This exception will be thrown by FileDownloader objects if they detect
|
||||
multiple files would have to be downloaded to the same file on disk.
|
||||
"""
|
||||
pass
|
||||
This exception will be thrown by FileDownloader objects if they detect
|
||||
multiple files would have to be downloaded to the same file on disk.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class PostProcessingError(Exception):
|
||||
"""Post Processing exception.
|
||||
"""Post Processing exception.
|
||||
|
||||
This exception may be raised by PostProcessor's .run() method to
|
||||
indicate an error in the postprocessing task.
|
||||
"""
|
||||
pass
|
||||
This exception may be raised by PostProcessor's .run() method to
|
||||
indicate an error in the postprocessing task.
|
||||
"""
|
||||
pass
|
||||
|
||||
class MaxDownloadsReached(Exception):
|
||||
""" --max-downloads limit has been reached. """
|
||||
pass
|
||||
""" --max-downloads limit has been reached. """
|
||||
pass
|
||||
|
||||
|
||||
class UnavailableVideoError(Exception):
|
||||
"""Unavailable Format exception.
|
||||
"""Unavailable Format exception.
|
||||
|
||||
This exception will be thrown when a video is requested
|
||||
in a format that is not available for that video.
|
||||
"""
|
||||
pass
|
||||
This exception will be thrown when a video is requested
|
||||
in a format that is not available for that video.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class ContentTooShortError(Exception):
|
||||
"""Content Too Short exception.
|
||||
"""Content Too Short exception.
|
||||
|
||||
This exception may be raised by FileDownloader objects when a file they
|
||||
download is too small for what the server announced first, indicating
|
||||
the connection was probably interrupted.
|
||||
"""
|
||||
# Both in bytes
|
||||
downloaded = None
|
||||
expected = None
|
||||
This exception may be raised by FileDownloader objects when a file they
|
||||
download is too small for what the server announced first, indicating
|
||||
the connection was probably interrupted.
|
||||
"""
|
||||
# Both in bytes
|
||||
downloaded = None
|
||||
expected = None
|
||||
|
||||
def __init__(self, downloaded, expected):
|
||||
self.downloaded = downloaded
|
||||
self.expected = expected
|
||||
def __init__(self, downloaded, expected):
|
||||
self.downloaded = downloaded
|
||||
self.expected = expected
|
||||
|
||||
|
||||
class Trouble(Exception):
|
||||
"""Trouble helper exception
|
||||
"""Trouble helper exception
|
||||
|
||||
This is an exception to be handled with
|
||||
FileDownloader.trouble
|
||||
"""
|
||||
This is an exception to be handled with
|
||||
FileDownloader.trouble
|
||||
"""
|
||||
|
||||
class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
|
||||
"""Handler for HTTP requests and responses.
|
||||
"""Handler for HTTP requests and responses.
|
||||
|
||||
This class, when installed with an OpenerDirector, automatically adds
|
||||
the standard headers to every HTTP request and handles gzipped and
|
||||
deflated responses from web servers. If compression is to be avoided in
|
||||
a particular request, the original request in the program code only has
|
||||
to include the HTTP header "Youtubedl-No-Compression", which will be
|
||||
removed before making the real request.
|
||||
This class, when installed with an OpenerDirector, automatically adds
|
||||
the standard headers to every HTTP request and handles gzipped and
|
||||
deflated responses from web servers. If compression is to be avoided in
|
||||
a particular request, the original request in the program code only has
|
||||
to include the HTTP header "Youtubedl-No-Compression", which will be
|
||||
removed before making the real request.
|
||||
|
||||
Part of this code was copied from:
|
||||
Part of this code was copied from:
|
||||
|
||||
http://techknack.net/python-urllib2-handlers/
|
||||
http://techknack.net/python-urllib2-handlers/
|
||||
|
||||
Andrew Rowls, the author of that code, agreed to release it to the
|
||||
public domain.
|
||||
"""
|
||||
Andrew Rowls, the author of that code, agreed to release it to the
|
||||
public domain.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def deflate(data):
|
||||
try:
|
||||
return zlib.decompress(data, -zlib.MAX_WBITS)
|
||||
except zlib.error:
|
||||
return zlib.decompress(data)
|
||||
@staticmethod
|
||||
def deflate(data):
|
||||
try:
|
||||
return zlib.decompress(data, -zlib.MAX_WBITS)
|
||||
except zlib.error:
|
||||
return zlib.decompress(data)
|
||||
|
||||
@staticmethod
|
||||
def addinfourl_wrapper(stream, headers, url, code):
|
||||
if hasattr(compat_urllib_request.addinfourl, 'getcode'):
|
||||
return compat_urllib_request.addinfourl(stream, headers, url, code)
|
||||
ret = compat_urllib_request.addinfourl(stream, headers, url)
|
||||
ret.code = code
|
||||
return ret
|
||||
@staticmethod
|
||||
def addinfourl_wrapper(stream, headers, url, code):
|
||||
if hasattr(compat_urllib_request.addinfourl, 'getcode'):
|
||||
return compat_urllib_request.addinfourl(stream, headers, url, code)
|
||||
ret = compat_urllib_request.addinfourl(stream, headers, url)
|
||||
ret.code = code
|
||||
return ret
|
||||
|
||||
def http_request(self, req):
|
||||
for h in std_headers:
|
||||
if h in req.headers:
|
||||
del req.headers[h]
|
||||
req.add_header(h, std_headers[h])
|
||||
if 'Youtubedl-no-compression' in req.headers:
|
||||
if 'Accept-encoding' in req.headers:
|
||||
del req.headers['Accept-encoding']
|
||||
del req.headers['Youtubedl-no-compression']
|
||||
return req
|
||||
def http_request(self, req):
|
||||
for h in std_headers:
|
||||
if h in req.headers:
|
||||
del req.headers[h]
|
||||
req.add_header(h, std_headers[h])
|
||||
if 'Youtubedl-no-compression' in req.headers:
|
||||
if 'Accept-encoding' in req.headers:
|
||||
del req.headers['Accept-encoding']
|
||||
del req.headers['Youtubedl-no-compression']
|
||||
return req
|
||||
|
||||
def http_response(self, req, resp):
|
||||
old_resp = resp
|
||||
# gzip
|
||||
if resp.headers.get('Content-encoding', '') == 'gzip':
|
||||
gz = gzip.GzipFile(fileobj=io.BytesIO(resp.read()), mode='r')
|
||||
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
|
||||
resp.msg = old_resp.msg
|
||||
# deflate
|
||||
if resp.headers.get('Content-encoding', '') == 'deflate':
|
||||
gz = io.BytesIO(self.deflate(resp.read()))
|
||||
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
|
||||
resp.msg = old_resp.msg
|
||||
return resp
|
||||
def http_response(self, req, resp):
|
||||
old_resp = resp
|
||||
# gzip
|
||||
if resp.headers.get('Content-encoding', '') == 'gzip':
|
||||
gz = gzip.GzipFile(fileobj=io.BytesIO(resp.read()), mode='r')
|
||||
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
|
||||
resp.msg = old_resp.msg
|
||||
# deflate
|
||||
if resp.headers.get('Content-encoding', '') == 'deflate':
|
||||
gz = io.BytesIO(self.deflate(resp.read()))
|
||||
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
|
||||
resp.msg = old_resp.msg
|
||||
return resp
|
||||
|
|
|
|||
Loading…
Reference in a new issue