w/youtube-dl
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Implement a different adult sites checking algorithm

Browse source
This commit is contained in:
Filippo Valsorda 2014-01-14 16:01:00 -05:00
parent 4950f30890
commit a45ea17042
1 changed files with 32 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
devscripts/check-porn.py
View file

@ -3,6 +3,9 @@
""" """
This script employs a VERY basic heuristic ('porn' in webpage.lower()) to check This script employs a VERY basic heuristic ('porn' in webpage.lower()) to check
if we are not 'age_limit' tagging some porn site if we are not 'age_limit' tagging some porn site
A second approach implemented relies on a list of porn domains, to activate it
pass the list filename as the only argument
""" """
# Allow direct execution # Allow direct execution
@ -11,25 +14,42 @@ import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from test.helper import get_testcases from test.helper import get_testcases
from youtube_dl.utils import compat_urllib_parse_urlparse
from youtube_dl.utils import compat_urllib_request from youtube_dl.utils import compat_urllib_request
if len(sys.argv) > 1:
METHOD = 'LIST'
LIST = open(sys.argv[1]).read().decode('utf8').strip()
else:
METHOD = 'EURISTIC'
for test in get_testcases(): for test in get_testcases():
try: if METHOD == 'EURISTIC':
webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read() try:
except: webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()
print('\nFail: {0}'.format(test['name'])) except:
continue print('\nFail: {0}'.format(test['name']))
continue
webpage = webpage.decode('utf8', 'replace') webpage = webpage.decode('utf8', 'replace')
if 'porn' in webpage.lower() and ('info_dict' not in test RESULT = 'porn' in webpage.lower()
or 'age_limit' not in test['info_dict']
or test['info_dict']['age_limit'] != 18): elif METHOD == 'LIST':
domain = compat_urllib_parse_urlparse(test['url']).netloc
if not domain:
print('\nFail: {0}'.format(test['name']))
continue
domain = '.'.join(domain.split('.')[-2:])
RESULT = ('.' + domain + '\n' in LIST or '\n' + domain + '\n' in LIST)
if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict']
or test['info_dict']['age_limit'] != 18):
print('\nPotential missing age_limit check: {0}'.format(test['name'])) print('\nPotential missing age_limit check: {0}'.format(test['name']))
elif 'porn' not in webpage.lower() and ('info_dict' in test and elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict']
'age_limit' in test['info_dict'] and and test['info_dict']['age_limit'] == 18):
test['info_dict']['age_limit'] == 18):
print('\nPotential false negative: {0}'.format(test['name'])) print('\nPotential false negative: {0}'.format(test['name']))
else: else: