handle unknown language when parsing dependencies

This commit is contained in:
Alan Hamlett 2014-12-23 05:30:40 -06:00
parent 0f76334fc3
commit 78640aebf2

View file

@ -42,8 +42,10 @@ class TokenParser(object):
self._save_dependency(dep, truncate=truncate)
def _extract_tokens(self):
if self.lexer:
with open(self.source_file, 'r', encoding='utf-8') as fh:
return self.lexer.get_tokens_unprocessed(fh.read(512000))
return []
def _save_dependency(self, dep, truncate=True):
dep = dep.strip().split('.')[0].strip() if truncate else dep.strip()
@ -60,6 +62,7 @@ class DependencyParser(object):
self.source_file = source_file
self.lexer = lexer
if self.lexer:
try:
module_name = self.lexer.__module__.split('.')[-1]
class_name = self.lexer.__class__.__name__.replace('Lexer', 'Parser', 1)