handle unknown language when parsing dependencies
This commit is contained in:
parent
0f76334fc3
commit
78640aebf2
1 changed files with 12 additions and 9 deletions
|
@ -42,8 +42,10 @@ class TokenParser(object):
|
||||||
self._save_dependency(dep, truncate=truncate)
|
self._save_dependency(dep, truncate=truncate)
|
||||||
|
|
||||||
def _extract_tokens(self):
|
def _extract_tokens(self):
|
||||||
|
if self.lexer:
|
||||||
with open(self.source_file, 'r', encoding='utf-8') as fh:
|
with open(self.source_file, 'r', encoding='utf-8') as fh:
|
||||||
return self.lexer.get_tokens_unprocessed(fh.read(512000))
|
return self.lexer.get_tokens_unprocessed(fh.read(512000))
|
||||||
|
return []
|
||||||
|
|
||||||
def _save_dependency(self, dep, truncate=True):
|
def _save_dependency(self, dep, truncate=True):
|
||||||
dep = dep.strip().split('.')[0].strip() if truncate else dep.strip()
|
dep = dep.strip().split('.')[0].strip() if truncate else dep.strip()
|
||||||
|
@ -60,6 +62,7 @@ class DependencyParser(object):
|
||||||
self.source_file = source_file
|
self.source_file = source_file
|
||||||
self.lexer = lexer
|
self.lexer = lexer
|
||||||
|
|
||||||
|
if self.lexer:
|
||||||
try:
|
try:
|
||||||
module_name = self.lexer.__module__.split('.')[-1]
|
module_name = self.lexer.__module__.split('.')[-1]
|
||||||
class_name = self.lexer.__class__.__name__.replace('Lexer', 'Parser', 1)
|
class_name = self.lexer.__class__.__name__.replace('Lexer', 'Parser', 1)
|
||||||
|
|
Loading…
Reference in a new issue