From 67c5f02a8085c4aa5d3af03e50066e70142ddadc Mon Sep 17 00:00:00 2001 From: Alan Hamlett Date: Mon, 22 Dec 2014 23:12:57 -0600 Subject: [PATCH] dependency parser base models --- wakatime/languages/__init__.py | 67 ++++++++++++++++++++++++++++++++++ wakatime/stats.py | 22 +++++------ 2 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 wakatime/languages/__init__.py diff --git a/wakatime/languages/__init__.py b/wakatime/languages/__init__.py new file mode 100644 index 0000000..7dc3921 --- /dev/null +++ b/wakatime/languages/__init__.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +""" + wakatime.languages + ~~~~~~~~~~~~~~~~~~ + + Parse dependencies from a source code file. + + :copyright: (c) 2013 Alan Hamlett. + :license: BSD, see LICENSE for more details. +""" + +from ..compat import open, import_module + + +class TokenParser(object): + source_file = None + lexer = None + dependencies = [] + tokens = [] + + def __init__(self, source_file, lexer=None): + self.source_file = source_file + self.lexer = lexer + + def parse(self, tokens=[]): + """ Should return a list of dependencies. + """ + if not tokens and not self.tokens: + self.tokens = self._extract_tokens() + raise Exception('Not yet implemented.') + + def append(self, dep): + self._save_dependency(dep) + + def _extract_tokens(self): + with open(self.source_file, 'r', encoding='utf-8') as fh: + return self.lexer.get_tokens_unprocessed(fh.read(512000)) + + def _save_dependency(self, dep): + dep = dep.strip().split('.')[0].strip() + if dep: + self.dependencies.append(dep) + + +class DependencyParser(object): + source_file = None + lexer = None + parser = None + + def __init__(self, source_file, lexer): + self.source_file = source_file + self.lexer = lexer + + try: + module_name = self.lexer.__module__.split('.')[-1] + class_name = self.lexer.__class__.__name__.replace('Lexer', 'Parser', 1) + module = import_module('.%s' % module_name, package=__package__) + self.parser = getattr(module, class_name) + except ImportError: + pass + + def parse(self): + if self.parser: + plugin = self.parser(self.source_file, lexer=self.lexer) + dependencies = plugin.parse() + return list(set(dependencies)) + return [] diff --git a/wakatime/stats.py b/wakatime/stats.py index 8afff9a..50ff4a4 100644 --- a/wakatime/stats.py +++ b/wakatime/stats.py @@ -14,6 +14,7 @@ import os import sys from .compat import u, open +from .languages import DependencyParser if sys.version_info[0] == 2: sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'packages', 'pygments_py2')) @@ -46,20 +47,17 @@ TRANSLATIONS = { def guess_language(file_name): - if file_name: - language = guess_language_from_extension(file_name.rsplit('.', 1)[-1]) - if language: - return language - lexer = None + language, lexer = None, None try: with open(file_name, 'r', encoding='utf-8') as fh: lexer = guess_lexer_for_filename(file_name, fh.read(512000)) except: pass - if lexer: - return translate_language(u(lexer.name)) - else: - return None + if file_name: + language = guess_language_from_extension(file_name.rsplit('.', 1)[-1]) + if lexer and language is None: + language = translate_language(u(lexer.name)) + return language, lexer def guess_language_from_extension(extension): @@ -89,9 +87,11 @@ def number_lines_in_file(file_name): def get_file_stats(file_name): - dependencies = [] + language, lexer = guess_language(file_name) + parser = DependencyParser(file_name, lexer) + dependencies = parser.parse() stats = { - 'language': guess_language(file_name), + 'language': language, 'dependencies': dependencies, 'lines': number_lines_in_file(file_name), }