dependency parser base models

wakatime/languages/__init__.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages
+    ~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from a source code file.
+
+    :copyright: (c) 2013 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from ..compat import open, import_module
+
+
+class TokenParser(object):
+    source_file = None
+    lexer = None
+    dependencies = []
+    tokens = []
+
+    def __init__(self, source_file, lexer=None):
+        self.source_file = source_file
+        self.lexer = lexer
+
+    def parse(self, tokens=[]):
+        """ Should return a list of dependencies.
+        """
+        if not tokens and not self.tokens:
+            self.tokens = self._extract_tokens()
+        raise Exception('Not yet implemented.')
+
+    def append(self, dep):
+        self._save_dependency(dep)
+
+    def _extract_tokens(self):
+        with open(self.source_file, 'r', encoding='utf-8') as fh:
+            return self.lexer.get_tokens_unprocessed(fh.read(512000))
+
+    def _save_dependency(self, dep):
+        dep = dep.strip().split('.')[0].strip()
+        if dep:
+            self.dependencies.append(dep)
+
+
+class DependencyParser(object):
+    source_file = None
+    lexer = None
+    parser = None
+
+    def __init__(self, source_file, lexer):
+        self.source_file = source_file
+        self.lexer = lexer
+
+        try:
+            module_name = self.lexer.__module__.split('.')[-1]
+            class_name = self.lexer.__class__.__name__.replace('Lexer', 'Parser', 1)
+            module = import_module('.%s' % module_name, package=__package__)
+            self.parser = getattr(module, class_name)
+        except ImportError:
+            pass
+
+    def parse(self):
+        if self.parser:
+            plugin = self.parser(self.source_file, lexer=self.lexer)
+            dependencies = plugin.parse()
+            return list(set(dependencies))
+        return []

wakatime/stats.py

@@ -14,6 +14,7 @@ import os
 import sys
 
 from .compat import u, open
+from .languages import DependencyParser
 
 if sys.version_info[0] == 2:
     sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'packages', 'pygments_py2'))
@@ -46,20 +47,17 @@ TRANSLATIONS = {
 
 
 def guess_language(file_name):
-    if file_name:
-        language = guess_language_from_extension(file_name.rsplit('.', 1)[-1])
-        if language:
-            return language
-    lexer = None
+    language, lexer = None, None
     try:
         with open(file_name, 'r', encoding='utf-8') as fh:
             lexer = guess_lexer_for_filename(file_name, fh.read(512000))
     except:
         pass
-    if lexer:
-        return translate_language(u(lexer.name))
-    else:
-        return None
+    if file_name:
+        language = guess_language_from_extension(file_name.rsplit('.', 1)[-1])
+    if lexer and language is None:
+        language = translate_language(u(lexer.name))
+    return language, lexer
 
 
 def guess_language_from_extension(extension):
@@ -89,9 +87,11 @@ def number_lines_in_file(file_name):
 
 
 def get_file_stats(file_name):
-    dependencies = []
+    language, lexer = guess_language(file_name)
+    parser = DependencyParser(file_name, lexer)
+    dependencies = parser.parse()
     stats = {
-        'language': guess_language(file_name),
+        'language': language,
         'dependencies': dependencies,
         'lines': number_lines_in_file(file_name),
     }