tests for c and cpp dependency detection. change api of TokenParser class.

wakatime/languages/__init__.py

@@ -29,17 +29,21 @@ class TokenParser(object):
     exclude = []
 
     def __init__(self, source_file, lexer=None):
-        self.tokens = []
+        self._tokens = None
         self.dependencies = []
         self.source_file = source_file
         self.lexer = lexer
         self.exclude = [re.compile(x, re.IGNORECASE) for x in self.exclude]
 
+    @property
+    def tokens(self):
+        if self._tokens is None:
+            self._tokens = self._extract_tokens()
+        return self._tokens
+
     def parse(self, tokens=[]):
         """ Should return a list of dependencies.
         """
-        if not tokens and not self.tokens:
-            self.tokens = self._extract_tokens()
         raise NotYetImplemented()
 
     def append(self, dep, truncate=False, separator=None, truncate_to=None,

wakatime/languages/c_cpp.py

@@ -13,10 +13,14 @@ from . import TokenParser
 
 
 class CppParser(TokenParser):
+    exclude = [
+        r'^stdio\.h$',
+        r'^stdlib\.h$',
+        r'^string\.h$',
+        r'^time\.h$',
+    ]
 
-    def parse(self, tokens=[]):
-        if not tokens and not self.tokens:
-            self.tokens = self._extract_tokens()
+    def parse(self):
         for index, token, content in self.tokens:
             self._process_token(token, content)
         return self.dependencies
@@ -29,7 +33,35 @@ class CppParser(TokenParser):
 
     def _process_preproc(self, token, content):
         if content.strip().startswith('include ') or content.strip().startswith("include\t"):
-            content = content.replace('include', '', 1).strip()
+            content = content.replace('include', '', 1).strip().strip('"').strip('<').strip('>').strip()
+            self.append(content)
+
+    def _process_other(self, token, content):
+        pass
+
+
+class CParser(TokenParser):
+    exclude = [
+        r'^stdio\.h$',
+        r'^stdlib\.h$',
+        r'^string\.h$',
+        r'^time\.h$',
+    ]
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Preproc':
+            self._process_preproc(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_preproc(self, token, content):
+        if content.strip().startswith('include ') or content.strip().startswith("include\t"):
+            content = content.replace('include', '', 1).strip().strip('"').strip('<').strip('>').strip()
             self.append(content)
 
     def _process_other(self, token, content):

wakatime/languages/data.py

@@ -26,10 +26,8 @@ class JsonParser(TokenParser):
     state = None
     level = 0
 
-    def parse(self, tokens=[]):
+    def parse(self):
         self._process_file_name(os.path.basename(self.source_file))
-        if not tokens and not self.tokens:
-            self.tokens = self._extract_tokens()
         for index, token, content in self.tokens:
             self._process_token(token, content)
         return self.dependencies

wakatime/languages/dotnet.py

@@ -14,9 +14,7 @@ from . import TokenParser
 
 class CSharpParser(TokenParser):
 
-    def parse(self, tokens=[]):
-        if not tokens and not self.tokens:
-            self.tokens = self._extract_tokens()
+    def parse(self):
         for index, token, content in self.tokens:
             self._process_token(token, content)
         return self.dependencies

wakatime/languages/jvm.py

@@ -25,9 +25,7 @@ class JavaParser(TokenParser):
     state = None
     buffer = u('')
 
-    def parse(self, tokens=[]):
-        if not tokens and not self.tokens:
-            self.tokens = self._extract_tokens()
+    def parse(self):
         for index, token, content in self.tokens:
             self._process_token(token, content)
         return self.dependencies

wakatime/languages/php.py

@@ -17,9 +17,7 @@ class PhpParser(TokenParser):
     state = None
     parens = 0
 
-    def parse(self, tokens=[]):
-        if not tokens and not self.tokens:
-            self.tokens = self._extract_tokens()
+    def parse(self):
         for index, token, content in self.tokens:
             self._process_token(token, content)
         return self.dependencies

wakatime/languages/python.py

@@ -16,10 +16,12 @@ class PythonParser(TokenParser):
     state = None
     parens = 0
     nonpackage = False
+    exclude = [
+        r'^os$',
+        r'^sys\.',
+    ]
 
-    def parse(self, tokens=[]):
-        if not tokens and not self.tokens:
-            self.tokens = self._extract_tokens()
+    def parse(self):
         for index, token, content in self.tokens:
             self._process_token(token, content)
         return self.dependencies

wakatime/languages/templates.py

@@ -71,9 +71,7 @@ KEYWORDS = [
 
 class LassoJavascriptParser(TokenParser):
 
-    def parse(self, tokens=[]):
-        if not tokens and not self.tokens:
-            self.tokens = self._extract_tokens()
+    def parse(self):
         for index, token, content in self.tokens:
             self._process_token(token, content)
         return self.dependencies
@@ -99,9 +97,7 @@ class HtmlDjangoParser(TokenParser):
     current_attr = None
     current_attr_value = None
 
-    def parse(self, tokens=[]):
-        if not tokens and not self.tokens:
-            self.tokens = self._extract_tokens()
+    def parse(self):
         for index, token, content in self.tokens:
             self._process_token(token, content)
         return self.dependencies

wakatime/languages/unknown.py

@@ -22,7 +22,7 @@ FILES = {
 
 class UnknownParser(TokenParser):
 
-    def parse(self, tokens=[]):
+    def parse(self):
         self._process_file_name(os.path.basename(self.source_file))
         return self.dependencies