move wakatime-cli to parent folder for faster startup on Windows

packages/wakatime/dependencies/__init__.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from a source code file.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import logging
+import re
+import sys
+
+from ..compat import u, open, import_module
+from ..exceptions import NotYetImplemented
+
+
+log = logging.getLogger('WakaTime')
+
+
+class TokenParser(object):
+    """The base class for all dependency parsers. To add support for your
+    language, inherit from this class and implement the :meth:`parse` method
+    to return a list of dependency strings.
+    """
+    exclude = []
+
+    def __init__(self, source_file, lexer=None):
+        self._tokens = None
+        self.dependencies = []
+        self.source_file = source_file
+        self.lexer = lexer
+        self.exclude = [re.compile(x, re.IGNORECASE) for x in self.exclude]
+
+    @property
+    def tokens(self):
+        if self._tokens is None:
+            self._tokens = self._extract_tokens()
+        return self._tokens
+
+    def parse(self, tokens=[]):
+        """ Should return a list of dependencies.
+        """
+        raise NotYetImplemented()
+
+    def append(self, dep, truncate=False, separator=None, truncate_to=None,
+               strip_whitespace=True):
+        self._save_dependency(
+            dep,
+            truncate=truncate,
+            truncate_to=truncate_to,
+            separator=separator,
+            strip_whitespace=strip_whitespace,
+        )
+
+    def partial(self, token):
+        return u(token).split('.')[-1]
+
+    def _extract_tokens(self):
+        if self.lexer:
+            try:
+                with open(self.source_file, 'r', encoding='utf-8') as fh:
+                    return self.lexer.get_tokens_unprocessed(fh.read(512000))
+            except:
+                pass
+            try:
+                with open(self.source_file, 'r', encoding=sys.getfilesystemencoding()) as fh:
+                    return self.lexer.get_tokens_unprocessed(fh.read(512000))  # pragma: nocover
+            except:
+                pass
+        return []
+
+    def _save_dependency(self, dep, truncate=False, separator=None,
+                         truncate_to=None, strip_whitespace=True):
+        if truncate:
+            if separator is None:
+                separator = u('.')
+            separator = u(separator)
+            dep = dep.split(separator)
+            if truncate_to is None or truncate_to < 1:
+                truncate_to = 1
+            if truncate_to > len(dep):
+                truncate_to = len(dep)
+            dep = dep[0] if len(dep) == 1 else separator.join(dep[:truncate_to])
+        if strip_whitespace:
+            dep = dep.strip()
+        if dep and (not separator or not dep.startswith(separator)):
+            should_exclude = False
+            for compiled in self.exclude:
+                if compiled.search(dep):
+                    should_exclude = True
+                    break
+            if not should_exclude:
+                self.dependencies.append(dep)
+
+
+class DependencyParser(object):
+    source_file = None
+    lexer = None
+    parser = None
+
+    def __init__(self, source_file, lexer):
+        self.source_file = source_file
+        self.lexer = lexer
+
+        if self.lexer:
+            module_name = self.root_lexer.__module__.rsplit('.', 1)[-1]
+            class_name = self.root_lexer.__class__.__name__.replace('Lexer', 'Parser', 1)
+        else:
+            module_name = 'unknown'
+            class_name = 'UnknownParser'
+
+        try:
+            module = import_module('.%s' % module_name, package=__package__)
+            try:
+                self.parser = getattr(module, class_name)
+            except AttributeError:
+                log.debug('Parsing dependencies not supported for {0}.{1}'.format(module_name, class_name))
+        except ImportError:
+            log.debug('Parsing dependencies not supported for {0}.{1}'.format(module_name, class_name))
+
+    @property
+    def root_lexer(self):
+        if hasattr(self.lexer, 'root_lexer'):
+            return self.lexer.root_lexer
+        return self.lexer
+
+    def parse(self):
+        if self.parser:
+            plugin = self.parser(self.source_file, lexer=self.lexer)
+            dependencies = plugin.parse()
+            return list(set(dependencies))
+        return []

packages/wakatime/dependencies/c_cpp.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.c_cpp
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from C++ code.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class CParser(TokenParser):
+    exclude = [
+        r'^stdio\.h$',
+        r'^stdlib\.h$',
+        r'^string\.h$',
+        r'^time\.h$',
+    ]
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Preproc' or self.partial(token) == 'PreprocFile':
+            self._process_preproc(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_preproc(self, token, content):
+        if self.state == 'include':
+            if content != '\n' and content != '#':
+                content = content.strip().strip('"').strip('<').strip('>').strip()
+                self.append(content, truncate=True, separator='/')
+            self.state = None
+        elif content.strip().startswith('include'):
+            self.state = 'include'
+        else:
+            self.state = None
+
+    def _process_other(self, token, content):
+        pass
+
+
+class CppParser(CParser):
+    pass

packages/wakatime/dependencies/data.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.data
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from data files.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import os
+
+from . import TokenParser
+from ..compat import u
+
+
+FILES = {
+    'bower.json': {'exact': True, 'dependency': 'bower'},
+    'component.json': {'exact': True, 'dependency': 'bower'},
+    'package.json': {'exact': True, 'dependency': 'npm'},
+}
+
+
+class JsonParser(TokenParser):
+    state = None
+    level = 0
+
+    def parse(self):
+        self._process_file_name(os.path.basename(self.source_file))
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_file_name(self, file_name):
+        for key, value in FILES.items():
+            found = (key == file_name) if value.get('exact') else (key.lower() in file_name.lower())
+            if found:
+                self.append(value['dependency'])
+
+    def _process_token(self, token, content):
+        if u(token) == 'Token.Name.Tag':
+            self._process_tag(token, content)
+        elif u(token) == 'Token.Literal.String.Single' or u(token) == 'Token.Literal.String.Double':
+            self._process_literal_string(token, content)
+        elif u(token) == 'Token.Punctuation':
+            self._process_punctuation(token, content)
+
+    def _process_tag(self, token, content):
+        if content.strip('"').strip("'") == 'dependencies' or content.strip('"').strip("'") == 'devDependencies':
+            self.state = 'dependencies'
+        elif self.state == 'dependencies' and self.level == 2:
+            self.append(content.strip('"').strip("'"))
+
+    def _process_literal_string(self, token, content):
+        pass
+
+    def _process_punctuation(self, token, content):
+        if content == '{':
+            self.level += 1
+        elif content == '}':
+            self.level -= 1
+            if self.state is not None and self.level <= 1:
+                self.state = None

packages/wakatime/dependencies/dotnet.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.dotnet
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from .NET code.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+from ..compat import u
+
+
+class CSharpParser(TokenParser):
+    exclude = [
+        r'^system$',
+        r'^microsoft$',
+    ]
+    state = None
+    buffer = u('')
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        if self.partial(token) == 'Namespace' or self.partial(token) == 'Name':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Punctuation':
+            self._process_punctuation(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_keyword(self, token, content):
+        if content == 'using':
+            self.state = 'import'
+            self.buffer = u('')
+
+    def _process_namespace(self, token, content):
+        if self.state == 'import':
+            if u(content) != u('import') and u(content) != u('package') and u(content) != u('namespace') and u(content) != u('static'):
+                if u(content) == u(';'):  # pragma: nocover
+                    self._process_punctuation(token, content)
+                else:
+                    self.buffer += u(content)
+
+    def _process_punctuation(self, token, content):
+        if self.state == 'import':
+            if u(content) == u(';'):
+                self.append(self.buffer, truncate=True)
+                self.buffer = u('')
+                self.state = None
+            elif u(content) == u('='):
+                self.buffer = u('')
+            else:
+                self.buffer += u(content)
+
+    def _process_other(self, token, content):
+        pass

packages/wakatime/dependencies/elm.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.elm
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Elm code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class ElmParser(TokenParser):
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        elif self.partial(token) == 'Class':
+            self._process_class(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_namespace(self, token, content):
+        self.state = content.strip()
+
+    def _process_class(self, token, content):
+        if self.state == 'import':
+            self.append(self._format(content))
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_other(self, token, content):
+        self.state = None
+
+    def _format(self, content):
+        return content.strip().split('.')[0].strip()

packages/wakatime/dependencies/go.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.go
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Go code.
+
+    :copyright: (c) 2016 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class GoParser(TokenParser):
+    state = None
+    parens = 0
+    aliases = 0
+    exclude = [
+        r'^"fmt"$',
+    ]
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Punctuation':
+            self._process_punctuation(token, content)
+        elif self.partial(token) == 'String':
+            self._process_string(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        elif self.partial(token) == 'Other':
+            self._process_other(token, content)
+        else:
+            self._process_misc(token, content)
+
+    def _process_namespace(self, token, content):
+        self.state = content
+        self.parens = 0
+        self.aliases = 0
+
+    def _process_string(self, token, content):
+        if self.state == 'import':
+            self.append(content, truncate=False)
+
+    def _process_punctuation(self, token, content):
+        if content == '(':
+            self.parens += 1
+        elif content == ')':
+            self.parens -= 1
+        elif content == '.':
+            self.aliases += 1
+        else:
+            self.state = None
+
+    def _process_text(self, token, content):
+        if self.state == 'import':
+            if content == "\n" and self.parens <= 0:
+                self.state = None
+                self.parens = 0
+                self.aliases = 0
+        else:
+            self.state = None
+
+    def _process_other(self, token, content):
+        if self.state == 'import':
+            self.aliases += 1
+        else:
+            self.state = None
+
+    def _process_misc(self, token, content):
+        self.state = None

packages/wakatime/dependencies/haskell.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.haskell
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Haskell code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class HaskellParser(TokenParser):
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Reserved':
+            self._process_reserved(token, content)
+        elif self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_reserved(self, token, content):
+        self.state = content.strip()
+
+    def _process_namespace(self, token, content):
+        if self.state == 'import':
+            self.append(self._format(content))
+
+    def _process_keyword(self, token, content):
+        if self.state != 'import' or content.strip() != 'qualified':
+            self.state = None
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_other(self, token, content):
+        self.state = None
+
+    def _format(self, content):
+        return content.strip().split('.')[0].strip()

packages/wakatime/dependencies/haxe.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.haxe
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Haxe code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class HaxeParser(TokenParser):
+    exclude = [
+        r'^haxe$',
+    ]
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_namespace(self, token, content):
+        if self.state == 'import':
+            self.append(self._format(content))
+            self.state = None
+        else:
+            self.state = content
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_other(self, token, content):
+        self.state = None
+
+    def _format(self, content):
+        return content.strip()

packages/wakatime/dependencies/html.py

@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.html
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from HTML.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+from ..compat import u
+
+
+""" If these keywords are found in the source file, treat them as a dependency.
+Must be lower-case strings.
+"""
+KEYWORDS = [
+    '_',
+    '$',
+    'angular',
+    'assert',  # probably mocha
+    'backbone',
+    'batman',
+    'c3',
+    'can',
+    'casper',
+    'chai',
+    'chaplin',
+    'd3',
+    'define',  # probably require
+    'describe',  # mocha or jasmine
+    'eco',
+    'ember',
+    'espresso',
+    'expect',  # probably jasmine
+    'exports',  # probably npm
+    'express',
+    'gulp',
+    'handlebars',
+    'highcharts',
+    'jasmine',
+    'jquery',
+    'jstz',
+    'ko',  # probably knockout
+    'm',  # probably mithril
+    'marionette',
+    'meteor',
+    'moment',
+    'monitorio',
+    'mustache',
+    'phantom',
+    'pickadate',
+    'pikaday',
+    'qunit',
+    'react',
+    'reactive',
+    'require',  # probably the commonjs spec
+    'ripple',
+    'rivets',
+    'socketio',
+    'spine',
+    'thorax',
+    'underscore',
+    'vue',
+    'way',
+    'zombie',
+]
+
+
+class HtmlParser(TokenParser):
+    tags = []
+    opening_tag = False
+    getting_attrs = False
+    current_attr = None
+    current_attr_value = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if u(token) == 'Token.Punctuation':
+            self._process_punctuation(token, content)
+        elif u(token) == 'Token.Name.Tag':
+            self._process_tag(token, content)
+        elif u(token) == 'Token.Literal.String':
+            self._process_string(token, content)
+        elif u(token) == 'Token.Name.Attribute':
+            self._process_attribute(token, content)
+
+    @property
+    def current_tag(self):
+        return None if len(self.tags) == 0 else self.tags[0]
+
+    def _process_punctuation(self, token, content):
+        if content.startswith('</') or content.startswith('/'):
+            try:
+                self.tags.pop(0)
+            except IndexError:
+                # ignore errors from malformed markup
+                pass
+            self.opening_tag = False
+            self.getting_attrs = False
+        elif content.startswith('<'):
+            self.opening_tag = True
+        elif content.startswith('>'):
+            self.opening_tag = False
+            self.getting_attrs = False
+
+    def _process_tag(self, token, content):
+        if self.opening_tag:
+            self.tags.insert(0, content.replace('<', '', 1).strip().lower())
+            self.getting_attrs = True
+        self.current_attr = None
+
+    def _process_attribute(self, token, content):
+        if self.getting_attrs:
+            self.current_attr = content.lower().strip('=')
+        self.current_attr_value = None
+
+    def _process_string(self, token, content):
+        if self.getting_attrs and self.current_attr is not None:
+            if content.endswith('"') or content.endswith("'"):
+                if self.current_attr_value is not None:
+                    self.current_attr_value += content
+                    if self.current_tag == 'script' and self.current_attr == 'src':
+                        self.append(self.current_attr_value)
+                    self.current_attr = None
+                    self.current_attr_value = None
+                else:
+                    if len(content) == 1:
+                        self.current_attr_value = content
+                    else:
+                        if self.current_tag == 'script' and self.current_attr == 'src':
+                            self.append(content)
+                        self.current_attr = None
+                        self.current_attr_value = None
+            elif content.startswith('"') or content.startswith("'"):
+                if self.current_attr_value is None:
+                    self.current_attr_value = content

packages/wakatime/dependencies/javascript.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.javascript
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from JavaScript code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import re
+
+from . import TokenParser
+
+
+class JavascriptParser(TokenParser):
+    state = None
+    extension = re.compile(r'\.\w{1,4}$')
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Reserved':
+            self._process_reserved(token, content)
+        elif self.partial(token) == 'Single':
+            self._process_string(token, content)
+        elif self.partial(token) == 'Punctuation':
+            self._process_punctuation(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_reserved(self, token, content):
+        if self.state is None:
+            self.state = content
+
+    def _process_string(self, token, content):
+        if self.state == 'import':
+            self.append(self._format_module(content))
+        self.state = None
+
+    def _process_punctuation(self, token, content):
+        if content == ';':
+            self.state = None
+
+    def _process_other(self, token, content):
+        pass
+
+    def _format_module(self, content):
+        content = content.strip().strip('"').strip("'").strip()
+        content = content.split('/')[-1].split('\\')[-1]
+        content = self.extension.sub('', content, count=1)
+        return content
+
+
+class TypeScriptParser(JavascriptParser):
+    pass

packages/wakatime/dependencies/jvm.py

@@ -0,0 +1,182 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.java
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Java code.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+from ..compat import u
+
+
+class JavaParser(TokenParser):
+    exclude = [
+        r'^java\.',
+        r'^javax\.',
+        r'^import$',
+        r'^package$',
+        r'^namespace$',
+        r'^static$',
+    ]
+    state = None
+    buffer = u('')
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        if self.partial(token) == 'Name':
+            self._process_name(token, content)
+        elif self.partial(token) == 'Attribute':
+            self._process_attribute(token, content)
+        elif self.partial(token) == 'Operator':
+            self._process_operator(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_namespace(self, token, content):
+        if u(content).split() and u(content).split()[0] == u('import'):
+            self.state = 'import'
+
+        elif self.state == 'import':
+            keywords = [
+                u('package'),
+                u('namespace'),
+                u('static'),
+            ]
+            if u(content) in keywords:
+                return
+            self.buffer = u('{0}{1}').format(self.buffer, u(content))
+
+        elif self.state == 'import-finished':
+            content = content.split(u('.'))
+
+            if len(content) == 1:
+                self.append(content[0])
+
+            elif len(content) > 1:
+                if len(content[0]) == 3:
+                    content = content[1:]
+                if content[-1] == u('*'):
+                    content = content[:len(content) - 1]
+
+                if len(content) == 1:
+                    self.append(content[0])
+                elif len(content) > 1:
+                    self.append(u('.').join(content[:2]))
+
+            self.state = None
+
+    def _process_name(self, token, content):
+        if self.state == 'import':
+            self.buffer = u('{0}{1}').format(self.buffer, u(content))
+
+    def _process_attribute(self, token, content):
+        if self.state == 'import':
+            self.buffer = u('{0}{1}').format(self.buffer, u(content))
+
+    def _process_operator(self, token, content):
+        if u(content) == u(';'):
+            self.state = 'import-finished'
+            self._process_namespace(token, self.buffer)
+            self.state = None
+            self.buffer = u('')
+        elif self.state == 'import':
+            self.buffer = u('{0}{1}').format(self.buffer, u(content))
+
+    def _process_other(self, token, content):
+        pass
+
+
+class KotlinParser(TokenParser):
+    state = None
+    exclude = [
+        r'^java\.',
+    ]
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        elif self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_keyword(self, token, content):
+        self.state = content
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_namespace(self, token, content):
+        if self.state == 'import':
+            self.append(self._format(content))
+        self.state = None
+
+    def _process_other(self, token, content):
+        self.state = None
+
+    def _format(self, content):
+        content = content.split(u('.'))
+
+        if content[-1] == u('*'):
+            content = content[:len(content) - 1]
+
+        if len(content) == 0:
+            return None
+
+        if len(content) == 1:
+            return content[0]
+
+        return u('.').join(content[:2])
+
+
+class ScalaParser(TokenParser):
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        elif self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_keyword(self, token, content):
+        self.state = content
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_namespace(self, token, content):
+        if self.state == 'import':
+            self.append(self._format(content))
+        self.state = None
+
+    def _process_other(self, token, content):
+        self.state = None
+
+    def _format(self, content):
+        return content.strip().lstrip('__root__').strip('_').strip('.')

packages/wakatime/dependencies/objective.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.objective
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Objective-C and Swift code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import re
+
+from . import TokenParser
+
+
+class SwiftParser(TokenParser):
+    state = None
+    exclude = [
+        r'^foundation$',
+    ]
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Declaration':
+            self._process_declaration(token, content)
+        elif self.partial(token) == 'Class':
+            self._process_class(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_declaration(self, token, content):
+        if self.state is None:
+            self.state = content
+
+    def _process_class(self, token, content):
+        if self.state == 'import':
+            self.append(content)
+        self.state = None
+
+    def _process_other(self, token, content):
+        pass
+
+
+class ObjectiveCParser(TokenParser):
+    state = None
+    extension = re.compile(r'\.[mh]$')
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Preproc':
+            self._process_preproc(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_preproc(self, token, content):
+        if self.state:
+            self._process_import(token, content)
+
+        self.state = content
+
+    def _process_import(self, token, content):
+        if self.state == '#' and content.startswith('import '):
+            self.append(self._format(content))
+        self.state = None
+
+    def _process_other(self, token, content):
+        pass
+
+    def _format(self, content):
+        content = content.strip().lstrip('import ').strip()
+        content = content.strip('"').strip("'").strip()
+        content = content.strip('<').strip('>').strip()
+        content = content.split('/')[0]
+        content = self.extension.sub('', content, count=1)
+        return content

packages/wakatime/dependencies/php.py

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.php
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from PHP code.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+from ..compat import u
+
+
+class PhpParser(TokenParser):
+    state = None
+    parens = 0
+    exclude = [
+        r'^app$',
+        r'app\.php$',
+    ]
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        elif u(token) == 'Token.Literal.String.Single' or u(token) == 'Token.Literal.String.Double':
+            self._process_literal_string(token, content)
+        elif u(token) == 'Token.Name.Other':
+            self._process_name(token, content)
+        elif u(token) == 'Token.Name.Function':
+            self._process_function(token, content)
+        elif self.partial(token) == 'Punctuation':
+            self._process_punctuation(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_name(self, token, content):
+        if self.state == 'use':
+            self.append(content, truncate=True, separator=u("\\"))
+
+    def _process_function(self, token, content):
+        if self.state == 'use function':
+            self.append(content, truncate=True, separator=u("\\"))
+            self.state = 'use'
+
+    def _process_keyword(self, token, content):
+        if content == 'include' or content == 'include_once' or content == 'require' or content == 'require_once':
+            self.state = 'include'
+        elif content == 'use':
+            self.state = 'use'
+        elif content == 'as':
+            self.state = 'as'
+        elif self.state == 'use' and content == 'function':
+            self.state = 'use function'
+        else:
+            self.state = None
+
+    def _process_literal_string(self, token, content):
+        if self.state == 'include':
+            if content != '"' and content != "'":
+                content = content.strip()
+                if u(token) == 'Token.Literal.String.Double':
+                    content = u("'{0}'").format(content)
+                self.append(content)
+                self.state = None
+
+    def _process_punctuation(self, token, content):
+        if content == '(':
+            self.parens += 1
+        elif content == ')':
+            self.parens -= 1
+        elif (self.state == 'use' or self.state == 'as') and content == ',':
+            self.state = 'use'
+        else:
+            self.state = None
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_other(self, token, content):
+        self.state = None

packages/wakatime/dependencies/python.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.python
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Python code.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class PythonParser(TokenParser):
+    state = None
+    parens = 0
+    nonpackage = False
+    exclude = [
+        r'^os$',
+        r'^sys$',
+        r'^sys\.',
+        r'^__future__$',
+    ]
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Operator':
+            self._process_operator(token, content)
+        elif self.partial(token) == 'Punctuation':
+            self._process_punctuation(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_namespace(self, token, content):
+        if self.state is None:
+            self.state = content
+        else:
+            if content == 'as':
+                self.nonpackage = True
+            else:
+                self._process_import(token, content)
+
+    def _process_operator(self, token, content):
+        pass
+
+    def _process_punctuation(self, token, content):
+        if content == '(':
+            self.parens += 1
+        elif content == ')':
+            self.parens -= 1
+        self.nonpackage = False
+
+    def _process_text(self, token, content):
+        if self.state is not None:
+            if content == "\n" and self.parens == 0:
+                self.state = None
+                self.nonpackage = False
+
+    def _process_other(self, token, content):
+        pass
+
+    def _process_import(self, token, content):
+        if not self.nonpackage:
+            if self.state == 'from':
+                self.append(content, truncate=True, truncate_to=1)
+                self.state = 'from-2'
+            elif self.state == 'import':
+                self.append(content, truncate=True, truncate_to=1)
+                self.state = 'import-2'
+            elif self.state == 'import-2':
+                self.append(content, truncate=True, truncate_to=1)
+            else:
+                self.state = None
+        self.nonpackage = False

packages/wakatime/dependencies/rust.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.rust
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Rust code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class RustParser(TokenParser):
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        elif self.partial(token) == 'Whitespace':
+            self._process_whitespace(token, content)
+        elif self.partial(token) == 'Name':
+            self._process_name(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_keyword(self, token, content):
+        if self.state == 'extern' and content == 'crate':
+            self.state = 'extern crate'
+        else:
+            self.state = content
+
+    def _process_whitespace(self, token, content):
+        pass
+
+    def _process_name(self, token, content):
+        if self.state == 'extern crate':
+            self.append(content)
+        self.state = None
+
+    def _process_other(self, token, content):
+        self.state = None

packages/wakatime/dependencies/unknown.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.unknown
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from files of unknown language.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import os
+
+from . import TokenParser
+
+
+FILES = {
+    'bower': {'exact': False, 'dependency': 'bower'},
+    'grunt': {'exact': False, 'dependency': 'grunt'},
+}
+
+
+class UnknownParser(TokenParser):
+
+    def parse(self):
+        self._process_file_name(os.path.basename(self.source_file))
+        return self.dependencies
+
+    def _process_file_name(self, file_name):
+        for key, value in FILES.items():
+            found = (key == file_name) if value.get('exact') else (key.lower() in file_name.lower())
+            if found:
+                self.append(value['dependency'])