From 7d4d50ee629d6565eae21377960552ed597bae7c Mon Sep 17 00:00:00 2001 From: Alan Hamlett Date: Tue, 23 Dec 2014 05:39:07 -0600 Subject: [PATCH] upgrade wakatime-cli to v3.0.1 --- packages/wakatime/HISTORY.rst | 12 ++ packages/wakatime/wakatime/__init__.py | 12 +- packages/wakatime/wakatime/compat.py | 35 ++++++ .../wakatime/wakatime/languages/__init__.py | 79 ++++++++++++ packages/wakatime/wakatime/languages/c_cpp.py | 37 ++++++ .../wakatime/wakatime/languages/dotnet.py | 38 ++++++ packages/wakatime/wakatime/languages/jvm.py | 38 ++++++ packages/wakatime/wakatime/languages/php.py | 66 ++++++++++ .../wakatime/wakatime/languages/python.py | 116 ++++++++++++++++++ packages/wakatime/wakatime/stats.py | 22 ++-- 10 files changed, 438 insertions(+), 17 deletions(-) create mode 100644 packages/wakatime/wakatime/languages/__init__.py create mode 100644 packages/wakatime/wakatime/languages/c_cpp.py create mode 100644 packages/wakatime/wakatime/languages/dotnet.py create mode 100644 packages/wakatime/wakatime/languages/jvm.py create mode 100644 packages/wakatime/wakatime/languages/php.py create mode 100644 packages/wakatime/wakatime/languages/python.py diff --git a/packages/wakatime/HISTORY.rst b/packages/wakatime/HISTORY.rst index 843cfe3..d7bb5b5 100644 --- a/packages/wakatime/HISTORY.rst +++ b/packages/wakatime/HISTORY.rst @@ -3,6 +3,18 @@ History ------- +3.0.1 (2014-12-23) +++++++++++++++++++ + +- handle unknown language when parsing dependencies + + +3.0.0 (2014-12-23) +++++++++++++++++++ + +- detect libraries and frameworks for C++, Java, .NET, PHP, and Python files + + 2.1.11 (2014-12-22) +++++++++++++++++++ diff --git a/packages/wakatime/wakatime/__init__.py b/packages/wakatime/wakatime/__init__.py index d13f109..5f12f14 100644 --- a/packages/wakatime/wakatime/__init__.py +++ b/packages/wakatime/wakatime/__init__.py @@ -13,7 +13,7 @@ from __future__ import print_function __title__ = 'wakatime' -__version__ = '2.1.11' +__version__ = '3.0.1' __author__ = 'Alan Hamlett' __license__ = 'BSD' __copyright__ = 'Copyright 2014 Alan Hamlett' @@ -40,7 +40,7 @@ except ImportError: sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'packages')) -from .compat import u, open, is_py2, is_py3 +from .compat import u, open, is_py3 from .queue import Queue from .log import setup_logging from .project import find_project @@ -251,10 +251,10 @@ def get_user_agent(plugin): return user_agent -def send_action(project=None, branch=None, stats={}, key=None, targetFile=None, +def send_heartbeat(project=None, branch=None, stats={}, key=None, targetFile=None, timestamp=None, isWrite=None, plugin=None, offline=None, hidefilenames=None, **kwargs): - url = 'https://wakatime.com/api/v1/actions' + url = 'https://wakatime.com/api/v1/heartbeats' log.debug('Sending heartbeat to api at %s' % url) data = { 'time': timestamp, @@ -401,7 +401,7 @@ def main(argv=None): branch = project.branch() project_name = project.name() - if send_action( + if send_heartbeat( project=project_name, branch=branch, stats=stats, @@ -412,7 +412,7 @@ def main(argv=None): heartbeat = queue.pop() if heartbeat is None: break - sent = send_action(project=heartbeat['project'], + sent = send_heartbeat(project=heartbeat['project'], targetFile=heartbeat['file'], timestamp=heartbeat['time'], branch=heartbeat['branch'], diff --git a/packages/wakatime/wakatime/compat.py b/packages/wakatime/wakatime/compat.py index 958afa1..38b8a90 100644 --- a/packages/wakatime/wakatime/compat.py +++ b/packages/wakatime/wakatime/compat.py @@ -40,3 +40,38 @@ elif is_py3: return str(text) open = open basestring = (str, bytes) + +try: + from importlib import import_module +except ImportError: + def _resolve_name(name, package, level): + """Return the absolute name of the module to be imported.""" + if not hasattr(package, 'rindex'): + raise ValueError("'package' not set to a string") + dot = len(package) + for x in xrange(level, 1, -1): + try: + dot = package.rindex('.', 0, dot) + except ValueError: + raise ValueError("attempted relative import beyond top-level " + "package") + return "%s.%s" % (package[:dot], name) + + def import_module(name, package=None): + """Import a module. + The 'package' argument is required when performing a relative import. + It specifies the package to use as the anchor point from which to + resolve the relative import to an absolute import. + """ + if name.startswith('.'): + if not package: + raise TypeError("relative imports require the 'package' " + + "argument") + level = 0 + for character in name: + if character != '.': + break + level += 1 + name = _resolve_name(name[level:], package, level) + __import__(name) + return sys.modules[name] diff --git a/packages/wakatime/wakatime/languages/__init__.py b/packages/wakatime/wakatime/languages/__init__.py new file mode 100644 index 0000000..9178dbe --- /dev/null +++ b/packages/wakatime/wakatime/languages/__init__.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +""" + wakatime.languages + ~~~~~~~~~~~~~~~~~~ + + Parse dependencies from a source code file. + + :copyright: (c) 2014 Alan Hamlett. + :license: BSD, see LICENSE for more details. +""" + +import logging + +from ..compat import open, import_module + + +log = logging.getLogger('WakaTime') + + +class TokenParser(object): + """The base class for all dependency parsers. To add support for your + language, inherit from this class and implement the :meth:`parse` method + to return a list of dependency strings. + """ + source_file = None + lexer = None + dependencies = [] + tokens = [] + + def __init__(self, source_file, lexer=None): + self.source_file = source_file + self.lexer = lexer + + def parse(self, tokens=[]): + """ Should return a list of dependencies. + """ + if not tokens and not self.tokens: + self.tokens = self._extract_tokens() + raise Exception('Not yet implemented.') + + def append(self, dep, truncate=True): + self._save_dependency(dep, truncate=truncate) + + def _extract_tokens(self): + if self.lexer: + with open(self.source_file, 'r', encoding='utf-8') as fh: + return self.lexer.get_tokens_unprocessed(fh.read(512000)) + return [] + + def _save_dependency(self, dep, truncate=True): + dep = dep.strip().split('.')[0].strip() if truncate else dep.strip() + if dep: + self.dependencies.append(dep) + + +class DependencyParser(object): + source_file = None + lexer = None + parser = None + + def __init__(self, source_file, lexer): + self.source_file = source_file + self.lexer = lexer + + if self.lexer: + try: + module_name = self.lexer.__module__.split('.')[-1] + class_name = self.lexer.__class__.__name__.replace('Lexer', 'Parser', 1) + module = import_module('.%s' % module_name, package=__package__) + self.parser = getattr(module, class_name) + except ImportError as ex: + log.debug(ex) + + def parse(self): + if self.parser: + plugin = self.parser(self.source_file, lexer=self.lexer) + dependencies = plugin.parse() + return list(set(dependencies)) + return [] diff --git a/packages/wakatime/wakatime/languages/c_cpp.py b/packages/wakatime/wakatime/languages/c_cpp.py new file mode 100644 index 0000000..f14f4ae --- /dev/null +++ b/packages/wakatime/wakatime/languages/c_cpp.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +""" + wakatime.languages.c_cpp + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Parse dependencies from C++ code. + + :copyright: (c) 2014 Alan Hamlett. + :license: BSD, see LICENSE for more details. +""" + +from . import TokenParser +from ..compat import u + + +class CppParser(TokenParser): + + def parse(self, tokens=[]): + if not tokens and not self.tokens: + self.tokens = self._extract_tokens() + for index, token, content in self.tokens: + self._process_token(token, content) + return self.dependencies + + def _process_token(self, token, content): + if u(token).split('.')[-1] == 'Preproc': + self._process_preproc(token, content) + else: + self._process_other(token, content) + + def _process_preproc(self, token, content): + if content.strip().startswith('include ') or content.strip().startswith("include\t"): + content = content.replace('include', '', 1).strip() + self.append(content, truncate=False) + + def _process_other(self, token, content): + pass diff --git a/packages/wakatime/wakatime/languages/dotnet.py b/packages/wakatime/wakatime/languages/dotnet.py new file mode 100644 index 0000000..a8ef993 --- /dev/null +++ b/packages/wakatime/wakatime/languages/dotnet.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +""" + wakatime.languages.dotnet + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + Parse dependencies from .NET code. + + :copyright: (c) 2014 Alan Hamlett. + :license: BSD, see LICENSE for more details. +""" + +from . import TokenParser +from ..compat import u + + +class CSharpParser(TokenParser): + + def parse(self, tokens=[]): + if not tokens and not self.tokens: + self.tokens = self._extract_tokens() + for index, token, content in self.tokens: + self._process_token(token, content) + return self.dependencies + + def _process_token(self, token, content): + if u(token).split('.')[-1] == 'Namespace': + self._process_namespace(token, content) + else: + self._process_other(token, content) + + def _process_namespace(self, token, content): + if content != 'import' and content != 'package' and content != 'namespace': + content = content.split('.') + content = content[0] if len(content) == 1 else '.'.join(content[0:len(content)-1]) + self.append(content, truncate=False) + + def _process_other(self, token, content): + pass diff --git a/packages/wakatime/wakatime/languages/jvm.py b/packages/wakatime/wakatime/languages/jvm.py new file mode 100644 index 0000000..ef9ca7d --- /dev/null +++ b/packages/wakatime/wakatime/languages/jvm.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +""" + wakatime.languages.java + ~~~~~~~~~~~~~~~~~~~~~~~ + + Parse dependencies from Java code. + + :copyright: (c) 2014 Alan Hamlett. + :license: BSD, see LICENSE for more details. +""" + +from . import TokenParser +from ..compat import u + + +class JavaParser(TokenParser): + + def parse(self, tokens=[]): + if not tokens and not self.tokens: + self.tokens = self._extract_tokens() + for index, token, content in self.tokens: + self._process_token(token, content) + return self.dependencies + + def _process_token(self, token, content): + if u(token).split('.')[-1] == 'Namespace': + self._process_namespace(token, content) + else: + self._process_other(token, content) + + def _process_namespace(self, token, content): + if content != 'import' and content != 'package' and content != 'namespace': + content = content.split('.') + content = content[0] if len(content) == 1 else '.'.join(content[0:len(content)-1]) + self.append(content, truncate=False) + + def _process_other(self, token, content): + pass diff --git a/packages/wakatime/wakatime/languages/php.py b/packages/wakatime/wakatime/languages/php.py new file mode 100644 index 0000000..d0a9219 --- /dev/null +++ b/packages/wakatime/wakatime/languages/php.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +""" + wakatime.languages.php + ~~~~~~~~~~~~~~~~~~~~~~ + + Parse dependencies from PHP code. + + :copyright: (c) 2014 Alan Hamlett. + :license: BSD, see LICENSE for more details. +""" + +from . import TokenParser +from ..compat import u + + +class PhpParser(TokenParser): + state = None + parens = 0 + + def parse(self, tokens=[]): + if not tokens and not self.tokens: + self.tokens = self._extract_tokens() + for index, token, content in self.tokens: + self._process_token(token, content) + return self.dependencies + + def _process_token(self, token, content): + if u(token).split('.')[-1] == 'Keyword': + self._process_keyword(token, content) + elif u(token) == 'Token.Literal.String.Single' or u(token) == 'Token.Literal.String.Double': + self._process_literal_string(token, content) + elif u(token).split('.')[-1] == 'Punctuation': + self._process_punctuation(token, content) + elif u(token).split('.')[-1] == 'Text': + self._process_text(token, content) + else: + self._process_other(token, content) + + def _process_keyword(self, token, content): + if content == 'include' or content == 'include_once' or content == 'require' or content == 'require_once': + self.state = 'include' + else: + self.state = None + + def _process_literal_string(self, token, content): + if self.state == 'include': + if content != '"': + content = content.strip() + if u(token) == 'Token.Literal.String.Single': + content = content.strip("'") + self.append(content, truncate=False) + self.state = None + + def _process_punctuation(self, token, content): + if content == '(': + self.parens += 1 + elif content == ')': + self.parens -= 1 + else: + self.state = None + + def _process_text(self, token, content): + pass + + def _process_other(self, token, content): + self.state = None diff --git a/packages/wakatime/wakatime/languages/python.py b/packages/wakatime/wakatime/languages/python.py new file mode 100644 index 0000000..1b1a087 --- /dev/null +++ b/packages/wakatime/wakatime/languages/python.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +""" + wakatime.languages.python + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + Parse dependencies from Python code. + + :copyright: (c) 2013 Alan Hamlett. + :license: BSD, see LICENSE for more details. +""" + +from . import TokenParser +from ..compat import u + + +class PythonParser(TokenParser): + state = None + parens = 0 + nonpackage = False + + def parse(self, tokens=[]): + if not tokens and not self.tokens: + self.tokens = self._extract_tokens() + for index, token, content in self.tokens: + self._process_token(token, content) + return self.dependencies + + def _process_token(self, token, content): + if u(token).split('.')[-1] == 'Namespace': + self._process_namespace(token, content) + elif u(token).split('.')[-1] == 'Name': + self._process_name(token, content) + elif u(token).split('.')[-1] == 'Word': + self._process_word(token, content) + elif u(token).split('.')[-1] == 'Operator': + self._process_operator(token, content) + elif u(token).split('.')[-1] == 'Punctuation': + self._process_punctuation(token, content) + elif u(token).split('.')[-1] == 'Text': + self._process_text(token, content) + else: + self._process_other(token, content) + + def _process_namespace(self, token, content): + if self.state is None: + self.state = content + else: + self._process_import(token, content) + + def _process_name(self, token, content): + if self.state is not None: + if self.nonpackage: + self.nonpackage = False + else: + if self.state == 'from': + self.append(content) + if self.state == 'from-2' and content != 'import': + self.append(content) + elif self.state == 'import': + self.append(content) + elif self.state == 'import-2': + self.append(content) + else: + self.state = None + + def _process_word(self, token, content): + if self.state is not None: + if self.nonpackage: + self.nonpackage = False + else: + if self.state == 'from': + self.append(content) + if self.state == 'from-2' and content != 'import': + self.append(content) + elif self.state == 'import': + self.append(content) + elif self.state == 'import-2': + self.append(content) + else: + self.state = None + + def _process_operator(self, token, content): + if self.state is not None: + if content == '.': + self.nonpackage = True + + def _process_punctuation(self, token, content): + if content == '(': + self.parens += 1 + elif content == ')': + self.parens -= 1 + self.nonpackage = False + + def _process_text(self, token, content): + if self.state is not None: + if content == "\n" and self.parens == 0: + self.state = None + self.nonpackage = False + + def _process_other(self, token, content): + pass + + def _process_import(self, token, content): + self.nonpackage = False + if self.state == 'from': + self.append(content) + self.state = 'from-2' + elif self.state == 'from-2' and content != 'import': + self.append(content) + elif self.state == 'import': + self.append(content) + self.state = 'import-2' + elif self.state == 'import-2': + self.append(content) + else: + self.state = None diff --git a/packages/wakatime/wakatime/stats.py b/packages/wakatime/wakatime/stats.py index 8afff9a..50ff4a4 100644 --- a/packages/wakatime/wakatime/stats.py +++ b/packages/wakatime/wakatime/stats.py @@ -14,6 +14,7 @@ import os import sys from .compat import u, open +from .languages import DependencyParser if sys.version_info[0] == 2: sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'packages', 'pygments_py2')) @@ -46,20 +47,17 @@ TRANSLATIONS = { def guess_language(file_name): - if file_name: - language = guess_language_from_extension(file_name.rsplit('.', 1)[-1]) - if language: - return language - lexer = None + language, lexer = None, None try: with open(file_name, 'r', encoding='utf-8') as fh: lexer = guess_lexer_for_filename(file_name, fh.read(512000)) except: pass - if lexer: - return translate_language(u(lexer.name)) - else: - return None + if file_name: + language = guess_language_from_extension(file_name.rsplit('.', 1)[-1]) + if lexer and language is None: + language = translate_language(u(lexer.name)) + return language, lexer def guess_language_from_extension(extension): @@ -89,9 +87,11 @@ def number_lines_in_file(file_name): def get_file_stats(file_name): - dependencies = [] + language, lexer = guess_language(file_name) + parser = DependencyParser(file_name, lexer) + dependencies = parser.parse() stats = { - 'language': guess_language(file_name), + 'language': language, 'dependencies': dependencies, 'lines': number_lines_in_file(file_name), }