Upgrade wakatime-cli to v10.1.2

This commit is contained in:
Alan Hamlett 2018-03-15 01:31:17 -07:00
parent 9e102d7c5c
commit bc770515f0
21 changed files with 520 additions and 93 deletions

View File

@ -1,7 +1,7 @@
__title__ = 'wakatime'
__description__ = 'Common interface to the WakaTime api.'
__url__ = 'https://github.com/wakatime/wakatime'
__version_info__ = ('10', '1', '0')
__version_info__ = ('10', '1', '2')
__version__ = '.'.join(__version_info__)
__author__ = 'Alan Hamlett'
__author_email__ = 'alan@wakatime.com'

View File

@ -106,8 +106,8 @@ class DependencyParser(object):
self.lexer = lexer
if self.lexer:
module_name = self.lexer.__module__.rsplit('.', 1)[-1]
class_name = self.lexer.__class__.__name__.replace('Lexer', 'Parser', 1)
module_name = self.root_lexer.__module__.rsplit('.', 1)[-1]
class_name = self.root_lexer.__class__.__name__.replace('Lexer', 'Parser', 1)
else:
module_name = 'unknown'
class_name = 'UnknownParser'
@ -121,6 +121,12 @@ class DependencyParser(object):
except ImportError:
log.debug('Parsing dependencies not supported for {0}.{1}'.format(module_name, class_name))
@property
def root_lexer(self):
if hasattr(self.lexer, 'root_lexer'):
return self.lexer.root_lexer
return self.lexer
def parse(self):
if self.parser:
plugin = self.parser(self.source_file, lexer=self.lexer)

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.c_cpp
~~~~~~~~~~~~~~~~~~~~~~~~
wakatime.dependencies.c_cpp
~~~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from C++ code.

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.data
~~~~~~~~~~~~~~~~~~~~~~~
wakatime.dependencies.data
~~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from data files.

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.dotnet
~~~~~~~~~~~~~~~~~~~~~~~~~
wakatime.dependencies.dotnet
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from .NET code.

View File

@ -0,0 +1,47 @@
# -*- coding: utf-8 -*-
"""
wakatime.dependencies.elm
~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from Elm code.
:copyright: (c) 2018 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
from . import TokenParser
class ElmParser(TokenParser):
state = None
def parse(self):
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_token(self, token, content):
if self.partial(token) == 'Namespace':
self._process_namespace(token, content)
elif self.partial(token) == 'Text':
self._process_text(token, content)
elif self.partial(token) == 'Class':
self._process_class(token, content)
else:
self._process_other(token, content)
def _process_namespace(self, token, content):
self.state = content.strip()
def _process_class(self, token, content):
if self.state == 'import':
self.append(self._format(content))
def _process_text(self, token, content):
pass
def _process_other(self, token, content):
self.state = None
def _format(self, content):
return content.strip().split('.')[0].strip()

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.go
~~~~~~~~~~~~~~~~~~~~~
wakatime.dependencies.go
~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from Go code.

View File

@ -0,0 +1,53 @@
# -*- coding: utf-8 -*-
"""
wakatime.dependencies.haskell
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from Haskell code.
:copyright: (c) 2018 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
from . import TokenParser
class HaskellParser(TokenParser):
state = None
def parse(self):
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_token(self, token, content):
if self.partial(token) == 'Reserved':
self._process_reserved(token, content)
elif self.partial(token) == 'Namespace':
self._process_namespace(token, content)
elif self.partial(token) == 'Keyword':
self._process_keyword(token, content)
elif self.partial(token) == 'Text':
self._process_text(token, content)
else:
self._process_other(token, content)
def _process_reserved(self, token, content):
self.state = content.strip()
def _process_namespace(self, token, content):
if self.state == 'import':
self.append(self._format(content))
def _process_keyword(self, token, content):
if self.state != 'import' or content.strip() != 'qualified':
self.state = None
def _process_text(self, token, content):
pass
def _process_other(self, token, content):
self.state = None
def _format(self, content):
return content.strip().split('.')[0].strip()

View File

@ -0,0 +1,48 @@
# -*- coding: utf-8 -*-
"""
wakatime.dependencies.haxe
~~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from Haxe code.
:copyright: (c) 2018 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
from . import TokenParser
class HaxeParser(TokenParser):
exclude = [
r'^haxe$',
]
state = None
def parse(self):
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_token(self, token, content):
if self.partial(token) == 'Namespace':
self._process_namespace(token, content)
elif self.partial(token) == 'Text':
self._process_text(token, content)
else:
self._process_other(token, content)
def _process_namespace(self, token, content):
if self.state == 'import':
self.append(self._format(content))
self.state = None
else:
self.state = content
def _process_text(self, token, content):
pass
def _process_other(self, token, content):
self.state = None
def _format(self, content):
return content.strip()

View File

@ -1,9 +1,9 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.templates
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
wakatime.dependencies.html
~~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from Templates.
Parse dependencies from HTML.
:copyright: (c) 2014 Alan Hamlett.
:license: BSD, see LICENSE for more details.
@ -69,7 +69,7 @@ KEYWORDS = [
]
class HtmlDjangoParser(TokenParser):
class HtmlParser(TokenParser):
tags = []
opening_tag = False
getting_attrs = False
@ -141,63 +141,3 @@ class HtmlDjangoParser(TokenParser):
elif content.startswith('"') or content.startswith("'"):
if self.current_attr_value is None:
self.current_attr_value = content
class VelocityHtmlParser(HtmlDjangoParser):
pass
class MyghtyHtmlParser(HtmlDjangoParser):
pass
class MasonParser(HtmlDjangoParser):
pass
class MakoHtmlParser(HtmlDjangoParser):
pass
class CheetahHtmlParser(HtmlDjangoParser):
pass
class HtmlGenshiParser(HtmlDjangoParser):
pass
class RhtmlParser(HtmlDjangoParser):
pass
class HtmlPhpParser(HtmlDjangoParser):
pass
class HtmlSmartyParser(HtmlDjangoParser):
pass
class EvoqueHtmlParser(HtmlDjangoParser):
pass
class ColdfusionHtmlParser(HtmlDjangoParser):
pass
class LassoHtmlParser(HtmlDjangoParser):
pass
class HandlebarsHtmlParser(HtmlDjangoParser):
pass
class YamlJinjaParser(HtmlDjangoParser):
pass
class TwigHtmlParser(HtmlDjangoParser):
pass

View File

@ -0,0 +1,60 @@
# -*- coding: utf-8 -*-
"""
wakatime.dependencies.javascript
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from JavaScript code.
:copyright: (c) 2018 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
import re
from . import TokenParser
class JavascriptParser(TokenParser):
state = None
extension = re.compile(r'\.\w{1,4}$')
def parse(self):
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_token(self, token, content):
if self.partial(token) == 'Reserved':
self._process_reserved(token, content)
elif self.partial(token) == 'Single':
self._process_string(token, content)
elif self.partial(token) == 'Punctuation':
self._process_punctuation(token, content)
else:
self._process_other(token, content)
def _process_reserved(self, token, content):
if self.state is None:
self.state = content
def _process_string(self, token, content):
if self.state == 'import':
self.append(self._format_module(content))
self.state = None
def _process_punctuation(self, token, content):
if content == ';':
self.state = None
def _process_other(self, token, content):
pass
def _format_module(self, content):
content = content.strip().strip('"').strip("'").strip()
content = content.split('/')[-1].split('\\')[-1]
content = self.extension.sub('', content, count=1)
return content
class TypeScriptParser(JavascriptParser):
pass

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.java
~~~~~~~~~~~~~~~~~~~~~~~
wakatime.dependencies.java
~~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from Java code.
@ -94,3 +94,89 @@ class JavaParser(TokenParser):
def _process_other(self, token, content):
pass
class KotlinParser(TokenParser):
state = None
exclude = [
r'^java\.',
]
def parse(self):
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_token(self, token, content):
if self.partial(token) == 'Keyword':
self._process_keyword(token, content)
elif self.partial(token) == 'Text':
self._process_text(token, content)
elif self.partial(token) == 'Namespace':
self._process_namespace(token, content)
else:
self._process_other(token, content)
def _process_keyword(self, token, content):
self.state = content
def _process_text(self, token, content):
pass
def _process_namespace(self, token, content):
if self.state == 'import':
self.append(self._format(content))
self.state = None
def _process_other(self, token, content):
self.state = None
def _format(self, content):
content = content.split(u('.'))
if content[-1] == u('*'):
content = content[:len(content) - 1]
if len(content) == 0:
return None
if len(content) == 1:
return content[0]
return u('.').join(content[:2])
class ScalaParser(TokenParser):
state = None
def parse(self):
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_token(self, token, content):
if self.partial(token) == 'Keyword':
self._process_keyword(token, content)
elif self.partial(token) == 'Text':
self._process_text(token, content)
elif self.partial(token) == 'Namespace':
self._process_namespace(token, content)
else:
self._process_other(token, content)
def _process_keyword(self, token, content):
self.state = content
def _process_text(self, token, content):
pass
def _process_namespace(self, token, content):
if self.state == 'import':
self.append(self._format(content))
self.state = None
def _process_other(self, token, content):
self.state = None
def _format(self, content):
return content.strip().lstrip('__root__').strip('_').strip('.')

View File

@ -0,0 +1,84 @@
# -*- coding: utf-8 -*-
"""
wakatime.dependencies.objective
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from Objective-C and Swift code.
:copyright: (c) 2018 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
import re
from . import TokenParser
class SwiftParser(TokenParser):
state = None
exclude = [
r'^foundation$',
]
def parse(self):
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_token(self, token, content):
if self.partial(token) == 'Declaration':
self._process_declaration(token, content)
elif self.partial(token) == 'Class':
self._process_class(token, content)
else:
self._process_other(token, content)
def _process_declaration(self, token, content):
if self.state is None:
self.state = content
def _process_class(self, token, content):
if self.state == 'import':
self.append(content)
self.state = None
def _process_other(self, token, content):
pass
class ObjectiveCParser(TokenParser):
state = None
extension = re.compile(r'\.[mh]$')
def parse(self):
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_token(self, token, content):
if self.partial(token) == 'Preproc':
self._process_preproc(token, content)
else:
self._process_other(token, content)
def _process_preproc(self, token, content):
if self.state:
self._process_import(token, content)
self.state = content
def _process_import(self, token, content):
if self.state == '#' and content.startswith('import '):
self.append(self._format(content))
self.state = None
def _process_other(self, token, content):
pass
def _format(self, content):
content = content.strip().lstrip('import ').strip()
content = content.strip('"').strip("'").strip()
content = content.strip('<').strip('>').strip()
content = content.split('/')[0]
content = self.extension.sub('', content, count=1)
return content

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.php
~~~~~~~~~~~~~~~~~~~~~~
wakatime.dependencies.php
~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from PHP code.
@ -16,6 +16,10 @@ from ..compat import u
class PhpParser(TokenParser):
state = None
parens = 0
exclude = [
r'^app$',
r'app\.php$',
]
def parse(self):
for index, token, content in self.tokens:

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.python
~~~~~~~~~~~~~~~~~~~~~~~~~
wakatime.dependencies.python
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from Python code.

View File

@ -0,0 +1,48 @@
# -*- coding: utf-8 -*-
"""
wakatime.dependencies.rust
~~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from Rust code.
:copyright: (c) 2018 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
from . import TokenParser
class RustParser(TokenParser):
state = None
def parse(self):
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_token(self, token, content):
if self.partial(token) == 'Keyword':
self._process_keyword(token, content)
elif self.partial(token) == 'Whitespace':
self._process_whitespace(token, content)
elif self.partial(token) == 'Name':
self._process_name(token, content)
else:
self._process_other(token, content)
def _process_keyword(self, token, content):
if self.state == 'extern' and content == 'crate':
self.state = 'extern crate'
else:
self.state = content
def _process_whitespace(self, token, content):
pass
def _process_name(self, token, content):
if self.state == 'extern crate':
self.append(content)
self.state = None
def _process_other(self, token, content):
self.state = None

View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.unknown
~~~~~~~~~~~~~~~~~~~~~~~~~~
wakatime.dependencies.unknown
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from files of unknown language.

View File

@ -12,3 +12,8 @@
class NotYetImplemented(Exception):
"""This method needs to be implemented."""
class SkipHeartbeat(Exception):
"""Raised to prevent the current heartbeat from being sent."""
pass

View File

@ -12,6 +12,7 @@ import logging
import re
from .compat import u, json
from .exceptions import SkipHeartbeat
from .project import get_project_info
from .stats import get_file_stats
from .utils import get_user_agent, should_exclude, format_file_path, find_project_file
@ -77,12 +78,17 @@ class Heartbeat(object):
self.project = project
self.branch = branch
stats = get_file_stats(self.entity,
entity_type=self.type,
lineno=data.get('lineno'),
cursorpos=data.get('cursorpos'),
plugin=args.plugin,
language=data.get('language'))
try:
stats = get_file_stats(self.entity,
entity_type=self.type,
lineno=data.get('lineno'),
cursorpos=data.get('cursorpos'),
plugin=args.plugin,
language=data.get('language'))
except SkipHeartbeat as ex:
self.skip = u(ex) or 'Skipping'
return
else:
self.project = data.get('project')
self.branch = data.get('branch')

View File

@ -37,7 +37,7 @@ class JavascriptLexer(RegexLexer):
name = 'JavaScript'
aliases = ['js', 'javascript']
filenames = ['*.js', '*.jsm']
filenames = ['*.js', '*.jsm', '*.mjs']
mimetypes = ['application/javascript', 'application/x-javascript',
'text/x-javascript', 'text/javascript']
@ -1035,7 +1035,6 @@ class CoffeeScriptLexer(RegexLexer):
filenames = ['*.coffee']
mimetypes = ['text/coffeescript']
_operator_re = (
r'\+\+|~|&&|\band\b|\bor\b|\bis\b|\bisnt\b|\bnot\b|\?|:|'
r'\|\||\\(?=\n)|'
@ -1464,6 +1463,7 @@ class EarlGreyLexer(RegexLexer):
],
}
class JuttleLexer(RegexLexer):
"""
For `Juttle`_ source code.

View File

@ -17,6 +17,7 @@ import sys
from .compat import u, open
from .constants import MAX_FILE_SIZE_SUPPORTED
from .dependencies import DependencyParser
from .exceptions import SkipHeartbeat
from .language_priorities import LANGUAGES
from .packages.pygments.lexers import (
@ -53,6 +54,8 @@ def get_file_stats(file_name, entity_type='file', lineno=None, cursorpos=None,
if not language:
language, lexer = guess_language(file_name)
language = use_root_language(language, lexer)
parser = DependencyParser(file_name, lexer)
dependencies = parser.parse()
@ -118,6 +121,8 @@ def guess_lexer_using_filename(file_name, text):
try:
lexer = custom_pygments_guess_lexer_for_filename(file_name, text)
except SkipHeartbeat as ex:
raise SkipHeartbeat(u(ex))
except:
log.traceback(logging.DEBUG)
@ -167,7 +172,7 @@ def get_language_from_extension(file_name):
filepart, extension = os.path.splitext(file_name)
if re.match(r'\.h.*', extension, re.IGNORECASE) or re.match(r'\.c.*', extension, re.IGNORECASE):
if re.match(r'\.h.*$', extension, re.IGNORECASE) or re.match(r'\.c.*$', extension, re.IGNORECASE):
if os.path.exists(u('{0}{1}').format(u(filepart), u('.c'))) or os.path.exists(u('{0}{1}').format(u(filepart), u('.C'))):
return 'C'
@ -178,6 +183,18 @@ def get_language_from_extension(file_name):
if '.c' in available_extensions:
return 'C'
if os.path.exists(u('{0}{1}').format(u(filepart), u('.m'))) or os.path.exists(u('{0}{1}').format(u(filepart), u('.M'))):
return 'Objective-C'
if os.path.exists(u('{0}{1}').format(u(filepart), u('.mm'))) or os.path.exists(u('{0}{1}').format(u(filepart), u('.MM'))):
return 'Objective-C++'
if re.match(r'\.m$', extension, re.IGNORECASE) and (os.path.exists(u('{0}{1}').format(u(filepart), u('.h'))) or os.path.exists(u('{0}{1}').format(u(filepart), u('.H')))):
return 'Objective-C'
if re.match(r'\.mm$', extension, re.IGNORECASE) and (os.path.exists(u('{0}{1}').format(u(filepart), u('.h'))) or os.path.exists(u('{0}{1}').format(u(filepart), u('.H')))):
return 'Objective-C++'
return None
@ -236,6 +253,13 @@ def get_lexer(language):
return None
def use_root_language(language, lexer):
if lexer and hasattr(lexer, 'root_lexer'):
return u(lexer.root_lexer.name)
return language
def get_language_from_json(language, key):
"""Finds the given language in a json file."""
@ -299,6 +323,12 @@ def custom_pygments_guess_lexer_for_filename(_fn, _text, **options):
return lexer(**options)
result.append(customize_lexer_priority(_fn, rv, lexer))
matlab = list(filter(lambda x: x[2].name.lower() == 'matlab', result))
if len(matlab) > 0:
objc = list(filter(lambda x: x[2].name.lower() == 'objective-c', result))
if objc and objc[0][0] == matlab[0][0]:
raise SkipHeartbeat('Skipping because not enough language accuracy.')
def type_sort(t):
# sort by:
# - analyse score
@ -322,7 +352,17 @@ def customize_lexer_priority(file_name, accuracy, lexer):
elif lexer_name == 'matlab':
available_extensions = extensions_in_same_folder(file_name)
if '.mat' in available_extensions:
priority = 0.06
accuracy += 0.01
if '.h' not in available_extensions:
accuracy += 0.01
elif lexer_name == 'objective-c':
available_extensions = extensions_in_same_folder(file_name)
if '.mat' in available_extensions:
accuracy -= 0.01
else:
accuracy += 0.01
if '.h' in available_extensions:
accuracy += 0.01
return (accuracy, priority, lexer)