upgrade wakatime-cli to v10.2.4

packages/wakatime/__about__.py

@@ -1,7 +1,7 @@
 __title__ = 'wakatime'
 __description__ = 'Common interface to the WakaTime api.'
 __url__ = 'https://github.com/wakatime/wakatime'
-__version_info__ = ('10', '2', '1')
+__version_info__ = ('10', '2', '4')
 __version__ = '.'.join(__version_info__)
 __author__ = 'Alan Hamlett'
 __author_email__ = 'alan@wakatime.com'

packages/wakatime/arguments.py

@@ -20,7 +20,7 @@ import traceback
 from .__about__ import __version__
 from .compat import basestring
 from .configs import parseConfigFile
-from .constants import AUTH_ERROR
+from .constants import AUTH_ERROR, DEFAULT_SYNC_OFFLINE_ACTIVITY
 from .packages import argparse
 
 
@@ -116,6 +116,12 @@ def parse_arguments():
                         action=StoreWithoutQuotes,
                         help='Optional language name. If valid, takes ' +
                              'priority over auto-detected language.')
+    parser.add_argument('--local-file', dest='local_file', metavar='FILE',
+                        action=FileAction,
+                        help='Absolute path to local file for the ' +
+                             'heartbeat. When --entity is a remote file, ' +
+                             'this local file will be used for stats and ' +
+                             'just the value of --entity sent with heartbeat.')
     parser.add_argument('--hostname', dest='hostname',
                         action=StoreWithoutQuotes,
                         help='Hostname of current machine.')
@@ -126,13 +132,23 @@ def parse_arguments():
     parser.add_argument('--disableoffline', dest='offline_deprecated',
                         action='store_true',
                         help=argparse.SUPPRESS)
-    parser.add_argument('--hide-filenames', dest='hide_filenames',
+    parser.add_argument('--hide-file-names', dest='hide_file_names',
                         action='store_true',
                         help='Obfuscate filenames. Will not send file names ' +
                              'to api.')
+    parser.add_argument('--hide-filenames', dest='hide_filenames',
+                        action='store_true',
+                        help=argparse.SUPPRESS)
     parser.add_argument('--hidefilenames', dest='hidefilenames',
                         action='store_true',
                         help=argparse.SUPPRESS)
+    parser.add_argument('--hide-project-names', dest='hide_project_names',
+                        action='store_true',
+                        help='Obfuscate project names. When a project ' +
+                             'folder is detected instead of using the ' +
+                             'folder name as the project, a ' +
+                             '.wakatime-project file is created with a ' +
+                             'random project name.')
     parser.add_argument('--exclude', dest='exclude', action='append',
                         help='Filename patterns to exclude from logging. ' +
                              'POSIX regex syntax. Can be used more than once.')
@@ -170,6 +186,17 @@ def parse_arguments():
                         action=StoreWithoutQuotes,
                         help='Number of seconds to wait when sending ' +
                              'heartbeats to api. Defaults to 60 seconds.')
+    parser.add_argument('--sync-offline-activity',
+                        dest='sync_offline_activity',
+                        action=StoreWithoutQuotes,
+                        help='Amount of offline activity to sync from your ' +
+                             'local ~/.wakatime.db sqlite3 file to your ' +
+                             'WakaTime Dashboard before exiting. Can be ' +
+                             '"none" or a positive integer number. Defaults ' +
+                             'to 5, meaning for every heartbeat sent while ' +
+                             'online 5 offline heartbeats are synced. Can ' +
+                             'be used without --entity to only sync offline ' +
+                             'activity without generating new heartbeats.')
     parser.add_argument('--config', dest='config', action=StoreWithoutQuotes,
                         help='Defaults to ~/.wakatime.cfg.')
     parser.add_argument('--verbose', dest='verbose', action='store_true',
@@ -214,9 +241,20 @@ def parse_arguments():
     if not args.entity:
         if args.file:
             args.entity = args.file
-        else:
+        elif not args.sync_offline_activity or args.sync_offline_activity == 'none':
             parser.error('argument --entity is required')
 
+    if not args.sync_offline_activity:
+        args.sync_offline_activity = DEFAULT_SYNC_OFFLINE_ACTIVITY
+    if args.sync_offline_activity == 'none':
+        args.sync_offline_activity = 0
+    try:
+        args.sync_offline_activity = int(args.sync_offline_activity)
+        if args.sync_offline_activity < 0:
+            raise Exception('Error')
+    except:
+        parser.error('argument --sync-offline-activity must be "none" or an integer number')
+
     if not args.language and args.alternate_language:
         args.language = args.alternate_language
 
@@ -249,24 +287,8 @@ def parse_arguments():
             pass
     if not args.exclude_unknown_project and configs.has_option('settings', 'exclude_unknown_project'):
         args.exclude_unknown_project = configs.getboolean('settings', 'exclude_unknown_project')
-    if not args.hide_filenames and args.hidefilenames:
+    boolean_or_list('hide_file_names', args, configs, alternative_names=['hide_filenames', 'hidefilenames'])
-        args.hide_filenames = args.hidefilenames
+    boolean_or_list('hide_project_names', args, configs, alternative_names=['hide_projectnames', 'hideprojectnames'])
-    if args.hide_filenames:
-        args.hide_filenames = ['.*']
-    else:
-        args.hide_filenames = []
-        option = None
-        if configs.has_option('settings', 'hidefilenames'):
-            option = configs.get('settings', 'hidefilenames')
-        if configs.has_option('settings', 'hide_filenames'):
-            option = configs.get('settings', 'hide_filenames')
-        if option is not None:
-            if option.strip().lower() == 'true':
-                args.hide_filenames = ['.*']
-            elif option.strip().lower() != 'false':
-                for pattern in option.split("\n"):
-                    if pattern.strip() != '':
-                        args.hide_filenames.append(pattern)
     if args.offline_deprecated:
         args.offline = False
     if args.offline and configs.has_option('settings', 'offline'):
@@ -307,3 +329,30 @@ def parse_arguments():
             print(traceback.format_exc())
 
     return args, configs
+
+
+def boolean_or_list(config_name, args, configs, alternative_names=[]):
+    """Get a boolean or list of regexes from args and configs."""
+
+    # when argument flag present, set to wildcard regex
+    for key in alternative_names:
+        if hasattr(args, key) and getattr(args, key):
+            setattr(args, config_name, ['.*'])
+            return
+
+    setattr(args, config_name, [])
+
+    option = None
+    alternative_names.insert(0, config_name)
+    for key in alternative_names:
+        if configs.has_option('settings', key):
+            option = configs.get('settings', key)
+            break
+
+    if option is not None:
+        if option.strip().lower() == 'true':
+            setattr(args, config_name, ['.*'])
+        elif option.strip().lower() != 'false':
+            for pattern in option.split("\n"):
+                if pattern.strip() != '':
+                    getattr(args, config_name).append(pattern)

packages/wakatime/constants.py

@@ -45,3 +45,12 @@ Files larger than this in bytes will not have a line count stat for performance.
 Default is 2MB.
 """
 MAX_FILE_SIZE_SUPPORTED = 2000000
+
+""" Default number of offline heartbeats to sync before exiting."""
+DEFAULT_SYNC_OFFLINE_ACTIVITY = 100
+
+""" Number of heartbeats per api request.
+Even when sending more heartbeats, this is the number of heartbeats sent per
+individual https request to the WakaTime API.
+"""
+HEARTBEATS_PER_REQUEST = 10

packages/wakatime/heartbeat.py

@@ -42,6 +42,8 @@ class Heartbeat(object):
     cursorpos = None
     user_agent = None
 
+    _sensitive = ('dependencies', 'lines', 'lineno', 'cursorpos', 'branch')
+
     def __init__(self, data, args, configs, _clone=None):
         if not data:
             self.skip = u('Skipping because heartbeat data is missing.')
@@ -83,13 +85,16 @@ class Heartbeat(object):
                 return
             if self.type == 'file':
                 self.entity = format_file_path(self.entity)
-                if not self.entity or not os.path.isfile(self.entity):
+                if not self._file_exists():
                     self.skip = u('File does not exist; ignoring this heartbeat.')
                     return
                 if self._excluded_by_missing_project_file():
                     self.skip = u('Skipping because missing .wakatime-project file in parent path.')
                     return
 
+            if args.local_file and not os.path.isfile(args.local_file):
+                args.local_file = None
+
             project, branch = get_project_info(configs, self, data)
             self.project = project
             self.branch = branch
@@ -104,7 +109,8 @@ class Heartbeat(object):
                                        lineno=data.get('lineno'),
                                        cursorpos=data.get('cursorpos'),
                                        plugin=args.plugin,
-                                       language=data.get('language'))
+                                       language=data.get('language'),
+                                       local_file=args.local_file)
             except SkipHeartbeat as ex:
                 self.skip = u(ex) or 'Skipping'
                 return
@@ -132,7 +138,7 @@ class Heartbeat(object):
         Returns a Heartbeat.
         """
 
-        if not self.args.hide_filenames:
+        if not self.args.hide_file_names:
             return self
 
         if self.entity is None:
@@ -141,29 +147,12 @@ class Heartbeat(object):
         if self.type != 'file':
             return self
 
-        for pattern in self.args.hide_filenames:
+        if self.should_obfuscate_filename():
-            try:
+            self._sanitize_metadata()
-                compiled = re.compile(pattern, re.IGNORECASE)
+            extension = u(os.path.splitext(self.entity)[1])
-                if compiled.search(self.entity):
+            self.entity = u('HIDDEN{0}').format(extension)
-
+        elif self.should_obfuscate_project():
-                    sanitized = {}
+            self._sanitize_metadata()
-                    sensitive = ['dependencies', 'lines', 'lineno', 'cursorpos', 'branch']
-                    for key, val in self.items():
-                        if key in sensitive:
-                            sanitized[key] = None
-                        else:
-                            sanitized[key] = val
-
-                    extension = u(os.path.splitext(self.entity)[1])
-                    sanitized['entity'] = u('HIDDEN{0}').format(extension)
-
-                    return self.update(sanitized)
-
-            except re.error as ex:
-                log.warning(u('Regex error ({msg}) for include pattern: {pattern}').format(
-                    msg=u(ex),
-                    pattern=u(pattern),
-                ))
 
         return self
 
@@ -201,6 +190,38 @@ class Heartbeat(object):
             is_write=self.is_write,
         )
 
+    def should_obfuscate_filename(self):
+        """Returns True if hide_file_names is true or the entity file path
+        matches one in the list of obfuscated file paths."""
+
+        for pattern in self.args.hide_file_names:
+            try:
+                compiled = re.compile(pattern, re.IGNORECASE)
+                if compiled.search(self.entity):
+                    return True
+            except re.error as ex:
+                log.warning(u('Regex error ({msg}) for hide_file_names pattern: {pattern}').format(
+                    msg=u(ex),
+                    pattern=u(pattern),
+                ))
+        return False
+
+    def should_obfuscate_project(self):
+        """Returns True if hide_project_names is true or the entity file path
+        matches one in the list of obfuscated project paths."""
+
+        for pattern in self.args.hide_project_names:
+            try:
+                compiled = re.compile(pattern, re.IGNORECASE)
+                if compiled.search(self.entity):
+                    return True
+            except re.error as ex:
+                log.warning(u('Regex error ({msg}) for hide_project_names pattern: {pattern}').format(
+                    msg=u(ex),
+                    pattern=u(pattern),
+                ))
+        return False
+
     def _unicode(self, value):
         if value is None:
             return None
@@ -211,6 +232,10 @@ class Heartbeat(object):
             return None
         return [self._unicode(value) for value in values]
 
+    def _file_exists(self):
+        return (self.entity and os.path.isfile(self.entity) or
+            self.args.local_file and os.path.isfile(self.args.local_file))
+
     def _excluded_by_pattern(self):
         return should_exclude(self.entity, self.args.include, self.args.exclude)
 
@@ -224,6 +249,10 @@ class Heartbeat(object):
             return False
         return find_project_file(self.entity) is None
 
+    def _sanitize_metadata(self):
+        for key in self._sensitive:
+            setattr(self, key, None)
+
     def __repr__(self):
         return self.json()
 

packages/wakatime/languages/default.json

@@ -27,7 +27,7 @@
   "f#": "F#",
   "fortran": "Fortran",
   "go": "Go",
-  "gous": "Gosu", 
+  "gosu": "Gosu",
   "groovy": "Groovy",
   "haml": "Haml",
   "haskell": "Haskell",

packages/wakatime/main.py

@@ -24,7 +24,7 @@ from .__about__ import __version__
 from .api import send_heartbeats
 from .arguments import parse_arguments
 from .compat import u, json
-from .constants import SUCCESS, UNKNOWN_ERROR
+from .constants import SUCCESS, UNKNOWN_ERROR, HEARTBEATS_PER_REQUEST
 from .logger import setup_logging
 
 log = logging.getLogger('WakaTime')
@@ -63,12 +63,22 @@ def execute(argv=None):
                     msg=u(ex),
                 ))
 
-        retval = send_heartbeats(heartbeats, args, configs)
+        retval = SUCCESS
+        while heartbeats:
+            retval = send_heartbeats(heartbeats[:HEARTBEATS_PER_REQUEST], args, configs)
+            heartbeats = heartbeats[HEARTBEATS_PER_REQUEST:]
+            if retval != SUCCESS:
+                break
+
+        if heartbeats:
+            Queue(args, configs).push_many(heartbeats)
+
         if retval == SUCCESS:
             queue = Queue(args, configs)
-            offline_heartbeats = queue.pop_many()
+            for offline_heartbeats in queue.pop_many(args.sync_offline_activity):
-            if len(offline_heartbeats) > 0:
                 retval = send_heartbeats(offline_heartbeats, args, configs)
+                if retval != SUCCESS:
+                    break
 
         return retval
 

packages/wakatime/offlinequeue.py

@@ -15,6 +15,7 @@ import os
 from time import sleep
 
 from .compat import json
+from .constants import DEFAULT_SYNC_OFFLINE_ACTIVITY, HEARTBEATS_PER_REQUEST
 from .heartbeat import Heartbeat
 
 
@@ -104,19 +105,23 @@ class Queue(object):
 
     def pop_many(self, limit=None):
         if limit is None:
-            limit = 5
+            limit = DEFAULT_SYNC_OFFLINE_ACTIVITY
 
         heartbeats = []
 
         count = 0
-        while limit == 0 or count < limit:
+        while count < limit:
             heartbeat = self.pop()
             if not heartbeat:
                 break
             heartbeats.append(heartbeat)
             count += 1
+            if count % HEARTBEATS_PER_REQUEST == 0:
+                yield heartbeats
+                heartbeats = []
 
-        return heartbeats
+        if heartbeats:
+            yield heartbeats
 
     def _get_db_file(self):
         home = '~'

packages/wakatime/project.py

@@ -9,8 +9,11 @@
     :license: BSD, see LICENSE for more details.
 """
 
+import os
 import logging
+import random
 
+from .compat import open
 from .projects.git import Git
 from .projects.mercurial import Mercurial
 from .projects.projectfile import ProjectFile
@@ -65,6 +68,8 @@ def get_project_info(configs, heartbeat, data):
     if project_name is None:
         project_name = data.get('project') or heartbeat.args.project
 
+    hide_project = heartbeat.should_obfuscate_project()
+
     if project_name is None or branch_name is None:
 
         for plugin_cls in REV_CONTROL_PLUGINS:
@@ -76,9 +81,18 @@ def get_project_info(configs, heartbeat, data):
             if project.process():
                 project_name = project_name or project.name()
                 branch_name = branch_name or project.branch()
+                if hide_project:
+                    branch_name = None
+                    project_name = generate_project_name()
+                    project_file = os.path.join(project.folder(), '.wakatime-project')
+                    try:
+                        with open(project_file, 'w') as fh:
+                            fh.write(project_name)
+                    except IOError:
+                        project_name = None
                 break
 
-    if project_name is None:
+    if project_name is None and not hide_project:
         project_name = data.get('alternate_project') or heartbeat.args.alternate_project
 
     return project_name, branch_name
@@ -88,3 +102,42 @@ def get_configs_for_plugin(plugin_name, configs):
     if configs and configs.has_section(plugin_name):
         return dict(configs.items(plugin_name))
     return None
+
+
+def generate_project_name():
+    """Generates a random project name."""
+
+    adjectives = [
+        'aged', 'ancient', 'autumn', 'billowing', 'bitter', 'black', 'blue', 'bold',
+        'broad', 'broken', 'calm', 'cold', 'cool', 'crimson', 'curly', 'damp',
+        'dark', 'dawn', 'delicate', 'divine', 'dry', 'empty', 'falling', 'fancy',
+        'flat', 'floral', 'fragrant', 'frosty', 'gentle', 'green', 'hidden', 'holy',
+        'icy', 'jolly', 'late', 'lingering', 'little', 'lively', 'long', 'lucky',
+        'misty', 'morning', 'muddy', 'mute', 'nameless', 'noisy', 'odd', 'old',
+        'orange', 'patient', 'plain', 'polished', 'proud', 'purple', 'quiet', 'rapid',
+        'raspy', 'red', 'restless', 'rough', 'round', 'royal', 'shiny', 'shrill',
+        'shy', 'silent', 'small', 'snowy', 'soft', 'solitary', 'sparkling', 'spring',
+        'square', 'steep', 'still', 'summer', 'super', 'sweet', 'throbbing', 'tight',
+        'tiny', 'twilight', 'wandering', 'weathered', 'white', 'wild', 'winter', 'wispy',
+        'withered', 'yellow', 'young'
+    ]
+    nouns = [
+        'art', 'band', 'bar', 'base', 'bird', 'block', 'boat', 'bonus',
+        'bread', 'breeze', 'brook', 'bush', 'butterfly', 'cake', 'cell', 'cherry',
+        'cloud', 'credit', 'darkness', 'dawn', 'dew', 'disk', 'dream', 'dust',
+        'feather', 'field', 'fire', 'firefly', 'flower', 'fog', 'forest', 'frog',
+        'frost', 'glade', 'glitter', 'grass', 'hall', 'hat', 'haze', 'heart',
+        'hill', 'king', 'lab', 'lake', 'leaf', 'limit', 'math', 'meadow',
+        'mode', 'moon', 'morning', 'mountain', 'mouse', 'mud', 'night', 'paper',
+        'pine', 'poetry', 'pond', 'queen', 'rain', 'recipe', 'resonance', 'rice',
+        'river', 'salad', 'scene', 'sea', 'shadow', 'shape', 'silence', 'sky',
+        'smoke', 'snow', 'snowflake', 'sound', 'star', 'sun', 'sun', 'sunset',
+        'surf', 'term', 'thunder', 'tooth', 'tree', 'truth', 'union', 'unit',
+        'violet', 'voice', 'water', 'waterfall', 'wave', 'wildflower', 'wind', 'wood'
+    ]
+    numbers = [str(x) for x in range(10)]
+    return ' '.join([
+        random.choice(adjectives).capitalize(),
+        random.choice(nouns).capitalize(),
+        random.choice(numbers) + random.choice(numbers),
+    ])

packages/wakatime/projects/base.py

@@ -43,3 +43,8 @@ class BaseProject(object):
         """ Returns the current branch.
         """
         raise NotYetImplemented()
+
+    def folder(self):
+        """ Returns the project's top folder path.
+        """
+        raise NotYetImplemented()

packages/wakatime/projects/git.py

@@ -25,6 +25,7 @@ class Git(BaseProject):
     _submodule = None
     _project_name = None
     _head_file = None
+    _project_folder = None
 
     def process(self):
         return self._find_git_config_file(self.path)
@@ -40,6 +41,9 @@ class Git(BaseProject):
                 return self._get_branch_from_head_file(line)
         return u('master')
 
+    def folder(self):
+        return self._project_folder
+
     def _find_git_config_file(self, path):
         path = os.path.realpath(path)
         if os.path.isfile(path):
@@ -47,6 +51,7 @@ class Git(BaseProject):
         if os.path.isfile(os.path.join(path, '.git', 'config')):
             self._project_name = os.path.basename(path)
             self._head_file = os.path.join(path, '.git', 'HEAD')
+            self._project_folder = path
             return True
 
         link_path = self._path_from_gitdir_link_file(path)
@@ -56,12 +61,14 @@ class Git(BaseProject):
             if self._is_worktree(link_path):
                 self._project_name = self._project_from_worktree(link_path)
                 self._head_file = os.path.join(link_path, 'HEAD')
+                self._project_folder = path
                 return True
 
             # next check if this is a submodule
             if self._submodules_supported_for_path(path):
                 self._project_name = os.path.basename(path)
                 self._head_file = os.path.join(link_path, 'HEAD')
+                self._project_folder = path
                 return True
 
         split_path = os.path.split(path)

packages/wakatime/projects/mercurial.py

@@ -47,6 +47,11 @@ class Mercurial(BaseProject):
                 log.traceback(logging.WARNING)
         return u('default')
 
+    def folder(self):
+        if self.configDir:
+            return os.path.dirname(self.configDir)
+        return None
+
     def _find_hg_config_dir(self, path):
         path = os.path.realpath(path)
         if os.path.isfile(path):

packages/wakatime/projects/subversion.py

@@ -41,6 +41,11 @@ class Subversion(BaseProject):
             return None  # pragma: nocover
         return u(self.info['URL'].split('/')[-1].split('\\')[-1])
 
+    def folder(self):
+        if 'Repository Root' not in self.info:
+            return None
+        return self.info['Repository Root']
+
     def _find_binary(self):
         if self.binary_location:
             return self.binary_location

packages/wakatime/stats.py

@@ -40,7 +40,7 @@ log = logging.getLogger('WakaTime')
 
 
 def get_file_stats(file_name, entity_type='file', lineno=None, cursorpos=None,
-                   plugin=None, language=None):
+                   plugin=None, language=None, local_file=None):
     if entity_type != 'file':
         stats = {
             'language': None,
@@ -52,24 +52,24 @@ def get_file_stats(file_name, entity_type='file', lineno=None, cursorpos=None,
     else:
         language, lexer = standardize_language(language, plugin)
         if not language:
-            language, lexer = guess_language(file_name)
+            language, lexer = guess_language(file_name, local_file)
 
         language = use_root_language(language, lexer)
 
-        parser = DependencyParser(file_name, lexer)
+        parser = DependencyParser(local_file or file_name, lexer)
         dependencies = parser.parse()
 
         stats = {
             'language': language,
             'dependencies': dependencies,
-            'lines': number_lines_in_file(file_name),
+            'lines': number_lines_in_file(local_file or file_name),
             'lineno': lineno,
             'cursorpos': cursorpos,
         }
     return stats
 
 
-def guess_language(file_name):
+def guess_language(file_name, local_file):
     """Guess lexer and language for a file.
 
     Returns a tuple of (language_str, lexer_obj).
@@ -81,14 +81,14 @@ def guess_language(file_name):
     if language:
         lexer = get_lexer(language)
     else:
-        lexer = smart_guess_lexer(file_name)
+        lexer = smart_guess_lexer(file_name, local_file)
         if lexer:
             language = u(lexer.name)
 
     return language, lexer
 
 
-def smart_guess_lexer(file_name):
+def smart_guess_lexer(file_name, local_file):
     """Guess Pygments lexer for a file.
 
     Looks for a vim modeline in file contents, then compares the accuracy
@@ -99,7 +99,7 @@ def smart_guess_lexer(file_name):
 
     text = get_file_head(file_name)
 
-    lexer1, accuracy1 = guess_lexer_using_filename(file_name, text)
+    lexer1, accuracy1 = guess_lexer_using_filename(local_file or file_name, text)
     lexer2, accuracy2 = guess_lexer_using_modeline(text)
 
     if lexer1: