enable bulk api endpoint

tests/samples/output/configs_test_good_config_file

@@ -1,5 +1,5 @@
 Traceback (most recent call last):
-  File "{file}", line {lineno}, in parseArguments
+  File "{file}", line {lineno}, in parse_arguments
     args.timeout = int(configs.get('settings', 'timeout'))
 ValueError: invalid literal for int() with base 10: 'abc'
 

tests/test_arguments.py

@@ -11,14 +11,13 @@ import shutil
 import sys
 import uuid
 from testfixtures import log_capture
-from wakatime.arguments import parseArguments
+from wakatime.arguments import parse_arguments
 from wakatime.compat import u
 from wakatime.constants import (
-    API_ERROR,
     AUTH_ERROR,
     SUCCESS,
-    MALFORMED_HEARTBEAT_ERROR,
 )
+from wakatime.utils import get_user_agent
 from wakatime.packages.requests.models import Response
 from . import utils
 
@@ -27,9 +26,9 @@ try:
 except (ImportError, SyntaxError):
     import json
 try:
-    from mock import ANY, call
+    from mock import ANY
 except ImportError:
-    from unittest.mock import ANY, call
+    from unittest.mock import ANY
 
 
 class ArgumentsTestCase(utils.TestCase):
@@ -83,44 +82,57 @@ class ArgumentsTestCase(utils.TestCase):
             self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
             self.patched['wakatime.offlinequeue.Queue.pop'].assert_called_once_with()
 
-    def test_argument_parsing_strips_quotes(self):
+    @log_capture()
+    def test_argument_parsing_strips_quotes(self, logs):
+        logging.disable(logging.NOTSET)
+
         response = Response()
-        response.status_code = 500
+        response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
         now = u(int(time.time()))
         config = 'tests/samples/configs/good_config.cfg'
         entity = 'tests/samples/codefiles/python.py'
-        plugin = '"abcplugin\\"withquotes"'
+        plugin = '"abc plugin\\"with quotes"'
         args = ['--file', '"' + entity + '"', '--config', config, '--time', now, '--plugin', plugin]
 
         retval = execute(args)
-        self.assertEquals(retval, API_ERROR)
+        self.assertEquals(retval, SUCCESS)
+        self.assertNothingPrinted()
+        self.assertNothingLogged(logs)
 
-        expected = 'abcplugin"withquotes'
+        ua = get_user_agent().replace('Unknown/0', 'abc plugin"with quotes')
-        self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][2], expected)
+        heartbeat = {
+            'entity': os.path.realpath(entity),
+            'project': os.path.basename(os.path.abspath('.')),
+            'branch': ANY,
+            'time': float(now),
+            'type': 'file',
+            'cursorpos': None,
+            'dependencies': ['sqlalchemy', 'jinja', 'simplejson', 'flask', 'app', 'django', 'pygments', 'unittest', 'mock'],
+            'language': u('Python'),
+            'lineno': None,
+            'lines': 37,
+            'is_write': False,
+            'user_agent': ua,
+        }
+        self.assertHeartbeatSent(heartbeat)
+
+        self.assertHeartbeatNotSavedOffline()
+        self.assertOfflineHeartbeatsSynced()
+        self.assertSessionCacheSaved()
+
+    @log_capture()
+    def test_lineno_and_cursorpos(self, logs):
+        logging.disable(logging.NOTSET)
 
-    def test_lineno_and_cursorpos(self):
         response = Response()
-        response.status_code = 0
+        response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
         entity = 'tests/samples/codefiles/twolinefile.txt'
         config = 'tests/samples/configs/good_config.cfg'
         now = u(int(time.time()))
-
-        args = ['--entity', entity, '--config', config, '--time', now, '--lineno', '3', '--cursorpos', '4', '--verbose']
-        retval = execute(args)
-
-        self.assertEquals(sys.stdout.getvalue(), '')
-        self.assertEquals(sys.stderr.getvalue(), '')
-
-        self.assertEquals(retval, API_ERROR)
-
-        self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-        self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-        self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
         heartbeat = {
             'language': 'Text only',
             'lines': 2,
@@ -128,23 +140,27 @@ class ArgumentsTestCase(utils.TestCase):
             'project': os.path.basename(os.path.abspath('.')),
             'cursorpos': '4',
             'lineno': '3',
-            'branch': 'master',
+            'branch': ANY,
             'time': float(now),
+            'is_write': False,
             'type': 'file',
-        }
+            'dependencies': [],
-        stats = {
+            'user_agent': ANY,
-            u('cursorpos'): '4',
-            u('dependencies'): [],
-            u('language'): u('Text only'),
-            u('lineno'): '3',
-            u('lines'): 2,
         }
 
-        self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+        args = ['--entity', entity, '--config', config, '--time', now, '--lineno', '3', '--cursorpos', '4', '--verbose']
-        for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
+        retval = execute(args)
-            self.assertEquals(heartbeat[key], val)
+
-        self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
+        self.assertEquals(retval, SUCCESS)
-        self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
+        self.assertNothingPrinted()
+        actual = self.getLogOutput(logs)
+        self.assertIn('WakaTime DEBUG Sending heartbeats to api', actual)
+
+        self.assertHeartbeatSent(heartbeat)
+
+        self.assertHeartbeatNotSavedOffline()
+        self.assertOfflineHeartbeatsSynced()
+        self.assertSessionCacheSaved()
 
     def test_invalid_timeout_passed_via_command_line(self):
         response = Response()
@@ -185,19 +201,16 @@ class ArgumentsTestCase(utils.TestCase):
         args = ['--file', entity, '--config', config, '--verbose']
         retval = execute(args)
         self.assertEquals(retval, SUCCESS)
-        self.assertEquals(sys.stdout.getvalue(), '')
+        self.assertNothingPrinted()
-        self.assertEquals(sys.stderr.getvalue(), '')
+        actual = self.getLogOutput(logs)
-
-        log_output = u("\n").join([u(' ').join(x) for x in logs.actual()])
         expected = 'WakaTime DEBUG File does not exist; ignoring this heartbeat.'
-        self.assertEquals(log_output, expected)
+        self.assertIn(expected, actual)
 
-        self.patched['wakatime.session_cache.SessionCache.get'].assert_not_called()
+        self.assertHeartbeatNotSent()
-        self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()
-        self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
 
-        self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
+        self.assertHeartbeatNotSavedOffline()
-        self.patched['wakatime.offlinequeue.Queue.pop'].assert_called_once_with()
+        self.assertOfflineHeartbeatsSynced()
+        self.assertSessionCacheUntouched()
 
     @log_capture()
     def test_missing_entity_argument(self, logs):
@@ -294,7 +307,7 @@ class ArgumentsTestCase(utils.TestCase):
         logging.disable(logging.NOTSET)
 
         response = Response()
-        response.status_code = 0
+        response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
         with utils.TemporaryDirectory() as tempdir:
@@ -308,39 +321,28 @@ class ArgumentsTestCase(utils.TestCase):
             args = ['--file', entity, '--key', key, '--time', now, '--config', 'fake-foobar']
 
             retval = execute(args)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertEquals(retval, SUCCESS)
-            self.assertEquals(sys.stderr.getvalue(), '')
+            self.assertNothingPrinted()
-
+            self.assertNothingLogged(logs)
-            log_output = u("\n").join([u(' ').join(x) for x in logs.actual()])
-            self.assertEquals(log_output, '')
-
-            self.assertEquals(retval, API_ERROR)
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
 
             heartbeat = {
                 'language': 'Text only',
                 'lines': 0,
                 'entity': os.path.realpath(entity),
-                'project': os.path.basename(os.path.abspath('.')),
+                'project': None,
                 'time': float(now),
                 'type': 'file',
+                'cursorpos': None,
+                'dependencies': [],
+                'lineno': None,
+                'is_write': False,
+                'user_agent': ANY,
             }
-            stats = {
+            self.assertHeartbeatSent(heartbeat)
-                u('cursorpos'): None,
-                u('dependencies'): [],
-                u('language'): u('Text only'),
-                u('lineno'): None,
-                u('lines'): 0,
-            }
 
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+            self.assertHeartbeatNotSavedOffline()
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
+            self.assertOfflineHeartbeatsSynced()
-                self.assertEquals(heartbeat[key], val)
+            self.assertSessionCacheSaved()
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_proxy_argument(self):
         response = Response()
@@ -395,9 +397,12 @@ class ArgumentsTestCase(utils.TestCase):
 
             self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_called_once_with(ANY, cert=None, proxies=ANY, stream=False, timeout=60, verify=False)
 
-    def test_write_argument(self):
+    @log_capture()
+    def test_write_argument(self, logs):
+        logging.disable(logging.NOTSET)
+
         response = Response()
-        response.status_code = 0
+        response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
         with utils.TemporaryDirectory() as tempdir:
@@ -406,45 +411,39 @@ class ArgumentsTestCase(utils.TestCase):
             entity = os.path.realpath(os.path.join(tempdir, 'emptyfile.txt'))
             now = u(int(time.time()))
             key = str(uuid.uuid4())
+            heartbeat = {
+                'language': 'Text only',
+                'lines': 0,
+                'entity': entity,
+                'project': None,
+                'time': float(now),
+                'type': 'file',
+                'is_write': True,
+                'dependencies': [],
+                'user_agent': ANY,
+            }
 
             args = ['--file', entity, '--key', key, '--write', '--verbose',
                     '--config', 'tests/samples/configs/good_config.cfg', '--time', now]
 
             retval = execute(args)
-            self.assertEquals(retval, API_ERROR)
+            self.assertEquals(retval, SUCCESS)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
+            actual = self.getLogOutput(logs)
+            self.assertIn('WakaTime DEBUG Sending heartbeats to api', actual)
 
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
+            self.assertHeartbeatSent(heartbeat)
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
 
-            heartbeat = {
+            self.assertHeartbeatNotSavedOffline()
-                'language': 'Text only',
+            self.assertOfflineHeartbeatsSynced()
-                'lines': 0,
+            self.assertSessionCacheSaved()
-                'entity': entity,
-                'project': os.path.basename(os.path.abspath('.')),
-                'time': float(now),
-                'type': 'file',
-                'is_write': True,
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('dependencies'): [],
-                u('language'): u('Text only'),
-                u('lineno'): None,
-                u('lines'): 0,
-            }
 
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+    @log_capture()
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
+    def test_entity_type_domain(self, logs):
-                self.assertEquals(heartbeat[key], val)
+        logging.disable(logging.NOTSET)
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
-    def test_entity_type_domain(self):
         response = Response()
-        response.status_code = 0
+        response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
         entity = 'google.com'
@@ -454,34 +453,34 @@ class ArgumentsTestCase(utils.TestCase):
         args = ['--entity', entity, '--entity-type', 'domain', '--config', config, '--time', now]
         retval = execute(args)
 
-        self.assertEquals(retval, API_ERROR)
+        self.assertEquals(retval, SUCCESS)
-        self.assertEquals(sys.stdout.getvalue(), '')
+        self.assertNothingPrinted()
-        self.assertEquals(sys.stderr.getvalue(), '')
+        self.assertNothingLogged(logs)
-
-        self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-        self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-        self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
 
         heartbeat = {
             'entity': u(entity),
             'time': float(now),
             'type': 'domain',
+            'cursorpos': None,
+            'language': None,
+            'lineno': None,
+            'lines': None,
+            'is_write': False,
+            'dependencies': [],
+            'user_agent': ANY,
         }
-        stats = {
+        self.assertHeartbeatSent(heartbeat)
-            u('cursorpos'): None,
-            u('dependencies'): [],
-            u('language'): None,
-            u('lineno'): None,
-            u('lines'): None,
-        }
 
-        self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(heartbeat, ANY, None)
+        self.assertHeartbeatNotSavedOffline()
-        self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
+        self.assertOfflineHeartbeatsSynced()
-        self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
+        self.assertSessionCacheSaved()
+
+    @log_capture()
+    def test_entity_type_app(self, logs):
+        logging.disable(logging.NOTSET)
 
-    def test_entity_type_app(self):
         response = Response()
-        response.status_code = 0
+        response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
         entity = 'Firefox'
@@ -491,48 +490,197 @@ class ArgumentsTestCase(utils.TestCase):
         args = ['--entity', entity, '--entity-type', 'app', '--config', config, '--time', now]
         retval = execute(args)
 
-        self.assertEquals(retval, API_ERROR)
+        self.assertEquals(retval, SUCCESS)
-        self.assertEquals(sys.stdout.getvalue(), '')
+        self.assertNothingPrinted()
-        self.assertEquals(sys.stderr.getvalue(), '')
+        self.assertNothingLogged(logs)
-
-        self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-        self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-        self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
 
         heartbeat = {
             'entity': u(entity),
             'time': float(now),
             'type': 'app',
+            'cursorpos': None,
+            'dependencies': [],
+            'language': None,
+            'lineno': None,
+            'lines': None,
+            'is_write': False,
+            'user_agent': ANY,
         }
-        stats = {
+        self.assertHeartbeatSent(heartbeat)
-            u('cursorpos'): None,
-            u('dependencies'): [],
-            u('language'): None,
-            u('lineno'): None,
-            u('lines'): None,
-        }
 
-        self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(heartbeat, ANY, None)
+        self.assertHeartbeatNotSavedOffline()
-        self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
+        self.assertOfflineHeartbeatsSynced()
-        self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
+        self.assertSessionCacheSaved()
+
+    @log_capture()
+    def test_old_alternate_language_argument_still_supported(self, logs):
+        logging.disable(logging.NOTSET)
 
-    def test_old_alternate_language_argument_still_supported(self):
         response = Response()
-        response.status_code = 500
+        response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
+        language = 'Java'
         now = u(int(time.time()))
         config = 'tests/samples/configs/good_config.cfg'
         entity = 'tests/samples/codefiles/python.py'
-        args = ['--file', entity, '--config', config, '--time', now, '--alternate-language', 'JAVA']
+        args = ['--file', entity, '--config', config, '--time', now, '--alternate-language', language.upper()]
 
         retval = execute(args)
-        self.assertEquals(retval, API_ERROR)
+        self.assertEquals(retval, SUCCESS)
+        self.assertNothingPrinted()
+        self.assertNothingLogged(logs)
 
-        language = u('Java')
+        heartbeat = {
-        self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('language'), language)
+            'entity': os.path.realpath(entity),
+            'project': os.path.basename(os.path.abspath('.')),
+            'branch': ANY,
+            'time': float(now),
+            'type': 'file',
+            'cursorpos': None,
+            'dependencies': [],
+            'language': u(language),
+            'lineno': None,
+            'lines': 37,
+            'is_write': False,
+            'user_agent': ANY,
+        }
+        self.assertHeartbeatSent(heartbeat)
+
+        self.assertHeartbeatNotSavedOffline()
+        self.assertOfflineHeartbeatsSynced()
+        self.assertSessionCacheSaved()
+
+    @log_capture()
+    def test_extra_heartbeats_alternate_project_not_used(self, logs):
+        logging.disable(logging.NOTSET)
+
+        response = Response()
+        response.status_code = 201
+        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+
+        now1 = u(int(time.time()))
+        project1 = os.path.basename(os.path.abspath('.'))
+        project_not_used = 'xyz'
+        entity1 = os.path.abspath('tests/samples/codefiles/emptyfile.txt')
+        entity2 = os.path.abspath('tests/samples/codefiles/twolinefile.txt')
+        config = 'tests/samples/configs/good_config.cfg'
+        args = ['--time', now1, '--file', entity1, '--config', config, '--extra-heartbeats']
+
+        with utils.mock.patch('wakatime.main.sys.stdin') as mock_stdin:
+            now2 = int(time.time())
+            heartbeats = json.dumps([{
+                'timestamp': now2,
+                'entity': entity2,
+                'entity_type': 'file',
+                'alternate_project': project_not_used,
+                'is_write': True,
+            }])
+            mock_stdin.readline.return_value = heartbeats
+
+            retval = execute(args)
+
+            self.assertEquals(retval, SUCCESS)
+            self.assertNothingPrinted()
+            self.assertNothingLogged(logs)
+
+            heartbeat = {
+                'language': 'Text only',
+                'lines': 0,
+                'entity': entity1,
+                'project': project1,
+                'branch': ANY,
+                'time': float(now1),
+                'is_write': False,
+                'type': 'file',
+                'dependencies': [],
+                'user_agent': ANY,
+            }
+            extra_heartbeats = [{
+                'language': 'Text only',
+                'lines': 2,
+                'entity': entity2,
+                'project': project1,
+                'branch': ANY,
+                'time': float(now2),
+                'is_write': True,
+                'type': 'file',
+                'dependencies': [],
+                'user_agent': ANY,
+            }]
+            self.assertHeartbeatSent(heartbeat, extra_heartbeats=extra_heartbeats)
+
+            self.assertHeartbeatNotSavedOffline()
+            self.assertOfflineHeartbeatsSynced()
+            self.assertSessionCacheSaved()
+
+    @log_capture()
+    def test_extra_heartbeats_using_project_from_editor(self, logs):
+        logging.disable(logging.NOTSET)
+
+        response = Response()
+        response.status_code = 201
+        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+
+        now1 = u(int(time.time()))
+        project1 = os.path.basename(os.path.abspath('.'))
+        project2 = 'xyz'
+        entity1 = os.path.abspath('tests/samples/codefiles/emptyfile.txt')
+        entity2 = os.path.abspath('tests/samples/codefiles/twolinefile.txt')
+        config = 'tests/samples/configs/good_config.cfg'
+        args = ['--time', now1, '--file', entity1, '--config', config, '--extra-heartbeats']
+
+        with utils.mock.patch('wakatime.main.sys.stdin') as mock_stdin:
+            now2 = int(time.time())
+            heartbeats = json.dumps([{
+                'timestamp': now2,
+                'entity': entity2,
+                'entity_type': 'file',
+                'project': project2,
+                'is_write': True,
+            }])
+            mock_stdin.readline.return_value = heartbeats
+
+            retval = execute(args)
+
+            self.assertEquals(retval, SUCCESS)
+            self.assertNothingPrinted()
+            self.assertNothingLogged(logs)
+
+            heartbeat = {
+                'language': 'Text only',
+                'lines': 0,
+                'entity': entity1,
+                'project': project1,
+                'branch': ANY,
+                'time': float(now1),
+                'is_write': False,
+                'type': 'file',
+                'dependencies': [],
+                'user_agent': ANY,
+            }
+            extra_heartbeats = [{
+                'language': 'Text only',
+                'lines': 2,
+                'entity': entity2,
+                'project': project2,
+                'branch': ANY,
+                'time': float(now2),
+                'is_write': True,
+                'type': 'file',
+                'dependencies': [],
+                'user_agent': ANY,
+            }]
+            self.assertHeartbeatSent(heartbeat, extra_heartbeats=extra_heartbeats)
+
+            self.assertHeartbeatNotSavedOffline()
+            self.assertOfflineHeartbeatsSynced()
+            self.assertSessionCacheSaved()
+
+    @log_capture()
+    def test_extra_heartbeats_when_project_not_detected(self, logs):
+        logging.disable(logging.NOTSET)
 
-    def test_extra_heartbeats_argument(self):
         response = Response()
         response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
@@ -540,22 +688,20 @@ class ArgumentsTestCase(utils.TestCase):
         with utils.TemporaryDirectory() as tempdir:
             entity = 'tests/samples/codefiles/twolinefile.txt'
             shutil.copy(entity, os.path.join(tempdir, 'twolinefile.txt'))
-            entity = os.path.realpath(os.path.join(tempdir, 'twolinefile.txt'))
 
+            now1 = u(int(time.time()))
             project1 = os.path.basename(os.path.abspath('.'))
-            project2 = 'xyz'
             entity1 = os.path.abspath('tests/samples/codefiles/emptyfile.txt')
-            entity2 = os.path.abspath('tests/samples/codefiles/twolinefile.txt')
+            entity2 = os.path.realpath(os.path.join(tempdir, 'twolinefile.txt'))
             config = 'tests/samples/configs/good_config.cfg'
-            args = ['--file', entity1, '--config', config, '--extra-heartbeats']
+            args = ['--time', now1, '--file', entity1, '--config', config, '--extra-heartbeats']
 
             with utils.mock.patch('wakatime.main.sys.stdin') as mock_stdin:
-                now = int(time.time())
+                now2 = int(time.time())
                 heartbeats = json.dumps([{
-                    'timestamp': now,
+                    'timestamp': now2,
                     'entity': entity2,
                     'entity_type': 'file',
-                    'project': project2,
                     'is_write': True,
                 }])
                 mock_stdin.readline.return_value = heartbeats
@@ -563,27 +709,103 @@ class ArgumentsTestCase(utils.TestCase):
                 retval = execute(args)
 
                 self.assertEquals(retval, SUCCESS)
-                self.assertEquals(sys.stdout.getvalue(), '')
+                self.assertNothingPrinted()
-                self.assertEquals(sys.stderr.getvalue(), '')
+                self.assertNothingLogged(logs)
 
-                self.patched['wakatime.session_cache.SessionCache.get'].assert_has_calls([call(), call()])
+                heartbeat = {
-                self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()
+                    'language': 'Text only',
-                self.patched['wakatime.session_cache.SessionCache.save'].assert_has_calls([call(ANY), call(ANY)])
+                    'lines': 0,
+                    'entity': entity1,
+                    'project': project1,
+                    'branch': ANY,
+                    'time': float(now1),
+                    'is_write': False,
+                    'type': 'file',
+                    'dependencies': [],
+                    'user_agent': ANY,
+                }
+                extra_heartbeats = [{
+                    'language': 'Text only',
+                    'lines': 2,
+                    'entity': entity2,
+                    'project': None,
+                    'time': float(now2),
+                    'is_write': True,
+                    'type': 'file',
+                    'dependencies': [],
+                    'user_agent': ANY,
+                }]
+                self.assertHeartbeatSent(heartbeat, extra_heartbeats=extra_heartbeats)
 
-                self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
+                self.assertHeartbeatNotSavedOffline()
-                self.patched['wakatime.offlinequeue.Queue.pop'].assert_called_once_with()
+                self.assertOfflineHeartbeatsSynced()
+                self.assertSessionCacheSaved()
 
-                calls = self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].call_args_list
+    @log_capture()
+    def test_extra_heartbeats_when_project_not_detected_alternate_project_used(self, logs):
+        logging.disable(logging.NOTSET)
 
-                body = calls[0][0][0].body
+        response = Response()
-                data = json.loads(body)
+        response.status_code = 201
-                self.assertEquals(data.get('entity'), entity1)
+        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-                self.assertEquals(data.get('project'), project1)
 
-                body = calls[1][0][0].body
+        with utils.TemporaryDirectory() as tempdir:
-                data = json.loads(body)
+            entity = 'tests/samples/codefiles/twolinefile.txt'
-                self.assertEquals(data.get('entity'), entity2)
+            shutil.copy(entity, os.path.join(tempdir, 'twolinefile.txt'))
-                self.assertEquals(data.get('project'), project2)
+
+            now1 = u(int(time.time()))
+            project1 = os.path.basename(os.path.abspath('.'))
+            project2 = 'xyz'
+            entity1 = os.path.abspath('tests/samples/codefiles/emptyfile.txt')
+            entity2 = os.path.realpath(os.path.join(tempdir, 'twolinefile.txt'))
+            config = 'tests/samples/configs/good_config.cfg'
+            args = ['--time', now1, '--file', entity1, '--config', config, '--extra-heartbeats']
+
+            with utils.mock.patch('wakatime.main.sys.stdin') as mock_stdin:
+                now2 = int(time.time())
+                heartbeats = json.dumps([{
+                    'timestamp': now2,
+                    'entity': entity2,
+                    'alternate_project': project2,
+                    'entity_type': 'file',
+                    'is_write': True,
+                }])
+                mock_stdin.readline.return_value = heartbeats
+
+                retval = execute(args)
+
+                self.assertEquals(retval, SUCCESS)
+                self.assertNothingPrinted()
+                self.assertNothingLogged(logs)
+
+                heartbeat = {
+                    'language': 'Text only',
+                    'lines': 0,
+                    'entity': entity1,
+                    'project': project1,
+                    'branch': ANY,
+                    'time': float(now1),
+                    'is_write': False,
+                    'type': 'file',
+                    'dependencies': [],
+                    'user_agent': ANY,
+                }
+                extra_heartbeats = [{
+                    'language': 'Text only',
+                    'lines': 2,
+                    'entity': entity2,
+                    'project': project2,
+                    'time': float(now2),
+                    'is_write': True,
+                    'type': 'file',
+                    'dependencies': [],
+                    'user_agent': ANY,
+                }]
+                self.assertHeartbeatSent(heartbeat, extra_heartbeats=extra_heartbeats)
+
+                self.assertHeartbeatNotSavedOffline()
+                self.assertOfflineHeartbeatsSynced()
+                self.assertSessionCacheSaved()
 
     @log_capture()
     def test_extra_heartbeats_with_malformed_json(self, logs):
@@ -607,20 +829,16 @@ class ArgumentsTestCase(utils.TestCase):
                 mock_stdin.readline.return_value = heartbeats
 
                 retval = execute(args)
+                self.assertEquals(retval, SUCCESS)
+                self.assertNothingPrinted()
+                actual = self.getLogOutput(logs)
+                self.assertIn('WakaTime WARNING Malformed extra heartbeats json', actual)
 
-                self.assertEquals(retval, MALFORMED_HEARTBEAT_ERROR)
+                self.assertHeartbeatSent()
-                self.assertEquals(sys.stdout.getvalue(), '')
-                self.assertEquals(sys.stderr.getvalue(), '')
 
-                log_output = u("\n").join([u(' ').join(x) for x in logs.actual()])
+                self.assertHeartbeatNotSavedOffline()
-                self.assertEquals(log_output, '')
+                self.assertOfflineHeartbeatsSynced()
-
+                self.assertSessionCacheSaved()
-                self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-                self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()
-                self.patched['wakatime.session_cache.SessionCache.save'].assert_called_once_with(ANY)
-
-                self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
-                self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_uses_wakatime_home_env_variable(self):
         with utils.TemporaryDirectory() as tempdir:
@@ -634,11 +852,11 @@ class ArgumentsTestCase(utils.TestCase):
             args = ['--file', entity, '--key', key, '--config', config]
 
             with utils.mock.patch.object(sys, 'argv', ['wakatime'] + args):
-                args, configs = parseArguments()
+                args, configs = parse_arguments()
                 self.assertEquals(args.logfile, None)
 
                 with utils.mock.patch('os.environ.get') as mock_env:
                     mock_env.return_value = os.path.realpath(tempdir)
 
-                    args, configs = parseArguments()
+                    args, configs = parse_arguments()
                     self.assertEquals(args.logfile, logfile)

tests/test_configs.py

@@ -15,22 +15,13 @@ import uuid
 from testfixtures import log_capture
 from wakatime.compat import u, is_py3
 from wakatime.constants import (
-    API_ERROR,
     AUTH_ERROR,
     CONFIG_FILE_PARSE_ERROR,
     SUCCESS,
 )
 from wakatime.packages.requests.models import Response
 from . import utils
-
+from .utils import ANY
-try:
-    from .packages import simplejson as json
-except (ImportError, SyntaxError):
-    import json
-try:
-    from mock import ANY
-except ImportError:
-    from unittest.mock import ANY
 
 
 class ConfigsTestCase(utils.TestCase):
@@ -72,7 +63,10 @@ class ConfigsTestCase(utils.TestCase):
         self.assertEquals(sys.stderr.getvalue(), expected_stderr)
         self.patched['wakatime.session_cache.SessionCache.get'].assert_not_called()
 
-    def test_config_file_from_env(self):
+    @log_capture()
+    def test_config_file_from_env(self, logs):
+        logging.disable(logging.NOTSET)
+
         response = Response()
         response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
@@ -91,18 +85,17 @@ class ConfigsTestCase(utils.TestCase):
                 args = ['--file', entity, '--logfile', '~/.wakatime.log']
                 retval = execute(args)
                 self.assertEquals(retval, SUCCESS)
-                expected_stdout = open('tests/samples/output/main_test_good_config_file').read()
+                expected_stdout = open('tests/samples/output/configs_test_good_config_file').read()
                 traceback_file = os.path.realpath('wakatime/arguments.py')
                 lineno = int(re.search(r' line (\d+),', sys.stdout.getvalue()).group(1))
                 self.assertEquals(sys.stdout.getvalue(), expected_stdout.format(file=traceback_file, lineno=lineno))
                 self.assertEquals(sys.stderr.getvalue(), '')
 
-                self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+                self.assertHeartbeatSent(proxies=ANY, verify=ANY)
-                self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()
-                self.patched['wakatime.session_cache.SessionCache.save'].assert_called_once_with(ANY)
 
-                self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
+                self.assertHeartbeatNotSavedOffline()
-                self.patched['wakatime.offlinequeue.Queue.pop'].assert_called_once_with()
+                self.assertOfflineHeartbeatsSynced()
+                self.assertSessionCacheSaved()
 
     def test_missing_config_file(self):
         config = 'foo'
@@ -139,7 +132,7 @@ class ConfigsTestCase(utils.TestCase):
             args = ['--file', entity, '--config', config, '--logfile', '~/.wakatime.log']
             retval = execute(args)
             self.assertEquals(retval, SUCCESS)
-            expected_stdout = open('tests/samples/output/main_test_good_config_file').read()
+            expected_stdout = open('tests/samples/output/configs_test_good_config_file').read()
             traceback_file = os.path.realpath('wakatime/arguments.py')
             lineno = int(re.search(r' line (\d+),', sys.stdout.getvalue()).group(1))
             self.assertEquals(sys.stdout.getvalue(), expected_stdout.format(file=traceback_file, lineno=lineno))
@@ -209,9 +202,12 @@ class ConfigsTestCase(utils.TestCase):
             self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()
             self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
 
-    def test_non_hidden_filename(self):
+    @log_capture()
+    def test_non_hidden_filename(self, logs):
+        logging.disable(logging.NOTSET)
+
         response = Response()
-        response.status_code = 0
+        response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
         with utils.TemporaryDirectory() as tempdir:
@@ -225,39 +221,33 @@ class ConfigsTestCase(utils.TestCase):
             args = ['--file', entity, '--key', key, '--config', config, '--time', now, '--logfile', '~/.wakatime.log']
 
             retval = execute(args)
-            self.assertEquals(retval, API_ERROR)
+            self.assertEquals(retval, SUCCESS)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
+            self.assertNothingLogged(logs)
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
 
             heartbeat = {
-                'language': 'Text only',
-                'lines': 2,
                 'entity': os.path.realpath(entity),
-                'project': os.path.basename(os.path.abspath('.')),
+                'project': None,
+                'branch': None,
                 'time': float(now),
                 'type': 'file',
+                'cursorpos': None,
+                'dependencies': [],
+                'language': u('Text only'),
+                'lineno': None,
+                'lines': 2,
+                'is_write': False,
+                'user_agent': ANY,
             }
-            stats = {
+            self.assertHeartbeatSent(heartbeat)
-                u('cursorpos'): None,
-                u('dependencies'): [],
-                u('language'): u('Text only'),
-                u('lineno'): None,
-                u('lines'): 2,
-            }
 
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+            self.assertHeartbeatNotSavedOffline()
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
+            self.assertOfflineHeartbeatsSynced()
-                self.assertEquals(heartbeat[key], val)
+            self.assertSessionCacheSaved()
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_hide_all_filenames(self):
         response = Response()
-        response.status_code = 0
+        response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
         with utils.TemporaryDirectory() as tempdir:
@@ -266,38 +256,35 @@ class ConfigsTestCase(utils.TestCase):
             entity = os.path.realpath(os.path.join(tempdir, 'python.py'))
             now = u(int(time.time()))
             config = 'tests/samples/configs/paranoid.cfg'
-            key = str(uuid.uuid4())
+            key = u(uuid.uuid4())
+            project = 'abcxyz'
 
-            args = ['--file', entity, '--key', key, '--config', config, '--time', now, '--logfile', '~/.wakatime.log']
+            args = ['--file', entity, '--key', key, '--config', config, '--time', now, '--logfile', '~/.wakatime.log', '--alternate-project', project]
 
             retval = execute(args)
-            self.assertEquals(retval, API_ERROR)
+            self.assertEquals(retval, SUCCESS)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
 
             heartbeat = {
                 'language': 'Python',
+                'lines': None,
                 'entity': 'HIDDEN.py',
-                'project': os.path.basename(os.path.abspath('.')),
+                'project': project,
                 'time': float(now),
+                'is_write': False,
                 'type': 'file',
+                'dependencies': None,
+                'user_agent': ANY,
             }
+            self.assertHeartbeatSent(heartbeat)
 
-            self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_called_once_with(
+            self.assertHeartbeatNotSavedOffline()
-                ANY, cert=None, proxies={}, stream=False, timeout=60, verify=True,
+            self.assertOfflineHeartbeatsSynced()
-            )
+            self.assertSessionCacheSaved()
-            body = self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].call_args[0][0].body
-            for key, val in json.loads(body).items():
-                self.assertEquals(val, heartbeat.get(key))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_hide_all_filenames_from_cli_arg(self):
         response = Response()
-        response.status_code = 0
+        response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
         with utils.TemporaryDirectory() as tempdir:
@@ -307,37 +294,34 @@ class ConfigsTestCase(utils.TestCase):
             now = u(int(time.time()))
             config = 'tests/samples/configs/good_config.cfg'
             key = str(uuid.uuid4())
+            project = 'abcxyz'
 
-            args = ['--file', entity, '--key', key, '--config', config, '--time', now, '--hidefilenames', '--logfile', '~/.wakatime.log']
+            args = ['--file', entity, '--key', key, '--config', config, '--time', now, '--hidefilenames', '--logfile', '~/.wakatime.log', '--alternate-project', project]
 
             retval = execute(args)
-            self.assertEquals(retval, API_ERROR)
+            self.assertEquals(retval, SUCCESS)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
 
             heartbeat = {
                 'language': 'Python',
+                'lines': None,
                 'entity': 'HIDDEN.py',
-                'project': os.path.basename(os.path.abspath('.')),
+                'project': project,
                 'time': float(now),
+                'is_write': False,
                 'type': 'file',
+                'dependencies': None,
+                'user_agent': ANY,
             }
+            self.assertHeartbeatSent(heartbeat)
 
-            self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_called_once_with(
+            self.assertHeartbeatNotSavedOffline()
-                ANY, cert=None, proxies={}, stream=False, timeout=60, verify=True,
+            self.assertOfflineHeartbeatsSynced()
-            )
+            self.assertSessionCacheSaved()
-            body = self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].call_args[0][0].body
-            for key, val in json.loads(body).items():
-                self.assertEquals(val, heartbeat.get(key))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_hide_matching_filenames(self):
         response = Response()
-        response.status_code = 0
+        response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
         with utils.TemporaryDirectory() as tempdir:
@@ -346,42 +330,38 @@ class ConfigsTestCase(utils.TestCase):
             entity = os.path.realpath(os.path.join(tempdir, 'python.py'))
             now = u(int(time.time()))
             config = 'tests/samples/configs/hide_file_names.cfg'
+            key = '033c47c9-0441-4eb5-8b3f-b51f27b31049'
+            project = 'abcxyz'
 
-            args = ['--file', entity, '--config', config, '--time', now, '--logfile', '~/.wakatime.log']
+            args = ['--file', entity, '--key', key, '--config', config, '--time', now, '--logfile', '~/.wakatime.log', '--alternate-project', project]
 
             retval = execute(args)
-            self.assertEquals(retval, API_ERROR)
+            self.assertEquals(retval, SUCCESS)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
 
             heartbeat = {
                 'language': 'Python',
+                'lines': None,
                 'entity': 'HIDDEN.py',
-                'project': os.path.basename(os.path.abspath('.')),
+                'project': project,
                 'time': float(now),
+                'is_write': False,
                 'type': 'file',
+                'dependencies': None,
+                'user_agent': ANY,
             }
+            headers = {
+                'Authorization': u('Basic {0}').format(u(base64.b64encode(str.encode(key) if is_py3 else key))),
+            }
+            self.assertHeartbeatSent(heartbeat, headers=headers)
 
-            self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_called_once_with(
+            self.assertHeartbeatNotSavedOffline()
-                ANY, cert=None, proxies={}, stream=False, timeout=60, verify=True,
+            self.assertOfflineHeartbeatsSynced()
-            )
+            self.assertSessionCacheSaved()
-            body = self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].call_args[0][0].body
-            for key, val in json.loads(body).items():
-                self.assertEquals(val, heartbeat.get(key))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
-
-            headers = self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].call_args[0][0].headers
-            key = '033c47c9-0441-4eb5-8b3f-b51f27b31049'
-            expected = u('Basic {0}').format(u(base64.b64encode(str.encode(key) if is_py3 else key)))
-            self.assertEquals(headers.get('Authorization'), expected)
 
     def test_does_not_hide_unmatching_filenames(self):
         response = Response()
-        response.status_code = 0
+        response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
         with utils.TemporaryDirectory() as tempdir:
@@ -390,48 +370,39 @@ class ConfigsTestCase(utils.TestCase):
             entity = os.path.realpath(os.path.join(tempdir, 'python.py'))
             now = u(int(time.time()))
             config = 'tests/samples/configs/hide_file_names_not_python.cfg'
-            key = str(uuid.uuid4())
+            key = u(uuid.uuid4())
             dependencies = ['sqlalchemy', 'jinja', 'simplejson', 'flask', 'app', 'django', 'pygments', 'unittest', 'mock']
+            project = 'abcxyz'
 
-            args = ['--file', entity, '--key', key, '--config', config, '--time', now, '--logfile', '~/.wakatime.log']
+            args = ['--file', entity, '--key', key, '--config', config, '--time', now, '--logfile', '~/.wakatime.log', '--alternate-project', project]
 
             retval = execute(args)
-            self.assertEquals(retval, API_ERROR)
+            self.assertEquals(retval, SUCCESS)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-            self.maxDiff = 10000
 
             heartbeat = {
                 'language': 'Python',
                 'lines': 37,
                 'entity': entity,
-                'project': os.path.basename(os.path.abspath('.')),
+                'project': project,
-                'dependencies': dependencies,
                 'time': float(now),
+                'is_write': False,
                 'type': 'file',
+                'dependencies': dependencies,
+                'user_agent': ANY,
             }
+            self.assertHeartbeatSent(heartbeat)
 
-            self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_called_once_with(
+            self.assertHeartbeatNotSavedOffline()
-                ANY, cert=None, proxies={}, stream=False, timeout=60, verify=True,
+            self.assertOfflineHeartbeatsSynced()
-            )
+            self.assertSessionCacheSaved()
-            body = self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].call_args[0][0].body
-            for key, val in json.loads(body).items():
-                if key == 'dependencies':
-                    self.assertEquals(sorted(val), sorted(heartbeat[key]))
-                else:
-                    self.assertEquals(val, heartbeat.get(key))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     @log_capture()
     def test_does_not_hide_filenames_from_invalid_regex(self, logs):
         logging.disable(logging.NOTSET)
 
         response = Response()
-        response.status_code = 0
+        response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
         with utils.TemporaryDirectory() as tempdir:
@@ -445,41 +416,34 @@ class ConfigsTestCase(utils.TestCase):
             args = ['--file', entity, '--key', key, '--config', config, '--time', now, '--logfile', '~/.wakatime.log']
 
             retval = execute(args)
-            self.assertEquals(retval, API_ERROR)
+            self.assertEquals(retval, SUCCESS)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
 
-            log_output = u("\n").join([u(' ').join(x) for x in logs.actual()])
+            actual = self.getLogOutput(logs)
             expected = u('WakaTime WARNING Regex error (unbalanced parenthesis) for include pattern: invalid(regex')
             if self.isPy35OrNewer:
                 expected = 'WakaTime WARNING Regex error (missing ), unterminated subpattern at position 7) for include pattern: invalid(regex'
-            self.assertEquals(expected, log_output)
+            self.assertEquals(expected, actual)
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
 
             heartbeat = {
                 'language': 'Text only',
                 'lines': 0,
-                'entity': entity,
+                'entity': os.path.realpath(entity),
-                'project': os.path.basename(os.path.abspath('.')),
+                'project': None,
+                'cursorpos': None,
+                'lineno': None,
                 'time': float(now),
+                'is_write': False,
                 'type': 'file',
-            }
+                'dependencies': [],
-            stats = {
+                'user_agent': ANY,
-                u('cursorpos'): None,
-                u('dependencies'): [],
-                u('language'): u('Text only'),
-                u('lineno'): None,
-                u('lines'): 0,
             }
 
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+            self.assertHeartbeatSent(heartbeat)
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
+
-                self.assertEquals(heartbeat[key], val)
+            self.assertHeartbeatNotSavedOffline()
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
+            self.assertOfflineHeartbeatsSynced()
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
+            self.assertSessionCacheSaved()
 
     @log_capture()
     def test_exclude_file(self, logs):
@@ -498,20 +462,16 @@ class ConfigsTestCase(utils.TestCase):
             args = ['--file', entity, '--config', config, '--exclude', 'empty', '--verbose', '--logfile', '~/.wakatime.log']
             retval = execute(args)
             self.assertEquals(retval, SUCCESS)
-
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stdout.getvalue(), '')
+            actual = self.getLogOutput(logs)
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            log_output = u("\n").join([u(' ').join(x) for x in logs.actual()])
             expected = 'WakaTime DEBUG Skipping because matches exclude pattern: empty'
-            self.assertEquals(log_output, expected)
+            self.assertEquals(actual, expected)
 
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_not_called()
+            self.assertHeartbeatNotSent()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
 
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
+            self.assertHeartbeatNotSavedOffline()
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_called_once_with()
+            self.assertOfflineHeartbeatsSynced()
+            self.assertSessionCacheUntouched()
 
     def test_hostname_set_from_config_file(self):
         response = Response()
@@ -528,16 +488,16 @@ class ConfigsTestCase(utils.TestCase):
             args = ['--file', entity, '--config', config, '--timeout', '15', '--logfile', '~/.wakatime.log']
             retval = execute(args)
             self.assertEquals(retval, SUCCESS)
+            self.assertNothingPrinted()
 
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+            headers = {
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()
+                'X-Machine-Name': hostname.encode('utf-8') if is_py3 else hostname,
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_called_once_with(ANY)
+            }
+            self.assertHeartbeatSent(headers=headers, proxies=ANY, timeout=15)
 
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
+            self.assertHeartbeatNotSavedOffline()
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_called_once_with()
+            self.assertOfflineHeartbeatsSynced()
-
+            self.assertSessionCacheSaved()
-            headers = self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].call_args[0][0].headers
-            self.assertEquals(headers.get('X-Machine-Name'), hostname.encode('utf-8') if is_py3 else hostname)
 
     def test_no_ssl_verify_from_config_file(self):
         response = Response()
@@ -553,14 +513,10 @@ class ConfigsTestCase(utils.TestCase):
             args = ['--file', entity, '--config', config, '--timeout', '15', '--logfile', '~/.wakatime.log']
             retval = execute(args)
             self.assertEquals(retval, SUCCESS)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
 
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+            self.assertHeartbeatSent(proxies=ANY, timeout=15, verify=False)
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_called_once_with(ANY)
 
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
+            self.assertHeartbeatNotSavedOffline()
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_called_once_with()
+            self.assertOfflineHeartbeatsSynced()
-
+            self.assertSessionCacheSaved()
-            self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_called_once_with(ANY, cert=None, proxies=ANY, stream=False, timeout=15, verify=False)

tests/test_dependencies.py

@@ -8,24 +8,16 @@ import logging
 import os
 import time
 import shutil
-import sys
 from testfixtures import log_capture
 from wakatime.compat import u
+from wakatime.constants import SUCCESS
 from wakatime.exceptions import NotYetImplemented
 from wakatime.dependencies import DependencyParser, TokenParser
 from wakatime.packages.pygments.lexers import ClassNotFound, PythonLexer
 from wakatime.packages.requests.models import Response
 from wakatime.stats import get_lexer_by_name
 from . import utils
-
+from .utils import ANY
-try:
-    from .packages import simplejson as json
-except (ImportError, SyntaxError):
-    import json
-try:
-    from mock import ANY
-except ImportError:
-    from unittest.mock import ANY
 
 
 class DependenciesTestCase(utils.TestCase):
@@ -40,6 +32,42 @@ class DependenciesTestCase(utils.TestCase):
         ['wakatime.session_cache.SessionCache.connect', None],
     ]
 
+    def shared(self, expected_dependencies=[], expected_language=ANY, expected_lines=ANY, entity='', config='good_config.cfg', extra_args=[]):
+        response = Response()
+        response.status_code = 201
+        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+
+        config = os.path.join('tests/samples/configs', config)
+
+        with utils.TemporaryDirectory() as tempdir:
+            shutil.copy(os.path.join('tests/samples/codefiles', entity), os.path.join(tempdir, os.path.basename(entity)))
+            entity = os.path.realpath(os.path.join(tempdir, os.path.basename(entity)))
+
+            now = u(int(time.time()))
+            args = ['--file', entity, '--config', config, '--time', now] + extra_args
+
+            retval = execute(args)
+            self.assertEquals(retval, SUCCESS)
+            self.assertNothingPrinted()
+
+            heartbeat = {
+                'language': expected_language,
+                'lines': expected_lines,
+                'entity': os.path.realpath(entity),
+                'project': ANY,
+                'branch': ANY,
+                'dependencies': expected_dependencies,
+                'time': float(now),
+                'type': 'file',
+                'is_write': False,
+                'user_agent': ANY,
+            }
+            self.assertHeartbeatSent(heartbeat)
+
+            self.assertHeartbeatNotSavedOffline()
+            self.assertOfflineHeartbeatsSynced()
+            self.assertSessionCacheSaved()
+
     def test_token_parser(self):
         with self.assertRaises(NotYetImplemented):
             source_file = 'tests/samples/codefiles/c_only/non_empty.h'
@@ -79,8 +107,7 @@ class DependenciesTestCase(utils.TestCase):
         log_output = u("\n").join([u(' ').join(x) for x in logs.actual()])
         self.assertEquals(log_output, '')
 
-        self.assertEquals(sys.stdout.getvalue(), '')
+        self.assertNothingPrinted()
-        self.assertEquals(sys.stderr.getvalue(), '')
 
         expected = []
         self.assertEquals(dependencies, expected)
@@ -104,8 +131,7 @@ class DependenciesTestCase(utils.TestCase):
         expected = 'WakaTime DEBUG Parsing dependencies not supported for python.FooClass'
         self.assertEquals(log_output, expected)
 
-        self.assertEquals(sys.stdout.getvalue(), '')
+        self.assertNothingPrinted()
-        self.assertEquals(sys.stderr.getvalue(), '')
 
         expected = []
         self.assertEquals(dependencies, expected)
@@ -132,162 +158,43 @@ class DependenciesTestCase(utils.TestCase):
         expected = 'WakaTime DEBUG Parsing dependencies not supported for python.FooClass'
         self.assertEquals(log_output, expected)
 
-        self.assertEquals(sys.stdout.getvalue(), '')
+        self.assertNothingPrinted()
-        self.assertEquals(sys.stderr.getvalue(), '')
 
         expected = []
         self.assertEquals(dependencies, expected)
 
     def test_io_error_suppressed_when_parsing_dependencies(self):
-        response = Response()
+        with utils.mock.patch('wakatime.dependencies.open') as mock_open:
-        response.status_code = 0
+            mock_open.side_effect = IOError('')
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
-        with utils.TemporaryDirectory() as tempdir:
+            self.shared(
-            entity = 'tests/samples/codefiles/python.py'
+                expected_dependencies=[],
-            shutil.copy(entity, os.path.join(tempdir, 'python.py'))
+                expected_language='Python',
-            entity = os.path.realpath(os.path.join(tempdir, 'python.py'))
+                expected_lines=37,
-
+                entity='python.py',
-            now = u(int(time.time()))
+            )
-            config = 'tests/samples/configs/good_config.cfg'
-
-            args = ['--file', entity, '--config', config, '--time', now]
-
-            with utils.mock.patch('wakatime.dependencies.open') as mock_open:
-                mock_open.side_effect = IOError('')
-                retval = execute(args)
-
-            self.assertEquals(retval, 102)
-            self.assertEquals(sys.stdout.getvalue(), '')
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
-            heartbeat = {
-                'language': u('Python'),
-                'lines': 37,
-                'entity': os.path.realpath(entity),
-                'project': u(os.path.basename(os.path.realpath('.'))),
-                'time': float(now),
-                'type': 'file',
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('dependencies'): [],
-                u('language'): u('Python'),
-                u('lineno'): None,
-                u('lines'): 37,
-            }
-            expected_dependencies = []
-
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
-                self.assertEquals(heartbeat[key], val)
-            dependencies = self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('dependencies', [])
-            self.assertListsEqual(dependencies, expected_dependencies)
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_classnotfound_error_raised_when_passing_none_to_pygments(self):
         with self.assertRaises(ClassNotFound):
             get_lexer_by_name(None)
 
     def test_classnotfound_error_suppressed_when_parsing_dependencies(self):
-        response = Response()
+        with utils.mock.patch('wakatime.stats.guess_lexer_using_filename') as mock_guess:
-        response.status_code = 0
+            mock_guess.return_value = (None, None)
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
-        with utils.TemporaryDirectory() as tempdir:
+            with utils.mock.patch('wakatime.stats.get_filetype_from_buffer') as mock_filetype:
-            entity = 'tests/samples/codefiles/python.py'
+                mock_filetype.return_value = 'foo'
-            shutil.copy(entity, os.path.join(tempdir, 'python.py'))
-            entity = os.path.realpath(os.path.join(tempdir, 'python.py'))
 
-            now = u(int(time.time()))
+                self.shared(
-            config = 'tests/samples/configs/good_config.cfg'
+                    expected_dependencies=[],
-
+                    expected_language=None,
-            args = ['--file', entity, '--config', config, '--time', now]
+                    expected_lines=37,
-
+                    entity='python.py',
-            with utils.mock.patch('wakatime.stats.guess_lexer_using_filename') as mock_guess:
+                )
-                mock_guess.return_value = (None, None)
-
-                with utils.mock.patch('wakatime.stats.get_filetype_from_buffer') as mock_filetype:
-                    mock_filetype.return_value = 'foo'
-                    retval = execute(args)
-
-            self.assertEquals(retval, 102)
-            self.assertEquals(sys.stdout.getvalue(), '')
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
-            heartbeat = {
-                'language': None,
-                'lines': 37,
-                'dependencies': [],
-                'entity': os.path.realpath(entity),
-                'project': u(os.path.basename(os.path.realpath('.'))),
-                'time': float(now),
-                'type': 'file',
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('language'): None,
-                u('lineno'): None,
-                u('lines'): 37,
-                u('dependencies'): [],
-            }
-
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
-                self.assertEquals(heartbeat[key], val)
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_python_dependencies_detected(self):
-        response = Response()
+        self.shared(
-        response.status_code = 0
+            expected_dependencies=[
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
-        with utils.TemporaryDirectory() as tempdir:
-            entity = 'tests/samples/codefiles/python.py'
-            shutil.copy(entity, os.path.join(tempdir, 'python.py'))
-            entity = os.path.realpath(os.path.join(tempdir, 'python.py'))
-
-            now = u(int(time.time()))
-            config = 'tests/samples/configs/good_config.cfg'
-
-            args = ['--file', entity, '--config', config, '--time', now]
-
-            retval = execute(args)
-            self.assertEquals(retval, 102)
-            self.assertEquals(sys.stdout.getvalue(), '')
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
-            heartbeat = {
-                'language': u('Python'),
-                'lines': 37,
-                'entity': os.path.realpath(entity),
-                'project': u(os.path.basename(os.path.realpath('.'))),
-                'dependencies': ANY,
-                'time': float(now),
-                'type': 'file',
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('dependencies'): ANY,
-                u('language'): u('Python'),
-                u('lineno'): None,
-                u('lines'): 37,
-            }
-            expected_dependencies = [
                 'app',
                 'django',
                 'flask',
@@ -297,361 +204,84 @@ class DependenciesTestCase(utils.TestCase):
                 'simplejson',
                 'sqlalchemy',
                 'unittest',
-            ]
+            ],
-
+            expected_language='Python',
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+            expected_lines=37,
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
+            entity='python.py',
-                self.assertEquals(heartbeat[key], val)
+        )
-            dependencies = self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['dependencies']
-            self.assertListsEqual(dependencies, expected_dependencies)
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_bower_dependencies_detected(self):
-        response = Response()
+        self.shared(
-        response.status_code = 0
+            expected_dependencies=[
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+                'bootstrap',
-
+                'bootstrap-daterangepicker',
-        with utils.TemporaryDirectory() as tempdir:
+                'moment',
-            entity = 'tests/samples/codefiles/bower.json'
+                'moment-timezone',
-            shutil.copy(entity, os.path.join(tempdir, 'bower.json'))
+                'bower',
-            entity = os.path.realpath(os.path.join(tempdir, 'bower.json'))
+                'animate.css',
-
+            ],
-            now = u(int(time.time()))
+            expected_language='JSON',
-            config = 'tests/samples/configs/good_config.cfg'
+            expected_lines=11,
-
+            entity='bower.json',
-            args = ['--file', entity, '--config', config, '--time', now]
+        )
-
-            retval = execute(args)
-            self.assertEquals(sys.stdout.getvalue(), '')
-            self.assertEquals(sys.stderr.getvalue(), '')
-            self.assertEquals(retval, 102)
-
-            heartbeat = {
-                'language': u('JSON'),
-                'lines': 11,
-                'entity': os.path.realpath(entity),
-                'project': u(os.path.basename(os.path.realpath('.'))),
-                'dependencies': ANY,
-                'time': float(now),
-                'type': 'file',
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('dependencies'): ANY,
-                u('language'): u('JSON'),
-                u('lineno'): None,
-                u('lines'): 11,
-            }
-            expected_dependencies = ['animate.css', 'moment', 'moment-timezone']
-
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
-                self.assertEquals(heartbeat[key], val)
-            for dep in expected_dependencies:
-                self.assertIn(dep, self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['dependencies'])
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_grunt_dependencies_detected(self):
-        response = Response()
+        self.shared(
-        response.status_code = 0
+            expected_dependencies=[
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+                'grunt',
-
+            ],
-        with utils.TemporaryDirectory() as tempdir:
+            expected_language=None,
-            entity = 'tests/samples/codefiles/Gruntfile'
+            expected_lines=23,
-            shutil.copy(entity, os.path.join(tempdir, 'Gruntfile'))
+            entity='Gruntfile',
-            entity = os.path.realpath(os.path.join(tempdir, 'Gruntfile'))
+        )
-
-            now = u(int(time.time()))
-            config = 'tests/samples/configs/good_config.cfg'
-
-            args = ['--file', entity, '--config', config, '--time', now]
-
-            retval = execute(args)
-            self.assertEquals(sys.stdout.getvalue(), '')
-            self.assertEquals(sys.stderr.getvalue(), '')
-            self.assertEquals(retval, 102)
-
-            heartbeat = {
-                'language': None,
-                'lines': 23,
-                'entity': os.path.realpath(entity),
-                'project': u(os.path.basename(os.path.realpath('.'))),
-                'dependencies': ANY,
-                'time': float(now),
-                'type': 'file',
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('dependencies'): ANY,
-                u('language'): None,
-                u('lineno'): None,
-                u('lines'): 23,
-            }
-            expected_dependencies = ['grunt']
-
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
-                self.assertEquals(heartbeat[key], val)
-            for dep in expected_dependencies:
-                self.assertIn(dep, self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['dependencies'])
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_java_dependencies_detected(self):
-        response = Response()
+        self.shared(
-        response.status_code = 0
+            expected_dependencies=[
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
-        with utils.TemporaryDirectory() as tempdir:
-            entity = 'tests/samples/codefiles/java.java'
-            shutil.copy(entity, os.path.join(tempdir, 'java.java'))
-            entity = os.path.realpath(os.path.join(tempdir, 'java.java'))
-
-            now = u(int(time.time()))
-            config = 'tests/samples/configs/good_config.cfg'
-
-            args = ['--file', entity, '--config', config, '--time', now]
-
-            retval = execute(args)
-            self.assertEquals(retval, 102)
-            self.assertEquals(sys.stdout.getvalue(), '')
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
-            heartbeat = {
-                'language': u('Java'),
-                'lines': 20,
-                'entity': os.path.realpath(entity),
-                'project': u(os.path.basename(os.path.realpath('.'))),
-                'dependencies': ANY,
-                'time': float(now),
-                'type': 'file',
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('dependencies'): ANY,
-                u('language'): u('Java'),
-                u('lineno'): None,
-                u('lines'): 20,
-            }
-            expected_dependencies = [
                 'colorfulwolf.webcamapplet',
                 'foobar',
-            ]
+            ],
-
+            expected_language='Java',
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+            expected_lines=20,
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
+            entity='java.java',
-                self.assertEquals(heartbeat[key], val)
+        )
-            dependencies = self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['dependencies']
-            self.assertListsEqual(dependencies, expected_dependencies)
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_c_dependencies_detected(self):
-        response = Response()
+        self.shared(
-        response.status_code = 0
+            expected_dependencies=[
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
-        with utils.TemporaryDirectory() as tempdir:
-            entity = 'tests/samples/codefiles/c_only/non_empty.c'
-            shutil.copy(entity, os.path.join(tempdir, 'see.c'))
-            entity = os.path.realpath(os.path.join(tempdir, 'see.c'))
-
-            now = u(int(time.time()))
-            config = 'tests/samples/configs/good_config.cfg'
-
-            args = ['--file', entity, '--config', config, '--time', now]
-
-            retval = execute(args)
-            self.assertEquals(retval, 102)
-            self.assertEquals(sys.stdout.getvalue(), '')
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
-            heartbeat = {
-                'language': u('C'),
-                'lines': 8,
-                'entity': os.path.realpath(entity),
-                'project': u(os.path.basename(os.path.realpath('.'))),
-                'dependencies': ANY,
-                'time': float(now),
-                'type': 'file',
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('dependencies'): ANY,
-                u('language'): u('C'),
-                u('lineno'): None,
-                u('lines'): 8,
-            }
-            expected_dependencies = [
                 'openssl',
-            ]
+            ],
-
+            expected_language='C',
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+            expected_lines=8,
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
+            entity='c_only/non_empty.c',
-                self.assertEquals(heartbeat[key], val)
+        )
-            dependencies = self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['dependencies']
-            self.assertListsEqual(dependencies, expected_dependencies)
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_cpp_dependencies_detected(self):
-        response = Response()
+        self.shared(
-        response.status_code = 0
+            expected_dependencies=[
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
-        with utils.TemporaryDirectory() as tempdir:
-            entity = 'tests/samples/codefiles/c_and_cpp/non_empty.cpp'
-            shutil.copy(entity, os.path.join(tempdir, 'non_empty.cpp'))
-            entity = os.path.realpath(os.path.join(tempdir, 'non_empty.cpp'))
-
-            now = u(int(time.time()))
-            config = 'tests/samples/configs/good_config.cfg'
-
-            args = ['--file', entity, '--config', config, '--time', now]
-
-            retval = execute(args)
-            self.assertEquals(retval, 102)
-            self.assertEquals(sys.stdout.getvalue(), '')
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
-            heartbeat = {
-                'language': u('C++'),
-                'lines': 8,
-                'entity': os.path.realpath(entity),
-                'project': u(os.path.basename(os.path.realpath('.'))),
-                'dependencies': ANY,
-                'time': float(now),
-                'type': 'file',
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('dependencies'): ANY,
-                u('language'): u('C++'),
-                u('lineno'): None,
-                u('lines'): 8,
-            }
-            expected_dependencies = [
                 'openssl',
-            ]
+            ],
-
+            expected_language='C++',
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+            expected_lines=8,
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
+            entity='c_and_cpp/non_empty.cpp',
-                self.assertEquals(heartbeat[key], val)
+        )
-            dependencies = self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['dependencies']
-            self.assertListsEqual(dependencies, expected_dependencies)
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_csharp_dependencies_detected(self):
-        response = Response()
+        self.shared(
-        response.status_code = 0
+            expected_dependencies=[
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
-        with utils.TemporaryDirectory() as tempdir:
-            entity = 'tests/samples/codefiles/csharp/seesharp.cs'
-            shutil.copy(entity, os.path.join(tempdir, 'seesharp.cs'))
-            entity = os.path.realpath(os.path.join(tempdir, 'seesharp.cs'))
-
-            now = u(int(time.time()))
-            config = 'tests/samples/configs/good_config.cfg'
-
-            args = ['--file', entity, '--config', config, '--time', now]
-
-            retval = execute(args)
-            self.assertEquals(retval, 102)
-            self.assertEquals(sys.stdout.getvalue(), '')
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
-            heartbeat = {
-                'language': u('C#'),
-                'lines': 18,
-                'entity': os.path.realpath(entity),
-                'dependencies': ANY,
-                'project': u(os.path.basename(os.path.realpath('.'))),
-                'time': float(now),
-                'type': 'file',
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('dependencies'): ANY,
-                u('language'): u('C#'),
-                u('lineno'): None,
-                u('lines'): 18,
-            }
-            expected_dependencies = [
                 'Proper',
                 'Fart',
                 'Math',
                 'WakaTime',
-            ]
+            ],
-
+            expected_language='C#',
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+            expected_lines=18,
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
+            entity='csharp/seesharp.cs',
-                self.assertEquals(heartbeat[key], val)
+        )
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            dependencies = self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['dependencies']
-            self.assertListsEqual(dependencies, expected_dependencies)
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_php_dependencies_detected(self):
-        response = Response()
+        self.shared(
-        response.status_code = 0
+            expected_dependencies=[
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
-        with utils.TemporaryDirectory() as tempdir:
-            entity = 'tests/samples/codefiles/php.php'
-            shutil.copy(entity, os.path.join(tempdir, 'php.php'))
-            entity = os.path.realpath(os.path.join(tempdir, 'php.php'))
-
-            now = u(int(time.time()))
-            config = 'tests/samples/configs/good_config.cfg'
-
-            args = ['--file', entity, '--config', config, '--time', now]
-
-            retval = execute(args)
-            self.assertEquals(retval, 102)
-            self.assertEquals(sys.stdout.getvalue(), '')
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
-            heartbeat = {
-                'language': u('PHP'),
-                'lines': ANY,
-                'entity': os.path.realpath(entity),
-                'dependencies': ANY,
-                'project': u(os.path.basename(os.path.realpath('.'))),
-                'time': float(now),
-                'type': 'file',
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('dependencies'): ANY,
-                u('language'): u('PHP'),
-                u('lineno'): None,
-                u('lines'): ANY,
-            }
-            expected_dependencies = [
                 'Interop',
                 'FooBarOne',
                 'FooBarTwo',
@@ -662,161 +292,35 @@ class DependenciesTestCase(utils.TestCase):
                 'ArrayObject',
                 "'ServiceLocator.php'",
                 "'ServiceLocatorTwo.php'",
-            ]
+            ],
-
+            expected_language='PHP',
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+            expected_lines=116,
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
+            entity='php.php',
-                self.assertEquals(heartbeat[key], val)
+        )
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            dependencies = self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['dependencies']
-            self.assertListsEqual(dependencies, expected_dependencies)
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_php_in_html_dependencies_detected(self):
-        response = Response()
+        self.shared(
-        response.status_code = 0
+            expected_dependencies=[
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
-        with utils.TemporaryDirectory() as tempdir:
-            entity = 'tests/samples/codefiles/html-with-php.html'
-            shutil.copy(entity, os.path.join(tempdir, 'html-with-php.html'))
-            entity = os.path.realpath(os.path.join(tempdir, 'html-with-php.html'))
-
-            now = u(int(time.time()))
-            config = 'tests/samples/configs/good_config.cfg'
-
-            args = ['--file', entity, '--config', config, '--time', now]
-
-            retval = execute(args)
-            self.assertEquals(retval, 102)
-            self.assertEquals(sys.stdout.getvalue(), '')
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
-            heartbeat = {
-                'language': u('HTML+PHP'),
-                'lines': ANY,
-                'dependencies': ANY,
-                'entity': os.path.realpath(entity),
-                'project': u(os.path.basename(os.path.realpath('.'))),
-                'time': float(now),
-                'type': 'file',
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('dependencies'): ANY,
-                u('language'): u('HTML+PHP'),
-                u('lineno'): None,
-                u('lines'): ANY,
-            }
-            expected_dependencies = [
                 '"https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"',
-            ]
+            ],
-
+            expected_language='HTML+PHP',
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+            expected_lines=22,
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
+            entity='html-with-php.html',
-                self.assertEquals(heartbeat[key], val)
+        )
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            dependencies = self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['dependencies']
-            self.assertListsEqual(dependencies, expected_dependencies)
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_html_django_dependencies_detected(self):
-        response = Response()
+        self.shared(
-        response.status_code = 0
+            expected_dependencies=[
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
-        with utils.TemporaryDirectory() as tempdir:
-            entity = 'tests/samples/codefiles/html-django.html'
-            shutil.copy(entity, os.path.join(tempdir, 'html-django.html'))
-            entity = os.path.realpath(os.path.join(tempdir, 'html-django.html'))
-
-            now = u(int(time.time()))
-            config = 'tests/samples/configs/good_config.cfg'
-
-            args = ['--file', entity, '--config', config, '--time', now]
-
-            retval = execute(args)
-            self.assertEquals(retval, 102)
-            self.assertEquals(sys.stdout.getvalue(), '')
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
-            heartbeat = {
-                'language': u('HTML+Django/Jinja'),
-                'lines': ANY,
-                'dependencies': ANY,
-                'entity': os.path.realpath(entity),
-                'project': u(os.path.basename(os.path.realpath('.'))),
-                'time': float(now),
-                'type': 'file',
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('dependencies'): ANY,
-                u('language'): u('HTML+Django/Jinja'),
-                u('lineno'): None,
-                u('lines'): ANY,
-            }
-            expected_dependencies = [
                 '"libs/json2.js"',
-            ]
+            ],
-
+            expected_language='HTML+Django/Jinja',
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+            expected_lines=40,
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
+            entity='html-django.html',
-                self.assertEquals(heartbeat[key], val)
+        )
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            dependencies = self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['dependencies']
-            self.assertListsEqual(dependencies, expected_dependencies)
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_go_dependencies_detected(self):
-        response = Response()
+        self.shared(
-        response.status_code = 0
+            expected_dependencies=[
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
-        with utils.TemporaryDirectory() as tempdir:
-            entity = 'tests/samples/codefiles/go.go'
-            shutil.copy(entity, os.path.join(tempdir, 'go.go'))
-            entity = os.path.realpath(os.path.join(tempdir, 'go.go'))
-
-            now = u(int(time.time()))
-            config = 'tests/samples/configs/good_config.cfg'
-
-            args = ['--file', entity, '--config', config, '--time', now]
-
-            retval = execute(args)
-            self.assertEquals(retval, 102)
-            self.assertEquals(sys.stdout.getvalue(), '')
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
-            heartbeat = {
-                'language': u('Go'),
-                'lines': 24,
-                'entity': os.path.realpath(entity),
-                'project': u(os.path.basename(os.path.realpath('.'))),
-                'dependencies': ANY,
-                'time': float(now),
-                'type': 'file',
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('dependencies'): ANY,
-                u('language'): u('Go'),
-                u('lineno'): None,
-                u('lines'): 24,
-            }
-            expected_dependencies = [
                 '"compress/gzip"',
                 '"direct"',
                 '"foobar"',
@@ -827,61 +331,18 @@ class DependenciesTestCase(utils.TestCase):
                 '"oldname"',
                 '"os"',
                 '"supress"',
-            ]
+            ],
-
+            expected_language='Go',
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+            expected_lines=24,
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
+            entity='go.go',
-                self.assertEquals(heartbeat[key], val)
+        )
-            dependencies = self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['dependencies']
-            self.assertListsEqual(dependencies, expected_dependencies)
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
     def test_dependencies_still_detected_when_alternate_language_used(self):
-        response = Response()
+        with utils.mock.patch('wakatime.stats.smart_guess_lexer') as mock_guess_lexer:
-        response.status_code = 0
+            mock_guess_lexer.return_value = None
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
-        with utils.TemporaryDirectory() as tempdir:
+            self.shared(
-            entity = 'tests/samples/codefiles/python.py'
+                expected_dependencies=[
-            shutil.copy(entity, os.path.join(tempdir, 'python.py'))
-            entity = os.path.realpath(os.path.join(tempdir, 'python.py'))
-
-            now = u(int(time.time()))
-            config = 'tests/samples/configs/good_config.cfg'
-
-            args = ['--file', entity, '--config', config, '--time', now, '--alternate-language', 'PYTHON']
-
-            with utils.mock.patch('wakatime.stats.smart_guess_lexer') as mock_guess_lexer:
-                mock_guess_lexer.return_value = None
-
-                retval = execute(args)
-
-                self.assertEquals(retval, 102)
-                self.assertEquals(sys.stdout.getvalue(), '')
-                self.assertEquals(sys.stderr.getvalue(), '')
-
-                self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-                self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-                self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
-                heartbeat = {
-                    'language': u('Python'),
-                    'lines': 37,
-                    'entity': os.path.realpath(entity),
-                    'project': u(os.path.basename(os.path.realpath('.'))),
-                    'dependencies': ANY,
-                    'time': float(now),
-                    'type': 'file',
-                }
-                stats = {
-                    u('cursorpos'): None,
-                    u('dependencies'): ANY,
-                    u('language'): u('Python'),
-                    u('lineno'): None,
-                    u('lines'): 37,
-                }
-                expected_dependencies = [
                     'app',
                     'django',
                     'flask',
@@ -891,12 +352,9 @@ class DependenciesTestCase(utils.TestCase):
                     'simplejson',
                     'sqlalchemy',
                     'unittest',
-                ]
+                ],
-
+                expected_language='Python',
-                self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+                expected_lines=37,
-                for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
+                entity='python.py',
-                    self.assertEquals(heartbeat[key], val)
+                extra_args=['--alternate-language', 'PYTHON'],
-                dependencies = self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['dependencies']
+            )
-                self.assertListsEqual(dependencies, expected_dependencies)
-                self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-                self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()

tests/test_languages.py

@@ -4,11 +4,14 @@
 from wakatime.main import execute
 from wakatime.packages import requests
 
+import os
 import time
 from wakatime.compat import u
+from wakatime.constants import SUCCESS
 from wakatime.packages.requests.models import Response
 from wakatime.stats import guess_language
 from . import utils
+from .utils import ANY
 
 
 class LanguagesTestCase(utils.TestCase):
@@ -23,53 +26,56 @@ class LanguagesTestCase(utils.TestCase):
         ['wakatime.session_cache.SessionCache.connect', None],
     ]
 
-    def test_c_language_detected_for_header_with_c_files_in_folder(self):
+    def shared(self, expected_language='', entity='', extra_args=[]):
         response = Response()
-        response.status_code = 500
+        response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
-        now = u(int(time.time()))
         config = 'tests/samples/configs/good_config.cfg'
-        entity = 'tests/samples/codefiles/c_only/see.h'
+        entity = os.path.join('tests/samples/codefiles', entity)
-        args = ['--file', entity, '--config', config, '--time', now]
+
+        now = u(int(time.time()))
+        args = ['--file', entity, '--config', config, '--time', now] + extra_args
 
         retval = execute(args)
-        self.assertEquals(retval, 102)
+        self.assertEquals(retval, SUCCESS)
+        self.assertNothingPrinted()
 
-        language = u('C')
+        heartbeat = {
-        self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('language'), language)
+            'language': expected_language,
+            'lines': ANY,
+            'entity': os.path.realpath(entity),
+            'project': ANY,
+            'branch': ANY,
+            'dependencies': ANY,
+            'time': float(now),
+            'type': 'file',
+            'is_write': False,
+            'user_agent': ANY,
+        }
+        self.assertHeartbeatSent(heartbeat)
+
+        self.assertHeartbeatNotSavedOffline()
+        self.assertOfflineHeartbeatsSynced()
+        self.assertSessionCacheSaved()
+
+    def test_c_language_detected_for_header_with_c_files_in_folder(self):
+        self.shared(
+            expected_language='C',
+            entity='c_only/see.h',
+        )
 
     def test_cpp_language_detected_for_header_with_c_and_cpp_files_in_folder(self):
-        response = Response()
+        self.shared(
-        response.status_code = 500
+            expected_language='C++',
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+            entity='c_and_cpp/empty.h',
-
+        )
-        now = u(int(time.time()))
-        config = 'tests/samples/configs/good_config.cfg'
-        entity = 'tests/samples/codefiles/c_and_cpp/empty.h'
-        args = ['--file', entity, '--config', config, '--time', now]
-
-        retval = execute(args)
-        self.assertEquals(retval, 102)
-
-        language = u('C++')
-        self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('language'), language)
 
     def test_c_not_detected_for_non_header_with_c_files_in_folder(self):
-        response = Response()
+        self.shared(
-        response.status_code = 500
+            expected_language='Python',
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+            entity='c_and_python/see.py',
-
+        )
-        now = u(int(time.time()))
-        config = 'tests/samples/configs/good_config.cfg'
-        entity = 'tests/samples/codefiles/c_and_python/see.py'
-        args = ['--file', entity, '--config', config, '--time', now]
-
-        retval = execute(args)
-        self.assertEquals(retval, 102)
-
-        language = u('Python')
-        self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('language'), language)
 
     def test_guess_language(self):
         with utils.mock.patch('wakatime.stats.smart_guess_lexer') as mock_guess_lexer:
@@ -80,208 +86,94 @@ class LanguagesTestCase(utils.TestCase):
             self.assertEquals(result, (None, None))
 
     def test_guess_language_from_vim_modeline(self):
-        response = Response()
+        self.shared(
-        response.status_code = 500
+            expected_language='Python',
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+            entity='python_without_extension',
+        )
 
-        now = u(int(time.time()))
+    def test_language_arg_takes_priority_over_detected_language(self):
-        config = 'tests/samples/configs/good_config.cfg'
+        self.shared(
-        entity = 'tests/samples/codefiles/python_without_extension'
+            expected_language='Java',
-        args = ['--file', entity, '--config', config, '--time', now]
+            entity='python.py',
-
+            extra_args=['--language', 'JAVA']
-        retval = execute(args)
+        )
-        self.assertEquals(retval, 102)
-
-        language = u('Python')
-        self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('language'), language)
-
-    def test_alternate_language_takes_priority_over_detected_language(self):
-        response = Response()
-        response.status_code = 500
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
-        now = u(int(time.time()))
-        config = 'tests/samples/configs/good_config.cfg'
-        entity = 'tests/samples/codefiles/python.py'
-        args = ['--file', entity, '--config', config, '--time', now, '--language', 'JAVA']
-
-        retval = execute(args)
-        self.assertEquals(retval, 102)
-
-        language = u('Java')
-        self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('language'), language)
-
-    def test_alternate_language_is_used_when_not_guessed(self):
-        response = Response()
-        response.status_code = 500
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
-        with utils.mock.patch('wakatime.stats.smart_guess_lexer') as mock_guess_lexer:
-            mock_guess_lexer.return_value = None
-            language = u('Java')
-
-            now = u(int(time.time()))
-            config = 'tests/samples/configs/good_config.cfg'
-            entity = 'tests/samples/codefiles/python.py'
-            args = ['--file', entity, '--config', config, '--time', now, '--language', language.upper()]
-
-            retval = execute(args)
-            self.assertEquals(retval, 102)
-
-            self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('language'), language)
-
-    def test_vim_alternate_language_is_used_when_not_guessed(self):
-        response = Response()
-        response.status_code = 500
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
+    def test_language_arg_is_used_when_not_guessed(self):
         with utils.mock.patch('wakatime.stats.smart_guess_lexer') as mock_guess_lexer:
             mock_guess_lexer.return_value = None
 
-            now = u(int(time.time()))
+            self.shared(
-            config = 'tests/samples/configs/good_config.cfg'
+                expected_language='Java',
-            entity = 'tests/samples/codefiles/python.py'
+                entity='python.py',
-            args = ['--file', entity, '--config', config, '--time', now, '--language', 'java', '--plugin', 'NeoVim/703 vim-wakatime/4.0.9']
+                extra_args=['--language', 'JAVA']
+            )
 
-            retval = execute(args)
+    def test_vim_language_arg_is_used_when_not_guessed(self):
-            self.assertEquals(retval, 102)
+        with utils.mock.patch('wakatime.stats.smart_guess_lexer') as mock_guess_lexer:
+            mock_guess_lexer.return_value = None
 
-            language = u('Java')
+            self.shared(
-            self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('language'), language)
+                expected_language='Java',
+                entity='python.py',
+                extra_args=['--language', 'java', '--plugin', 'NeoVim/703 vim-wakatime/4.0.9']
+            )
 
     def test_alternate_language_not_used_when_invalid(self):
-        response = Response()
-        response.status_code = 500
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
         with utils.mock.patch('wakatime.stats.smart_guess_lexer') as mock_guess_lexer:
             mock_guess_lexer.return_value = None
 
-            now = u(int(time.time()))
+            self.shared(
-            config = 'tests/samples/configs/good_config.cfg'
+                expected_language=None,
-            entity = 'tests/samples/codefiles/python.py'
+                entity='python.py',
-            args = ['--file', entity, '--config', config, '--time', now, '--language', 'foo', '--plugin', 'NeoVim/703 vim-wakatime/4.0.9']
+                extra_args=['--language', 'foo', '--plugin', 'NeoVim/703 vim-wakatime/4.0.9']
-
+            )
-            retval = execute(args)
-            self.assertEquals(retval, 102)
-
-            language = None
-            self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('language'), language)
 
     def test_error_reading_alternate_language_json_map_file(self):
-        response = Response()
-        response.status_code = 500
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
         with utils.mock.patch('wakatime.stats.smart_guess_lexer') as mock_guess_lexer:
             mock_guess_lexer.return_value = None
 
             with utils.mock.patch('wakatime.stats.open') as mock_open:
                 mock_open.side_effect = IOError('')
 
-                now = u(int(time.time()))
+                self.shared(
-                config = 'tests/samples/configs/good_config.cfg'
+                    expected_language=None,
-                entity = 'tests/samples/codefiles/python.py'
+                    entity='python.py',
-                args = ['--file', entity, '--config', config, '--time', now, '--language', 'foo', '--plugin', 'NeoVim/703 vim-wakatime/4.0.9']
+                    extra_args=['--language', 'foo', '--plugin', 'NeoVim/703 vim-wakatime/4.0.9']
-
+                )
-                retval = execute(args)
-                self.assertEquals(retval, 102)
-
-                language = None
-                self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('language'), language)
 
     def test_typescript_detected_over_typoscript(self):
-        response = Response()
+        self.shared(
-        response.status_code = 500
+            expected_language='TypeScript',
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+            entity='typescript.ts',
-
+            extra_args=['--language', 'foo', '--plugin', 'NeoVim/703 vim-wakatime/4.0.9']
-        now = u(int(time.time()))
+        )
-        config = 'tests/samples/configs/good_config.cfg'
-        entity = 'tests/samples/codefiles/typescript.ts'
-        args = ['--file', entity, '--config', config, '--time', now]
-
-        retval = execute(args)
-        self.assertEquals(retval, 102)
-
-        language = u('TypeScript')
-        self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('language'), language)
 
     def test_perl_detected_over_prolog(self):
-        response = Response()
+        self.shared(
-        response.status_code = 500
+            expected_language='Perl',
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+            entity='perl.pl',
-
+        )
-        now = u(int(time.time()))
-        config = 'tests/samples/configs/good_config.cfg'
-        entity = 'tests/samples/codefiles/perl.pl'
-        args = ['--file', entity, '--config', config, '--time', now]
-
-        retval = execute(args)
-        self.assertEquals(retval, 102)
-
-        language = u('Perl')
-        self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('language'), language)
 
     def test_fsharp_detected_over_forth(self):
-        response = Response()
+        self.shared(
-        response.status_code = 500
+            expected_language='F#',
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+            entity='fsharp.fs',
-
+        )
-        now = u(int(time.time()))
-        config = 'tests/samples/configs/good_config.cfg'
-        entity = 'tests/samples/codefiles/fsharp.fs'
-        args = ['--file', entity, '--config', config, '--time', now]
-
-        retval = execute(args)
-        self.assertEquals(retval, 102)
-
-        language = u('F#')
-        self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('language'), language)
 
     def test_objectivec_detected_over_matlab_when_file_empty(self):
-        response = Response()
+        self.shared(
-        response.status_code = 500
+            expected_language='Objective-C',
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+            entity='matlab/empty.m',
-
+        )
-        now = u(int(time.time()))
-        config = 'tests/samples/configs/good_config.cfg'
-        entity = 'tests/samples/codefiles/matlab/empty.m'
-        args = ['--file', entity, '--config', config, '--time', now]
-
-        retval = execute(args)
-        self.assertEquals(retval, 102)
-
-        language = u('Objective-C')
-        self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('language'), language)
 
     def test_matlab_detected(self):
-        response = Response()
+        self.shared(
-        response.status_code = 500
+            expected_language='Matlab',
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+            entity='matlab/matlab.m',
-
+        )
-        now = u(int(time.time()))
-        config = 'tests/samples/configs/good_config.cfg'
-        entity = 'tests/samples/codefiles/matlab/matlab.m'
-        args = ['--file', entity, '--config', config, '--time', now]
-
-        retval = execute(args)
-        self.assertEquals(retval, 102)
-
-        language = u('Matlab')
-        self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('language'), language)
 
     def test_matlab_detected_over_objectivec_when_mat_file_in_folder(self):
-        response = Response()
+        self.shared(
-        response.status_code = 500
+            expected_language='Matlab',
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+            entity='matlab/with_mat_files/empty.m',
-
+        )
-        now = u(int(time.time()))
-        config = 'tests/samples/configs/good_config.cfg'
-        entity = 'tests/samples/codefiles/matlab/with_mat_files/empty.m'
-        args = ['--file', entity, '--config', config, '--time', now]
-
-        retval = execute(args)
-        self.assertEquals(retval, 102)
-
-        language = u('Matlab')
-        self.assertEqual(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('language'), language)

tests/test_logging.py

@@ -10,7 +10,6 @@ import logging
 import os
 import time
 import shutil
-import sys
 from testfixtures import log_capture
 from . import utils
 
@@ -42,8 +41,7 @@ class LoggingTestCase(utils.TestCase):
 
         retval = execute(args)
         self.assertEquals(retval, 102)
-        self.assertEquals(sys.stdout.getvalue(), '')
+        self.assertNothingPrinted()
-        self.assertEquals(sys.stderr.getvalue(), '')
 
         self.assertEquals(logging.WARNING, logging.getLogger('WakaTime').level)
         logfile = os.path.realpath(os.path.expanduser('~/.wakatime.log'))
@@ -77,8 +75,7 @@ class LoggingTestCase(utils.TestCase):
 
             retval = execute(args)
             self.assertEquals(retval, 102)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
 
             self.assertEquals(logging.WARNING, logging.getLogger('WakaTime').level)
             self.assertEquals(logfile, logging.getLogger('WakaTime').handlers[0].baseFilename)
@@ -112,8 +109,7 @@ class LoggingTestCase(utils.TestCase):
 
                 retval = execute(args)
                 self.assertEquals(retval, 102)
-                self.assertEquals(sys.stdout.getvalue(), '')
+                self.assertNothingPrinted()
-                self.assertEquals(sys.stderr.getvalue(), '')
 
                 self.assertEquals(logging.WARNING, logging.getLogger('WakaTime').level)
                 logfile = os.path.realpath(logging.getLogger('WakaTime').handlers[0].baseFilename)
@@ -135,8 +131,7 @@ class LoggingTestCase(utils.TestCase):
 
         retval = execute(args)
         self.assertEquals(retval, 102)
-        self.assertEquals(sys.stdout.getvalue(), '')
+        self.assertNothingPrinted()
-        self.assertEquals(sys.stderr.getvalue(), '')
 
         self.assertEquals(logging.DEBUG, logging.getLogger('WakaTime').level)
         logfile = os.path.realpath(os.path.expanduser('~/.wakatime.log'))
@@ -150,7 +145,7 @@ class LoggingTestCase(utils.TestCase):
         if self.isPy35OrNewer:
             expected = u('WakaTime WARNING Regex error (unbalanced parenthesis at position 15) for exclude pattern: \\(invalid regex)')
         self.assertEquals(output[1], expected)
-        self.assertEquals(output[2], u('WakaTime DEBUG Sending heartbeat to api at https://api.wakatime.com/api/v1/heartbeats'))
+        self.assertEquals(output[2], u('WakaTime DEBUG Sending heartbeats to api at https://api.wakatime.com/api/v1/heartbeats.bulk'))
         self.assertIn('Python', output[3])
         self.assertIn('response_code', output[4])
 
@@ -172,8 +167,7 @@ class LoggingTestCase(utils.TestCase):
 
             retval = execute(args)
             self.assertEquals(retval, 102)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
 
             log_output = u("\n").join([u(' ').join(x) for x in logs.actual()])
             self.assertIn(u('WakaTime DEBUG Traceback (most recent call last):'), log_output)
@@ -197,8 +191,7 @@ class LoggingTestCase(utils.TestCase):
 
             retval = execute(args)
             self.assertEquals(retval, 102)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
 
             log_output = u("\n").join([u(' ').join(x) for x in logs.actual()])
             self.assertEquals(u(''), log_output)

tests/test_main.py

@@ -18,19 +18,11 @@ from wakatime.constants import (
     MAX_FILE_SIZE_SUPPORTED,
     SUCCESS,
 )
+from wakatime.packages import tzlocal
 from wakatime.packages.requests.exceptions import RequestException
 from wakatime.packages.requests.models import Response
 from . import utils
-
+from .utils import ANY
-try:
-    from .packages import simplejson as json
-except (ImportError, SyntaxError):
-    import json
-try:
-    from mock import ANY
-except ImportError:
-    from unittest.mock import ANY
-from wakatime.packages import tzlocal
 
 
 class MainTestCase(utils.TestCase):
@@ -56,44 +48,26 @@ class MainTestCase(utils.TestCase):
             entity = os.path.realpath(os.path.join(tempdir, 'twolinefile.txt'))
             now = u(int(time.time()))
             key = str(uuid.uuid4())
+            heartbeat = {
+                'language': 'Text only',
+                'entity': 'HIDDEN.txt',
+                'project': None,
+                'time': float(now),
+                'type': 'file',
+                'is_write': False,
+                'user_agent': ANY,
+            }
 
             args = ['--file', entity, '--key', key,
                     '--config', 'tests/samples/configs/paranoid.cfg', '--time', now]
 
             retval = execute(args)
             self.assertEquals(retval, API_ERROR)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
+            self.assertHeartbeatSent(heartbeat)
-
+            self.assertHeartbeatSavedOffline()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
+            self.assertOfflineHeartbeatsNotSynced()
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+            self.assertSessionCacheDeleted()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
-            heartbeat = {
-                'language': 'Text only',
-                'lines': 2,
-                'entity': 'HIDDEN.txt',
-                'project': os.path.basename(os.path.abspath('.')),
-                'time': float(now),
-                'type': 'file',
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('dependencies'): [],
-                u('language'): u('Text only'),
-                u('lineno'): None,
-                u('lines'): 2,
-            }
-
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
-                self.assertEquals(heartbeat[key], val)
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
-
-            self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_called_once_with(
-                ANY, cert=None, proxies={}, stream=False, timeout=60, verify=True,
-            )
 
     def test_400_response(self):
         response = Response()
@@ -106,25 +80,26 @@ class MainTestCase(utils.TestCase):
             entity = os.path.realpath(os.path.join(tempdir, 'twolinefile.txt'))
             now = u(int(time.time()))
             key = str(uuid.uuid4())
+            heartbeat = {
+                'language': 'Text only',
+                'entity': 'HIDDEN.txt',
+                'project': None,
+                'time': float(now),
+                'type': 'file',
+                'is_write': False,
+                'user_agent': ANY,
+            }
 
             args = ['--file', entity, '--key', key,
                     '--config', 'tests/samples/configs/paranoid.cfg', '--time', now]
 
             retval = execute(args)
             self.assertEquals(retval, API_ERROR)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
+            self.assertHeartbeatSent(heartbeat)
-
+            self.assertHeartbeatNotSavedOffline()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
+            self.assertOfflineHeartbeatsNotSynced()
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+            self.assertSessionCacheDeleted()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
-
-            self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_called_once_with(
-                ANY, cert=None, proxies={}, stream=False, timeout=60, verify=True,
-            )
 
     def test_401_response(self):
         response = Response()
@@ -137,44 +112,27 @@ class MainTestCase(utils.TestCase):
             entity = os.path.realpath(os.path.join(tempdir, 'twolinefile.txt'))
             now = u(int(time.time()))
             key = str(uuid.uuid4())
+            heartbeat = {
+                'language': 'Text only',
+                'lines': None,
+                'entity': 'HIDDEN.txt',
+                'project': None,
+                'time': float(now),
+                'type': 'file',
+                'is_write': False,
+                'user_agent': ANY,
+            }
 
             args = ['--file', entity, '--key', key,
                     '--config', 'tests/samples/configs/paranoid.cfg', '--time', now]
 
             retval = execute(args)
             self.assertEquals(retval, AUTH_ERROR)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
+            self.assertHeartbeatSent(heartbeat)
-
+            self.assertHeartbeatSavedOffline()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
+            self.assertOfflineHeartbeatsNotSynced()
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+            self.assertSessionCacheDeleted()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
-            heartbeat = {
-                'language': 'Text only',
-                'lines': 2,
-                'entity': 'HIDDEN.txt',
-                'project': os.path.basename(os.path.abspath('.')),
-                'time': float(now),
-                'type': 'file',
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('dependencies'): [],
-                u('language'): u('Text only'),
-                u('lineno'): None,
-                u('lines'): 2,
-            }
-
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
-                self.assertEquals(heartbeat[key], val)
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
-
-            self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_called_once_with(
-                ANY, cert=None, proxies={}, stream=False, timeout=60, verify=True,
-            )
 
     @log_capture()
     def test_500_response_without_offline_logging(self, logs):
@@ -193,35 +151,39 @@ class MainTestCase(utils.TestCase):
             entity = os.path.realpath(os.path.join(tempdir, 'twolinefile.txt'))
             now = u(int(time.time()))
             key = str(uuid.uuid4())
+            heartbeat = {
+                'language': 'Text only',
+                'lines': 2,
+                'entity': entity,
+                'project': None,
+                'time': float(now),
+                'type': 'file',
+                'is_write': False,
+                'user_agent': ANY,
+                'dependencies': [],
+            }
 
             args = ['--file', entity, '--key', key, '--disableoffline',
                     '--config', 'tests/samples/configs/good_config.cfg', '--time', now]
 
             retval = execute(args)
             self.assertEquals(retval, API_ERROR)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
 
-            log_output = u("\n").join([u(' ').join(x) for x in logs.actual()])
+            actual = self.getLogOutput(logs)
             expected = "WakaTime ERROR {'response_code': 500, 'response_content': u'fake content'}"
-            if log_output[-2] == '0':
+            if actual[-2] == '0':
                 expected = "WakaTime ERROR {'response_content': u'fake content', 'response_code': 500}"
             if is_py3:
                 expected = "WakaTime ERROR {'response_code': 500, 'response_content': 'fake content'}"
-                if log_output[-2] == '0':
+                if actual[-2] == '0':
                     expected = "WakaTime ERROR {'response_content': 'fake content', 'response_code': 500}"
-            self.assertEquals(expected, log_output)
+            self.assertEquals(expected, actual)
 
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
+            self.assertHeartbeatSent(heartbeat)
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+            self.assertHeartbeatNotSavedOffline()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
+            self.assertOfflineHeartbeatsNotSynced()
-
+            self.assertSessionCacheDeleted()
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
-
-            self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_called_once_with(
-                ANY, cert=None, proxies={}, stream=False, timeout=60, verify=True,
-            )
 
     @log_capture()
     def test_requests_exception(self, logs):
@@ -235,54 +197,39 @@ class MainTestCase(utils.TestCase):
             entity = os.path.realpath(os.path.join(tempdir, 'twolinefile.txt'))
             now = u(int(time.time()))
             key = str(uuid.uuid4())
+            heartbeat = {
+                'language': 'Text only',
+                'lines': 2,
+                'entity': entity,
+                'project': None,
+                'time': float(now),
+                'type': 'file',
+                'is_write': False,
+                'user_agent': ANY,
+                'dependencies': [],
+            }
 
             args = ['--file', entity, '--key', key, '--verbose',
                     '--config', 'tests/samples/configs/good_config.cfg', '--time', now]
 
             retval = execute(args)
             self.assertEquals(retval, API_ERROR)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
 
-            log_output = u("\n").join([u(' ').join(x) for x in logs.actual()])
+            actual = self.getLogOutput(logs)
             expected = 'Parsing dependencies not supported for special.TextParser'
-            self.assertIn(expected, log_output)
+            self.assertIn(expected, actual)
-            expected = 'WakaTime DEBUG Sending heartbeat to api at https://api.wakatime.com/api/v1/heartbeats'
+            expected = 'WakaTime DEBUG Sending heartbeats to api at https://api.wakatime.com/api/v1/heartbeats.bulk'
-            self.assertIn(expected, log_output)
+            self.assertIn(expected, actual)
             expected = "RequestException': u'requests exception'"
             if is_py3:
                 expected = "RequestException': 'requests exception'"
-            self.assertIn(expected, log_output)
+            self.assertIn(expected, actual)
 
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
+            self.assertHeartbeatSent(heartbeat)
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+            self.assertHeartbeatSavedOffline()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
+            self.assertOfflineHeartbeatsNotSynced()
-
+            self.assertSessionCacheDeleted()
-            heartbeat = {
-                'language': 'Text only',
-                'lines': 2,
-                'entity': entity,
-                'project': os.path.basename(os.path.abspath('.')),
-                'time': float(now),
-                'type': 'file',
-            }
-            stats = {
-                u('cursorpos'): None,
-                u('dependencies'): [],
-                u('language'): u('Text only'),
-                u('lineno'): None,
-                u('lines'): 2,
-            }
-
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
-                self.assertEquals(heartbeat[key], val)
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
-
-            self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_called_once_with(
-                ANY, cert=None, proxies={}, stream=False, timeout=60, verify=True,
-            )
 
     @log_capture()
     def test_requests_exception_without_offline_logging(self, logs):
@@ -302,8 +249,7 @@ class MainTestCase(utils.TestCase):
 
             retval = execute(args)
             self.assertEquals(retval, API_ERROR)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
 
             log_output = u("\n").join([u(' ').join(x) for x in logs.actual()])
             expected = "WakaTime ERROR {'RequestException': u'requests exception'}"
@@ -311,16 +257,10 @@ class MainTestCase(utils.TestCase):
                 expected = "WakaTime ERROR {'RequestException': 'requests exception'}"
             self.assertEquals(expected, log_output)
 
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
+            self.assertHeartbeatSent()
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+            self.assertHeartbeatNotSavedOffline()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
+            self.assertOfflineHeartbeatsNotSynced()
-
+            self.assertSessionCacheDeleted()
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
-
-            self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_called_once_with(
-                ANY, cert=None, proxies={}, stream=False, timeout=60, verify=True,
-            )
 
     @log_capture()
     def test_invalid_api_key(self, logs):
@@ -345,14 +285,10 @@ class MainTestCase(utils.TestCase):
         expected = ''
         self.assertEquals(log_output, expected)
 
-        self.patched['wakatime.session_cache.SessionCache.get'].assert_not_called()
+        self.assertHeartbeatNotSent()
-        self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()
+        self.assertHeartbeatNotSavedOffline()
-        self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
+        self.assertOfflineHeartbeatsNotSynced()
-
+        self.assertSessionCacheUntouched()
-        self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
-        self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
-
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_not_called()
 
     def test_nonascii_hostname(self):
         response = Response()
@@ -373,18 +309,15 @@ class MainTestCase(utils.TestCase):
                 args = ['--file', entity, '--config', config]
                 retval = execute(args)
                 self.assertEquals(retval, SUCCESS)
-                self.assertEquals(sys.stdout.getvalue(), '')
+                self.assertNothingPrinted()
-                self.assertEquals(sys.stderr.getvalue(), '')
 
-                self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+                headers = {
-                self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()
+                    'X-Machine-Name': hostname.encode('utf-8') if is_py3 else hostname,
-                self.patched['wakatime.session_cache.SessionCache.save'].assert_called_once_with(ANY)
+                }
-
+                self.assertHeartbeatSent(headers=headers)
-                self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
+                self.assertHeartbeatNotSavedOffline()
-                self.patched['wakatime.offlinequeue.Queue.pop'].assert_called_once_with()
+                self.assertOfflineHeartbeatsSynced()
-
+                self.assertSessionCacheSaved()
-                headers = self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].call_args[0][0].headers
-                self.assertEquals(headers.get('X-Machine-Name'), hostname.encode('utf-8') if is_py3 else hostname)
 
     def test_nonascii_timezone(self):
         response = Response()
@@ -406,19 +339,19 @@ class MainTestCase(utils.TestCase):
                 mock_getlocalzone.return_value = timezone
 
                 config = 'tests/samples/configs/has_everything.cfg'
-                args = ['--file', entity, '--config', config, '--timeout', '15']
+                timeout = 15
+                args = ['--file', entity, '--config', config, '--timeout', u(timeout)]
                 retval = execute(args)
                 self.assertEquals(retval, SUCCESS)
+                self.assertNothingPrinted()
 
-                self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+                headers = {
-                self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()
+                    'TimeZone': u(timezone.zone).encode('utf-8') if is_py3 else timezone.zone,
-                self.patched['wakatime.session_cache.SessionCache.save'].assert_called_once_with(ANY)
+                }
-
+                self.assertHeartbeatSent(headers=headers, proxies=ANY, timeout=timeout)
-                self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
+                self.assertHeartbeatNotSavedOffline()
-                self.patched['wakatime.offlinequeue.Queue.pop'].assert_called_once_with()
+                self.assertOfflineHeartbeatsSynced()
-
+                self.assertSessionCacheSaved()
-                headers = self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].call_args[0][0].headers
-                self.assertEquals(headers.get('TimeZone'), u(timezone.zone).encode('utf-8') if is_py3 else timezone.zone)
 
     def test_timezone_with_invalid_encoding(self):
         response = Response()
@@ -442,21 +375,20 @@ class MainTestCase(utils.TestCase):
             with utils.mock.patch('wakatime.packages.tzlocal.get_localzone') as mock_getlocalzone:
                 mock_getlocalzone.return_value = timezone
 
+                timeout = 15
                 config = 'tests/samples/configs/has_everything.cfg'
-                args = ['--file', entity, '--config', config, '--timeout', '15']
+                args = ['--file', entity, '--config', config, '--timeout', u(timeout)]
                 retval = execute(args)
                 self.assertEquals(retval, SUCCESS)
+                self.assertNothingPrinted()
 
-                self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+                headers = {
-                self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()
+                    'TimeZone': u(bytes('\xab', 'utf-16') if is_py3 else '\xab').encode('utf-8'),
-                self.patched['wakatime.session_cache.SessionCache.save'].assert_called_once_with(ANY)
+                }
-
+                self.assertHeartbeatSent(headers=headers, proxies=ANY, timeout=timeout)
-                self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
+                self.assertHeartbeatNotSavedOffline()
-                self.patched['wakatime.offlinequeue.Queue.pop'].assert_called_once_with()
+                self.assertOfflineHeartbeatsSynced()
-
+                self.assertSessionCacheSaved()
-                headers = self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].call_args[0][0].headers
-                expected_tz = u(bytes('\xab', 'utf-16') if is_py3 else '\xab').encode('utf-8')
-                self.assertEquals(headers.get('TimeZone'), expected_tz)
 
     def test_tzlocal_exception(self):
         response = Response()
@@ -471,20 +403,20 @@ class MainTestCase(utils.TestCase):
             with utils.mock.patch('wakatime.packages.tzlocal.get_localzone') as mock_getlocalzone:
                 mock_getlocalzone.side_effect = Exception('tzlocal exception')
 
+                timeout = 15
                 config = 'tests/samples/configs/has_everything.cfg'
-                args = ['--file', entity, '--config', config, '--timeout', '15']
+                args = ['--file', entity, '--config', config, '--timeout', u(timeout)]
                 retval = execute(args)
                 self.assertEquals(retval, SUCCESS)
+                self.assertNothingPrinted()
 
-                self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+                headers = {
-                self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()
+                    'TimeZone': None,
-                self.patched['wakatime.session_cache.SessionCache.save'].assert_called_once_with(ANY)
+                }
-
+                self.assertHeartbeatSent(headers=headers, proxies=ANY, timeout=timeout)
-                self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
+                self.assertHeartbeatNotSavedOffline()
-                self.patched['wakatime.offlinequeue.Queue.pop'].assert_called_once_with()
+                self.assertOfflineHeartbeatsSynced()
-
+                self.assertSessionCacheSaved()
-                headers = self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].call_args[0][0].headers
-                self.assertEquals(headers.get('TimeZone'), None)
 
     def test_timezone_header(self):
         response = Response()
@@ -500,26 +432,23 @@ class MainTestCase(utils.TestCase):
             args = ['--file', entity, '--config', config]
             retval = execute(args)
             self.assertEquals(retval, SUCCESS)
-            self.assertEquals(sys.stdout.getvalue(), '')
+            self.assertNothingPrinted()
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_called_once_with(ANY)
-
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_called_once_with()
 
             timezone = tzlocal.get_localzone()
-            headers = self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].call_args[0][0].headers
+            headers = {
-            self.assertEquals(headers.get('TimeZone'), u(timezone.zone).encode('utf-8') if is_py3 else timezone.zone)
+                'TimeZone': u(timezone.zone).encode('utf-8') if is_py3 else timezone.zone,
+            }
+            self.assertHeartbeatSent(headers=headers)
+            self.assertHeartbeatNotSavedOffline()
+            self.assertOfflineHeartbeatsSynced()
+            self.assertSessionCacheSaved()
 
     @log_capture()
     def test_nonascii_filename(self, logs):
         logging.disable(logging.NOTSET)
 
         response = Response()
-        response.status_code = 0
+        response.status_code = 201
         self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
         with utils.TemporaryDirectory() as tempdir:
@@ -530,54 +459,38 @@ class MainTestCase(utils.TestCase):
             now = u(int(time.time()))
             config = 'tests/samples/configs/good_config.cfg'
             key = str(uuid.uuid4())
-
-            args = ['--file', entity, '--key', key, '--config', config, '--time', now]
-
-            retval = execute(args)
-            self.assertEquals(retval, API_ERROR)
-            self.assertEquals(sys.stdout.getvalue(), '')
-            self.assertEquals(sys.stderr.getvalue(), '')
-
-            log_output = u("\n").join([u(' ').join(x) for x in logs.actual()])
-            self.assertEquals(log_output, '')
-
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
-            self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
-
             heartbeat = {
                 'language': 'Text only',
                 'lines': 0,
                 'entity': os.path.realpath(entity),
-                'project': os.path.basename(os.path.abspath('.')),
+                'project': None,
                 'time': float(now),
                 'type': 'file',
-            }
+                'is_write': False,
-            stats = {
+                'user_agent': ANY,
-                u('cursorpos'): None,
+                'dependencies': [],
-                u('dependencies'): [],
-                u('language'): u('Text only'),
-                u('lineno'): None,
-                u('lines'): 0,
             }
 
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+            args = ['--file', entity, '--key', key, '--config', config, '--time', now]
-            for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
-                self.assertEquals(heartbeat[key], val)
-            self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
-            self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_called_once_with(
+            retval = execute(args)
-                ANY, cert=None, proxies={}, stream=False, timeout=60, verify=True,
+            self.assertEquals(retval, SUCCESS)
-            )
+            self.assertNothingPrinted()
+            self.assertNothingLogged(logs)
+
+            self.assertHeartbeatSent(heartbeat)
+
+            self.assertHeartbeatNotSavedOffline()
+            self.assertOfflineHeartbeatsSynced()
+            self.assertSessionCacheSaved()
 
     @log_capture()
     def test_unhandled_exception(self, logs):
         logging.disable(logging.NOTSET)
 
-        with utils.mock.patch('wakatime.main.process_heartbeat') as mock_process_heartbeat:
+        with utils.mock.patch('wakatime.main.send_heartbeats') as mock_send:
             ex_msg = 'testing unhandled exception'
-            mock_process_heartbeat.side_effect = RuntimeError(ex_msg)
+            mock_send.side_effect = RuntimeError(ex_msg)
 
             entity = 'tests/samples/codefiles/twolinefile.txt'
             config = 'tests/samples/configs/good_config.cfg'
@@ -593,11 +506,11 @@ class MainTestCase(utils.TestCase):
             log_output = u("\n").join([u(' ').join(x) for x in logs.actual()])
             self.assertIn(ex_msg, log_output)
 
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
+            self.assertHeartbeatNotSent()
-            self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
-            self.patched['wakatime.session_cache.SessionCache.get'].assert_not_called()
 
-            self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_not_called()
+            self.assertHeartbeatNotSavedOffline()
+            self.assertOfflineHeartbeatsNotSynced()
+            self.assertSessionCacheUntouched()
 
     def test_large_file_skips_lines_count(self):
         response = Response()
@@ -607,6 +520,20 @@ class MainTestCase(utils.TestCase):
         entity = 'tests/samples/codefiles/twolinefile.txt'
         config = 'tests/samples/configs/good_config.cfg'
         now = u(int(time.time()))
+        heartbeat = {
+            'language': 'Text only',
+            'lines': None,
+            'entity': os.path.realpath(entity),
+            'project': os.path.basename(os.path.abspath('.')),
+            'cursorpos': None,
+            'lineno': None,
+            'branch': ANY,
+            'time': float(now),
+            'type': 'file',
+            'is_write': False,
+            'user_agent': ANY,
+            'dependencies': [],
+        }
 
         args = ['--entity', entity, '--config', config, '--time', now]
 
@@ -615,39 +542,10 @@ class MainTestCase(utils.TestCase):
 
             retval = execute(args)
             self.assertEquals(retval, API_ERROR)
+            self.assertNothingPrinted()
 
-            self.assertEquals(sys.stdout.getvalue(), '')
+        self.assertHeartbeatSent(heartbeat)
-            self.assertEquals(sys.stderr.getvalue(), '')
 
-        self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+        self.assertHeartbeatSavedOffline()
-        self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
+        self.assertOfflineHeartbeatsNotSynced()
-        self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
+        self.assertSessionCacheDeleted()
-
-        heartbeat = {
-            'language': 'Text only',
-            'lines': None,
-            'entity': os.path.realpath(entity),
-            'project': os.path.basename(os.path.abspath('.')),
-            'cursorpos': None,
-            'lineno': None,
-            'branch': 'master',
-            'time': float(now),
-            'type': 'file',
-        }
-        stats = {
-            u('cursorpos'): None,
-            u('dependencies'): [],
-            u('language'): u('Text only'),
-            u('lineno'): None,
-            u('lines'): None,
-        }
-
-        self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
-        for key, val in self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].items():
-            self.assertEquals(heartbeat[key], val)
-        self.assertEquals(stats, json.loads(self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][1]))
-        self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
-
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_called_once_with(
-            ANY, cert=None, proxies={}, stream=False, timeout=60, verify=True,
-        )

tests/test_offlinequeue.py

@@ -8,20 +8,13 @@ from wakatime.packages import requests
 import logging
 import os
 import sqlite3
-import sys
 import time
 from testfixtures import log_capture
 from wakatime.compat import u
-from wakatime.constants import (
+from wakatime.constants import SUCCESS
-    AUTH_ERROR,
-    SUCCESS,
-)
 from wakatime.packages.requests.models import Response
 from . import utils
-try:
+from .utils import json
-    from .packages import simplejson as json
-except (ImportError, SyntaxError):
-    import json
 
 
 class OfflineQueueTestCase(utils.TestCase):
@@ -35,7 +28,7 @@ class OfflineQueueTestCase(utils.TestCase):
 
     def test_heartbeat_saved_from_error_response(self):
         with utils.NamedTemporaryFile() as fh:
-            with utils.mock.patch('wakatime.offlinequeue.Queue.get_db_file') as mock_db_file:
+            with utils.mock.patch('wakatime.offlinequeue.Queue._get_db_file') as mock_db_file:
                 mock_db_file.return_value = fh.name
 
                 response = Response()
@@ -49,13 +42,13 @@ class OfflineQueueTestCase(utils.TestCase):
                 args = ['--file', entity, '--config', config, '--time', now]
                 execute(args)
 
-                queue = Queue()
+                queue = Queue(None, None)
                 saved_heartbeat = queue.pop()
                 self.assertEquals(os.path.realpath(entity), saved_heartbeat['entity'])
 
     def test_heartbeat_discarded_from_400_response(self):
         with utils.NamedTemporaryFile() as fh:
-            with utils.mock.patch('wakatime.offlinequeue.Queue.get_db_file') as mock_db_file:
+            with utils.mock.patch('wakatime.offlinequeue.Queue._get_db_file') as mock_db_file:
                 mock_db_file.return_value = fh.name
 
                 response = Response()
@@ -69,13 +62,13 @@ class OfflineQueueTestCase(utils.TestCase):
                 args = ['--file', entity, '--config', config, '--time', now]
                 execute(args)
 
-                queue = Queue()
+                queue = Queue(None, None)
                 saved_heartbeat = queue.pop()
                 self.assertEquals(None, saved_heartbeat)
 
     def test_offline_heartbeat_sent_after_success_response(self):
         with utils.NamedTemporaryFile() as fh:
-            with utils.mock.patch('wakatime.offlinequeue.Queue.get_db_file') as mock_db_file:
+            with utils.mock.patch('wakatime.offlinequeue.Queue._get_db_file') as mock_db_file:
                 mock_db_file.return_value = fh.name
 
                 response = Response()
@@ -92,13 +85,13 @@ class OfflineQueueTestCase(utils.TestCase):
                 response.status_code = 201
                 execute(args)
 
-                queue = Queue()
+                queue = Queue(None, None)
                 saved_heartbeat = queue.pop()
                 self.assertEquals(None, saved_heartbeat)
 
     def test_all_offline_heartbeats_sent_after_success_response(self):
         with utils.NamedTemporaryFile() as fh:
-            with utils.mock.patch('wakatime.offlinequeue.Queue.get_db_file') as mock_db_file:
+            with utils.mock.patch('wakatime.offlinequeue.Queue._get_db_file') as mock_db_file:
                 mock_db_file.return_value = fh.name
 
                 response = Response()
@@ -130,45 +123,45 @@ class OfflineQueueTestCase(utils.TestCase):
                 execute(args)
 
                 # offline queue should be empty
-                queue = Queue()
+                queue = Queue(None, None)
                 saved_heartbeat = queue.pop()
                 self.assertEquals(None, saved_heartbeat)
 
                 calls = self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].call_args_list
 
                 body = calls[0][0][0].body
-                data = json.loads(body)
+                data = json.loads(body)[0]
                 self.assertEquals(data.get('entity'), os.path.abspath(entity1))
                 self.assertEquals(data.get('project'), project1)
                 self.assertEquals(u(int(data.get('time'))), now1)
 
                 body = calls[1][0][0].body
-                data = json.loads(body)
+                data = json.loads(body)[0]
                 self.assertEquals(data.get('entity'), os.path.abspath(entity2))
                 self.assertEquals(data.get('project'), project2)
                 self.assertEquals(u(int(data.get('time'))), now2)
 
                 body = calls[2][0][0].body
-                data = json.loads(body)
+                data = json.loads(body)[0]
                 self.assertEquals(data.get('entity'), os.path.abspath(entity3))
                 self.assertEquals(data.get('project'), project3)
                 self.assertEquals(u(int(data.get('time'))), now3)
 
                 body = calls[3][0][0].body
-                data = json.loads(body)
+                data = json.loads(body)[0]
                 self.assertEquals(data.get('entity'), os.path.abspath(entity1))
                 self.assertEquals(data.get('project'), project1)
                 self.assertEquals(u(int(data.get('time'))), now1)
 
-                body = calls[4][0][0].body
+                body = calls[3][0][0].body
-                data = json.loads(body)
+                data = json.loads(body)[1]
                 self.assertEquals(data.get('entity'), os.path.abspath(entity2))
                 self.assertEquals(data.get('project'), project2)
                 self.assertEquals(u(int(data.get('time'))), now2)
 
     def test_auth_error_when_sending_offline_heartbeats(self):
         with utils.NamedTemporaryFile() as fh:
-            with utils.mock.patch('wakatime.offlinequeue.Queue.get_db_file') as mock_db_file:
+            with utils.mock.patch('wakatime.offlinequeue.Queue._get_db_file') as mock_db_file:
                 mock_db_file.return_value = fh.name
 
                 response = Response()
@@ -214,18 +207,17 @@ class OfflineQueueTestCase(utils.TestCase):
                 self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
 
                 retval = execute(args)
-                self.assertEquals(retval, AUTH_ERROR)
+                self.assertEquals(retval, SUCCESS)
 
                 # offline queue should be empty
-                queue = Queue()
+                queue = Queue(None, None)
                 saved_heartbeat = queue.pop()
-                self.assertEquals(sys.stdout.getvalue(), '')
+                self.assertNothingPrinted()
-                self.assertEquals(sys.stderr.getvalue(), '')
+                self.assertIsNone(saved_heartbeat)
-                self.assertEquals(os.path.realpath(entity2), saved_heartbeat['entity'])
 
     def test_500_error_when_sending_offline_heartbeats(self):
         with utils.NamedTemporaryFile() as fh:
-            with utils.mock.patch('wakatime.offlinequeue.Queue.get_db_file') as mock_db_file:
+            with utils.mock.patch('wakatime.offlinequeue.Queue._get_db_file') as mock_db_file:
                 mock_db_file.return_value = fh.name
 
                 response = Response()
@@ -274,15 +266,14 @@ class OfflineQueueTestCase(utils.TestCase):
                 self.assertEquals(retval, SUCCESS)
 
                 # offline queue should be empty
-                queue = Queue()
+                queue = Queue(None, None)
                 saved_heartbeat = queue.pop()
-                self.assertEquals(sys.stdout.getvalue(), '')
+                self.assertNothingPrinted()
-                self.assertEquals(sys.stderr.getvalue(), '')
+                self.assertIsNone(saved_heartbeat)
-                self.assertEquals(os.path.realpath(entity2), saved_heartbeat['entity'])
 
     def test_empty_project_can_be_saved(self):
         with utils.NamedTemporaryFile() as fh:
-            with utils.mock.patch('wakatime.offlinequeue.Queue.get_db_file') as mock_db_file:
+            with utils.mock.patch('wakatime.offlinequeue.Queue._get_db_file') as mock_db_file:
                 mock_db_file.return_value = fh.name
 
                 response = Response()
@@ -296,15 +287,14 @@ class OfflineQueueTestCase(utils.TestCase):
                 args = ['--file', entity, '--config', config, '--time', now]
                 execute(args)
 
-                queue = Queue()
+                queue = Queue(None, None)
                 saved_heartbeat = queue.pop()
-                self.assertEquals(sys.stdout.getvalue(), '')
+                self.assertNothingPrinted()
-                self.assertEquals(sys.stderr.getvalue(), '')
                 self.assertEquals(os.path.realpath(entity), saved_heartbeat['entity'])
 
     def test_get_handles_connection_exception(self):
         with utils.NamedTemporaryFile() as fh:
-            with utils.mock.patch('wakatime.offlinequeue.Queue.get_db_file') as mock_db_file:
+            with utils.mock.patch('wakatime.offlinequeue.Queue._get_db_file') as mock_db_file:
                 mock_db_file.return_value = fh.name
 
                 response = Response()
@@ -324,17 +314,17 @@ class OfflineQueueTestCase(utils.TestCase):
                     response.status_code = 201
                     execute(args)
 
-                    queue = Queue()
+                    queue = Queue(None, None)
                     saved_heartbeat = queue.pop()
                     self.assertEquals(None, saved_heartbeat)
 
-                queue = Queue()
+                queue = Queue(None, None)
                 saved_heartbeat = queue.pop()
                 self.assertEquals(os.path.realpath(entity), saved_heartbeat['entity'])
 
     def test_push_handles_connection_exception(self):
         with utils.NamedTemporaryFile() as fh:
-            with utils.mock.patch('wakatime.offlinequeue.Queue.get_db_file') as mock_db_file:
+            with utils.mock.patch('wakatime.offlinequeue.Queue._get_db_file') as mock_db_file:
                 mock_db_file.return_value = fh.name
 
                 response = Response()
@@ -354,18 +344,18 @@ class OfflineQueueTestCase(utils.TestCase):
                     response.status_code = 201
                     execute(args)
 
-                queue = Queue()
+                queue = Queue(None, None)
                 saved_heartbeat = queue.pop()
                 self.assertEquals(None, saved_heartbeat)
 
     def test_uses_home_folder_by_default(self):
-        queue = Queue()
+        queue = Queue(None, None)
-        db_file = queue.get_db_file()
+        db_file = queue._get_db_file()
         expected = os.path.join(os.path.expanduser('~'), '.wakatime.db')
         self.assertEquals(db_file, expected)
 
     def test_uses_wakatime_home_env_variable(self):
-        queue = Queue()
+        queue = Queue(None, None)
 
         with utils.TemporaryDirectory() as tempdir:
             expected = os.path.realpath(os.path.join(tempdir, '.wakatime.db'))
@@ -373,7 +363,7 @@ class OfflineQueueTestCase(utils.TestCase):
             with utils.mock.patch('os.environ.get') as mock_env:
                 mock_env.return_value = os.path.realpath(tempdir)
 
-                actual = queue.get_db_file()
+                actual = queue._get_db_file()
                 self.assertEquals(actual, expected)
 
     @log_capture()
@@ -381,7 +371,7 @@ class OfflineQueueTestCase(utils.TestCase):
         logging.disable(logging.NOTSET)
 
         with utils.NamedTemporaryFile() as fh:
-            with utils.mock.patch('wakatime.offlinequeue.Queue.get_db_file') as mock_db_file:
+            with utils.mock.patch('wakatime.offlinequeue.Queue._get_db_file') as mock_db_file:
                 mock_db_file.return_value = fh.name
 
                 exception_msg = u("Oops, requests raised an exception. This is a test.")
@@ -394,12 +384,11 @@ class OfflineQueueTestCase(utils.TestCase):
                 args = ['--file', entity, '--config', config, '--time', now]
                 execute(args)
 
-                queue = Queue()
+                queue = Queue(None, None)
                 saved_heartbeat = queue.pop()
                 self.assertEquals(os.path.realpath(entity), saved_heartbeat['entity'])
 
-                self.assertEquals(sys.stdout.getvalue(), '')
+                self.assertNothingPrinted()
-                self.assertEquals(sys.stderr.getvalue(), '')
 
                 output = [u(' ').join(x) for x in logs.actual()]
                 self.assertIn(exception_msg, output[0])

tests/test_project.py

@@ -14,10 +14,11 @@ import tempfile
 import time
 from testfixtures import log_capture
 from wakatime.compat import u
-from wakatime.constants import API_ERROR
+from wakatime.constants import API_ERROR, SUCCESS
 from wakatime.exceptions import NotYetImplemented
 from wakatime.projects.base import BaseProject
 from . import utils
+from .utils import ANY, json
 
 
 class ProjectTestCase(utils.TestCase):
@@ -32,6 +33,40 @@ class ProjectTestCase(utils.TestCase):
         ['wakatime.session_cache.SessionCache.connect', None],
     ]
 
+    def shared(self, expected_project='', expected_branch=ANY, entity='', config='good_config.cfg', extra_args=[]):
+        response = Response()
+        response.status_code = 201
+        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+
+        config = os.path.join('tests/samples/configs', config)
+        if not os.path.exists(entity):
+            entity = os.path.realpath(os.path.join('tests/samples', entity))
+
+        now = u(int(time.time()))
+        args = ['--file', entity, '--config', config, '--time', now] + extra_args
+
+        retval = execute(args)
+        self.assertEquals(retval, SUCCESS)
+        self.assertNothingPrinted()
+
+        heartbeat = {
+            'language': ANY,
+            'lines': ANY,
+            'entity': os.path.realpath(entity),
+            'project': expected_project,
+            'branch': expected_branch,
+            'dependencies': ANY,
+            'time': float(now),
+            'type': 'file',
+            'is_write': False,
+            'user_agent': ANY,
+        }
+        self.assertHeartbeatSent(heartbeat)
+
+        self.assertHeartbeatNotSavedOffline()
+        self.assertOfflineHeartbeatsSynced()
+        self.assertSessionCacheSaved()
+
     def test_project_base(self):
         path = 'tests/samples/codefiles/see.h'
         project = BaseProject(path)
@@ -110,98 +145,63 @@ class ProjectTestCase(utils.TestCase):
         args = ['--file', entity, '--config', config, '--time', now, '--alternate-project', 'alt-project']
         execute(args)
 
-        calls = self.patched['wakatime.offlinequeue.Queue.push'].call_args_list
+        calls = self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].call_args_list
-        self.assertEquals(None, calls[0][0][0].get('project'))
+
-        self.assertEquals('alt-project', calls[1][0][0]['project'])
+        body = calls[0][0][0].body
+        data = json.loads(body)[0]
+        self.assertEquals(None, data.get('project'))
+
+        body = calls[1][0][0].body
+        data = json.loads(body)[0]
+        self.assertEquals('alt-project', data['project'])
 
     def test_wakatime_project_file(self):
-        response = Response()
+        self.shared(
-        response.status_code = 0
+            expected_project='waka-project-file',
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+            entity='projects/wakatime_project_file/emptyfile.txt',
-
+        )
-        now = u(int(time.time()))
-        entity = 'tests/samples/projects/wakatime_project_file/emptyfile.txt'
-        config = 'tests/samples/configs/good_config.cfg'
-
-        args = ['--file', entity, '--config', config, '--time', now]
-
-        execute(args)
-
-        self.assertEquals('waka-project-file', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['project'])
 
     def test_git_project_detected(self):
-        response = Response()
-        response.status_code = 0
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
         tempdir = tempfile.mkdtemp()
         shutil.copytree('tests/samples/projects/git', os.path.join(tempdir, 'git'))
         shutil.move(os.path.join(tempdir, 'git', 'dot_git'), os.path.join(tempdir, 'git', '.git'))
 
-        now = u(int(time.time()))
+        self.shared(
-        entity = os.path.join(tempdir, 'git', 'emptyfile.txt')
+            expected_project='git',
-        config = 'tests/samples/configs/good_config.cfg'
+            expected_branch='master',
-
+            entity=os.path.join(tempdir, 'git', 'emptyfile.txt'),
-        args = ['--file', entity, '--config', config, '--time', now]
+        )
-
-        execute(args)
-
-        self.assertEquals('git', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['project'])
-        self.assertEquals('master', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['branch'])
 
     def test_ioerror_when_reading_git_branch(self):
-        response = Response()
-        response.status_code = 0
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
         tempdir = tempfile.mkdtemp()
         shutil.copytree('tests/samples/projects/git', os.path.join(tempdir, 'git'))
         shutil.move(os.path.join(tempdir, 'git', 'dot_git'), os.path.join(tempdir, 'git', '.git'))
 
-        now = u(int(time.time()))
         entity = os.path.join(tempdir, 'git', 'emptyfile.txt')
-        config = 'tests/samples/configs/good_config.cfg'
-
-        args = ['--file', entity, '--config', config, '--time', now]
 
         with utils.mock.patch('wakatime.projects.git.open') as mock_open:
             mock_open.side_effect = IOError('')
-            execute(args)
 
-        self.assertEquals('git', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['project'])
+            self.shared(
-        self.assertEquals('master', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('branch'))
+                expected_project='git',
+                expected_branch='master',
+                entity=entity,
+            )
 
     def test_git_detached_head_not_used_as_branch(self):
-        response = Response()
-        response.status_code = 0
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
         tempdir = tempfile.mkdtemp()
         shutil.copytree('tests/samples/projects/git-with-detached-head', os.path.join(tempdir, 'git'))
         shutil.move(os.path.join(tempdir, 'git', 'dot_git'), os.path.join(tempdir, 'git', '.git'))
 
-        now = u(int(time.time()))
         entity = os.path.join(tempdir, 'git', 'emptyfile.txt')
-        config = 'tests/samples/configs/good_config.cfg'
 
-        args = ['--file', entity, '--config', config, '--time', now]
+        self.shared(
-
+            expected_project='git',
-        execute(args)
+            expected_branch=None,
-
+            entity=entity,
-        self.assertEquals('git', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['project'])
+        )
-        self.assertNotIn('branch', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0])
 
     def test_svn_project_detected(self):
-        response = Response()
-        response.status_code = 0
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
-        now = u(int(time.time()))
-        entity = 'tests/samples/projects/svn/afolder/emptyfile.txt'
-        config = 'tests/samples/configs/good_config.cfg'
-
-        args = ['--file', entity, '--config', config, '--time', now]
-
         with utils.mock.patch('wakatime.projects.git.Git.process') as mock_git:
             mock_git.return_value = False
 
@@ -213,25 +213,16 @@ class ProjectTestCase(utils.TestCase):
                     stderr = ''
                     mock_popen.return_value = utils.DynamicIterable((stdout, stderr), max_calls=1)
 
-                    execute(args)
+                    expected = None if platform.system() == 'Windows' else 'svn'
-
+                    self.shared(
-        expected = None if platform.system() == 'Windows' else 'svn'
+                        expected_project=expected,
-        self.assertEquals(expected, self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0].get('project'))
+                        entity='projects/svn/afolder/emptyfile.txt',
+                    )
 
     def test_svn_exception_handled(self):
-        response = Response()
-        response.status_code = 0
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
         with utils.mock.patch('wakatime.projects.git.Git.process') as mock_git:
             mock_git.return_value = False
 
-            now = u(int(time.time()))
-            entity = 'tests/samples/projects/svn/afolder/emptyfile.txt'
-            config = 'tests/samples/configs/good_config.cfg'
-
-            args = ['--file', entity, '--config', config, '--time', now]
-
             with utils.mock.patch('wakatime.projects.subversion.Subversion._has_xcode_tools') as mock_has_xcode:
                 mock_has_xcode.return_value = True
 
@@ -241,21 +232,12 @@ class ProjectTestCase(utils.TestCase):
                     with utils.mock.patch('wakatime.projects.subversion.Popen.communicate') as mock_communicate:
                         mock_communicate.side_effect = OSError('')
 
-                        execute(args)
+                        self.shared(
-
+                            expected_project=None,
-            self.assertNotIn('project', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0])
+                            entity='projects/svn/afolder/emptyfile.txt',
+                        )
 
     def test_svn_on_mac_without_xcode_tools_installed(self):
-        response = Response()
-        response.status_code = 0
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
-        now = u(int(time.time()))
-        entity = 'tests/samples/projects/svn/afolder/emptyfile.txt'
-        config = 'tests/samples/configs/good_config.cfg'
-
-        args = ['--file', entity, '--config', config, '--time', now]
-
         with utils.mock.patch('wakatime.projects.git.Git.process') as mock_git:
             mock_git.return_value = False
 
@@ -267,9 +249,10 @@ class ProjectTestCase(utils.TestCase):
                     stderr = ''
                     mock_popen.return_value = utils.DynamicIterable((stdout, stderr), raise_on_calls=[OSError('')])
 
-                    execute(args)
+                    self.shared(
-
+                        expected_project=None,
-            self.assertNotIn('project', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0])
+                        entity='projects/svn/afolder/emptyfile.txt',
+                    )
 
     def test_svn_on_mac_with_xcode_tools_installed(self):
         response = Response()
@@ -352,109 +335,78 @@ class ProjectTestCase(utils.TestCase):
             self.assertEquals('default', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['branch'])
 
     def test_git_submodule_detected(self):
-        response = Response()
-        response.status_code = 0
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
         tempdir = tempfile.mkdtemp()
         shutil.copytree('tests/samples/projects/git-with-submodule', os.path.join(tempdir, 'git'))
         shutil.move(os.path.join(tempdir, 'git', 'dot_git'), os.path.join(tempdir, 'git', '.git'))
         shutil.move(os.path.join(tempdir, 'git', 'asubmodule', 'dot_git'), os.path.join(tempdir, 'git', 'asubmodule', '.git'))
 
-        now = u(int(time.time()))
         entity = os.path.join(tempdir, 'git', 'asubmodule', 'emptyfile.txt')
-        config = 'tests/samples/configs/good_config.cfg'
 
-        args = ['--file', entity, '--config', config, '--time', now]
+        self.shared(
-
+            expected_project='asubmodule',
-        execute(args)
+            expected_branch='asubbranch',
-
+            entity=entity,
-        self.assertEquals('asubmodule', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['project'])
+        )
-        self.assertNotIn('asubbranch', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0])
 
     def test_git_submodule_detected_and_enabled_globally(self):
-        response = Response()
-        response.status_code = 0
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
         tempdir = tempfile.mkdtemp()
         shutil.copytree('tests/samples/projects/git-with-submodule', os.path.join(tempdir, 'git'))
         shutil.move(os.path.join(tempdir, 'git', 'dot_git'), os.path.join(tempdir, 'git', '.git'))
         shutil.move(os.path.join(tempdir, 'git', 'asubmodule', 'dot_git'), os.path.join(tempdir, 'git', 'asubmodule', '.git'))
 
-        now = u(int(time.time()))
         entity = os.path.join(tempdir, 'git', 'asubmodule', 'emptyfile.txt')
-        config = 'tests/samples/configs/git-submodules-enabled.cfg'
 
-        args = ['--file', entity, '--config', config, '--time', now]
+        self.shared(
-
+            expected_project='asubmodule',
-        execute(args)
+            expected_branch='asubbranch',
-
+            entity=entity,
-        self.assertEquals('asubmodule', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['project'])
+            config='git-submodules-enabled.cfg',
-        self.assertNotIn('asubbranch', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0])
+        )
 
     def test_git_submodule_detected_but_disabled_globally(self):
-        response = Response()
-        response.status_code = 0
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
         tempdir = tempfile.mkdtemp()
         shutil.copytree('tests/samples/projects/git-with-submodule', os.path.join(tempdir, 'git'))
         shutil.move(os.path.join(tempdir, 'git', 'dot_git'), os.path.join(tempdir, 'git', '.git'))
         shutil.move(os.path.join(tempdir, 'git', 'asubmodule', 'dot_git'), os.path.join(tempdir, 'git', 'asubmodule', '.git'))
 
-        now = u(int(time.time()))
         entity = os.path.join(tempdir, 'git', 'asubmodule', 'emptyfile.txt')
-        config = 'tests/samples/configs/git-submodules-disabled.cfg'
 
-        args = ['--file', entity, '--config', config, '--time', now]
+        self.shared(
-
+            expected_project='git',
-        execute(args)
+            expected_branch='master',
-
+            entity=entity,
-        self.assertEquals('git', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['project'])
+            config='git-submodules-disabled.cfg',
-        self.assertNotIn('master', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0])
+        )
 
     def test_git_submodule_detected_but_disabled_using_regex(self):
-        response = Response()
-        response.status_code = 0
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
         tempdir = tempfile.mkdtemp()
         shutil.copytree('tests/samples/projects/git-with-submodule', os.path.join(tempdir, 'git'))
         shutil.move(os.path.join(tempdir, 'git', 'dot_git'), os.path.join(tempdir, 'git', '.git'))
         shutil.move(os.path.join(tempdir, 'git', 'asubmodule', 'dot_git'), os.path.join(tempdir, 'git', 'asubmodule', '.git'))
 
-        now = u(int(time.time()))
         entity = os.path.join(tempdir, 'git', 'asubmodule', 'emptyfile.txt')
-        config = 'tests/samples/configs/git-submodules-disabled-using-regex.cfg'
 
-        args = ['--file', entity, '--config', config, '--time', now]
+        self.shared(
-
+            expected_project='git',
-        execute(args)
+            expected_branch='master',
-
+            entity=entity,
-        self.assertEquals('git', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['project'])
+            config='git-submodules-disabled-using-regex.cfg',
-        self.assertNotIn('master', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0])
+        )
 
     def test_git_submodule_detected_but_enabled_using_regex(self):
-        response = Response()
-        response.status_code = 0
-        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
-
         tempdir = tempfile.mkdtemp()
         shutil.copytree('tests/samples/projects/git-with-submodule', os.path.join(tempdir, 'git'))
         shutil.move(os.path.join(tempdir, 'git', 'dot_git'), os.path.join(tempdir, 'git', '.git'))
         shutil.move(os.path.join(tempdir, 'git', 'asubmodule', 'dot_git'), os.path.join(tempdir, 'git', 'asubmodule', '.git'))
 
-        now = u(int(time.time()))
         entity = os.path.join(tempdir, 'git', 'asubmodule', 'emptyfile.txt')
-        config = 'tests/samples/configs/git-submodules-enabled-using-regex.cfg'
 
-        args = ['--file', entity, '--config', config, '--time', now]
+        self.shared(
-
+            expected_project='asubmodule',
-        execute(args)
+            expected_branch='asubbranch',
-
+            entity=entity,
-        self.assertEquals('asubmodule', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0]['project'])
+            config='git-submodules-enabled-using-regex.cfg',
-        self.assertNotIn('asubbranch', self.patched['wakatime.offlinequeue.Queue.push'].call_args[0][0])
+        )
 
     @log_capture()
     def test_project_map(self, logs):

tests/test_proxy.py

@@ -174,7 +174,7 @@ class ProxyTestCase(utils.TestCase):
             self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
             self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
 
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY)
             self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
             expected_calls = [
@@ -212,7 +212,7 @@ class ProxyTestCase(utils.TestCase):
             self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
             self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
 
-            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY, ANY, None)
+            self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY)
             self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
 
             expected_calls = [

tests/test_session_cache.py

@@ -33,7 +33,7 @@ class SessionCacheTestCase(utils.TestCase):
         with utils.NamedTemporaryFile() as fh:
             cache = SessionCache()
 
-            with utils.mock.patch('wakatime.session_cache.SessionCache.get_db_file') as mock_dbfile:
+            with utils.mock.patch('wakatime.session_cache.SessionCache._get_db_file') as mock_dbfile:
                 mock_dbfile.return_value = fh.name
 
                 session = cache.get()
@@ -49,7 +49,7 @@ class SessionCacheTestCase(utils.TestCase):
         with utils.NamedTemporaryFile() as fh:
             cache = SessionCache()
 
-            with utils.mock.patch('wakatime.session_cache.SessionCache.get_db_file') as mock_dbfile:
+            with utils.mock.patch('wakatime.session_cache.SessionCache._get_db_file') as mock_dbfile:
                 mock_dbfile.return_value = fh.name
 
                 with utils.mock.patch('wakatime.session_cache.SessionCache.connect') as mock_connect:
@@ -69,12 +69,12 @@ class SessionCacheTestCase(utils.TestCase):
             expected = os.path.realpath(os.path.join(os.path.expanduser('~'), '.wakatime.db'))
 
             cache = SessionCache()
-            actual = cache.get_db_file()
+            actual = cache._get_db_file()
             self.assertEquals(actual, expected)
 
             with utils.mock.patch('os.environ.get') as mock_env:
                 mock_env.return_value = os.path.realpath(tempdir)
 
                 expected = os.path.realpath(os.path.join(tempdir, '.wakatime.db'))
-                actual = cache.get_db_file()
+                actual = cache._get_db_file()
                 self.assertEquals(actual, expected)

tests/utils.py

@@ -10,14 +10,20 @@ from wakatime.compat import u
 
 try:
     import mock
+    from mock import ANY
 except ImportError:
     import unittest.mock as mock
+    from unittest.mock import ANY
 try:
     # Python 2.6
     import unittest2 as unittest
 except ImportError:
     # Python >= 2.7
     import unittest
+try:
+    from .packages import simplejson as json
+except (ImportError, SyntaxError):
+    import json
 
 
 class TestCase(unittest.TestCase):
@@ -27,6 +33,8 @@ class TestCase(unittest.TestCase):
         # disable logging while testing
         logging.disable(logging.CRITICAL)
 
+        self.maxDiff = 1000
+
         self.patched = {}
         if hasattr(self, 'patch_these'):
             for patch_this in self.patch_these:
@@ -49,8 +57,85 @@ class TestCase(unittest.TestCase):
     def normalize_list(self, items):
         return sorted([u(x) for x in items])
 
-    def assertListsEqual(self, first_list, second_list):
+    def assertListsEqual(self, first_list, second_list, message=None):
-        self.assertEquals(self.normalize_list(first_list), self.normalize_list(second_list))
+        if isinstance(first_list, list) and isinstance(second_list, list):
+            if message:
+                self.assertEquals(self.normalize_list(first_list), self.normalize_list(second_list), message)
+            else:
+                self.assertEquals(self.normalize_list(first_list), self.normalize_list(second_list))
+        else:
+            if message:
+                self.assertEquals(first_list, second_list, message)
+            else:
+                self.assertEquals(first_list, second_list)
+
+    def assertHeartbeatNotSent(self):
+        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_not_called()
+
+    def assertHeartbeatSent(self, heartbeat=None, extra_heartbeats=[], headers=None, cert=None, proxies={}, stream=False, timeout=60, verify=True):
+        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].assert_called_once_with(
+            ANY, cert=cert, proxies=proxies, stream=stream, timeout=timeout, verify=verify,
+        )
+
+        body = json.loads(self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].call_args[0][0].body)
+        self.assertIsInstance(body, list)
+
+        if headers:
+            actual_headers = self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].call_args[0][0].headers
+            for key, val in headers.items():
+                self.assertEquals(actual_headers.get(key), val, u('Expected api request to have header {0}={1}, instead {0}={2}').format(u(key), u(actual_headers.get(key)), u(val)))
+
+        if heartbeat:
+            keys = list(body[0].keys()) + list(heartbeat.keys())
+            for key in keys:
+                if isinstance(heartbeat.get(key), list):
+                    self.assertListsEqual(heartbeat.get(key), body[0].get(key), u('Expected heartbeat to be sent with {0}={1}, instead {0}={2}').format(u(key), u(heartbeat.get(key)), u(body[0].get(key))))
+                else:
+                    self.assertEquals(heartbeat.get(key), body[0].get(key), u('Expected heartbeat to be sent with {0}={1}, instead {0}={2}').format(u(key), u(heartbeat.get(key)), u(body[0].get(key))))
+
+        if extra_heartbeats:
+            for i in range(len(extra_heartbeats)):
+                keys = list(body[i + 1].keys()) + list(extra_heartbeats[i].keys())
+                for key in keys:
+                    self.assertEquals(extra_heartbeats[i].get(key), body[i + 1].get(key), u('Expected extra heartbeat {3} to be sent with {0}={1}, instead {0}={2}').format(u(key), u(extra_heartbeats[i].get(key)), u(body[i + 1].get(key)), i))
+
+    def assertSessionCacheUntouched(self):
+        self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()
+        self.patched['wakatime.session_cache.SessionCache.get'].assert_not_called()
+        self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
+
+    def assertSessionCacheDeleted(self):
+        self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
+        self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+        self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
+
+    def assertSessionCacheSaved(self):
+        self.patched['wakatime.session_cache.SessionCache.save'].assert_called_once_with(ANY)
+        self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+        self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()
+
+    def assertHeartbeatSavedOffline(self):
+        self.patched['wakatime.offlinequeue.Queue.push'].assert_called_once_with(ANY)
+        self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
+
+    def assertHeartbeatNotSavedOffline(self):
+        self.patched['wakatime.offlinequeue.Queue.push'].assert_not_called()
+
+    def assertOfflineHeartbeatsSynced(self):
+        self.patched['wakatime.offlinequeue.Queue.pop'].assert_called()
+
+    def assertOfflineHeartbeatsNotSynced(self):
+        self.patched['wakatime.offlinequeue.Queue.pop'].assert_not_called()
+
+    def assertNothingPrinted(self):
+        self.assertEquals(sys.stdout.getvalue(), '')
+        self.assertEquals(sys.stderr.getvalue(), '')
+
+    def assertNothingLogged(self, logs):
+        self.assertEquals(self.getLogOutput(logs), '')
+
+    def getLogOutput(self, logs):
+        return u("\n").join([u(' ').join(x) for x in logs.actual()])
 
     @property
     def isPy35OrNewer(self):
@@ -65,6 +150,7 @@ try:
 except ImportError:
     # Python < 3
     import shutil
+
     class TemporaryDirectory(object):
         """Context manager for tempfile.mkdtemp().
 
@@ -111,8 +197,10 @@ class DynamicIterable(object):
         self.raise_on_calls = raise_on_calls
         self.index = 0
         self.data = data
+
     def __iter__(self):
         return self
+
     def __next__(self):
         if self.raise_on_calls and self.called < len(self.raise_on_calls) and self.raise_on_calls[self.called]:
             raise self.raise_on_calls[self.called]
@@ -125,5 +213,6 @@ class DynamicIterable(object):
         if not self.max_calls or self.called <= self.max_calls:
             return val
         return None
+
     def next(self):
         return self.__next__()

wakatime/api.py

@@ -0,0 +1,177 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.api
+    ~~~~~~~~~~~~
+
+    :copyright: (c) 2017 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+
+from __future__ import print_function
+
+import base64
+import logging
+import sys
+import traceback
+
+from .compat import u, is_py3, json
+from .constants import (
+    API_ERROR,
+    AUTH_ERROR,
+    SUCCESS,
+    UNKNOWN_ERROR,
+)
+
+from .offlinequeue import Queue
+from .packages.requests.exceptions import RequestException
+from .session_cache import SessionCache
+from .utils import get_hostname, get_user_agent
+from .packages import tzlocal
+
+
+log = logging.getLogger('WakaTime')
+
+
+try:
+    from .packages import requests
+except ImportError:
+    log.traceback(logging.ERROR)
+    print(traceback.format_exc())
+    log.error('Please upgrade Python to the latest version.')
+    print('Please upgrade Python to the latest version.')
+    sys.exit(UNKNOWN_ERROR)
+
+
+def send_heartbeats(heartbeats, args, configs, use_ntlm_proxy=False):
+    """Send heartbeats to WakaTime API.
+
+    Returns `SUCCESS` when heartbeat was sent, otherwise returns an error code.
+    """
+
+    if len(heartbeats) == 0:
+        return SUCCESS
+
+    api_url = args.api_url
+    if not api_url:
+        api_url = 'https://api.wakatime.com/api/v1/heartbeats.bulk'
+    log.debug('Sending heartbeats to api at %s' % api_url)
+    timeout = args.timeout
+    if not timeout:
+        timeout = 60
+
+    data = [h.sanitize().dict() for h in heartbeats]
+    log.debug(data)
+
+    # setup api request
+    request_body = json.dumps(data)
+    api_key = u(base64.b64encode(str.encode(args.key) if is_py3 else args.key))
+    auth = u('Basic {api_key}').format(api_key=api_key)
+    headers = {
+        'User-Agent': get_user_agent(args.plugin),
+        'Content-Type': 'application/json',
+        'Accept': 'application/json',
+        'Authorization': auth,
+    }
+
+    hostname = get_hostname(args)
+    if hostname:
+        headers['X-Machine-Name'] = u(hostname).encode('utf-8')
+
+    # add Olson timezone to request
+    try:
+        tz = tzlocal.get_localzone()
+    except:
+        tz = None
+    if tz:
+        headers['TimeZone'] = u(tz.zone).encode('utf-8')
+
+    session_cache = SessionCache()
+    session = session_cache.get()
+
+    should_try_ntlm = False
+    proxies = {}
+    if args.proxy:
+        if use_ntlm_proxy:
+            from .packages.requests_ntlm import HttpNtlmAuth
+            username = args.proxy.rsplit(':', 1)
+            password = ''
+            if len(username) == 2:
+                password = username[1]
+            username = username[0]
+            session.auth = HttpNtlmAuth(username, password, session)
+        else:
+            should_try_ntlm = '\\' in args.proxy
+            proxies['https'] = args.proxy
+
+    # send request to api
+    response, code = None, None
+    try:
+        response = session.post(api_url, data=request_body, headers=headers,
+                                proxies=proxies, timeout=timeout,
+                                verify=not args.nosslverify)
+    except RequestException:
+        if should_try_ntlm:
+            return send_heartbeats(heartbeats, args, configs, use_ntlm_proxy=True)
+        else:
+            exception_data = {
+                sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
+            }
+            if log.isEnabledFor(logging.DEBUG):
+                exception_data['traceback'] = traceback.format_exc()
+            if args.offline:
+                queue = Queue(args, configs)
+                queue.push_many(heartbeats)
+                if log.isEnabledFor(logging.DEBUG):
+                    log.warn(exception_data)
+            else:
+                log.error(exception_data)
+
+    except:  # delete cached session when requests raises unknown exception
+        if should_try_ntlm:
+            return send_heartbeats(heartbeats, args, configs, use_ntlm_proxy=True)
+        else:
+            exception_data = {
+                sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
+                'traceback': traceback.format_exc(),
+            }
+            if args.offline:
+                queue = Queue(args, configs)
+                queue.push_many(heartbeats)
+                log.warn(exception_data)
+
+    else:
+        code = response.status_code if response is not None else None
+        content = response.text if response is not None else None
+        if code == requests.codes.created or code == requests.codes.accepted:
+            log.debug({
+                'response_code': code,
+            })
+            session_cache.save(session)
+            return SUCCESS
+
+        if should_try_ntlm:
+            return send_heartbeats(heartbeats, args, configs, use_ntlm_proxy=True)
+        else:
+            if args.offline:
+                if code == 400:
+                    log.error({
+                        'response_code': code,
+                        'response_content': content,
+                    })
+                else:
+                    if log.isEnabledFor(logging.DEBUG):
+                        log.warn({
+                            'response_code': code,
+                            'response_content': content,
+                        })
+                    queue = Queue(args, configs)
+                    queue.push_many(heartbeats)
+            else:
+                log.error({
+                    'response_code': code,
+                    'response_content': content,
+                })
+
+    session_cache.delete()
+    return AUTH_ERROR if code == 401 else API_ERROR

wakatime/arguments.py

@@ -45,7 +45,7 @@ class StoreWithoutQuotes(argparse.Action):
         setattr(namespace, self.dest, values)
 
 
-def parseArguments():
+def parse_arguments():
     """Parse command line arguments and configs from ~/.wakatime.cfg.
     Command line arguments take precedence over config file settings.
     Returns instances of ArgumentParser and SafeConfigParser.

wakatime/compat.py

@@ -9,6 +9,7 @@
     :license: BSD, see LICENSE for more details.
 """
 
+
 import codecs
 import sys
 
@@ -91,3 +92,9 @@ except ImportError:  # pragma: nocover
             name = _resolve_name(name[level:], package, level)
         __import__(name)
         return sys.modules[name]
+
+
+try:
+    from .packages import simplejson as json
+except (ImportError, SyntaxError):
+    import json

wakatime/constants.py

@@ -34,11 +34,6 @@ Exit code used when there was an unhandled exception.
 """
 UNKNOWN_ERROR = 105
 
-""" Malformed Heartbeat Error
-Exit code used when the JSON input from `--extra-heartbeats` is malformed.
-"""
-MALFORMED_HEARTBEAT_ERROR = 106
-
 """ Connection Error
 Exit code used when there was proxy or other problem connecting to the WakaTime
 API servers.

wakatime/heartbeat.py

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.heartbeat
+    ~~~~~~~~~~~~~~~~~~
+    :copyright: (c) 2017 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+
+import os
+import logging
+import re
+
+from .compat import u, json
+from .project import get_project_info
+from .stats import get_file_stats
+from .utils import get_user_agent, should_exclude, format_file_path
+
+
+log = logging.getLogger('WakaTime')
+
+
+class Heartbeat(object):
+    """Heartbeat data for sending to API or storing in offline cache."""
+
+    skip = False
+    args = None
+    configs = None
+
+    time = None
+    entity = None
+    type = None
+    is_write = None
+    project = None
+    branch = None
+    language = None
+    dependencies = None
+    lines = None
+    lineno = None
+    cursorpos = None
+    user_agent = None
+
+    def __init__(self, data, args, configs, _clone=None):
+        self.args = args
+        self.configs = configs
+
+        self.entity = data.get('entity')
+        self.time = data.get('time', data.get('timestamp'))
+        self.is_write = data.get('is_write')
+        self.user_agent = data.get('user_agent') or get_user_agent(args.plugin)
+
+        self.type = data.get('type', data.get('entity_type'))
+        if self.type not in ['file', 'domain', 'app']:
+            self.type = 'file'
+
+        if not _clone:
+            exclude = self._excluded_by_pattern()
+            if exclude:
+                self.skip = u('Skipping because matches exclude pattern: {pattern}').format(
+                    pattern=u(exclude),
+                )
+                return
+            if self.type == 'file':
+                self.entity = format_file_path(self.entity)
+            if self.type == 'file' and not os.path.isfile(self.entity):
+                self.skip = u('File does not exist; ignoring this heartbeat.')
+                return
+
+            project, branch = get_project_info(configs, self, data)
+            self.project = project
+            self.branch = branch
+
+            stats = get_file_stats(self.entity,
+                                   entity_type=self.type,
+                                   lineno=data.get('lineno'),
+                                   cursorpos=data.get('cursorpos'),
+                                   plugin=args.plugin,
+                                   language=data.get('language'))
+        else:
+            self.project = data.get('project')
+            self.branch = data.get('branch')
+            stats = data
+
+        for key in ['language', 'dependencies', 'lines', 'lineno', 'cursorpos']:
+            if stats.get(key) is not None:
+                setattr(self, key, stats[key])
+
+    def update(self, attrs):
+        """Return a copy of the current Heartbeat with updated attributes."""
+
+        data = self.dict()
+        data.update(attrs)
+        heartbeat = Heartbeat(data, self.args, self.configs, _clone=True)
+        heartbeat.skip = self.skip
+        return heartbeat
+
+    def sanitize(self):
+        """Removes sensitive data including file names and dependencies.
+
+        Returns a Heartbeat.
+        """
+
+        if not self.args.hidefilenames:
+            return self
+
+        if self.entity is None:
+            return self
+
+        if self.type != 'file':
+            return self
+
+        for pattern in self.args.hidefilenames:
+            try:
+                compiled = re.compile(pattern, re.IGNORECASE)
+                if compiled.search(self.entity):
+
+                    sanitized = {}
+                    sensitive = ['dependencies', 'lines', 'lineno', 'cursorpos', 'branch']
+                    for key, val in self.items():
+                        if key in sensitive:
+                            sanitized[key] = None
+                        else:
+                            sanitized[key] = val
+
+                    extension = u(os.path.splitext(self.entity)[1])
+                    sanitized['entity'] = u('HIDDEN{0}').format(extension)
+
+                    return self.update(sanitized)
+
+            except re.error as ex:
+                log.warning(u('Regex error ({msg}) for include pattern: {pattern}').format(
+                    msg=u(ex),
+                    pattern=u(pattern),
+                ))
+
+        return self
+
+    def json(self):
+        return json.dumps(self.dict())
+
+    def dict(self):
+        return {
+            'time': self.time,
+            'entity': self.entity,
+            'type': self.type,
+            'is_write': self.is_write,
+            'project': self.project,
+            'branch': self.branch,
+            'language': self.language,
+            'dependencies': self.dependencies,
+            'lines': self.lines,
+            'lineno': self.lineno,
+            'cursorpos': self.cursorpos,
+            'user_agent': self.user_agent,
+        }
+
+    def items(self):
+        return self.dict().items()
+
+    def get_id(self):
+        return u('{h.time}-{h.type}-{h.project}-{h.branch}-{h.entity}-{h.is_write}').format(
+            h=self,
+        )
+
+    def _excluded_by_pattern(self):
+        return should_exclude(self.entity, self.args.include, self.args.exclude)
+
+    def __repr__(self):
+        return self.json()
+
+    def __bool__(self):
+        return not self.skip
+
+    def __nonzero__(self):
+        return self.__bool__()
+
+    def __getitem__(self, key):
+        return self.dict()[key]

wakatime/main.py

@@ -11,387 +11,67 @@
 
 from __future__ import print_function
 
-import base64
 import logging
 import os
-import re
 import sys
 import traceback
-import socket
 
 pwd = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, os.path.dirname(pwd))
 sys.path.insert(0, os.path.join(pwd, 'packages'))
 
 from .__about__ import __version__
-from .arguments import parseArguments
+from .api import send_heartbeats
-from .compat import u, is_py3
+from .arguments import parse_arguments
+from .compat import u, json
 from .constants import (
-    API_ERROR,
-    AUTH_ERROR,
     SUCCESS,
     UNKNOWN_ERROR,
-    MALFORMED_HEARTBEAT_ERROR,
 )
 from .logger import setup_logging
 
 log = logging.getLogger('WakaTime')
 
-try:
+from .heartbeat import Heartbeat
-    from .packages import requests
-except ImportError:
-    log.traceback(logging.ERROR)
-    print(traceback.format_exc())
-    log.error('Please upgrade Python to the latest version.')
-    print('Please upgrade Python to the latest version.')
-    sys.exit(UNKNOWN_ERROR)
-
 from .offlinequeue import Queue
-from .packages.requests.exceptions import RequestException
-from .project import get_project_info
-from .session_cache import SessionCache
-from .stats import get_file_stats
-from .utils import get_user_agent, should_exclude, format_file_path
-try:
-    from .packages import simplejson as json  # pragma: nocover
-except (ImportError, SyntaxError):  # pragma: nocover
-    import json
-from .packages import tzlocal
-
-
-def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None,
-                   entity=None, timestamp=None, is_write=None, plugin=None,
-                   offline=None, entity_type='file', hidefilenames=None,
-                   proxy=None, nosslverify=None, api_url=None, timeout=None,
-                   use_ntlm_proxy=False, **kwargs):
-    """Sends heartbeat as POST request to WakaTime api server.
-
-    Returns `SUCCESS` when heartbeat was sent, otherwise returns an
-    error code constant.
-    """
-
-    if not api_url:
-        api_url = 'https://api.wakatime.com/api/v1/heartbeats'
-    if not timeout:
-        timeout = 60
-    log.debug('Sending heartbeat to api at %s' % api_url)
-    data = {
-        'time': timestamp,
-        'entity': entity,
-        'type': entity_type,
-    }
-    if stats.get('lines'):
-        data['lines'] = stats['lines']
-    if stats.get('language'):
-        data['language'] = stats['language']
-    if stats.get('dependencies'):
-        data['dependencies'] = stats['dependencies']
-    if stats.get('lineno'):
-        data['lineno'] = stats['lineno']
-    if stats.get('cursorpos'):
-        data['cursorpos'] = stats['cursorpos']
-    if is_write:
-        data['is_write'] = is_write
-    if project:
-        data['project'] = project
-    if branch:
-        data['branch'] = branch
-
-    if hidefilenames and entity is not None and entity_type == 'file':
-        for pattern in hidefilenames:
-            try:
-                compiled = re.compile(pattern, re.IGNORECASE)
-                if compiled.search(entity):
-                    extension = u(os.path.splitext(data['entity'])[1])
-                    data['entity'] = u('HIDDEN{0}').format(extension)
-
-                    # also delete any sensitive info when hiding file names
-                    sensitive = ['dependencies', 'lines', 'lineno', 'cursorpos', 'branch']
-                    for sensitiveKey in sensitive:
-                        if sensitiveKey in data:
-                            del data[sensitiveKey]
-
-                    break
-            except re.error as ex:
-                log.warning(u('Regex error ({msg}) for include pattern: {pattern}').format(
-                    msg=u(ex),
-                    pattern=u(pattern),
-                ))
-
-    log.debug(data)
-
-    # setup api request
-    request_body = json.dumps(data)
-    api_key = u(base64.b64encode(str.encode(key) if is_py3 else key))
-    auth = u('Basic {api_key}').format(api_key=api_key)
-    headers = {
-        'User-Agent': get_user_agent(plugin),
-        'Content-Type': 'application/json',
-        'Accept': 'application/json',
-        'Authorization': auth,
-    }
-    if hostname:
-        headers['X-Machine-Name'] = u(hostname).encode('utf-8')
-
-    # add Olson timezone to request
-    try:
-        tz = tzlocal.get_localzone()
-    except:
-        tz = None
-    if tz:
-        headers['TimeZone'] = u(tz.zone).encode('utf-8')
-
-    session_cache = SessionCache()
-    session = session_cache.get()
-
-    should_try_ntlm = False
-    proxies = {}
-    if proxy:
-        if use_ntlm_proxy:
-            from .packages.requests_ntlm import HttpNtlmAuth
-            username = proxy.rsplit(':', 1)
-            password = ''
-            if len(username) == 2:
-                password = username[1]
-            username = username[0]
-            session.auth = HttpNtlmAuth(username, password, session)
-        else:
-            should_try_ntlm = '\\' in proxy
-            proxies['https'] = proxy
-
-    # send request to api
-    response = None
-    try:
-        response = session.post(api_url, data=request_body, headers=headers,
-                                proxies=proxies, timeout=timeout,
-                                verify=not nosslverify)
-    except RequestException:
-        if should_try_ntlm:
-            return send_heartbeat(
-                project=project,
-                entity=entity,
-                timestamp=timestamp,
-                branch=branch,
-                hostname=hostname,
-                stats=stats,
-                key=key,
-                is_write=is_write,
-                plugin=plugin,
-                offline=offline,
-                hidefilenames=hidefilenames,
-                entity_type=entity_type,
-                proxy=proxy,
-                api_url=api_url,
-                timeout=timeout,
-                use_ntlm_proxy=True,
-            )
-        else:
-            exception_data = {
-                sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
-            }
-            if log.isEnabledFor(logging.DEBUG):
-                exception_data['traceback'] = traceback.format_exc()
-            if offline:
-                queue = Queue()
-                queue.push(data, json.dumps(stats), plugin)
-                if log.isEnabledFor(logging.DEBUG):
-                    log.warn(exception_data)
-            else:
-                log.error(exception_data)
-
-    except:  # delete cached session when requests raises unknown exception
-        if should_try_ntlm:
-            return send_heartbeat(
-                project=project,
-                entity=entity,
-                timestamp=timestamp,
-                branch=branch,
-                hostname=hostname,
-                stats=stats,
-                key=key,
-                is_write=is_write,
-                plugin=plugin,
-                offline=offline,
-                hidefilenames=hidefilenames,
-                entity_type=entity_type,
-                proxy=proxy,
-                api_url=api_url,
-                timeout=timeout,
-                use_ntlm_proxy=True,
-            )
-        else:
-            exception_data = {
-                sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
-                'traceback': traceback.format_exc(),
-            }
-            if offline:
-                queue = Queue()
-                queue.push(data, json.dumps(stats), plugin)
-                log.warn(exception_data)
-
-    else:
-        code = response.status_code if response is not None else None
-        content = response.text if response is not None else None
-        if code == requests.codes.created or code == requests.codes.accepted:
-            log.debug({
-                'response_code': code,
-            })
-            session_cache.save(session)
-            return SUCCESS
-        if should_try_ntlm:
-            return send_heartbeat(
-                project=project,
-                entity=entity,
-                timestamp=timestamp,
-                branch=branch,
-                hostname=hostname,
-                stats=stats,
-                key=key,
-                is_write=is_write,
-                plugin=plugin,
-                offline=offline,
-                hidefilenames=hidefilenames,
-                entity_type=entity_type,
-                proxy=proxy,
-                api_url=api_url,
-                timeout=timeout,
-                use_ntlm_proxy=True,
-            )
-        else:
-            if offline:
-                if code != 400:
-                    queue = Queue()
-                    queue.push(data, json.dumps(stats), plugin)
-                    if code == 401:
-                        log.error({
-                            'response_code': code,
-                            'response_content': content,
-                        })
-                        session_cache.delete()
-                        return AUTH_ERROR
-                    elif log.isEnabledFor(logging.DEBUG):
-                        log.warn({
-                            'response_code': code,
-                            'response_content': content,
-                        })
-                else:
-                    log.error({
-                        'response_code': code,
-                        'response_content': content,
-                    })
-            else:
-                log.error({
-                    'response_code': code,
-                    'response_content': content,
-                })
-    session_cache.delete()
-    return API_ERROR
-
-
-def sync_offline_heartbeats(args, hostname):
-    """Sends all heartbeats which were cached in the offline Queue."""
-
-    queue = Queue()
-    while True:
-        heartbeat = queue.pop()
-        if heartbeat is None:
-            break
-        status = send_heartbeat(
-            project=heartbeat['project'],
-            entity=heartbeat['entity'],
-            timestamp=heartbeat['time'],
-            branch=heartbeat['branch'],
-            hostname=hostname,
-            stats=json.loads(heartbeat['stats']),
-            key=args.key,
-            is_write=heartbeat['is_write'],
-            plugin=heartbeat['plugin'],
-            offline=args.offline,
-            hidefilenames=args.hidefilenames,
-            entity_type=heartbeat['type'],
-            proxy=args.proxy,
-            api_url=args.api_url,
-            timeout=args.timeout,
-        )
-        if status != SUCCESS:
-            if status == AUTH_ERROR:
-                return AUTH_ERROR
-            break
-    return SUCCESS
-
-
-def process_heartbeat(args, configs, hostname, heartbeat):
-    exclude = should_exclude(heartbeat['entity'], args.include, args.exclude)
-    if exclude is not False:
-        log.debug(u('Skipping because matches exclude pattern: {pattern}').format(
-            pattern=u(exclude),
-        ))
-        return SUCCESS
-
-    if heartbeat.get('entity_type') not in ['file', 'domain', 'app']:
-        heartbeat['entity_type'] = 'file'
-
-    if heartbeat['entity_type'] == 'file':
-        heartbeat['entity'] = format_file_path(heartbeat['entity'])
-
-    if heartbeat['entity_type'] != 'file' or os.path.isfile(heartbeat['entity']):
-
-        stats = get_file_stats(heartbeat['entity'],
-                               entity_type=heartbeat['entity_type'],
-                               lineno=heartbeat.get('lineno'),
-                               cursorpos=heartbeat.get('cursorpos'),
-                               plugin=args.plugin,
-                               language=heartbeat.get('language'))
-
-        project = heartbeat.get('project') or heartbeat.get('alternate_project')
-        branch = None
-        if heartbeat['entity_type'] == 'file':
-            project, branch = get_project_info(configs, heartbeat)
-
-        heartbeat['project'] = project
-        heartbeat['branch'] = branch
-        heartbeat['stats'] = stats
-        heartbeat['hostname'] = hostname
-        heartbeat['timeout'] = args.timeout
-        heartbeat['key'] = args.key
-        heartbeat['plugin'] = args.plugin
-        heartbeat['offline'] = args.offline
-        heartbeat['hidefilenames'] = args.hidefilenames
-        heartbeat['proxy'] = args.proxy
-        heartbeat['nosslverify'] = args.nosslverify
-        heartbeat['api_url'] = args.api_url
-
-        return send_heartbeat(**heartbeat)
-
-    else:
-        log.debug('File does not exist; ignoring this heartbeat.')
-        return SUCCESS
 
 
 def execute(argv=None):
     if argv:
         sys.argv = ['wakatime'] + argv
 
-    args, configs = parseArguments()
+    args, configs = parse_arguments()
 
     setup_logging(args, __version__)
 
     try:
+        heartbeats = []
 
-        hostname = args.hostname or socket.gethostname()
+        hb = Heartbeat(vars(args), args, configs)
-
+        if hb:
-        heartbeat = vars(args)
+            heartbeats.append(hb)
-        retval = process_heartbeat(args, configs, hostname, heartbeat)
+        else:
+            log.debug(hb.skip)
 
         if args.extra_heartbeats:
             try:
-                for heartbeat in json.loads(sys.stdin.readline()):
+                for extra_data in json.loads(sys.stdin.readline()):
-                    retval = process_heartbeat(args, configs, hostname, heartbeat)
+                    hb = Heartbeat(extra_data, args, configs)
-            except json.JSONDecodeError:
+                    if hb:
-                retval = MALFORMED_HEARTBEAT_ERROR
+                        heartbeats.append(hb)
+                    else:
+                        log.debug(hb.skip)
+            except json.JSONDecodeError as ex:
+                log.warning(u('Malformed extra heartbeats json: {msg}').format(
+                    msg=u(ex),
+                ))
 
+        retval = send_heartbeats(heartbeats, args, configs)
         if retval == SUCCESS:
-            retval = sync_offline_heartbeats(args, hostname)
+            queue = Queue(args, configs)
+            offline_heartbeats = queue.pop_many()
+            if len(offline_heartbeats) > 0:
+                retval = send_heartbeats(offline_heartbeats, args, configs)
 
         return retval
 

wakatime/offlinequeue.py

@@ -14,77 +14,68 @@ import logging
 import os
 from time import sleep
 
+from .compat import json
+from .heartbeat import Heartbeat
+
+
 try:
     import sqlite3
     HAS_SQL = True
 except ImportError:  # pragma: nocover
     HAS_SQL = False
 
-from .compat import u
-
 
 log = logging.getLogger('WakaTime')
 
 
 class Queue(object):
     db_file = '.wakatime.db'
-    table_name = 'heartbeat_1'
+    table_name = 'heartbeat_2'
 
-    def get_db_file(self):
+    args = None
-        home = '~'
+    configs = None
-        if os.environ.get('WAKATIME_HOME'):
+
-            home = os.environ.get('WAKATIME_HOME')
+    def __init__(self, args, configs):
-        return os.path.join(os.path.expanduser(home), '.wakatime.db')
+        self.args = args
+        self.configs = configs
 
     def connect(self):
-        conn = sqlite3.connect(self.get_db_file(), isolation_level=None)
+        conn = sqlite3.connect(self._get_db_file(), isolation_level=None)
         c = conn.cursor()
         c.execute('''CREATE TABLE IF NOT EXISTS {0} (
-            entity text,
+            id text,
-            type text,
+            heartbeat text)
-            time real,
-            project text,
-            branch text,
-            is_write integer,
-            stats text,
-            misc text,
-            plugin text)
         '''.format(self.table_name))
         return (conn, c)
 
-    def push(self, data, stats, plugin, misc=None):
+    def push(self, heartbeat):
-        if not HAS_SQL:  # pragma: nocover
+        if not HAS_SQL:
             return
         try:
             conn, c = self.connect()
-            heartbeat = {
+            data = {
-                'entity': u(data.get('entity')),
+                'id': heartbeat.get_id(),
-                'type': u(data.get('type')),
+                'heartbeat': heartbeat.json(),
-                'time': data.get('time'),
-                'project': u(data.get('project')),
-                'branch': u(data.get('branch')),
-                'is_write': 1 if data.get('is_write') else 0,
-                'stats': u(stats),
-                'misc': u(misc),
-                'plugin': u(plugin),
             }
-            c.execute('INSERT INTO {0} VALUES (:entity,:type,:time,:project,:branch,:is_write,:stats,:misc,:plugin)'.format(self.table_name), heartbeat)
+            c.execute('INSERT INTO {0} VALUES (:id,:heartbeat)'.format(self.table_name), data)
             conn.commit()
             conn.close()
         except sqlite3.Error:
             log.traceback()
 
     def pop(self):
-        if not HAS_SQL:  # pragma: nocover
+        if not HAS_SQL:
             return None
         tries = 3
         wait = 0.1
-        heartbeat = None
         try:
             conn, c = self.connect()
         except sqlite3.Error:
             log.traceback(logging.DEBUG)
             return None
+
+        heartbeat = None
+
         loop = True
         while loop and tries > -1:
             try:
@@ -92,40 +83,43 @@ class Queue(object):
                 c.execute('SELECT * FROM {0} LIMIT 1'.format(self.table_name))
                 row = c.fetchone()
                 if row is not None:
-                    values = []
+                    id = row[0]
-                    clauses = []
+                    heartbeat = Heartbeat(json.loads(row[1]), self.args, self.configs, _clone=True)
-                    index = 0
+                    c.execute('DELETE FROM {0} WHERE id=?'.format(self.table_name), [id])
-                    for row_name in ['entity', 'type', 'time', 'project', 'branch', 'is_write']:
-                        if row[index] is not None:
-                            clauses.append('{0}=?'.format(row_name))
-                            values.append(row[index])
-                        else:  # pragma: nocover
-                            clauses.append('{0} IS NULL'.format(row_name))
-                        index += 1
-                    if len(values) > 0:
-                        c.execute('DELETE FROM {0} WHERE {1}'.format(self.table_name, ' AND '.join(clauses)), values)
-                    else:  # pragma: nocover
-                        c.execute('DELETE FROM {0} WHERE {1}'.format(self.table_name, ' AND '.join(clauses)))
                 conn.commit()
-                if row is not None:
-                    heartbeat = {
-                        'entity': row[0],
-                        'type': row[1],
-                        'time': row[2],
-                        'project': row[3],
-                        'branch': row[4],
-                        'is_write': True if row[5] is 1 else False,
-                        'stats': row[6],
-                        'misc': row[7],
-                        'plugin': row[8],
-                    }
                 loop = False
-            except sqlite3.Error:  # pragma: nocover
+            except sqlite3.Error:
                 log.traceback(logging.DEBUG)
                 sleep(wait)
                 tries -= 1
         try:
             conn.close()
-        except sqlite3.Error:  # pragma: nocover
+        except sqlite3.Error:
             log.traceback(logging.DEBUG)
         return heartbeat
+
+    def push_many(self, heartbeats):
+        for heartbeat in heartbeats:
+            self.push(heartbeat)
+
+    def pop_many(self, limit=None):
+        if limit is None:
+            limit = 100
+
+        heartbeats = []
+
+        count = 0
+        while limit == 0 or count < limit:
+            heartbeat = self.pop()
+            if not heartbeat:
+                break
+            heartbeats.append(heartbeat)
+            count += 1
+
+        return heartbeats
+
+    def _get_db_file(self):
+        home = '~'
+        if os.environ.get('WAKATIME_HOME'):
+            home = os.environ.get('WAKATIME_HOME')
+        return os.path.join(os.path.expanduser(home), '.wakatime.db')

wakatime/project.py

@@ -33,7 +33,7 @@ REV_CONTROL_PLUGINS = [
 ]
 
 
-def get_project_info(configs, heartbeat):
+def get_project_info(configs, heartbeat, data):
     """Find the current project and branch.
 
     First looks for a .wakatime-project file. Second, uses the --project arg.
@@ -43,21 +43,27 @@ def get_project_info(configs, heartbeat):
     Returns a project, branch tuple.
     """
 
-    project_name, branch_name = None, None
+    project_name, branch_name = heartbeat.project, heartbeat.branch
 
-    for plugin_cls in CONFIG_PLUGINS:
+    if heartbeat.type != 'file':
+        project_name = project_name or heartbeat.args.project or heartbeat.args.alternate_project
+        return project_name, branch_name
 
-        plugin_name = plugin_cls.__name__.lower()
+    if project_name is None or branch_name is None:
-        plugin_configs = get_configs_for_plugin(plugin_name, configs)
 
-        project = plugin_cls(heartbeat['entity'], configs=plugin_configs)
+        for plugin_cls in CONFIG_PLUGINS:
-        if project.process():
+
-            project_name = project_name or project.name()
+            plugin_name = plugin_cls.__name__.lower()
-            branch_name = project.branch()
+            plugin_configs = get_configs_for_plugin(plugin_name, configs)
-            break
+
+            project = plugin_cls(heartbeat.entity, configs=plugin_configs)
+            if project.process():
+                project_name = project_name or project.name()
+                branch_name = project.branch()
+                break
 
     if project_name is None:
-        project_name = heartbeat.get('project')
+        project_name = data.get('project') or heartbeat.args.project
 
     if project_name is None or branch_name is None:
 
@@ -66,14 +72,14 @@ def get_project_info(configs, heartbeat):
             plugin_name = plugin_cls.__name__.lower()
             plugin_configs = get_configs_for_plugin(plugin_name, configs)
 
-            project = plugin_cls(heartbeat['entity'], configs=plugin_configs)
+            project = plugin_cls(heartbeat.entity, configs=plugin_configs)
             if project.process():
                 project_name = project_name or project.name()
                 branch_name = branch_name or project.branch()
                 break
 
     if project_name is None:
-        project_name = heartbeat.get('alternate_project')
+        project_name = data.get('alternate_project') or heartbeat.args.alternate_project
 
     return project_name, branch_name
 

wakatime/session_cache.py

@@ -33,14 +33,8 @@ class SessionCache(object):
     db_file = '.wakatime.db'
     table_name = 'session'
 
-    def get_db_file(self):
-        home = '~'
-        if os.environ.get('WAKATIME_HOME'):
-            home = os.environ.get('WAKATIME_HOME')
-        return os.path.join(os.path.expanduser(home), '.wakatime.db')
-
     def connect(self):
-        conn = sqlite3.connect(self.get_db_file(), isolation_level=None)
+        conn = sqlite3.connect(self._get_db_file(), isolation_level=None)
         c = conn.cursor()
         c.execute('''CREATE TABLE IF NOT EXISTS {0} (
             value BLOB)
@@ -110,3 +104,9 @@ class SessionCache(object):
             conn.close()
         except:
             log.traceback(logging.DEBUG)
+
+    def _get_db_file(self):
+        home = '~'
+        if os.environ.get('WAKATIME_HOME'):
+            home = os.environ.get('WAKATIME_HOME')
+        return os.path.join(os.path.expanduser(home), '.wakatime.db')

wakatime/utils.py

@@ -14,6 +14,7 @@ import platform
 import logging
 import os
 import re
+import socket
 import sys
 
 from .__about__ import __version__
@@ -48,7 +49,7 @@ def should_exclude(entity, include, exclude):
     return False
 
 
-def get_user_agent(plugin):
+def get_user_agent(plugin=None):
     ver = sys.version_info
     python_version = '%d.%d.%d.%s.%d' % (ver[0], ver[1], ver[2], ver[3], ver[4])
     user_agent = u('wakatime/{ver} ({platform}) Python{py_ver}').format(
@@ -77,3 +78,7 @@ def format_file_path(filepath):
     except:  # pragma: nocover
         pass
     return filepath
+
+
+def get_hostname(args):
+    return args.hostname or socket.gethostname()