support unicode heartbeats in offline cache

tests/test_main.py

@@ -484,6 +484,47 @@ class MainTestCase(utils.TestCase):
             self.assertOfflineHeartbeatsSynced()
             self.assertSessionCacheSaved()
 
+    @log_capture()
+    def test_nonascii_filename_saved_when_offline(self, logs):
+        logging.disable(logging.NOTSET)
+
+        response = Response()
+        response.status_code = 500
+        self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+
+        with utils.TemporaryDirectory() as tempdir:
+            filename = list(filter(lambda x: x.endswith('.txt'), os.listdir(u('tests/samples/codefiles/unicode'))))[0]
+            entity = os.path.join('tests/samples/codefiles/unicode', filename)
+            shutil.copy(entity, os.path.join(tempdir, filename))
+            entity = os.path.realpath(os.path.join(tempdir, filename))
+            now = u(int(time.time()))
+            config = 'tests/samples/configs/good_config.cfg'
+            key = str(uuid.uuid4())
+            heartbeat = {
+                'language': 'Text only',
+                'lines': 0,
+                'entity': os.path.realpath(entity),
+                'project': None,
+                'time': float(now),
+                'type': 'file',
+                'is_write': False,
+                'user_agent': ANY,
+                'dependencies': [],
+            }
+
+            args = ['--file', entity, '--key', key, '--config', config, '--time', now]
+
+            retval = execute(args)
+            self.assertEquals(retval, API_ERROR)
+            self.assertNothingPrinted()
+            self.assertNothingLogged(logs)
+
+            self.assertHeartbeatSent(heartbeat)
+
+            self.assertHeartbeatSavedOffline()
+            self.assertOfflineHeartbeatsNotSynced()
+            self.assertSessionCacheDeleted()
+
     @log_capture()
     def test_unhandled_exception(self, logs):
         logging.disable(logging.NOTSET)

tests/test_offlinequeue.py

@@ -555,3 +555,94 @@ class OfflineQueueTestCase(utils.TestCase):
                 self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
                 self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
                 self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
+
+    """@log_capture()
+    def non_ascii_heartbeat_saved(self, logs):
+        logging.disable(logging.NOTSET)
+
+        with utils.NamedTemporaryFile() as fh:
+            with utils.mock.patch('wakatime.offlinequeue.Queue._get_db_file') as mock_db_file:
+                mock_db_file.return_value = fh.name
+
+                response = Response()
+                response.status_code = 500
+                self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+
+                now = u(int(time.time()))
+                entity = 'tests/samples/codefiles/twolinefile.txt'
+                config = 'tests/samples/configs/good_config.cfg'
+
+                return 'tz汉语' if is_py3 else 'tz\xe6\xb1\x89\xe8\xaf\xad'
+
+                args = ['--file', entity, '--config', config, '--time', now]
+                execute(args)
+
+                queue = Queue(None, None)
+                saved_heartbeat = queue.pop()
+                self.assertEquals(os.path.realpath(entity), saved_heartbeat['entity'])
+
+                self.assertNothingPrinted()
+                self.assertNothingLogged()
+
+                self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with()
+                self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with()
+                self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called()
+    """
+
+    @log_capture()
+    def test_nonascii_heartbeat_saved(self, logs):
+        logging.disable(logging.NOTSET)
+
+        with utils.NamedTemporaryFile() as fh:
+            with utils.mock.patch('wakatime.offlinequeue.Queue._get_db_file') as mock_db_file:
+                mock_db_file.return_value = fh.name
+
+                response = Response()
+                response.status_code = 500
+                self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response
+
+                with utils.TemporaryDirectory() as tempdir:
+                    filename = list(filter(lambda x: x.endswith('.txt'), os.listdir(u('tests/samples/codefiles/unicode'))))[0]
+                    entity = os.path.join('tests/samples/codefiles/unicode', filename)
+                    shutil.copy(entity, os.path.join(tempdir, filename))
+                    entity = os.path.realpath(os.path.join(tempdir, filename))
+                    now = u(int(time.time()))
+                    config = 'tests/samples/configs/good_config.cfg'
+                    key = str(uuid.uuid4())
+                    language = 'lang汉语' if self.isPy35OrNewer else 'lang\xe6\xb1\x89\xe8\xaf\xad'
+                    project = 'proj汉语' if self.isPy35OrNewer else 'proj\xe6\xb1\x89\xe8\xaf\xad'
+                    branch = 'branch汉语' if self.isPy35OrNewer else 'branch\xe6\xb1\x89\xe8\xaf\xad'
+                    heartbeat = {
+                        'language': u(language),
+                        'lines': 0,
+                        'entity': os.path.realpath(entity),
+                        'project': u(project),
+                        'branch': u(branch),
+                        'time': float(now),
+                        'type': 'file',
+                        'is_write': False,
+                        'user_agent': ANY,
+                        'dependencies': [],
+                    }
+
+                    args = ['--file', entity, '--key', key, '--config', config, '--time', now]
+
+                    with utils.mock.patch('wakatime.stats.standardize_language') as mock_language:
+                        mock_language.return_value = (language, None)
+
+                        with utils.mock.patch('wakatime.heartbeat.get_project_info') as mock_project:
+                            mock_project.return_value = (project, branch)
+
+                            execute(args)
+
+                    self.assertNothingPrinted()
+                    self.assertNothingLogged(logs)
+
+                    self.assertHeartbeatSent(heartbeat)
+
+                    queue = Queue(None, None)
+                    saved_heartbeat = queue.pop()
+                    self.assertEquals(os.path.realpath(entity), saved_heartbeat['entity'])
+                    self.assertEquals(u(language), saved_heartbeat['language'])
+                    self.assertEquals(u(project), saved_heartbeat['project'])
+                    self.assertEquals(u(branch), saved_heartbeat['branch'])

tests/utils.py

@@ -91,7 +91,7 @@ class TestCase(unittest.TestCase):
                 if isinstance(heartbeat.get(key), list):
                     self.assertListsEqual(heartbeat.get(key), body[0].get(key), u('Expected heartbeat to be sent with {0}={1}, instead {0}={2}').format(u(key), u(heartbeat.get(key)), u(body[0].get(key))))
                 else:
-                    self.assertEquals(heartbeat.get(key), body[0].get(key), u('Expected heartbeat to be sent with {1} {0}={2}, instead {3} {0}={4}').format(u(key), type(heartbeat.get(key)), u(heartbeat.get(key)), type(body[0].get(key)), u(body[0].get(key))))
+                    self.assertEquals(heartbeat.get(key), body[0].get(key), u('Expected heartbeat to be sent with {1} {0}={2}, instead {3} {0}={4}').format(u(key), type(heartbeat.get(key)).__name__, u(heartbeat.get(key)), type(body[0].get(key)).__name__, u(body[0].get(key))))
 
         if extra_heartbeats:
             for i in range(len(extra_heartbeats)):
@@ -100,7 +100,7 @@ class TestCase(unittest.TestCase):
                     if isinstance(extra_heartbeats[i].get(key), list):
                         self.assertListsEqual(extra_heartbeats[i].get(key), body[i + 1].get(key), u('Expected extra heartbeat {3} to be sent with {0}={1}, instead {0}={2}').format(u(key), u(extra_heartbeats[i].get(key)), u(body[i + 1].get(key)), i))
                     else:
-                        self.assertEquals(extra_heartbeats[i].get(key), body[i + 1].get(key), u('Expected extra heartbeat {5} to be sent with {1} {0}={2}, instead {3} {0}={4}').format(u(key), type(extra_heartbeats[i].get(key)), u(extra_heartbeats[i].get(key)), type(body[i + 1].get(key)), u(body[i + 1].get(key)), i))
+                        self.assertEquals(extra_heartbeats[i].get(key), body[i + 1].get(key), u('Expected extra heartbeat {5} to be sent with {1} {0}={2}, instead {3} {0}={4}').format(u(key), type(extra_heartbeats[i].get(key)).__name__, u(extra_heartbeats[i].get(key)), type(body[i + 1].get(key)).__name__, u(body[i + 1].get(key)), i))
 
     def assertSessionCacheUntouched(self):
         self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called()

wakatime/heartbeat.py

@@ -141,27 +141,42 @@ class Heartbeat(object):
     def dict(self):
         return {
             'time': self.time,
-            'entity': self.entity,
+            'entity': self._unicode(self.entity),
             'type': self.type,
             'is_write': self.is_write,
-            'project': self.project,
+            'project': self._unicode(self.project),
-            'branch': self.branch,
+            'branch': self._unicode(self.branch),
-            'language': self.language,
+            'language': self._unicode(self.language),
-            'dependencies': self.dependencies,
+            'dependencies': self._unicode_list(self.dependencies),
             'lines': self.lines,
             'lineno': self.lineno,
             'cursorpos': self.cursorpos,
-            'user_agent': self.user_agent,
+            'user_agent': self._unicode(self.user_agent),
         }
 
     def items(self):
         return self.dict().items()
 
     def get_id(self):
-        return u('{h.time}-{h.type}-{h.project}-{h.branch}-{h.entity}-{h.is_write}').format(
+        return u('{time}-{type}-{project}-{branch}-{entity}-{is_write}').format(
-            h=self,
+            time=self.time,
+            type=self.type,
+            project=self._unicode(self.project),
+            branch=self._unicode(self.branch),
+            entity=self._unicode(self.entity),
+            is_write=self.is_write,
         )
 
+    def _unicode(self, value):
+        if value is None:
+            return None
+        return u(value)
+
+    def _unicode_list(self, values):
+        if values is None:
+            return None
+        return [self._unicode(value) for value in values]
+
     def _excluded_by_pattern(self):
         return should_exclude(self.entity, self.args.include, self.args.exclude)