From 7b09f8f0c6b4a9e901a1641a8397f9341433b54d Mon Sep 17 00:00:00 2001 From: Alan Hamlett Date: Wed, 22 Nov 2017 12:58:29 -0800 Subject: [PATCH] support unicode heartbeats in offline cache --- tests/test_main.py | 41 +++++++++++++++++ tests/test_offlinequeue.py | 91 ++++++++++++++++++++++++++++++++++++++ tests/utils.py | 4 +- wakatime/heartbeat.py | 31 +++++++++---- 4 files changed, 157 insertions(+), 10 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 9bd0af7..d3a3cfd 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -484,6 +484,47 @@ class MainTestCase(utils.TestCase): self.assertOfflineHeartbeatsSynced() self.assertSessionCacheSaved() + @log_capture() + def test_nonascii_filename_saved_when_offline(self, logs): + logging.disable(logging.NOTSET) + + response = Response() + response.status_code = 500 + self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response + + with utils.TemporaryDirectory() as tempdir: + filename = list(filter(lambda x: x.endswith('.txt'), os.listdir(u('tests/samples/codefiles/unicode'))))[0] + entity = os.path.join('tests/samples/codefiles/unicode', filename) + shutil.copy(entity, os.path.join(tempdir, filename)) + entity = os.path.realpath(os.path.join(tempdir, filename)) + now = u(int(time.time())) + config = 'tests/samples/configs/good_config.cfg' + key = str(uuid.uuid4()) + heartbeat = { + 'language': 'Text only', + 'lines': 0, + 'entity': os.path.realpath(entity), + 'project': None, + 'time': float(now), + 'type': 'file', + 'is_write': False, + 'user_agent': ANY, + 'dependencies': [], + } + + args = ['--file', entity, '--key', key, '--config', config, '--time', now] + + retval = execute(args) + self.assertEquals(retval, API_ERROR) + self.assertNothingPrinted() + self.assertNothingLogged(logs) + + self.assertHeartbeatSent(heartbeat) + + self.assertHeartbeatSavedOffline() + self.assertOfflineHeartbeatsNotSynced() + self.assertSessionCacheDeleted() + @log_capture() def test_unhandled_exception(self, logs): logging.disable(logging.NOTSET) diff --git a/tests/test_offlinequeue.py b/tests/test_offlinequeue.py index 56b0da0..5afc423 100644 --- a/tests/test_offlinequeue.py +++ b/tests/test_offlinequeue.py @@ -555,3 +555,94 @@ class OfflineQueueTestCase(utils.TestCase): self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with() self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with() self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called() + + """@log_capture() + def non_ascii_heartbeat_saved(self, logs): + logging.disable(logging.NOTSET) + + with utils.NamedTemporaryFile() as fh: + with utils.mock.patch('wakatime.offlinequeue.Queue._get_db_file') as mock_db_file: + mock_db_file.return_value = fh.name + + response = Response() + response.status_code = 500 + self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response + + now = u(int(time.time())) + entity = 'tests/samples/codefiles/twolinefile.txt' + config = 'tests/samples/configs/good_config.cfg' + + return 'tz汉语' if is_py3 else 'tz\xe6\xb1\x89\xe8\xaf\xad' + + args = ['--file', entity, '--config', config, '--time', now] + execute(args) + + queue = Queue(None, None) + saved_heartbeat = queue.pop() + self.assertEquals(os.path.realpath(entity), saved_heartbeat['entity']) + + self.assertNothingPrinted() + self.assertNothingLogged() + + self.patched['wakatime.session_cache.SessionCache.get'].assert_called_once_with() + self.patched['wakatime.session_cache.SessionCache.delete'].assert_called_once_with() + self.patched['wakatime.session_cache.SessionCache.save'].assert_not_called() + """ + + @log_capture() + def test_nonascii_heartbeat_saved(self, logs): + logging.disable(logging.NOTSET) + + with utils.NamedTemporaryFile() as fh: + with utils.mock.patch('wakatime.offlinequeue.Queue._get_db_file') as mock_db_file: + mock_db_file.return_value = fh.name + + response = Response() + response.status_code = 500 + self.patched['wakatime.packages.requests.adapters.HTTPAdapter.send'].return_value = response + + with utils.TemporaryDirectory() as tempdir: + filename = list(filter(lambda x: x.endswith('.txt'), os.listdir(u('tests/samples/codefiles/unicode'))))[0] + entity = os.path.join('tests/samples/codefiles/unicode', filename) + shutil.copy(entity, os.path.join(tempdir, filename)) + entity = os.path.realpath(os.path.join(tempdir, filename)) + now = u(int(time.time())) + config = 'tests/samples/configs/good_config.cfg' + key = str(uuid.uuid4()) + language = 'lang汉语' if self.isPy35OrNewer else 'lang\xe6\xb1\x89\xe8\xaf\xad' + project = 'proj汉语' if self.isPy35OrNewer else 'proj\xe6\xb1\x89\xe8\xaf\xad' + branch = 'branch汉语' if self.isPy35OrNewer else 'branch\xe6\xb1\x89\xe8\xaf\xad' + heartbeat = { + 'language': u(language), + 'lines': 0, + 'entity': os.path.realpath(entity), + 'project': u(project), + 'branch': u(branch), + 'time': float(now), + 'type': 'file', + 'is_write': False, + 'user_agent': ANY, + 'dependencies': [], + } + + args = ['--file', entity, '--key', key, '--config', config, '--time', now] + + with utils.mock.patch('wakatime.stats.standardize_language') as mock_language: + mock_language.return_value = (language, None) + + with utils.mock.patch('wakatime.heartbeat.get_project_info') as mock_project: + mock_project.return_value = (project, branch) + + execute(args) + + self.assertNothingPrinted() + self.assertNothingLogged(logs) + + self.assertHeartbeatSent(heartbeat) + + queue = Queue(None, None) + saved_heartbeat = queue.pop() + self.assertEquals(os.path.realpath(entity), saved_heartbeat['entity']) + self.assertEquals(u(language), saved_heartbeat['language']) + self.assertEquals(u(project), saved_heartbeat['project']) + self.assertEquals(u(branch), saved_heartbeat['branch']) diff --git a/tests/utils.py b/tests/utils.py index 0e4d0b5..7034262 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -91,7 +91,7 @@ class TestCase(unittest.TestCase): if isinstance(heartbeat.get(key), list): self.assertListsEqual(heartbeat.get(key), body[0].get(key), u('Expected heartbeat to be sent with {0}={1}, instead {0}={2}').format(u(key), u(heartbeat.get(key)), u(body[0].get(key)))) else: - self.assertEquals(heartbeat.get(key), body[0].get(key), u('Expected heartbeat to be sent with {1} {0}={2}, instead {3} {0}={4}').format(u(key), type(heartbeat.get(key)), u(heartbeat.get(key)), type(body[0].get(key)), u(body[0].get(key)))) + self.assertEquals(heartbeat.get(key), body[0].get(key), u('Expected heartbeat to be sent with {1} {0}={2}, instead {3} {0}={4}').format(u(key), type(heartbeat.get(key)).__name__, u(heartbeat.get(key)), type(body[0].get(key)).__name__, u(body[0].get(key)))) if extra_heartbeats: for i in range(len(extra_heartbeats)): @@ -100,7 +100,7 @@ class TestCase(unittest.TestCase): if isinstance(extra_heartbeats[i].get(key), list): self.assertListsEqual(extra_heartbeats[i].get(key), body[i + 1].get(key), u('Expected extra heartbeat {3} to be sent with {0}={1}, instead {0}={2}').format(u(key), u(extra_heartbeats[i].get(key)), u(body[i + 1].get(key)), i)) else: - self.assertEquals(extra_heartbeats[i].get(key), body[i + 1].get(key), u('Expected extra heartbeat {5} to be sent with {1} {0}={2}, instead {3} {0}={4}').format(u(key), type(extra_heartbeats[i].get(key)), u(extra_heartbeats[i].get(key)), type(body[i + 1].get(key)), u(body[i + 1].get(key)), i)) + self.assertEquals(extra_heartbeats[i].get(key), body[i + 1].get(key), u('Expected extra heartbeat {5} to be sent with {1} {0}={2}, instead {3} {0}={4}').format(u(key), type(extra_heartbeats[i].get(key)).__name__, u(extra_heartbeats[i].get(key)), type(body[i + 1].get(key)).__name__, u(body[i + 1].get(key)), i)) def assertSessionCacheUntouched(self): self.patched['wakatime.session_cache.SessionCache.delete'].assert_not_called() diff --git a/wakatime/heartbeat.py b/wakatime/heartbeat.py index 31c5044..c60d188 100644 --- a/wakatime/heartbeat.py +++ b/wakatime/heartbeat.py @@ -141,27 +141,42 @@ class Heartbeat(object): def dict(self): return { 'time': self.time, - 'entity': self.entity, + 'entity': self._unicode(self.entity), 'type': self.type, 'is_write': self.is_write, - 'project': self.project, - 'branch': self.branch, - 'language': self.language, - 'dependencies': self.dependencies, + 'project': self._unicode(self.project), + 'branch': self._unicode(self.branch), + 'language': self._unicode(self.language), + 'dependencies': self._unicode_list(self.dependencies), 'lines': self.lines, 'lineno': self.lineno, 'cursorpos': self.cursorpos, - 'user_agent': self.user_agent, + 'user_agent': self._unicode(self.user_agent), } def items(self): return self.dict().items() def get_id(self): - return u('{h.time}-{h.type}-{h.project}-{h.branch}-{h.entity}-{h.is_write}').format( - h=self, + return u('{time}-{type}-{project}-{branch}-{entity}-{is_write}').format( + time=self.time, + type=self.type, + project=self._unicode(self.project), + branch=self._unicode(self.branch), + entity=self._unicode(self.entity), + is_write=self.is_write, ) + def _unicode(self, value): + if value is None: + return None + return u(value) + + def _unicode_list(self, values): + if values is None: + return None + return [self._unicode(value) for value in values] + def _excluded_by_pattern(self): return should_exclude(self.entity, self.args.include, self.args.exclude)