lavatech/rana-cli
2
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

try opening files with filesystem encoding when opening with utf-8 causes UnicodeDecodeError

Browse source
This commit is contained in:
Alan Hamlett 2015-08-23 18:49:34 -07:00
parent 31a353be7a
commit 43c461a7cc
6 changed files with 50 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

13
wakatime/languages/__init__.py
View file

@ -10,6 +10,7 @@
"""
import logging
import sys
import traceback
from ..compat import u, open, import_module
@ -53,8 +54,16 @@ class TokenParser(object):
def _extract_tokens(self):
if self.lexer:
with open(self.source_file, 'r', encoding='utf-8') as fh:
return self.lexer.get_tokens_unprocessed(fh.read(512000))
try:
with open(self.source_file, 'r', encoding='utf-8') as fh:
return self.lexer.get_tokens_unprocessed(fh.read(512000))
except:
pass
try:
with open(self.source_file, 'r', encoding=sys.getfilesystemencoding()) as fh:
return self.lexer.get_tokens_unprocessed(fh.read(512000))
except:
pass
return []
def _save_dependency(self, dep, truncate=False, separator=None,