try opening files with filesystem encoding when opening with utf-8 causes UnicodeDecodeError

wakatime/languages/__init__.py

@@ -10,6 +10,7 @@
 """
 
 import logging
+import sys
 import traceback
 
 from ..compat import u, open, import_module
@@ -53,8 +54,16 @@ class TokenParser(object):
 
     def _extract_tokens(self):
         if self.lexer:
-            with open(self.source_file, 'r', encoding='utf-8') as fh:
+            try:
-                return self.lexer.get_tokens_unprocessed(fh.read(512000))
+                with open(self.source_file, 'r', encoding='utf-8') as fh:
+                    return self.lexer.get_tokens_unprocessed(fh.read(512000))
+            except:
+                pass
+            try:
+                with open(self.source_file, 'r', encoding=sys.getfilesystemencoding()) as fh:
+                    return self.lexer.get_tokens_unprocessed(fh.read(512000))
+            except:
+                pass
         return []
 
     def _save_dependency(self, dep, truncate=False, separator=None,

wakatime/projects/git.py

@@ -11,6 +11,7 @@
 
 import logging
 import os
+import sys
 
 from .base import BaseProject
 from ..compat import u, open
@@ -38,8 +39,14 @@ class Git(BaseProject):
             try:
                 with open(head, 'r', encoding='utf-8') as fh:
                     return u(fh.readline().strip().rsplit('/', 1)[-1])
+            except UnicodeDecodeError:
+                try:
+                    with open(head, 'r', encoding=sys.getfilesystemencoding()) as fh:
+                        return u(fh.readline().strip().rsplit('/', 1)[-1])
+                except:
+                    log.exception("Exception:")
             except IOError:
-                pass
+                log.exception("Exception:")
         return None
 
     def _project_base(self):

wakatime/projects/mercurial.py

@@ -11,6 +11,7 @@
 
 import logging
 import os
+import sys
 
 from .base import BaseProject
 from ..compat import u, open
@@ -36,8 +37,14 @@ class Mercurial(BaseProject):
             try:
                 with open(branch_file, 'r', encoding='utf-8') as fh:
                     return u(fh.readline().strip().rsplit('/', 1)[-1])
+            except UnicodeDecodeError:
+                try:
+                    with open(branch_file, 'r', encoding=sys.getfilesystemencoding()) as fh:
+                        return u(fh.readline().strip().rsplit('/', 1)[-1])
+                except:
+                    log.exception("Exception:")
             except IOError:
-                pass
+                log.exception("Exception:")
         return u('default')
 
     def _find_hg_config_dir(self, path):

wakatime/projects/subversion.py

@@ -46,13 +46,13 @@ class Subversion(BaseProject):
             '/usr/local/bin/svn',
         ]
         for location in locations:
-            with open(os.devnull, 'wb') as DEVNULL:
+            try:
-                try:
+                with open(os.devnull, 'wb') as DEVNULL:
                     Popen([location, '--version'], stdout=DEVNULL, stderr=DEVNULL)
                     self.binary_location = location
                     return location
-                except:
+            except:
-                    pass
+                pass
         self.binary_location = 'svn'
         return 'svn'
 

wakatime/projects/wakatime_project_file.py

@@ -13,6 +13,7 @@
 
 import logging
 import os
+import sys
 
 from .base import BaseProject
 from ..compat import u, open
@@ -34,6 +35,13 @@ class WakaTimeProjectFile(BaseProject):
                 with open(self.config, 'r', encoding='utf-8') as fh:
                     self._project_name = u(fh.readline().strip())
                     self._project_branch = u(fh.readline().strip())
+            except UnicodeDecodeError:
+                try:
+                    with open(self.config, 'r', encoding=sys.getfilesystemencoding()) as fh:
+                        self._project_name = u(fh.readline().strip())
+                        self._project_branch = u(fh.readline().strip())
+                except:
+                    log.exception("Exception:")
             except IOError:
                 log.exception("Exception:")
 

wakatime/stats.py

@@ -144,7 +144,12 @@ def number_lines_in_file(file_name):
             for line in fh:
                 lines += 1
     except:
-        return None
+        try:
+            with open(file_name, 'r', encoding=sys.getfilesystemencoding()) as fh:
+                for line in fh:
+                    lines += 1
+        except:
+            return None
     return lines
 
 
@@ -180,5 +185,9 @@ def get_file_contents(file_name):
         with open(file_name, 'r', encoding='utf-8') as fh:
             text = fh.read(512000)
     except:
-        pass
+        try:
+            with open(file_name, 'r', encoding=sys.getfilesystemencoding()) as fh:
+                text = fh.read(512000)
+        except:
+            log.exception("Exception:")
     return text