From 43c461a7cc97da2b7a36db4c75f1e432b8a5b649 Mon Sep 17 00:00:00 2001
From: Alan Hamlett <alan.hamlett@gmail.com>
Date: Sun, 23 Aug 2015 18:49:34 -0700
Subject: [PATCH] try opening files with filesystem encoding when opening with
 utf-8 causes UnicodeDecodeError

---
 wakatime/languages/__init__.py             | 13 +++++++++++--
 wakatime/projects/git.py                   |  9 ++++++++-
 wakatime/projects/mercurial.py             |  9 ++++++++-
 wakatime/projects/subversion.py            |  8 ++++----
 wakatime/projects/wakatime_project_file.py |  8 ++++++++
 wakatime/stats.py                          | 13 +++++++++++--
 6 files changed, 50 insertions(+), 10 deletions(-)

diff --git a/wakatime/languages/__init__.py b/wakatime/languages/__init__.py
index 1795880..1982778 100644
--- a/wakatime/languages/__init__.py
+++ b/wakatime/languages/__init__.py
@@ -10,6 +10,7 @@
 """
 
 import logging
+import sys
 import traceback
 
 from ..compat import u, open, import_module
@@ -53,8 +54,16 @@ class TokenParser(object):
 
     def _extract_tokens(self):
         if self.lexer:
-            with open(self.source_file, 'r', encoding='utf-8') as fh:
-                return self.lexer.get_tokens_unprocessed(fh.read(512000))
+            try:
+                with open(self.source_file, 'r', encoding='utf-8') as fh:
+                    return self.lexer.get_tokens_unprocessed(fh.read(512000))
+            except:
+                pass
+            try:
+                with open(self.source_file, 'r', encoding=sys.getfilesystemencoding()) as fh:
+                    return self.lexer.get_tokens_unprocessed(fh.read(512000))
+            except:
+                pass
         return []
 
     def _save_dependency(self, dep, truncate=False, separator=None,
diff --git a/wakatime/projects/git.py b/wakatime/projects/git.py
index cf4d51e..db6c3f2 100644
--- a/wakatime/projects/git.py
+++ b/wakatime/projects/git.py
@@ -11,6 +11,7 @@
 
 import logging
 import os
+import sys
 
 from .base import BaseProject
 from ..compat import u, open
@@ -38,8 +39,14 @@ class Git(BaseProject):
             try:
                 with open(head, 'r', encoding='utf-8') as fh:
                     return u(fh.readline().strip().rsplit('/', 1)[-1])
+            except UnicodeDecodeError:
+                try:
+                    with open(head, 'r', encoding=sys.getfilesystemencoding()) as fh:
+                        return u(fh.readline().strip().rsplit('/', 1)[-1])
+                except:
+                    log.exception("Exception:")
             except IOError:
-                pass
+                log.exception("Exception:")
         return None
 
     def _project_base(self):
diff --git a/wakatime/projects/mercurial.py b/wakatime/projects/mercurial.py
index 5f222ee..11e15fa 100644
--- a/wakatime/projects/mercurial.py
+++ b/wakatime/projects/mercurial.py
@@ -11,6 +11,7 @@
 
 import logging
 import os
+import sys
 
 from .base import BaseProject
 from ..compat import u, open
@@ -36,8 +37,14 @@ class Mercurial(BaseProject):
             try:
                 with open(branch_file, 'r', encoding='utf-8') as fh:
                     return u(fh.readline().strip().rsplit('/', 1)[-1])
+            except UnicodeDecodeError:
+                try:
+                    with open(branch_file, 'r', encoding=sys.getfilesystemencoding()) as fh:
+                        return u(fh.readline().strip().rsplit('/', 1)[-1])
+                except:
+                    log.exception("Exception:")
             except IOError:
-                pass
+                log.exception("Exception:")
         return u('default')
 
     def _find_hg_config_dir(self, path):
diff --git a/wakatime/projects/subversion.py b/wakatime/projects/subversion.py
index e1ed061..cfc8233 100644
--- a/wakatime/projects/subversion.py
+++ b/wakatime/projects/subversion.py
@@ -46,13 +46,13 @@ class Subversion(BaseProject):
             '/usr/local/bin/svn',
         ]
         for location in locations:
-            with open(os.devnull, 'wb') as DEVNULL:
-                try:
+            try:
+                with open(os.devnull, 'wb') as DEVNULL:
                     Popen([location, '--version'], stdout=DEVNULL, stderr=DEVNULL)
                     self.binary_location = location
                     return location
-                except:
-                    pass
+            except:
+                pass
         self.binary_location = 'svn'
         return 'svn'
 
diff --git a/wakatime/projects/wakatime_project_file.py b/wakatime/projects/wakatime_project_file.py
index 97a4652..3b1b8db 100644
--- a/wakatime/projects/wakatime_project_file.py
+++ b/wakatime/projects/wakatime_project_file.py
@@ -13,6 +13,7 @@
 
 import logging
 import os
+import sys
 
 from .base import BaseProject
 from ..compat import u, open
@@ -34,6 +35,13 @@ class WakaTimeProjectFile(BaseProject):
                 with open(self.config, 'r', encoding='utf-8') as fh:
                     self._project_name = u(fh.readline().strip())
                     self._project_branch = u(fh.readline().strip())
+            except UnicodeDecodeError:
+                try:
+                    with open(self.config, 'r', encoding=sys.getfilesystemencoding()) as fh:
+                        self._project_name = u(fh.readline().strip())
+                        self._project_branch = u(fh.readline().strip())
+                except:
+                    log.exception("Exception:")
             except IOError:
                 log.exception("Exception:")
 
diff --git a/wakatime/stats.py b/wakatime/stats.py
index b04e7ac..db424c0 100644
--- a/wakatime/stats.py
+++ b/wakatime/stats.py
@@ -144,7 +144,12 @@ def number_lines_in_file(file_name):
             for line in fh:
                 lines += 1
     except:
-        return None
+        try:
+            with open(file_name, 'r', encoding=sys.getfilesystemencoding()) as fh:
+                for line in fh:
+                    lines += 1
+        except:
+            return None
     return lines
 
 
@@ -180,5 +185,9 @@ def get_file_contents(file_name):
         with open(file_name, 'r', encoding='utf-8') as fh:
             text = fh.read(512000)
     except:
-        pass
+        try:
+            with open(file_name, 'r', encoding=sys.getfilesystemencoding()) as fh:
+                text = fh.read(512000)
+        except:
+            log.exception("Exception:")
     return text