Merge pull request #33 from myint/latin1

Handle Latin-1 encoded text in diffs

.gitignore

@@ -1,7 +1,9 @@
+.*.swp
 *.pyc
 *~
 *.tmp
 /.coverage
+/.travis-solo/
 /htmlcov/
 /MANIFEST
 /build/

cdiff.py

@@ -36,6 +36,12 @@ import select
 import difflib
 
 
+try:
+    unicode
+except NameError:
+    unicode = str
+
+
 COLORS = {
     'reset'         : '\x1b[0m',
     'underline'     : '\x1b[4m',
@@ -248,7 +254,7 @@ class PatchStreamForwarder(object):
     def _forward_line(self):
         try:
             line = next(self._istream)
-            self._in.write(line.encode('utf-8'))
+            self._in.write(line)
         except StopIteration:
             self._in.close()
 
@@ -643,11 +649,17 @@ def revision_control_log(args):
 
 def decode(line):
     """Decode UTF-8 if necessary."""
-    try:
-        return line.decode('utf-8')
-    except (AttributeError, UnicodeDecodeError):
+    if isinstance(line, unicode):
         return line
 
+    for encoding in ['utf-8', 'latin1']:
+        try:
+            return line.decode(encoding)
+        except UnicodeDecodeError:
+            pass
+
+    return '*** cdiff: undecodable bytes ***\n'
+
 
 def main():
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)
@@ -715,7 +727,8 @@ def main():
             parser.print_help()
             return 1
     else:
-        diff_hdl = sys.stdin
+        diff_hdl = (sys.stdin.buffer if hasattr(sys.stdin, 'buffer')
+                    else sys.stdin)
 
     stream = PatchStream(diff_hdl)
 
@@ -730,8 +743,11 @@ def main():
         # pipe out stream untouched to make sure it is still a patch
         if sys.hexversion < 0x03000000:
             reload(sys).setdefaultencoding('utf8')
+
+        byte_output = (sys.stdout.buffer if hasattr(sys.stdout, 'buffer')
+                       else sys.stdout)
         for line in stream:
-            sys.stdout.write(decode(line))
+            byte_output.write(line)
 
     if diff_hdl is not sys.stdin:
         diff_hdl.close()

tests/latin1/in.diff

@@ -0,0 +1,9 @@
+diff --git a/test/latin1.cc b/test/latin1.cc
+index a556e5c..d81bb0c 100644
+--- a/test/latin1.cc
++++ b/test/latin1.cc
+@@ -1,4 +1,3 @@
+-// é
+ int latin1()
+ {
+ 	static int x;

tests/latin1/out.normal

@@ -0,0 +1,10 @@
+diff --git a/test/latin1.cc b/test/latin1.cc
+index a556e5c..d81bb0c 100644
+--- a/test/latin1.cc
++++ b/test/latin1.cc
+@@ -1,4 +1,3 @@
+-// é
+ int latin1()
+ {
+ 	static int x;
+

tests/latin1/out.side-by-side

@@ -0,0 +1,9 @@
+diff --git a/test/latin1.cc b/test/latin1.cc
+index a556e5c..d81bb0c 100644
+--- a/test/latin1.cc
++++ b/test/latin1.cc
+@@ -1,4 +1,3 @@
+1 // é   
+2 int latin1()                                                                     1 int latin1()
+3 {                                                                                2 {
+4         static int x;                                                            3         static int x;

tests/latin1/out.w70

@@ -0,0 +1,9 @@
+diff --git a/test/latin1.cc b/test/latin1.cc
+index a556e5c..d81bb0c 100644
+--- a/test/latin1.cc
++++ b/test/latin1.cc
+@@ -1,4 +1,3 @@
+1 // é   
+2 int latin1()                                                           1 int latin1()
+3 {                                                                      2 {
+4         static int x;                                                  3         static int x;

tests/test_cdiff.py

@@ -77,9 +77,13 @@ class DecodeTest(unittest.TestCase):
         utext = 'hello'.encode('utf-8')
         self.assertEqual('hello', cdiff.decode(utext))
 
-    def test_malformed_utf8(self):
+    def test_latin_1(self):
         text = '\x80\x02q\x01(U'
-        self.assertEqual(text, cdiff.decode(text))
+        if sys.version_info[0] == 2:
+            decoded_text = text.decode('latin-1')
+        else:
+            decoded_text = text
+        self.assertEqual(decoded_text, cdiff.decode(text))
 
 
 class HunkTest(unittest.TestCase):