BUG: when given unicode inputs, arg_split should return unicode outputs. Always use utf-8 to encode the string instead of relying on sys.stdin.encoding, which may not be able to accept the full range of Unicode characters. When given unicode strings, arg_split is probably not receiving input from a terminal.

rkern · rkern · commit 6aff23d2f423 · 2011-01-26T14:22:12.000-06:00
diff --git a/IPython/utils/process.py b/IPython/utils/process.py
@@ -114,11 +114,17 @@ def arg_split(s, posix=False):
     # http://bugs.python.org/issue1170
     # At least encoding the input when it's unicode seems to help, but there
     # may be more problems lurking.  Apparently this is fixed in python3.
+    is_unicode = False
     if isinstance(s, unicode):
-        s = s.encode(sys.stdin.encoding)
+        is_unicode = True
+        s = s.encode('utf-8')
     lex = shlex.shlex(s, posix=posix)
     lex.whitespace_split = True
-    return list(lex)
+    tokens = list(lex)
+    if is_unicode:
+        # Convert the tokens back to unicode.
+        tokens = [x.decode('utf-8') for x in tokens]
+    return tokens
 
 
 def abbrev_cwd():
diff --git a/IPython/utils/tests/test_process.py b/IPython/utils/tests/test_process.py
@@ -66,6 +66,9 @@ def test_arg_split():
     """Ensure that argument lines are correctly split like in a shell."""
     tests = [['hi', ['hi']],
              [u'hi', [u'hi']],
+             ['hello there', ['hello', 'there']],
+             [u'h\N{LATIN SMALL LETTER A WITH CARON}llo', [u'h\N{LATIN SMALL LETTER A WITH CARON}llo']],
+             ['something "with quotes"', ['something', '"with quotes"']],
              ]
     for argstr, argv in tests:
         nt.assert_equal(arg_split(argstr), argv)
