Issue #23374: Fixed pydoc failure with non-ASCII files when stdout encoding

serhiy-storchaka · serhiy-storchaka · commit 8d7fb404d969 · 2015-02-20T23:46:06.000+02:00
differs from file system encoding (e.g. on Mac OS).
diff --git a/Lib/pydoc.py b/Lib/pydoc.py
@@ -1407,9 +1407,6 @@ def bold(self, text):
 def pager(text):
     """The first time this is called, determine what kind of pager to use."""
     global pager
-    # Escape non-encodable characters to avoid encoding errors later
-    encoding = sys.getfilesystemencoding()
-    text = text.encode(encoding, 'backslashreplace').decode(encoding)
     pager = getpager()
     pager(text)
 
@@ -1452,27 +1449,34 @@ def plain(text):
 
 def pipepager(text, cmd):
     """Page through text by feeding it to another program."""
-    pipe = os.popen(cmd, 'w')
+    import subprocess
+    proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE)
     try:
-        pipe.write(text)
-        pipe.close()
+        with proc:
+            with io.TextIOWrapper(proc.stdin, errors='backslashreplace') as pipe:
+                pipe.write(text)
     except OSError:
         pass # Ignore broken pipes caused by quitting the pager program.
 
 def tempfilepager(text, cmd):
     """Page through text by invoking a program on a temporary file."""
     import tempfile
     filename = tempfile.mktemp()
-    with open(filename, 'w') as file:
+    with open(filename, 'w', errors='backslashreplace') as file:
         file.write(text)
     try:
         os.system(cmd + ' "' + filename + '"')
     finally:
         os.unlink(filename)
 
+def _escape_stdout(text):
+    # Escape non-encodable characters to avoid encoding errors later
+    encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8'
+    return text.encode(encoding, 'backslashreplace').decode(encoding)
+
 def ttypager(text):
     """Page through text on a text terminal."""
-    lines = plain(text).split('\n')
+    lines = plain(_escape_stdout(text)).split('\n')
     try:
         import tty
         fd = sys.stdin.fileno()
@@ -1516,7 +1520,7 @@ def ttypager(text):
 
 def plainpager(text):
     """Simply print unformatted text.  This is the ultimate fallback."""
-    sys.stdout.write(plain(text))
+    sys.stdout.write(plain(_escape_stdout(text)))
 
 def describe(thing):
     """Produce a short description of the given thing."""
diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py
@@ -35,6 +35,10 @@
 except ImportError:
     threading = None
 
+class nonascii:
+    'Це не латиниця'
+    pass
+
 if test.support.HAVE_DOCSTRINGS:
     expected_data_docstrings = (
         'dictionary for instance variables (if defined)',
@@ -474,6 +478,11 @@ def test_not_here(self):
         self.assertEqual(expected, result,
             "documentation for missing module found")
 
+    def test_not_ascii(self):
+        result = run_pydoc('test.test_pydoc.nonascii', PYTHONIOENCODING='ascii')
+        encoded = nonascii.__doc__.encode('ascii', 'backslashreplace')
+        self.assertIn(encoded, result)
+
     def test_input_strip(self):
         missing_module = " test.i_am_not_here "
         result = str(run_pydoc(missing_module), 'ascii')
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -13,6 +13,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #23374: Fixed pydoc failure with non-ASCII files when stdout encoding
+  differs from file system encoding (e.g. on Mac OS).
+
 - Issue #23481: Remove RC4 from the SSL module's default cipher list.
 
 - Issue #21548: Fix pydoc.synopsis() and pydoc.apropos() on modules with empty
