Skip to content

Commit 440e515

Browse files
pekkaklarckpekkaklarck
committed
Fix encoding detection on Windows with Python 3.6.
Also added tests and did a little related cleanup to encoding detection. Fixes robotframework#2829.
1 parent 680a0b7 commit 440e515

File tree

10 files changed

+85
-31
lines changed

10 files changed

+85
-31
lines changed

‎atest/resources/atest_variables.py‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,25 @@
11
from os.path import abspath, dirname, join, normpath
2+
import locale
23
import os
4+
import subprocess
5+
36
import robot
47

5-
__all__ = ['ROBOTPATH', 'ROBOT_VERSION', 'DATADIR', 'WINDOWS']
8+
9+
__all__ = ['ROBOTPATH', 'ROBOT_VERSION', 'DATADIR', 'SYSTEM_ENCODING',
10+
'CONSOLE_ENCODING']
11+
612

713
ROBOTPATH = dirname(abspath(robot.__file__))
814
ROBOT_VERSION = robot.version.get_version()
915
DATADIR = normpath(join(dirname(abspath(__file__)), '..', 'testdata'))
10-
WINDOWS = os.sep == '\\'
16+
17+
# FIXME: Always use locale.getpreferredencoding when atests are run on Py 3.6!
18+
if os.name == 'nt':
19+
SYSTEM_ENCODING = locale.getpreferredencoding(False)
20+
# FIXME: Add encoding when running atests on Py 3.6
21+
cp = subprocess.check_output('chcp', shell=True).split()[-1]
22+
CONSOLE_ENCODING = 'cp' + cp
23+
else:
24+
SYSTEM_ENCODING = 'UTF-8'
25+
CONSOLE_ENCODING = locale.getdefaultlocale()[-1]

‎atest/robot/standard_libraries/operating_system/create_file.robot‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
*** Settings ***
2-
Suite Setup Run Tests ${EMPTY} standard_libraries/operating_system/create_file.robot
2+
Suite Setup Run Tests
3+
... -v SYSTEM_ENCODING:${SYSTEM_ENCODING} -v CONSOLE_ENCODING:${CONSOLE_ENCODING}
4+
... standard_libraries/operating_system/create_file.robot
35
Resource atest_resource.robot
46

57
*** Test Cases ***
@@ -15,6 +17,12 @@ Create Non-ASCII File With Default Encoding
1517
Create File With Encoding
1618
Check Test Case ${TESTNAME}
1719

20+
Create File With System Encoding
21+
Check Test Case ${TESTNAME}
22+
23+
Create File With Console Encoding
24+
Check Test Case ${TESTNAME}
25+
1826
Create File With Non-ASCII Name
1927
Check Test Case ${TESTNAME}
2028

‎atest/robot/standard_libraries/operating_system/get_file.robot‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
*** Settings ***
2-
Suite Setup Run Tests ${EMPTY} standard_libraries/operating_system/get_file.robot
2+
Suite Setup Run Tests
3+
... -v SYSTEM_ENCODING:${SYSTEM_ENCODING} -v CONSOLE_ENCODING:${CONSOLE_ENCODING}
4+
... standard_libraries/operating_system/get_file.robot
35
Resource atest_resource.robot
46

57
*** Test Cases ***

‎atest/robot/test_libraries/print_logging.robot‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
*** Settings ***
22
Documentation Tests for logging using stdout/stderr
3-
Suite Setup Run Tests ${EMPTY} test_libraries/print_logging.robot
3+
Suite Setup Run Tests
4+
... -v CONSOLE_ENCODING:${CONSOLE_ENCODING}
5+
... test_libraries/print_logging.robot
46
Resource atest_resource.robot
57

68
*** Test Cases ***
@@ -45,15 +47,15 @@ Logging Non-ASCII As Bytes
4547
[Tags] no-ipy
4648
${tc} = Check Test Case ${TEST NAME}
4749
${expected} = Get Expected Bytes Hyvää päivää!
50+
Check Log Message ${tc.kws[1].msgs[0]} ${expected}
4851
Check Log Message ${tc.kws[2].msgs[0]} ${expected}
49-
Check Log Message ${tc.kws[3].msgs[0]} ${expected}
5052
Check Stderr Contains ${expected}
5153

5254
Logging Mixed Non-ASCII Unicode And Bytes
5355
[Tags] no-ipy
5456
${tc} = Check Test Case ${TEST NAME}
5557
${bytes} = Get Expected Bytes Hyvä byte!
56-
Check Log Message ${tc.kws[2].msgs[0]} ${bytes} Hyvä Unicode!
58+
Check Log Message ${tc.kws[1].msgs[0]} ${bytes} Hyvä Unicode!
5759

5860
Logging HTML
5961
${tc} = Check Test Case ${TEST NAME}
@@ -72,6 +74,5 @@ FAIL is not valid log level
7274
Get Expected Bytes
7375
[Arguments] ${string}
7476
Return From Keyword If ${INTERPRETER.is_py2} ${string}
75-
${encoding} = Evaluate robot.utils.encoding.SYSTEM_ENCODING modules=robot
76-
${bytes} = Encode String To Bytes ${string} ${encoding}
77+
${bytes} = Encode String To Bytes ${string} ${CONSOLE_ENCODING}
7778
[Return] b'${bytes}'

‎atest/testdata/standard_libraries/operating_system/create_file.robot‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@ Suite Teardown Remove Base Test Directory
33
Test Setup Create Base Test Directory
44
Resource os_resource.robot
55

6+
*** Variables ***
7+
${SYSTEM_ENCODING} ASCII # Should be overridden from CLI
8+
${CONSOLE_ENCODING} ASCII # Should be overridden from CLI
9+
610
*** Test Cases ***
711
Create File With Default Content
812
Create File ${TESTFILE}
@@ -26,10 +30,14 @@ Create File With Encoding
2630
Hyvää yötä! ISO-8859-1
2731
Спасибо UTF-8
2832
Спасибо ISO-8859-5
29-
Hyvää yötä! SYSTEM
30-
Hyvää yötä! sysTEM
31-
Hyvää yötä! console
32-
Hyvää yötä! CONsole
33+
34+
Create File With System Encoding
35+
Create File ${TESTFILE} Nön-ÄSCÏÏ Cöntënt encoding=SYSTEM
36+
Verify File ${TESTFILE} Nön-ÄSCÏÏ Cöntënt encoding=${SYSTEM_ENCODING}
37+
38+
Create File With Console Encoding
39+
Create File ${TESTFILE} Nön-ÄSCÏÏ Cöntënt encoding=CONSole
40+
Verify File ${TESTFILE} Nön-ÄSCÏÏ Cöntënt encoding=${CONSOLE_ENCODING}
3341

3442
Create File With Non-ASCII Name
3543
[Template] Create and Verify File

‎atest/testdata/standard_libraries/operating_system/get_file.robot‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ Resource os_resource.robot
55
Library String
66

77
*** Variables ***
8+
${SYSTEM_ENCODING} ASCII # Should be overridden from CLI
9+
${CONSOLE_ENCODING} ASCII # Should be overridden from CLI
810
${UTF-8 FILE} ${CURDIR}${/}files${/}utf-8.txt
911
${ASCII FILE} ${CURDIR}${/}files${/}ascii.txt
1012
${LATIN-1 FILE} ${CURDIR}${/}files${/}latin-1.txt
@@ -48,14 +50,12 @@ Get Latin-1 With Default Encoding
4850
Get File ${LATIN-1 FILE}
4951

5052
Get file with system encoding
51-
${encoding} = Evaluate robot.utils.SYSTEM_ENCODING modules=robot
52-
Create File ${TEST FILE} ${RESULT} encoding=${encoding}
53+
Create File ${TEST FILE} ${RESULT} encoding=${SYSTEM_ENCODING}
5354
${file} = Get file ${TEST FILE} encoding=SYStem
5455
Should Be Equal ${file} ${RESULT}
5556

5657
Get file with console encoding
57-
${encoding} = Evaluate robot.utils.CONSOLE_ENCODING modules=robot
58-
Create File ${TEST FILE} ${RESULT} encoding=${encoding}
58+
Create File ${TEST FILE} ${RESULT} encoding=${CONSOLE_ENCODING}
5959
${file} = Get file ${TEST FILE} encoding=COnsoLE
6060
Should Be Equal ${file} ${RESULT}
6161

‎atest/testdata/standard_libraries/process/process_resource.robot‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,7 @@ Custom stream should contain
5555
[Arguments] ${path} ${expected}
5656
Return From Keyword If not $path
5757
${path} = Normalize Path ${path}
58-
${encoding} = Evaluate robot.utils.encoding.CONSOLE_ENCODING robot
59-
${content} = Get File ${path} encoding=${encoding}
58+
${content} = Get File ${path} encoding=CONSOLE
6059
Should Be Equal ${content.rstrip()} ${expected}
6160
[Return] ${path}
6261

‎atest/testdata/test_libraries/print_logging.robot‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ Library ExampleLibrary
55
Library PrintLib.py
66
Library String
77

8+
*** Variables ***
9+
${CONSOLE_ENCODING} ASCII # Should be overridden from CLI
10+
811
*** Test Cases ***
912
Logging Using Stdout And Stderr
1013
Print Hello from Python Library!
@@ -29,14 +32,12 @@ Logging Non-ASCII As Unicode
2932
Print Hyvää päivää stderr! stderr
3033

3134
Logging Non-ASCII As Bytes
32-
${encoding} = Evaluate robot.utils.encoding.CONSOLE_ENCODING robot
33-
${bytes} = Encode String To Bytes Hyvää päivää! ${encoding}
35+
${bytes} = Encode String To Bytes Hyvää päivää! ${CONSOLE ENCODING}
3436
Print ${bytes}
3537
Print ${bytes} stderr
3638

3739
Logging Mixed Non-ASCII Unicode And Bytes
38-
${encoding} = Evaluate robot.utils.encoding.CONSOLE_ENCODING robot
39-
${bytes} = Encode String To Bytes Hyvä byte! ${encoding}
40+
${bytes} = Encode String To Bytes Hyvä byte! ${CONSOLE ENCODING}
4041
Print Many ${bytes} Hyvä Unicode!
4142

4243
Logging HTML

‎src/robot/utils/encodingsniffer.py‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,11 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515

16-
import sys
1716
import os
17+
import sys
18+
import locale
1819

19-
from .platform import JYTHON, WINDOWS, UNIXY
20+
from .platform import JYTHON, PY2, PY3, UNIXY, WINDOWS
2021

2122

2223
if UNIXY:
@@ -52,7 +53,13 @@ def _get_encoding(platform_getters, default):
5253

5354

5455
def _get_python_system_encoding():
55-
return sys.getfilesystemencoding()
56+
# `locale.getpreferredencoding(False)` returns exactly what we want, but
57+
# but it doesn't seem to work outside Windows on Python 2. Luckily on
58+
# those platforms `sys.getfilesystemencoding()` ought to do the right
59+
# thing as well.
60+
if PY2 and not WINDOWS:
61+
return sys.getfilesystemencoding()
62+
return locale.getpreferredencoding(False)
5663

5764

5865
def _get_java_system_encoding():
@@ -61,7 +68,10 @@ def _get_java_system_encoding():
6168

6269

6370
def _get_unixy_encoding():
64-
for name in 'LANG', 'LC_CTYPE', 'LANGUAGE', 'LC_ALL':
71+
# Cannot use `locale.getdefaultlocale()` because it raises ValueError
72+
# if encoding is invalid. Using same environment variables here anyway.
73+
# https://docs.python.org/3/library/locale.html#locale.getdefaultlocale
74+
for name in 'LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE':
6575
if name in os.environ:
6676
# Encoding can be in format like `UTF-8` or `en_US.UTF-8`
6777
encoding = os.environ[name].split('.')[-1]
@@ -71,7 +81,12 @@ def _get_unixy_encoding():
7181

7282

7383
def _get_stream_output_encoding():
74-
# Stream may not have encoding attribute if it is intercepted outside RF in
84+
# Python < 3.6 on Windows returns different encoding depending on are
85+
# outputs redirected or not, and Python >= 3.6 always use UTF-8. We
86+
# want the real console encoding regardless the platform.
87+
if WINDOWS and PY3:
88+
return None
89+
# Stream may not have encoding attribute if intercepted outside RF in
7590
# Python. Encoding is None if process output is redirected and Python < 3.
7691
for stream in sys.__stdout__, sys.__stderr__, sys.__stdin__:
7792
encoding = getattr(stream, 'encoding', None)
@@ -92,7 +107,7 @@ def _get_code_page(method_name):
92107
from ctypes import cdll
93108
try:
94109
method = getattr(cdll.kernel32, method_name)
95-
except TypeError: # Occurred few times with IronPython (mainly on CI).
110+
except TypeError: # Occurred few times with IronPython on CI.
96111
return None
97112
method.argtypes = () # Needed with Jython.
98113
return 'cp%s' % method()

‎utest/utils/test_encodingsniffer.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
import unittest
22
import sys
33

4-
from robot.utils.encodingsniffer import get_console_encoding
54
from robot.utils.asserts import assert_equal, assert_not_none
5+
from robot.utils.encodingsniffer import get_console_encoding
6+
from robot.utils import PY3, WINDOWS
67

78

89
class StreamStub(object):
@@ -11,7 +12,11 @@ def __init__(self, encoding):
1112
self.encoding = encoding
1213

1314

14-
class TestGetConsoleEncodingFromStandardStreams(unittest.TestCase):
15+
# We don't look at streams on Windows with Python 3. Cannot run tests either.
16+
BASE = unittest.TestCase if not (WINDOWS and PY3) else object
17+
18+
19+
class TestGetConsoleEncodingFromStandardStreams(BASE):
1520

1621
def setUp(self):
1722
self._orig_streams = sys.__stdout__, sys.__stderr__, sys.__stdin__

0 commit comments

Comments
 (0)