Properly decode html response data based on specified charset.

jweigel · jweigel · commit 2d356e70afaf · 2014-08-15T16:50:09.000-07:00
diff --git a/ebaysdk/__init__.py b/ebaysdk/__init__.py
@@ -25,6 +25,7 @@
 
 from ebaysdk.utils import xml2dict, dict2xml, list2xml, object_dict
 
+content_type_re = re.compile('^Content\-Type:\s([^;]+);charset=(.*?)$', re.I)
 
 def get_version():
     "Get the version."
@@ -357,6 +358,24 @@ def _process_http_request(self):
         self._response_reason = re.match(r'^HTTP.+? +\d+ +(.*) *$', self._response_status).group(1)
         response_data = self._response_body.getvalue()
 
+        # Look for charset specifier and decode the response_data
+        # based on it.
+        for header in self._response_header.getvalue().splitlines():
+            header_match = content_type_re.match(header)
+
+            try:
+                if header_match and header_match.group(2):
+                    response_data = response_data.decode(header_match.group(2))
+                    break
+            except Exception as e:
+                charset = ''
+                try:
+                    charset = header_match.group(2)
+                except Exception:
+                    pass
+
+                sys.stderr.write("Failed to decode response using charset %s (%s)\n" % (charset, str(e)))
+
         self._response_header = None
         self._response_body = None
         self._curl.close()
