diff --git a/CHANGES.rst b/CHANGES.rst
index 27f993b..60a3182 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,6 +1,13 @@
 Changes
 -------
 
+Unreleased
+~~~~~~~~~~
+
+* Fix bug where error handling code could raise an exception causing a
+  confusing exception to be returned.
+
+
 0.2.4 (2017-09-17)
 ~~~~~~~~~~~~~~~~~~
 
diff --git a/rapidjson.cpp b/rapidjson.cpp
index 64a8f59..057c671 100644
--- a/rapidjson.cpp
+++ b/rapidjson.cpp
@@ -1278,11 +1278,16 @@ do_decode(PyObject* decoder, const char* jsonStr, Py_ssize_t jsonStrLen,
             PyObject* etraceback;
             PyErr_Fetch(&etype, &evalue, &etraceback);
 
+            // Try to add the offset in the error message if the exception
+            // value is a string.  Otherwise, use the original exception since
+            // we can't be sure the exception type takes a single string.
             const char* emsg = msg;
-            if (PyUnicode_Check(evalue))
+            if (PyUnicode_Check(evalue)) {
                 emsg = PyUnicode_AsUTF8(evalue);
-
-            PyErr_Format(etype, fmt, offset, emsg);
+                PyErr_Format(etype, fmt, offset, emsg);
+            }
+            else
+                PyErr_Restore(etype, evalue, etraceback);
         }
         else
             PyErr_Format(PyExc_ValueError, fmt, offset, msg);
diff --git a/tests/test_unicode.py b/tests/test_unicode.py
index e2d006d..f2a2f25 100644
--- a/tests/test_unicode.py
+++ b/tests/test_unicode.py
@@ -46,3 +46,13 @@ def test_dump_surrogate(o, dumps):
 def test_load_surrogate(j, loads):
     with pytest.raises(ValueError, match="surrogate pair in string is invalid"):
         loads(j)
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize('j', [
+    '"\\udc00"',
+    '"\\udfff"',
+])
+def test_unicode_decode_error(j, loads):
+    with pytest.raises(UnicodeDecodeError, match="'utf-8' codec can't decode byte"):
+        loads(j)