Issue #22166: clear codec caches in test_codecs

ncoghlan · ncoghlan · commit 8fad1676a215 · 2014-09-15T23:50:44.000+12:00
diff --git a/Include/codecs.h b/Include/codecs.h
@@ -49,6 +49,10 @@ PyAPI_FUNC(int) PyCodec_Register(
 PyAPI_FUNC(PyObject *) _PyCodec_Lookup(
        const char *encoding
        );
+
+PyAPI_FUNC(int) _PyCodec_Forget(
+       const char *encoding
+       );
 #endif
 
 /* Codec registry encoding check API.
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
@@ -2578,6 +2578,14 @@ def _get_test_codec(codec_name):
     return _TEST_CODECS.get(codec_name)
 codecs.register(_get_test_codec) # Returns None, not usable as a decorator
 
+try:
+    # Issue #22166: Also need to clear the internal cache in CPython
+    from _codecs import _forget_codec
+except ImportError:
+    def _forget_codec(codec_name):
+        pass
+
+
 class ExceptionChainingTest(unittest.TestCase):
 
     def setUp(self):
@@ -2603,6 +2611,12 @@ def setUp(self):
 
     def tearDown(self):
         _TEST_CODECS.pop(self.codec_name, None)
+        # Issue #22166: Also pop from caches to avoid appearance of ref leaks
+        encodings._cache.pop(self.codec_name, None)
+        try:
+            _forget_codec(self.codec_name)
+        except KeyError:
+            pass
 
     def set_codec(self, encode, decode):
         codec_info = codecs.CodecInfo(encode, decode,
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -344,6 +344,10 @@ IDLE
 Tests
 -----
 
+- Issue #22166:  with the assistance of a new internal _codecs._forget_codec
+  helping function, test_codecs now clears the encoding caches to avoid the
+  appearance of a reference leak
+
 - Issue #22236: Tkinter tests now don't reuse default root window.  New root
   window is created for every test class.
 
diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c
@@ -42,6 +42,12 @@ Copyright (c) Corporation for National Research Initiatives.
 #include <windows.h>
 #endif
 
+/*[clinic input]
+module _codecs
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e1390e3da3cb9deb]*/
+
+
 /* --- Registry ----------------------------------------------------------- */
 
 PyDoc_STRVAR(register__doc__,
@@ -134,6 +140,53 @@ codec_decode(PyObject *self, PyObject *args)
 
 /* --- Helpers ------------------------------------------------------------ */
 
+/*[clinic input]
+_codecs._forget_codec
+
+    encoding: str
+    /
+
+Purge the named codec from the internal codec lookup cache
+[clinic start generated code]*/
+
+PyDoc_STRVAR(_codecs__forget_codec__doc__,
+"_forget_codec($module, encoding, /)\n"
+"--\n"
+"\n"
+"Purge the named codec from the internal codec lookup cache");
+
+#define _CODECS__FORGET_CODEC_METHODDEF    \
+    {"_forget_codec", (PyCFunction)_codecs__forget_codec, METH_VARARGS, _codecs__forget_codec__doc__},
+
+static PyObject *
+_codecs__forget_codec_impl(PyModuleDef *module, const char *encoding);
+
+static PyObject *
+_codecs__forget_codec(PyModuleDef *module, PyObject *args)
+{
+    PyObject *return_value = NULL;
+    const char *encoding;
+
+    if (!PyArg_ParseTuple(args,
+        "s:_forget_codec",
+        &encoding))
+        goto exit;
+    return_value = _codecs__forget_codec_impl(module, encoding);
+
+exit:
+    return return_value;
+}
+
+static PyObject *
+_codecs__forget_codec_impl(PyModuleDef *module, const char *encoding)
+/*[clinic end generated code: output=a75e631591702a5c input=18d5d92d0e386c38]*/
+{
+    if (_PyCodec_Forget(encoding) < 0) {
+        return NULL;
+    };
+    Py_RETURN_NONE;
+}
+
 static
 PyObject *codec_tuple(PyObject *unicode,
                       Py_ssize_t len)
@@ -1168,6 +1221,7 @@ static PyMethodDef _codecs_functions[] = {
         register_error__doc__},
     {"lookup_error",            lookup_error,                   METH_VARARGS,
         lookup_error__doc__},
+    _CODECS__FORGET_CODEC_METHODDEF
     {NULL, NULL}                /* sentinel */
 };
 
diff --git a/Python/codecs.c b/Python/codecs.c
@@ -185,6 +185,32 @@ PyObject *_PyCodec_Lookup(const char *encoding)
     return NULL;
 }
 
+int _PyCodec_Forget(const char *encoding)
+{
+    PyInterpreterState *interp;
+    PyObject *v;
+    int result;
+
+    interp = PyThreadState_GET()->interp;
+    if (interp->codec_search_path == NULL) {
+        return -1;
+    }
+
+    /* Convert the encoding to a normalized Python string: all
+       characters are converted to lower case, spaces and hyphens are
+       replaced with underscores. */
+    v = normalizestring(encoding);
+    if (v == NULL) {
+        return -1;
+    }
+
+    /* Drop the named codec from the internal cache */
+    result = PyDict_DelItem(interp->codec_search_cache, v);
+    Py_DECREF(v);
+
+    return result;
+}
+
 /* Codec registry encoding check API. */
 
 int PyCodec_KnownEncoding(const char *encoding)
