pythongh-91053: Add an optional callback that is invoked whenever a function is modified (python#98175)

mpage · web-flow · commit 3db0a21f731c · 2022-11-22T13:06:44.000+01:00
diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst
@@ -118,3 +118,63 @@ There are a few functions specific to Python functions.
    must be a dictionary or ``Py_None``.
 
    Raises :exc:`SystemError` and returns ``-1`` on failure.
+
+
+.. c:function:: int PyFunction_AddWatcher(PyFunction_WatchCallback callback)
+
+   Register *callback* as a function watcher for the current interpreter.
+   Return an ID which may be passed to :c:func:`PyFunction_ClearWatcher`.
+   In case of error (e.g. no more watcher IDs available),
+   return ``-1`` and set an exception.
+
+   .. versionadded:: 3.12
+
+
+.. c:function:: int PyFunction_ClearWatcher(int watcher_id)
+
+   Clear watcher identified by *watcher_id* previously returned from
+   :c:func:`PyFunction_AddWatcher` for the current interpreter.
+   Return ``0`` on success, or ``-1`` and set an exception on error
+   (e.g.  if the given *watcher_id* was never registered.)
+
+   .. versionadded:: 3.12
+
+
+.. c:type:: PyFunction_WatchEvent
+
+   Enumeration of possible function watcher events:
+   - ``PyFunction_EVENT_CREATE``
+   - ``PyFunction_EVENT_DESTROY``
+   - ``PyFunction_EVENT_MODIFY_CODE``
+   - ``PyFunction_EVENT_MODIFY_DEFAULTS``
+   - ``PyFunction_EVENT_MODIFY_KWDEFAULTS``
+
+   .. versionadded:: 3.12
+
+
+.. c:type:: int (*PyFunction_WatchCallback)(PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value)
+
+   Type of a function watcher callback function.
+
+   If *event* is ``PyFunction_EVENT_CREATE`` or ``PyFunction_EVENT_DESTROY``
+   then *new_value* will be ``NULL``. Otherwise, *new_value* will hold a
+   :term:`borrowed reference` to the new value that is about to be stored in
+   *func* for the attribute that is being modified.
+
+   The callback may inspect but must not modify *func*; doing so could have
+   unpredictable effects, including infinite recursion.
+
+   If *event* is ``PyFunction_EVENT_CREATE``, then the callback is invoked
+   after `func` has been fully initialized. Otherwise, the callback is invoked
+   before the modification to *func* takes place, so the prior state of *func*
+   can be inspected. The runtime is permitted to optimize away the creation of
+   function objects when possible. In such cases no event will be emitted.
+   Although this creates the possitibility of an observable difference of
+   runtime behavior depending on optimization decisions, it does not change
+   the semantics of the Python code being executed.
+
+   If the callback returns with an exception set, it must return ``-1``; this
+   exception will be printed as an unraisable exception using
+   :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``.
+
+   .. versionadded:: 3.12
diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h
@@ -131,6 +131,55 @@ PyAPI_DATA(PyTypeObject) PyStaticMethod_Type;
 PyAPI_FUNC(PyObject *) PyClassMethod_New(PyObject *);
 PyAPI_FUNC(PyObject *) PyStaticMethod_New(PyObject *);
 
+#define FOREACH_FUNC_EVENT(V) \
+    V(CREATE)                 \
+    V(DESTROY)                \
+    V(MODIFY_CODE)            \
+    V(MODIFY_DEFAULTS)        \
+    V(MODIFY_KWDEFAULTS)
+
+typedef enum {
+    #define DEF_EVENT(EVENT) PyFunction_EVENT_##EVENT,
+    FOREACH_FUNC_EVENT(DEF_EVENT)
+    #undef DEF_EVENT
+} PyFunction_WatchEvent;
+
+/*
+ * A callback that is invoked for different events in a function's lifecycle.
+ *
+ * The callback is invoked with a borrowed reference to func, after it is
+ * created and before it is modified or destroyed. The callback should not
+ * modify func.
+ *
+ * When a function's code object, defaults, or kwdefaults are modified the
+ * callback will be invoked with the respective event and new_value will
+ * contain a borrowed reference to the new value that is about to be stored in
+ * the function. Otherwise the third argument is NULL.
+ *
+ * If the callback returns with an exception set, it must return -1. Otherwise
+ * it should return 0.
+ */
+typedef int (*PyFunction_WatchCallback)(
+  PyFunction_WatchEvent event,
+  PyFunctionObject *func,
+  PyObject *new_value);
+
+/*
+ * Register a per-interpreter callback that will be invoked for function lifecycle
+ * events.
+ *
+ * Returns a handle that may be passed to PyFunction_ClearWatcher on success,
+ * or -1 and sets an error if no more handles are available.
+ */
+PyAPI_FUNC(int) PyFunction_AddWatcher(PyFunction_WatchCallback callback);
+
+/*
+ * Clear the watcher associated with the watcher_id handle.
+ *
+ * Returns 0 on success or -1 if no watcher exists for the supplied id.
+ */
+PyAPI_FUNC(int) PyFunction_ClearWatcher(int watcher_id);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h
@@ -8,6 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#define FUNC_MAX_WATCHERS 8
+
 struct _py_func_runtime_state {
     uint32_t next_version;
 };
diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
@@ -17,6 +17,7 @@ extern "C" {
 #include "pycore_dict_state.h"    // struct _Py_dict_state
 #include "pycore_exceptions.h"    // struct _Py_exc_state
 #include "pycore_floatobject.h"   // struct _Py_float_state
+#include "pycore_function.h"      // FUNC_MAX_WATCHERS
 #include "pycore_genobject.h"     // struct _Py_async_gen_state
 #include "pycore_gc.h"            // struct _gc_runtime_state
 #include "pycore_list.h"          // struct _Py_list_state
@@ -171,6 +172,11 @@ struct _is {
     // Initialized to _PyEval_EvalFrameDefault().
     _PyFrameEvalFunction eval_frame;
 
+    PyDict_WatchCallback dict_watchers[DICT_MAX_WATCHERS];
+    PyFunction_WatchCallback func_watchers[FUNC_MAX_WATCHERS];
+    // One bit is set for each non-NULL entry in func_watchers
+    uint8_t active_func_watchers;
+
     Py_ssize_t co_extra_user_count;
     freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS];
 
diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py
@@ -336,5 +336,98 @@ def test_no_more_ids_available(self):
                 self.add_watcher()
 
 
+class TestFuncWatchers(unittest.TestCase):
+    @contextmanager
+    def add_watcher(self, func):
+        wid = _testcapi.add_func_watcher(func)
+        try:
+            yield
+        finally:
+            _testcapi.clear_func_watcher(wid)
+
+    def test_func_events_dispatched(self):
+        events = []
+        def watcher(*args):
+            events.append(args)
+
+        with self.add_watcher(watcher):
+            def myfunc():
+                pass
+            self.assertIn((_testcapi.PYFUNC_EVENT_CREATE, myfunc, None), events)
+            myfunc_id = id(myfunc)
+
+            new_code = self.test_func_events_dispatched.__code__
+            myfunc.__code__ = new_code
+            self.assertIn((_testcapi.PYFUNC_EVENT_MODIFY_CODE, myfunc, new_code), events)
+
+            new_defaults = (123,)
+            myfunc.__defaults__ = new_defaults
+            self.assertIn((_testcapi.PYFUNC_EVENT_MODIFY_DEFAULTS, myfunc, new_defaults), events)
+
+            new_defaults = (456,)
+            _testcapi.set_func_defaults_via_capi(myfunc, new_defaults)
+            self.assertIn((_testcapi.PYFUNC_EVENT_MODIFY_DEFAULTS, myfunc, new_defaults), events)
+
+            new_kwdefaults = {"self": 123}
+            myfunc.__kwdefaults__ = new_kwdefaults
+            self.assertIn((_testcapi.PYFUNC_EVENT_MODIFY_KWDEFAULTS, myfunc, new_kwdefaults), events)
+
+            new_kwdefaults = {"self": 456}
+            _testcapi.set_func_kwdefaults_via_capi(myfunc, new_kwdefaults)
+            self.assertIn((_testcapi.PYFUNC_EVENT_MODIFY_KWDEFAULTS, myfunc, new_kwdefaults), events)
+
+            # Clear events reference to func
+            events = []
+            del myfunc
+            self.assertIn((_testcapi.PYFUNC_EVENT_DESTROY, myfunc_id, None), events)
+
+    def test_multiple_watchers(self):
+        events0 = []
+        def first_watcher(*args):
+            events0.append(args)
+
+        events1 = []
+        def second_watcher(*args):
+            events1.append(args)
+
+        with self.add_watcher(first_watcher):
+            with self.add_watcher(second_watcher):
+                def myfunc():
+                    pass
+
+                event = (_testcapi.PYFUNC_EVENT_CREATE, myfunc, None)
+                self.assertIn(event, events0)
+                self.assertIn(event, events1)
+
+    def test_watcher_raises_error(self):
+        class MyError(Exception):
+            pass
+
+        def watcher(*args):
+            raise MyError("testing 123")
+
+        with self.add_watcher(watcher):
+            with catch_unraisable_exception() as cm:
+                def myfunc():
+                    pass
+
+                self.assertIs(cm.unraisable.object, myfunc)
+                self.assertIsInstance(cm.unraisable.exc_value, MyError)
+
+    def test_clear_out_of_range_watcher_id(self):
+        with self.assertRaisesRegex(ValueError, r"invalid func watcher ID -1"):
+            _testcapi.clear_func_watcher(-1)
+        with self.assertRaisesRegex(ValueError, r"invalid func watcher ID 8"):
+            _testcapi.clear_func_watcher(8)  # FUNC_MAX_WATCHERS = 8
+
+    def test_clear_unassigned_watcher_id(self):
+        with self.assertRaisesRegex(ValueError, r"no func watcher set for ID 1"):
+            _testcapi.clear_func_watcher(1)
+
+    def test_allocate_too_many_watchers(self):
+        with self.assertRaisesRegex(RuntimeError, r"no more func watcher IDs"):
+            _testcapi.allocate_too_many_func_watchers()
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-44-52.gh-issue-91053.f5Bo3p.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-44-52.gh-issue-91053.f5Bo3p.rst
@@ -0,0 +1,4 @@
+Optimizing interpreters and JIT compilers may need to invalidate internal
+metadata when functions are modified. This change adds the ability to
+provide a callback that will be invoked each time a function is created,
+modified, or destroyed.
diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c
diff --git a/Objects/funcobject.c b/Objects/funcobject.c
diff --git a/Python/pystate.c b/Python/pystate.c
