Add capability for specializing CALL_FUNCTION.

python · markshannon · Oct 20, 2021 · Oct 11, 2021 · Oct 13, 2021 · Oct 13, 2021
commit e4160c954123bbdeccb73247011f68219954fdb2
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
@@ -308,6 +308,7 @@ int _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNI
 int _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
 int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr);
 int _Py_Specialize_BinaryAdd(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr);
+int _Py_Specialize_CallFunction(PyThreadState *tstate, PyObject *callable, int nargs, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache);
 
 #define PRINT_SPECIALIZATION_STATS 0
 #define PRINT_SPECIALIZATION_STATS_DETAILED 0

diff --git a/Include/opcode.h b/Include/opcode.h
diff --git a/Lib/opcode.py b/Lib/opcode.py
@@ -247,6 +247,7 @@ def jabs_op(name, op):
     "STORE_ATTR_INSTANCE_VALUE",
     "STORE_ATTR_SLOT",
     "STORE_ATTR_WITH_HINT",
+    "CALL_FUNCTION_ADAPTIVE",
     # Super instructions
     "LOAD_FAST__LOAD_FAST",
     "STORE_FAST__LOAD_FAST",

@@ -4598,6 +4598,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
 
         TARGET(CALL_FUNCTION) {
             PREDICTED(CALL_FUNCTION);
+            STAT_INC(CALL_FUNCTION, unquickened);
             PyObject *function;
             nargs = oparg;
             kwnames = NULL;
@@ -4655,6 +4656,28 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
             DISPATCH();
         }
 
+        TARGET(CALL_FUNCTION_ADAPTIVE) {
+            assert(cframe.use_tracing == 0);
+            SpecializedCacheEntry *cache = GET_CACHE();
+            nargs = cache->adaptive.original_oparg;
+            if (cache->adaptive.counter == 0) {
+                PyObject *callable = PEEK(nargs+1);
+                next_instr--;
+                if (_Py_Specialize_CallFunction(tstate, callable, nargs, next_instr, cache) < 0) {
+                    goto error;
+                }
+                DISPATCH();
+            }
+            else {
+                STAT_INC(CALL_FUNCTION, deferred);
+                cache->adaptive.counter--;
+                oparg = nargs;
+                kwnames = NULL;
+                postcall_shrink = 1;
+                goto call_function;
+            }
+        }
+
         TARGET(CALL_FUNCTION_EX) {
             PREDICTED(CALL_FUNCTION_EX);
             PyObject *func, *callargs, *kwargs = NULL, *result;

diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
diff --git a/Python/specialize.c b/Python/specialize.c
@@ -232,6 +232,7 @@ static uint8_t adaptive_opcodes[256] = {
     [BINARY_ADD] = BINARY_ADD_ADAPTIVE,
     [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE,
     [STORE_ATTR] = STORE_ATTR_ADAPTIVE,
+    [CALL_FUNCTION] = CALL_FUNCTION_ADAPTIVE,
 };
 
 /* The number of cache entries required for a "family" of instructions. */
@@ -242,6 +243,7 @@ static uint8_t cache_requirements[256] = {
     [BINARY_ADD] = 0,
     [BINARY_SUBSCR] = 0,
     [STORE_ATTR] = 2, /* _PyAdaptiveEntry and _PyAttrCache */
+    [CALL_FUNCTION] = 1 /* _PyAdaptiveEntry */
 };
 
 /* Return the oparg for the cache_offset and instruction index.
@@ -452,6 +454,11 @@ _Py_Quicken(PyCodeObject *code) {
 #define SPEC_FAIL_NON_FUNCTION_SCOPE 11
 #define SPEC_FAIL_DIFFERENT_TYPES 12
 
+/* Calls */
+#define SPEC_FAIL_BUILTIN_FUNCTION 7
+#define SPEC_FAIL_CLASS 8
+#define SPEC_FAIL_PYTHON_FUNCTION 9
+
 
 static int
 specialize_module_load_attr(
@@ -1188,3 +1195,57 @@ _Py_Specialize_BinaryAdd(PyObject *left, PyObject *right, _Py_CODEUNIT *instr)
     assert(!PyErr_Occurred());
     return 0;
 }
+
+static int specialize_c_call(
+    PyObject *callable, int nargs, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache)
+{
+    SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_BUILTIN_FUNCTION);
+    return -1;
+}
+
+static int specialize_class_call(
+    PyObject *callable, int nargs, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache)
+{
+    SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_CLASS);
+    return -1;
+}
+
+static int specialize_py_call(PyThreadState *tstate,
+    PyObject *callable, int nargs, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache)
+{
+    SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_PYTHON_FUNCTION);
+    return -1;
+}
+
+int
+_Py_Specialize_CallFunction(
+    PyThreadState *tstate, PyObject *callable, int nargs,
+    _Py_CODEUNIT *instr, SpecializedCacheEntry *cache)
+{
+    _PyAdaptiveEntry *cache0 = &cache->adaptive;
+    int fail;
+    if (PyCFunction_CheckExact(callable)) {
+        fail = specialize_c_call(callable, nargs, instr, cache);
+    }
+    else if (PyFunction_Check(callable)) {
+        fail = specialize_py_call(tstate, callable, nargs, instr, cache);
+    }
+    else if (PyType_Check(callable)) {
+        fail = specialize_class_call(callable, nargs, instr, cache);
+    }
+    else {
+        SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_OTHER);
+        fail = -1;
+    }
+    if (fail) {
+        STAT_INC(CALL_FUNCTION, specialization_failure);
+        assert(!PyErr_Occurred());
+        cache_backoff(cache0);
+    }
+    else {
+        STAT_INC(CALL_FUNCTION, specialization_success);
+        assert(!PyErr_Occurred());
+        cache0->counter = saturating_start();
+    }
+    return 0;
+}