From ed4b4fd6b841aedd1c9c13f42c153278089cb15f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 6 Aug 2026 17:12:05 +0300 Subject: [PATCH] gh-78416: Support required keyword-only arguments in PyArg_ParseTupleAndKeywords() The new format unit "$$" starts the keyword-only arguments which are required until "|", so that they can be mixed with optional arguments. Unlike "$", it does not depend on whether "|" was specified before it. It was previously rejected as "$ specified twice". Argument Clinic uses it if "$" cannot express the signature, so that such functions can now use the limited C API. It also no longer rejects a required keyword-only parameter after an optional positional one. --- Doc/c-api/arg.rst | 15 +++ Doc/whatsnew/3.16.rst | 8 +- Include/cpython/modsupport.h | 1 + Lib/test/test_capi/test_getargs.py | 77 +++++++++++++++- Lib/test/test_clinic.py | 77 ++++++++++++++++ ...6-08-06-17-11-54.gh-issue-78416.jsJUNi.rst | 3 + Modules/_testcapi/getargs.c | 40 ++++++++ Modules/_testclinic_limited.c | 22 +++++ Modules/clinic/_testclinic_limited.c.h | 34 ++++++- Python/getargs.c | 91 ++++++++++++++----- Tools/c-analyzer/cpython/ignored.tsv | 1 + Tools/clinic/libclinic/codegen.py | 5 + Tools/clinic/libclinic/converter.py | 27 +++++- Tools/clinic/libclinic/dsl_parser.py | 4 + Tools/clinic/libclinic/parse_args.py | 2 - 15 files changed, 377 insertions(+), 30 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-08-06-17-11-54.gh-issue-78416.jsJUNi.rst diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index a878b20c928ea3d..b6660290a9eed2f 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -402,18 +402,33 @@ inside nested parentheses. They are: For example, the format string ``"OO|OO"`` corresponds to the Python signature ``f(a, b, c=None, d=None)``. + :c:func:`PyArg_ParseTupleAndKeywords` only: + after ``$$`` it indicates that the remaining keyword-only arguments are optional. + ``$`` :c:func:`PyArg_ParseTupleAndKeywords` only: Indicates that the remaining arguments in the Python argument list are keyword-only. They are optional if ``|`` was specified before ``$``, and required otherwise. ``|`` cannot be specified after ``$``. + Use ``$$`` to mix required and optional keyword-only arguments. For example, the format string ``"O|O$O"`` corresponds to the Python signature ``f(a, b=None, *, c=None)``, and the format string ``"OO$OO"`` corresponds to ``f(a, b, *, c, d)``. .. versionadded:: 3.3 +``$$`` + :c:func:`PyArg_ParseTupleAndKeywords` only: + Indicates that the remaining arguments in the Python argument list are + keyword-only and required. + They become optional after ``|``. + Unlike ``$``, it does not depend on whether ``|`` was specified before it. + For example, the format string ``"O|O$$O|O"`` corresponds to the Python + signature ``f(a, b=None, *, c, d=None)``. + + .. versionadded:: next + ``:`` The list of format units ends here; the string after the colon is used as the function name in error messages (the "associated value" of the exception that diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 210bcafe65f5e9c..16fda4033391648 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -845,7 +845,13 @@ C API changes New features ------------ -* TODO +* :c:func:`PyArg_ParseTupleAndKeywords` now supports required keyword-only + arguments mixed with optional arguments. + The new format unit ``$$`` starts the keyword-only arguments which are + required until ``|``. + For example, the format string ``"O|O$$O|O"`` corresponds to the Python + signature ``f(a, b=None, *, c, d=None)``. + (Contributed by Serhiy Storchaka in :gh:`78416`.) Porting to Python 3.16 ---------------------- diff --git a/Include/cpython/modsupport.h b/Include/cpython/modsupport.h index cfeee6e8ab3414d..4aef6252a843dc1 100644 --- a/Include/cpython/modsupport.h +++ b/Include/cpython/modsupport.h @@ -31,6 +31,7 @@ typedef struct _PyArg_Parser { int pos; /* number of positional-only arguments */ int min; /* minimal number of arguments */ int max; /* maximal number of positional arguments */ + int minkw; /* index of the first optional keyword-only argument */ PyObject *kwtuple; /* tuple of keyword parameter names */ struct _PyArg_Parser *next; } _PyArg_Parser; diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index bbc09e50eb8e450..7f7f9295a501a04 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -1,6 +1,7 @@ import string import sys import unittest +from functools import partial from test import support from test.support import import_helper from test.support import script_helper @@ -825,6 +826,80 @@ def __hash__(self): getargs_keyword_only(1, 2, **{BadStr("monster"): 666}) +class RequiredKeywordOnly_TestCase(unittest.TestCase): + # "$$" marks the start of the keyword-only arguments, which are required + # until "|", so that required and optional ones can be mixed. + + def parse(self, format, args, kwargs, keywords=('a', 'b', 'c', 'd')): + return _testcapi.parse_tuple_and_keywords(args, kwargs, + format, list(keywords)) + + def test_required_after_optional(self): + # f(a, b=None, *, c, d=None) + parse = partial(self.parse, "O|O$$O|O") + self.assertEqual(parse((1, 2), {'c': 3, 'd': 4}), (1, 2, 3, 4)) + self.assertEqual(parse((1,), {'c': 3}), (1, None, 3, None)) + with self.assertRaisesRegex(TypeError, "missing required argument 'c'"): + parse((1,), {}) + with self.assertRaisesRegex(TypeError, "missing required argument 'c'"): + parse((1,), {'d': 4}) + with self.assertRaisesRegex(TypeError, "at most 2 positional"): + parse((1, 2, 3), {'c': 3}) + + def test_all_keyword_only_required(self): + # f(a, b=None, *, c, d) + parse = partial(self.parse, "O|O$$OO") + self.assertEqual(parse((1,), {'c': 3, 'd': 4}), (1, None, 3, 4)) + with self.assertRaisesRegex(TypeError, "missing required argument 'd'"): + parse((1,), {'c': 3}) + + def test_all_positional_required(self): + # f(a, b, *, c, d=None) + parse = partial(self.parse, "OO$$O|O") + self.assertEqual(parse((1, 2), {'c': 3}), (1, 2, 3, None)) + with self.assertRaisesRegex(TypeError, "missing required argument 'c'"): + parse((1, 2), {}) + + def test_cached_parser(self): + # The same format, parsed once and cached in a _PyArg_Parser. + f = _testcapi.getargs_fast_required_kwonly + self.assertEqual(f(1, 2, c=3, d=4), (1, 2, 3, 4)) + self.assertEqual(f(1, c=3), (1, None, 3, None)) + with self.assertRaisesRegex(TypeError, "missing required argument 'c'"): + f(1) + with self.assertRaisesRegex(TypeError, "missing required argument 'c'"): + f(1, d=4) + + def test_invalid_format(self): + for format, msg in ( + ("O$$O$$O", r"\$ specified twice"), + ("O$$O$O", r"\$ specified twice"), + ("O$$O|O|O", r"\| specified twice"), + ("O$O|O", r"\$ before \|"), + ): + with self.subTest(format=format): + n = format.count('O') + npos = len(format) - len(format.lstrip('O')) + args = tuple(range(npos)) + kwargs = {'abcd'[i]: i for i in range(npos, n)} + with self.assertRaisesRegex(SystemError, msg): + self.parse(format, args, kwargs, 'abcd'[:n]) + + def test_unchanged_meaning_of_dollar(self): + # A single "$" still inherits the state of the positional arguments. + parse = partial(self.parse, "O|O$O", keywords=('a', 'b', 'c')) + self.assertEqual(parse((1,), {}), (1, None, None)) + parse = partial(self.parse, "OO$O", keywords=('a', 'b', 'c')) + self.assertEqual(parse((1, 2), {'c': 3}), (1, 2, 3)) + with self.assertRaisesRegex(TypeError, "missing required argument 'c'"): + parse((1, 2), {}) + # The same with a cached parser. + f = _testcapi.getargs_fast_kwonly + self.assertEqual(f(1, 2, c=3, d=4), (1, 2, 3, 4)) + with self.assertRaisesRegex(TypeError, "missing required argument 'c'"): + f(1, 2) + + class PositionalOnlyAndKeywords_TestCase(unittest.TestCase): from _testcapi import getargs_positional_only_and_keywords as getargs @@ -1269,8 +1344,6 @@ def test_bad_use(self): (1,), {}, '||O', ['a']) self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, (1, 2), {}, '|O|O', ['a', 'b']) - self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, - (), {'a': 1}, '$$O', ['a']) self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, (), {'a': 1, 'b': 2}, '$O$O', ['a', 'b']) self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 1dc1c4eaaaba196..eb3f231d408bc62 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -2207,6 +2207,36 @@ def test_depr_slash_duplicate2(self): err = "Function 'bar': '/ [from 3.14]' must precede '/ [from 3.15]'" self.expect_failure(block, err, lineno=5) + def test_required_keyword_only_after_optional(self): + function = self.parse_function(""" + module foo + foo.bar + a: int + b: int = 0 + * + c: int + d: int = 0 + Docstring. + """) + _, a, b, c, d = function.parameters.values() + self.assertFalse(a.converter.is_optional()) + self.assertTrue(b.converter.is_optional()) + self.assertFalse(c.converter.is_optional()) + self.assertTrue(d.converter.is_optional()) + + def test_optional_before_required_keyword_only(self): + block = """ + module foo + foo.bar + * + a: int = 0 + b: int + Docstring. + """ + err = ("Can't have a parameter without a default ('b') " + "after a parameter with a default!") + self.expect_failure(block, err, lineno=4) + def test_single_slash(self): block = """ module foo @@ -4664,6 +4694,41 @@ def test_limited_capi_float(self): self.assertIn("float f;", generated) self.assertIn("f = (float) PyFloat_AsDouble", generated) + def test_limited_capi_required_keyword_only(self): + block = self.wrap_clinic_input(""" + func + a: object + b: object = None + * + c: object + d: object = None + """) + generated = self.clinic.parse(block) + # "$" cannot express this, only "$$" can. + self.assertIn('"O|O$$O|O:func"', generated) + + def test_limited_capi_optional_after_required_keyword_only(self): + block = self.wrap_clinic_input(""" + func + a: object + * + b: object + c: object = None + """) + generated = self.clinic.parse(block) + self.assertIn('"O$$O|O:func"', generated) + + def test_limited_capi_keyword_only(self): + # "$" is still used if it can express the signature. + block = self.wrap_clinic_input(""" + func + a: object + * + b: object + """) + generated = self.clinic.parse(block) + self.assertIn('"O$O:func"', generated) + def test_limited_capi_double(self): block = self.wrap_clinic_input(""" func @@ -4720,6 +4785,18 @@ def test_my_double_sum(self): with self.assertRaises(TypeError): func(1., "2") + def test_required_kwonly(self): + # test a required keyword-only parameter after an optional one + func = _testclinic_limited.required_kwonly + self.assertEqual(func(1, 2, c=3, d=4), (1, 2, 3, 4)) + self.assertEqual(func(1, c=3), (1, None, 3, None)) + with self.assertRaisesRegex(TypeError, "argument 'c'"): + func(1, 2) + with self.assertRaisesRegex(TypeError, "argument 'c'"): + func(1, 2, d=4) + with self.assertRaises(TypeError): + func(1, 2, 3) + def test_get_file_descriptor(self): # test 'file descriptor' converter: call PyObject_AsFileDescriptor() get_fd = _testclinic_limited.get_file_descriptor diff --git a/Misc/NEWS.d/next/C_API/2026-08-06-17-11-54.gh-issue-78416.jsJUNi.rst b/Misc/NEWS.d/next/C_API/2026-08-06-17-11-54.gh-issue-78416.jsJUNi.rst new file mode 100644 index 000000000000000..e5f678ca1a67b92 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-08-06-17-11-54.gh-issue-78416.jsJUNi.rst @@ -0,0 +1,3 @@ +:c:func:`PyArg_ParseTupleAndKeywords` now supports required keyword-only arguments mixed with optional arguments. +The new format unit ``$$`` starts the keyword-only arguments which are required until ``|``. +For example, the format string ``"O|O$$O|O"`` corresponds to the Python signature ``f(a, b=None, *, c, d=None)``. diff --git a/Modules/_testcapi/getargs.c b/Modules/_testcapi/getargs.c index ee04c760d272132..6a926f0f0621794 100644 --- a/Modules/_testcapi/getargs.c +++ b/Modules/_testcapi/getargs.c @@ -765,6 +765,42 @@ gh_99240_clear_args(PyObject *self, PyObject *args) Py_RETURN_NONE; } +/* f(a, b=None, *, c, d=None) parsed with a cached _PyArg_Parser. */ +static PyObject * +getargs_fast_required_kwonly(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static const char * const keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser parser = { + .format = "O|O$$O|O:getargs_fast_required_kwonly", + .keywords = keywords, + }; + PyObject *a, *b = Py_None, *c, *d = Py_None; + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &parser, + &a, &b, &c, &d)) + { + return NULL; + } + return Py_BuildValue("OOOO", a, b, c, d); +} + +/* f(a, b, *, c, d) parsed with a cached _PyArg_Parser. */ +static PyObject * +getargs_fast_kwonly(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static const char * const keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser parser = { + .format = "OO$OO:getargs_fast_kwonly", + .keywords = keywords, + }; + PyObject *a, *b, *c, *d; + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &parser, + &a, &b, &c, &d)) + { + return NULL; + } + return Py_BuildValue("OOOO", a, b, c, d); +} + static PyMethodDef test_methods[] = { {"get_args", get_args, METH_VARARGS}, {"get_kwargs", _PyCFunction_CAST(get_kwargs), METH_VARARGS|METH_KEYWORDS}, @@ -809,6 +845,10 @@ static PyMethodDef test_methods[] = { {"getargs_z_hash", getargs_z_hash, METH_VARARGS}, {"getargs_z_star", getargs_z_star, METH_VARARGS}, {"parse_tuple_and_keywords", parse_tuple_and_keywords, METH_VARARGS}, + {"getargs_fast_required_kwonly", + _PyCFunction_CAST(getargs_fast_required_kwonly), METH_VARARGS|METH_KEYWORDS}, + {"getargs_fast_kwonly", _PyCFunction_CAST(getargs_fast_kwonly), + METH_VARARGS|METH_KEYWORDS}, {"gh_99240_clear_args", gh_99240_clear_args, METH_VARARGS}, {"test_w_code_invalid", test_w_code_invalid, METH_NOARGS}, {NULL}, diff --git a/Modules/_testclinic_limited.c b/Modules/_testclinic_limited.c index 370433b3e2a0d94..f141499ad007f08 100644 --- a/Modules/_testclinic_limited.c +++ b/Modules/_testclinic_limited.c @@ -129,9 +129,31 @@ static PyMethodDef tester_methods[] = { MY_FLOAT_SUM_METHODDEF MY_DOUBLE_SUM_METHODDEF GET_FILE_DESCRIPTOR_METHODDEF + REQUIRED_KWONLY_METHODDEF {NULL, NULL} }; +/*[clinic input] +required_kwonly + + a: object + b: object = None + * + c: object + d: object = None + +Mix an optional parameter with a required keyword-only one. +[clinic start generated code]*/ + +static PyObject * +required_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyObject *d) +/*[clinic end generated code: output=8e9a974afa614cd6 input=297ec6487f6373dd]*/ +{ + return Py_BuildValue("OOOO", a, b, c, d); +} + + static struct PyModuleDef _testclinic_module = { PyModuleDef_HEAD_INIT, .m_name = "_testclinic_limited", diff --git a/Modules/clinic/_testclinic_limited.c.h b/Modules/clinic/_testclinic_limited.c.h index 94897f4c6dc4276..29f6b26237076a1 100644 --- a/Modules/clinic/_testclinic_limited.c.h +++ b/Modules/clinic/_testclinic_limited.c.h @@ -206,4 +206,36 @@ get_file_descriptor(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=03fd7811c056dc74 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(required_kwonly__doc__, +"required_kwonly($module, /, a, b=None, *, c, d=None)\n" +"--\n" +"\n" +"Mix an optional parameter with a required keyword-only one."); + +#define REQUIRED_KWONLY_METHODDEF \ + {"required_kwonly", (PyCFunction)(void(*)(void))required_kwonly, METH_VARARGS|METH_KEYWORDS, required_kwonly__doc__}, + +static PyObject * +required_kwonly_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyObject *d); + +static PyObject * +required_kwonly(PyObject *module, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + static char *_keywords[] = {"a", "b", "c", "d", NULL}; + PyObject *a; + PyObject *b = Py_None; + PyObject *c; + PyObject *d = Py_None; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O$$O|O:required_kwonly", _keywords, + &a, &b, &c, &d)) + goto exit; + return_value = required_kwonly_impl(module, a, b, c, d); + +exit: + return return_value; +} +/*[clinic end generated code: output=030f62e40d565db2 input=a9049054013a1b77]*/ diff --git a/Python/getargs.c b/Python/getargs.c index 3f423266bff7f47..c84f5ce4e33a586 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1669,6 +1669,15 @@ find_keyword_str(PyObject *kwnames, PyObject *const *kwstack, const char *key) #define IS_END_OF_FORMAT(c) (c == '\0' || c == ';' || c == ':') +/* Whether the keyword-only arguments start as required, i.e. the next + marker in the format string is "$$" rather than "$". */ +static int +starts_required_kwonly(const char *format) +{ + const char *marker = strchr(format, '$'); + return marker != NULL && marker[1] == '$'; +} + static int vgetargskeywords_impl(PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, PyObject *kwnames, @@ -1680,6 +1689,10 @@ vgetargskeywords_impl(PyObject *const *args, Py_ssize_t nargs, const char *fname, *msg, *custom_msg; int min = INT_MAX; int max = INT_MAX; + /* The index of the first optional keyword-only argument. */ + int minkw = INT_MAX; + /* Whether the keyword-only arguments start as required ("$$"). */ + int kwonly_required = 0; int i, pos, len; int skip = 0; Py_ssize_t nkwargs; @@ -1758,20 +1771,21 @@ vgetargskeywords_impl(PyObject *const *args, Py_ssize_t nargs, /* convert tuple args and keyword args in same loop, using kwlist to drive process */ for (i = 0; i < len; i++) { if (*format == '|') { - if (min != INT_MAX) { + if (max != INT_MAX && !kwonly_required) { PyErr_SetString(PyExc_SystemError, - "Invalid format string (| specified twice)"); + "Invalid format string ($ before |)"); return cleanreturn(0, &freelist); } - - min = i; - format++; - - if (max != INT_MAX) { + /* The optional arguments start here: the positional ones before + "$$", the keyword-only ones after it. */ + int *popt = kwonly_required ? &minkw : &min; + if (*popt != INT_MAX) { PyErr_SetString(PyExc_SystemError, - "Invalid format string ($ before |)"); + "Invalid format string (| specified twice)"); return cleanreturn(0, &freelist); } + *popt = i; + format++; } if (*format == '$') { if (max != INT_MAX) { @@ -1782,6 +1796,15 @@ vgetargskeywords_impl(PyObject *const *args, Py_ssize_t nargs, max = i; format++; + if (*format == '$') { + /* "$$" starts the required keyword-only arguments. */ + kwonly_required = 1; + format++; + } + else { + /* "$" inherits the state of the positional arguments. */ + minkw = min; + } if (max < pos) { PyErr_SetString(PyExc_SystemError, @@ -1854,7 +1877,7 @@ vgetargskeywords_impl(PyObject *const *args, Py_ssize_t nargs, continue; } - if (i < min) { + if (i < (i < max ? min : minkw)) { if (i < pos) { assert (min == INT_MAX); assert (max == INT_MAX); @@ -1877,7 +1900,9 @@ vgetargskeywords_impl(PyObject *const *args, Py_ssize_t nargs, * fulfilled and no keyword args left, with no further * validation. XXX Maybe skip this in debug build ? */ - if (!nkwargs && !skip) { + if (!nkwargs && !skip && + (i >= minkw || !starts_required_kwonly(format))) + { return cleanreturn(1, &freelist); } } @@ -2046,7 +2071,7 @@ scan_keywords(const char * const *keywords, int *ptotal, int *pposonly) static int parse_format(const char *format, int total, int npos, const char **pfname, const char **pcustommsg, - int *pmin, int *pmax) + int *pmin, int *pmax, int *pminkw) { /* grab the function name or custom error msg first (mutually exclusive) */ const char *custommsg; @@ -2064,19 +2089,26 @@ parse_format(const char *format, int total, int npos, int min = INT_MAX; int max = INT_MAX; + /* The index of the first optional keyword-only argument. */ + int minkw = INT_MAX; + /* Whether the keyword-only arguments start as required ("$$"). */ + int kwonly_required = 0; for (int i = 0; i < total; i++) { if (*format == '|') { - if (min != INT_MAX) { + if (max != INT_MAX && !kwonly_required) { PyErr_SetString(PyExc_SystemError, - "Invalid format string (| specified twice)"); + "Invalid format string ($ before |)"); return -1; } - if (max != INT_MAX) { + /* The optional arguments start here: the positional ones before + "$$", the keyword-only ones after it. */ + int *popt = kwonly_required ? &minkw : &min; + if (*popt != INT_MAX) { PyErr_SetString(PyExc_SystemError, - "Invalid format string ($ before |)"); + "Invalid format string (| specified twice)"); return -1; } - min = i; + *popt = i; format++; } if (*format == '$') { @@ -2092,6 +2124,15 @@ parse_format(const char *format, int total, int npos, } max = i; format++; + if (*format == '$') { + /* "$$" starts the required keyword-only arguments. */ + kwonly_required = 1; + format++; + } + else { + /* "$" inherits the state of the positional arguments. */ + minkw = min; + } } if (IS_END_OF_FORMAT(*format)) { PyErr_Format(PyExc_SystemError, @@ -2107,6 +2148,8 @@ parse_format(const char *format, int total, int npos, return -1; } } + /* Without a marker the optional arguments are not keyword-only. */ + minkw = Py_MIN((max == INT_MAX) ? min : minkw, total); min = Py_MIN(min, total); max = Py_MIN(max, total); @@ -2121,6 +2164,7 @@ parse_format(const char *format, int total, int npos, *pcustommsg = custommsg; *pmin = min; *pmax = max; + *pminkw = minkw; return 0; } @@ -2164,11 +2208,11 @@ _parser_init(void *arg) } const char *fname, *custommsg = NULL; - int min = 0, max = 0; + int min = 0, max = 0, minkw = 0; if (parser->format) { assert(parser->fname == NULL); if (parse_format(parser->format, len, pos, - &fname, &custommsg, &min, &max) < 0) { + &fname, &custommsg, &min, &max, &minkw) < 0) { return -1; } } @@ -2211,6 +2255,7 @@ _parser_init(void *arg) parser->custom_msg = custommsg; parser->min = min; parser->max = max; + parser->minkw = minkw; parser->kwtuple = kwtuple; parser->is_kwtuple_owned = owned; @@ -2380,7 +2425,11 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, if (*format == '|') { format++; } - if (*format == '$') { + while (*format == '$') { + format++; + } + if (*format == '|') { + /* Optional keyword-only arguments after "$$". */ format++; } assert(!IS_END_OF_FORMAT(*format)); @@ -2418,7 +2467,7 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, continue; } - if (i < parser->min) { + if (i < (i < parser->max ? parser->min : parser->minkw)) { /* Less arguments than required */ if (i < pos) { int min = Py_MIN(pos, parser->min); @@ -2446,7 +2495,7 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, * fulfilled and no keyword args left, with no further * validation. XXX Maybe skip this in debug build ? */ - if (!nkwargs) { + if (!nkwargs && i >= parser->minkw) { return cleanreturn(1, &freelist); } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 4c143164650a2ba..50c9967db33a195 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -785,6 +785,7 @@ Modules/_zstd/_zstdmodule.c - _zstdmodule - Modules/clinic/md5module.c.h _md5_md5 _keywords - Modules/clinic/grpmodule.c.h grp_getgrgid _keywords - Modules/clinic/grpmodule.c.h grp_getgrnam _keywords - +Modules/clinic/_testclinic_limited.c.h required_kwonly _keywords - Objects/object.c - constants static PyObject*[] Objects/dictobject.c - PyFrozenDict_Type - diff --git a/Tools/clinic/libclinic/codegen.py b/Tools/clinic/libclinic/codegen.py index b2f1db6f8ef8da7..556b839f24e4ec0 100644 --- a/Tools/clinic/libclinic/codegen.py +++ b/Tools/clinic/libclinic/codegen.py @@ -38,6 +38,11 @@ def __init__(self) -> None: # Should be individual strings that will get self.format_units: list[str] = [] + # The marker which starts the keyword-only parameters, + # and whether the optional ones already started. + self.kwonly_marker = '' + self.kwonly_optional = False + # The varargs arguments for PyArg_ParseTuple. self.parse_arguments: list[str] = [] diff --git a/Tools/clinic/libclinic/converter.py b/Tools/clinic/libclinic/converter.py index c10235237d4b716..8b8bd3956679fb9 100644 --- a/Tools/clinic/libclinic/converter.py +++ b/Tools/clinic/libclinic/converter.py @@ -327,10 +327,31 @@ def _render_non_self( data.keywords.append(parameter.name) # format_units - if self.is_optional() and '|' not in data.format_units: + if not parameter.is_keyword_only(): + if self.is_optional() and '|' not in data.format_units: + data.format_units.append('|') + elif not data.kwonly_marker: + # The first keyword-only parameter. "$" only inherits the + # state of the positional parameters, "$$" sets its own. + if self.is_optional(): + if '|' not in data.format_units: + data.format_units.append('|') + data.kwonly_marker = '$' + data.kwonly_optional = True + elif ('|' in data.format_units + or any(p.converter.is_optional() + for p in parameter.function.render_parameters + if p.is_keyword_only())): + # Required before or after an optional parameter: + # only "$$" can express this. + data.kwonly_marker = '$$' + else: + data.kwonly_marker = '$' + data.format_units.append(data.kwonly_marker) + elif self.is_optional() and not data.kwonly_optional: + # The first optional keyword-only parameter after "$$". + data.kwonly_optional = True data.format_units.append('|') - if parameter.is_keyword_only() and '$' not in data.format_units: - data.format_units.append('$') data.format_units.append(self.format_unit) # parse_arguments diff --git a/Tools/clinic/libclinic/dsl_parser.py b/Tools/clinic/libclinic/dsl_parser.py index 4dcbc815cc6f25b..824388bdd79a52e 100644 --- a/Tools/clinic/libclinic/dsl_parser.py +++ b/Tools/clinic/libclinic/dsl_parser.py @@ -1179,6 +1179,10 @@ def parse_star(self, function: Function, version: VersionTuple | None) -> None: self.check_previous_star() self.check_remaining_star() self.keyword_only = True + # A keyword-only parameter can be required even if + # a positional one is optional. + if self.parameter_state is ParamState.OPTIONAL: + self.parameter_state = ParamState.REQUIRED else: if self.keyword_only: fail(f"Function {function.name!r}: '* [from ...]' must precede '*'") diff --git a/Tools/clinic/libclinic/parse_args.py b/Tools/clinic/libclinic/parse_args.py index 2ad1e94ea2b4c79..e8e570d63bb028d 100644 --- a/Tools/clinic/libclinic/parse_args.py +++ b/Tools/clinic/libclinic/parse_args.py @@ -341,8 +341,6 @@ def init_limited_capi(self) -> None: self.limited_capi = self.codegen.limited_capi if self.limited_capi and ( (self.varpos and self.pos_only < len(self.parameters)) or - (any(p.is_optional() for p in self.parameters) and - any(p.is_keyword_only() and not p.is_optional() for p in self.parameters)) or any(c.broken_limited_capi for c in self.converters)): warn(f"Function {self.func.full_name} cannot use limited C API") self.limited_capi = False