Skip to content

Commit 80cb31b

Browse files
larryhastingslarryhastings
committed
Issue #20530: Argument Clinic's signature format has been revised again.
The new syntax is highly human readable while still preventing false positives. The syntax also extends Python syntax to denote "self" and positional-only parameters, allowing inspect.Signature objects to be totally accurate for all supported builtins in Python 3.4.
1 parent 5f69149 commit 80cb31b

28 files changed

Lines changed: 825 additions & 327 deletions

‎Include/object.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -496,8 +496,8 @@ PyAPI_FUNC(unsigned int) PyType_ClearCache(void);
496496
PyAPI_FUNC(void) PyType_Modified(PyTypeObject *);
497497

498498
#ifndef Py_LIMITED_API
499-
PyAPI_FUNC(PyObject *) _PyType_GetDocFromInternalDoc(const char *);
500-
PyAPI_FUNC(PyObject *) _PyType_GetTextSignatureFromInternalDoc(const char *);
499+
PyAPI_FUNC(PyObject *) _PyType_GetDocFromInternalDoc(const char *, const char *);
500+
PyAPI_FUNC(PyObject *) _PyType_GetTextSignatureFromInternalDoc(const char *, const char *);
501501
#endif
502502

503503
/* Generic operations on objects */

‎Lib/inspect.py‎

Lines changed: 92 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import re
4040
import sys
4141
import tokenize
42+
import token
4243
import types
4344
import warnings
4445
import functools
@@ -1648,25 +1649,88 @@ def _signature_get_bound_param(spec):
16481649
return spec[2:pos]
16491650

16501651

1652+
def _signature_strip_non_python_syntax(signature):
1653+
"""
1654+
Takes a signature in Argument Clinic's extended signature format.
1655+
Returns a tuple of three things:
1656+
* that signature re-rendered in standard Python syntax,
1657+
* the index of the "self" parameter (generally 0), or None if
1658+
the function does not have a "self" parameter, and
1659+
* the index of the last "positional only" parameter,
1660+
or None if the signature has no positional-only parameters.
1661+
"""
1662+
1663+
if not signature:
1664+
return signature, None, None
1665+
1666+
self_parameter = None
1667+
last_positional_only = None
1668+
1669+
lines = [l.encode('ascii') for l in signature.split('\n')]
1670+
generator = iter(lines).__next__
1671+
token_stream = tokenize.tokenize(generator)
1672+
1673+
delayed_comma = False
1674+
skip_next_comma = False
1675+
text = []
1676+
add = text.append
1677+
1678+
current_parameter = 0
1679+
OP = token.OP
1680+
ERRORTOKEN = token.ERRORTOKEN
1681+
1682+
# token stream always starts with ENCODING token, skip it
1683+
t = next(token_stream)
1684+
assert t.type == tokenize.ENCODING
1685+
1686+
for t in token_stream:
1687+
type, string = t.type, t.string
1688+
1689+
if type == OP:
1690+
if string == ',':
1691+
if skip_next_comma:
1692+
skip_next_comma = False
1693+
else:
1694+
assert not delayed_comma
1695+
delayed_comma = True
1696+
current_parameter += 1
1697+
continue
1698+
1699+
if string == '/':
1700+
assert not skip_next_comma
1701+
assert last_positional_only is None
1702+
skip_next_comma = True
1703+
last_positional_only = current_parameter - 1
1704+
continue
1705+
1706+
if (type == ERRORTOKEN) and (string == '$'):
1707+
assert self_parameter is None
1708+
self_parameter = current_parameter
1709+
continue
1710+
1711+
if delayed_comma:
1712+
delayed_comma = False
1713+
if not ((type == OP) and (string == ')')):
1714+
add(', ')
1715+
add(string)
1716+
if (string == ','):
1717+
add(' ')
1718+
clean_signature = ''.join(text)
1719+
return clean_signature, self_parameter, last_positional_only
1720+
1721+
16511722
def _signature_fromstr(cls, obj, s):
16521723
# Internal helper to parse content of '__text_signature__'
16531724
# and return a Signature based on it
16541725
Parameter = cls._parameter_cls
16551726

1656-
if s.endswith("/)"):
1657-
kind = Parameter.POSITIONAL_ONLY
1658-
s = s[:-2] + ')'
1659-
else:
1660-
kind = Parameter.POSITIONAL_OR_KEYWORD
1661-
1662-
first_parameter_is_self = s.startswith("($")
1663-
if first_parameter_is_self:
1664-
s = '(' + s[2:]
1727+
clean_signature, self_parameter, last_positional_only = \
1728+
_signature_strip_non_python_syntax(s)
16651729

1666-
s = "def foo" + s + ": pass"
1730+
program = "def foo" + clean_signature + ": pass"
16671731

16681732
try:
1669-
module = ast.parse(s)
1733+
module = ast.parse(program)
16701734
except SyntaxError:
16711735
module = None
16721736

@@ -1750,8 +1814,14 @@ def p(name_node, default_node, default=empty):
17501814
args = reversed(f.args.args)
17511815
defaults = reversed(f.args.defaults)
17521816
iter = itertools.zip_longest(args, defaults, fillvalue=None)
1753-
for name, default in reversed(list(iter)):
1817+
if last_positional_only is not None:
1818+
kind = Parameter.POSITIONAL_ONLY
1819+
else:
1820+
kind = Parameter.POSITIONAL_OR_KEYWORD
1821+
for i, (name, default) in enumerate(reversed(list(iter))):
17541822
p(name, default)
1823+
if i == last_positional_only:
1824+
kind = Parameter.POSITIONAL_OR_KEYWORD
17551825

17561826
# *args
17571827
if f.args.vararg:
@@ -1768,7 +1838,7 @@ def p(name_node, default_node, default=empty):
17681838
kind = Parameter.VAR_KEYWORD
17691839
p(f.args.kwarg, empty)
17701840

1771-
if first_parameter_is_self:
1841+
if self_parameter is not None:
17721842
assert parameters
17731843
if getattr(obj, '__self__', None):
17741844
# strip off self, it's already been bound
@@ -1861,12 +1931,13 @@ def signature(obj):
18611931
# At this point we know, that `obj` is a class, with no user-
18621932
# defined '__init__', '__new__', or class-level '__call__'
18631933

1864-
for base in obj.__mro__:
1934+
for base in obj.__mro__[:-1]:
18651935
# Since '__text_signature__' is implemented as a
18661936
# descriptor that extracts text signature from the
18671937
# class docstring, if 'obj' is derived from a builtin
18681938
# class, its own '__text_signature__' may be 'None'.
1869-
# Therefore, we go through the MRO to find the first
1939+
# Therefore, we go through the MRO (except the last
1940+
# class in there, which is 'object') to find the first
18701941
# class with non-empty text signature.
18711942
try:
18721943
text_sig = base.__text_signature__
@@ -1881,13 +1952,7 @@ def signature(obj):
18811952
# No '__text_signature__' was found for the 'obj' class.
18821953
# Last option is to check if its '__init__' is
18831954
# object.__init__ or type.__init__.
1884-
if type in obj.__mro__:
1885-
# 'obj' is a metaclass without user-defined __init__
1886-
# or __new__.
1887-
if obj.__init__ is type.__init__:
1888-
# Return a signature of 'type' builtin.
1889-
return signature(type)
1890-
else:
1955+
if type not in obj.__mro__:
18911956
# We have a class (not metaclass), but no user-defined
18921957
# __init__ or __new__ for it
18931958
if obj.__init__ is object.__init__:
@@ -1901,7 +1966,11 @@ def signature(obj):
19011966
# infinite recursion (and even potential segfault)
19021967
call = _signature_get_user_defined_method(type(obj), '__call__')
19031968
if call is not None:
1904-
sig = signature(call)
1969+
try:
1970+
sig = signature(call)
1971+
except ValueError as ex:
1972+
msg = 'no signature found for {!r}'.format(obj)
1973+
raise ValueError(msg) from ex
19051974

19061975
if sig is not None:
19071976
# For classes and objects we skip the first parameter of their

‎Lib/test/test_capi.py‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,20 +126,29 @@ def test_docstring_signature_parsing(self):
126126
self.assertEqual(_testcapi.docstring_no_signature.__text_signature__, None)
127127

128128
self.assertEqual(_testcapi.docstring_with_invalid_signature.__doc__,
129-
"sig= (module, boo)\n"
129+
"docstring_with_invalid_signature($module, /, boo)\n"
130130
"\n"
131131
"This docstring has an invalid signature."
132132
)
133133
self.assertEqual(_testcapi.docstring_with_invalid_signature.__text_signature__, None)
134134

135+
self.assertEqual(_testcapi.docstring_with_invalid_signature2.__doc__,
136+
"docstring_with_invalid_signature2($module, /, boo)\n"
137+
"\n"
138+
"--\n"
139+
"\n"
140+
"This docstring also has an invalid signature."
141+
)
142+
self.assertEqual(_testcapi.docstring_with_invalid_signature2.__text_signature__, None)
143+
135144
self.assertEqual(_testcapi.docstring_with_signature.__doc__,
136145
"This docstring has a valid signature.")
137-
self.assertEqual(_testcapi.docstring_with_signature.__text_signature__, "(module, sig)")
146+
self.assertEqual(_testcapi.docstring_with_signature.__text_signature__, "($module, /, sig)")
138147

139148
self.assertEqual(_testcapi.docstring_with_signature_and_extra_newlines.__doc__,
140-
"This docstring has a valid signature and some extra newlines.")
149+
"\nThis docstring has a valid signature and some extra newlines.")
141150
self.assertEqual(_testcapi.docstring_with_signature_and_extra_newlines.__text_signature__,
142-
"(module, parameter)")
151+
"($module, /, parameter)")
143152

144153

145154
@unittest.skipUnless(threading, 'Threading required for this test.')

‎Lib/test/test_inspect.py‎

Lines changed: 65 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1684,7 +1684,6 @@ def p(name): return signature.parameters[name].default
16841684
self.assertEqual(p('sys'), sys.maxsize)
16851685
self.assertEqual(p('exp'), sys.maxsize - 1)
16861686

1687-
test_callable(type)
16881687
test_callable(object)
16891688

16901689
# normal method
@@ -1710,9 +1709,12 @@ def p(name): return signature.parameters[name].default
17101709
# support for 'method-wrapper'
17111710
test_callable(min.__call__)
17121711

1713-
class ThisWorksNow:
1714-
__call__ = type
1715-
test_callable(ThisWorksNow())
1712+
# This doesn't work now.
1713+
# (We don't have a valid signature for "type" in 3.4)
1714+
with self.assertRaisesRegex(ValueError, "no signature found"):
1715+
class ThisWorksNow:
1716+
__call__ = type
1717+
test_callable(ThisWorksNow())
17161718

17171719
@cpython_only
17181720
@unittest.skipIf(MISSING_C_DOCSTRINGS,
@@ -2213,11 +2215,11 @@ class D(C): pass
22132215

22142216
# Test meta-classes without user-defined __init__ or __new__
22152217
class C(type): pass
2216-
self.assertEqual(str(inspect.signature(C)),
2217-
'(object_or_name, bases, dict)')
22182218
class D(C): pass
2219-
self.assertEqual(str(inspect.signature(D)),
2220-
'(object_or_name, bases, dict)')
2219+
with self.assertRaisesRegex(ValueError, "callable.*is not supported"):
2220+
self.assertEqual(inspect.signature(C), None)
2221+
with self.assertRaisesRegex(ValueError, "callable.*is not supported"):
2222+
self.assertEqual(inspect.signature(D), None)
22212223

22222224
@unittest.skipIf(MISSING_C_DOCSTRINGS,
22232225
"Signature information for builtins requires docstrings")
@@ -2768,6 +2770,61 @@ def test_signature_get_bound_param(self):
27682770
self.assertEqual(getter('($self, obj)'), 'self')
27692771
self.assertEqual(getter('($cls, /, obj)'), 'cls')
27702772

2773+
def _strip_non_python_syntax(self, input,
2774+
clean_signature, self_parameter, last_positional_only):
2775+
computed_clean_signature, \
2776+
computed_self_parameter, \
2777+
computed_last_positional_only = \
2778+
inspect._signature_strip_non_python_syntax(input)
2779+
self.assertEqual(computed_clean_signature, clean_signature)
2780+
self.assertEqual(computed_self_parameter, self_parameter)
2781+
self.assertEqual(computed_last_positional_only, last_positional_only)
2782+
2783+
def test_signature_strip_non_python_syntax(self):
2784+
self._strip_non_python_syntax(
2785+
"($module, /, path, mode, *, dir_fd=None, " +
2786+
"effective_ids=False,\n follow_symlinks=True)",
2787+
"(module, path, mode, *, dir_fd=None, " +
2788+
"effective_ids=False, follow_symlinks=True)",
2789+
0,
2790+
0)
2791+
2792+
self._strip_non_python_syntax(
2793+
"($module, word, salt, /)",
2794+
"(module, word, salt)",
2795+
0,
2796+
2)
2797+
2798+
self._strip_non_python_syntax(
2799+
"(x, y=None, z=None, /)",
2800+
"(x, y=None, z=None)",
2801+
None,
2802+
2)
2803+
2804+
self._strip_non_python_syntax(
2805+
"(x, y=None, z=None)",
2806+
"(x, y=None, z=None)",
2807+
None,
2808+
None)
2809+
2810+
self._strip_non_python_syntax(
2811+
"(x,\n y=None,\n z = None )",
2812+
"(x, y=None, z=None)",
2813+
None,
2814+
None)
2815+
2816+
self._strip_non_python_syntax(
2817+
"",
2818+
"",
2819+
None,
2820+
None)
2821+
2822+
self._strip_non_python_syntax(
2823+
None,
2824+
None,
2825+
None,
2826+
None)
2827+
27712828

27722829
class TestUnwrap(unittest.TestCase):
27732830

‎Misc/NEWS‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -200,18 +200,24 @@ Tests
200200
Tools/Demos
201201
-----------
202202

203-
- #Issue 20456: Argument Clinic now observes the C preprocessor conditional
203+
- Issue #20530: Argument Clinic's signature format has been revised again.
204+
The new syntax is highly human readable while still preventing false
205+
positives. The syntax also extends Python syntax to denote "self" and
206+
positional-only parameters, allowing inspect.Signature objects to be
207+
totally accurate for all supported builtins in Python 3.4.
208+
209+
- Issue #20456: Argument Clinic now observes the C preprocessor conditional
204210
compilation statements of the C files it parses. When a Clinic block is
205211
inside a conditional code, it adjusts its output to match, including
206212
automatically generating an empty methoddef macro.
207213

208-
- #Issue 20456: Cloned functions in Argument Clinic now use the correct
214+
- Issue #20456: Cloned functions in Argument Clinic now use the correct
209215
name, not the name of the function they were cloned from, for text
210216
strings inside generated code.
211217

212-
- #Issue 20456: Fixed Argument Clinic's test suite and "--converters" feature.
218+
- Issue #20456: Fixed Argument Clinic's test suite and "--converters" feature.
213219

214-
- #Issue 20456: Argument Clinic now allows specifying different names
220+
- Issue #20456: Argument Clinic now allows specifying different names
215221
for a parameter in Python and C, using "as" on the parameter line.
216222

217223
- Issue #20326: Argument Clinic now uses a simple, unique signature to

‎Modules/_cryptmodule.c‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@ results for a given *word*.
3030
[clinic start generated code]*/
3131

3232
PyDoc_STRVAR(crypt_crypt__doc__,
33-
"sig=($module, word, salt)\n"
33+
"crypt($module, word, salt, /)\n"
34+
"--\n"
35+
"\n"
3436
"Hash a *word* with the given *salt* and return the hashed password.\n"
3537
"\n"
3638
"*word* will usually be a user\'s password. *salt* (either a random 2 or 16\n"
@@ -63,7 +65,7 @@ crypt_crypt(PyModuleDef *module, PyObject *args)
6365

6466
static PyObject *
6567
crypt_crypt_impl(PyModuleDef *module, const char *word, const char *salt)
66-
/*[clinic end generated code: output=c7443257e03fca92 input=4d93b6d0f41fbf58]*/
68+
/*[clinic end generated code: output=3eaacdf994a6ff23 input=4d93b6d0f41fbf58]*/
6769
{
6870
/* On some platforms (AtheOS) crypt returns NULL for an invalid
6971
salt. Return None in that case. XXX Maybe raise an exception? */

‎Modules/_datetimemodule.c‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4159,7 +4159,9 @@ If no tz is specified, uses local timezone.
41594159
[clinic start generated code]*/
41604160

41614161
PyDoc_STRVAR(datetime_datetime_now__doc__,
4162-
"sig=($type, tz=None)\n"
4162+
"now($type, /, tz=None)\n"
4163+
"--\n"
4164+
"\n"
41634165
"Returns new datetime object representing current time local to tz.\n"
41644166
"\n"
41654167
" tz\n"
@@ -4192,7 +4194,7 @@ datetime_datetime_now(PyTypeObject *type, PyObject *args, PyObject *kwargs)
41924194

41934195
static PyObject *
41944196
datetime_datetime_now_impl(PyTypeObject *type, PyObject *tz)
4195-
/*[clinic end generated code: output=c8a47308483e579a input=80d09869c5267d00]*/
4197+
/*[clinic end generated code: output=583c5637e3c843fa input=80d09869c5267d00]*/
41964198
{
41974199
PyObject *self;
41984200

0 commit comments

Comments
 (0)