py/lexer: Add support for underscores in numeric literals.

dpgeorge · dpgeorge · commit 6a445b60fad8 · 2018-06-12T12:17:43.000+10:00
This is a very convenient feature introduced in Python 3.6 by PEP 515.
diff --git a/py/lexer.c b/py/lexer.c
@@ -590,6 +590,8 @@ void mp_lexer_to_next(mp_lexer_t *lex) {
                 }
                 vstr_add_char(&lex->vstr, CUR_CHAR(lex));
                 next_char(lex);
+            } else if (is_char(lex, '_')) {
+                next_char(lex);
             } else {
                 break;
             }
diff --git a/py/parsenum.c b/py/parsenum.c
@@ -83,6 +83,8 @@ mp_obj_t mp_parse_num_integer(const char *restrict str_, size_t len, int base, m
         mp_uint_t dig = *str;
         if ('0' <= dig && dig <= '9') {
             dig -= '0';
+        } else if (dig == '_') {
+            continue;
         } else {
             dig |= 0x20; // make digit lower-case
             if ('a' <= dig && dig <= 'z') {
@@ -273,6 +275,8 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool
             } else if (allow_imag && (dig | 0x20) == 'j') {
                 imag = true;
                 break;
+            } else if (dig == '_') {
+                continue;
             } else {
                 // unknown character
                 str--;
diff --git a/tests/basics/python36.py b/tests/basics/python36.py
@@ -0,0 +1,10 @@
+# tests for things that only Python 3.6 supports
+
+# underscores in numeric literals
+print(100_000)
+print(0b1010_0101)
+print(0xff_ff)
+
+# underscore supported by int constructor
+print(int('1_2_3'))
+print(int('0o1_2_3', 8))
diff --git a/tests/basics/python36.py.exp b/tests/basics/python36.py.exp
@@ -0,0 +1,5 @@
+100000
+165
+65535
+123
+83
diff --git a/tests/float/python36.py b/tests/float/python36.py
@@ -0,0 +1,10 @@
+# tests for things that only Python 3.6 supports, needing floats
+
+# underscores in numeric literals
+print(1_000.1_8)
+print('%.2g' % 1e1_2)
+
+# underscore supported by int/float constructors
+print(float('1_2_3'))
+print(float('1_2_3.4'))
+print('%.2g' % float('1e1_3'))
diff --git a/tests/float/python36.py.exp b/tests/float/python36.py.exp
@@ -0,0 +1,5 @@
+1000.18
+1e+12
+123.0
+123.4
+1e+13

Original file line number	Diff line number	Diff line change
`@@ -590,6 +590,8 @@ void mp_lexer_to_next(mp_lexer_t *lex) {`
`590`	`590`	`}`
`591`	`591`	`vstr_add_char(&lex->vstr, CUR_CHAR(lex));`
`592`	`592`	`next_char(lex);`
	`593`	`+ } else if (is_char(lex, '_')) {`
	`594`	`+ next_char(lex);`
`593`	`595`	`} else {`
`594`	`596`	`break;`
`595`	`597`	`}`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +100000
 +165
 +65535
 +123
 +83