py/lexer: Properly classify floats that look like hex numbers.

dpgeorge · dpgeorge · commit 2b000474d935 · 2015-09-07T17:33:44.000+01:00
Eg 0e0 almost looks like a hex number but in fact is a float.
diff --git a/py/lexer.c b/py/lexer.c
@@ -105,8 +105,9 @@ STATIC bool is_following_digit(mp_lexer_t *lex) {
     return unichar_isdigit(lex->chr1);
 }
 
-STATIC bool is_following_letter(mp_lexer_t *lex) {
-    return unichar_isalpha(lex->chr1);
+STATIC bool is_following_base_char(mp_lexer_t *lex) {
+    const unichar chr1 = lex->chr1 | 0x20;
+    return chr1 == 'b' || chr1 == 'o' || chr1 == 'x';
 }
 
 STATIC bool is_following_odigit(mp_lexer_t *lex) {
@@ -541,7 +542,7 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, bool first_token) {
             lex->tok_kind = MP_TOKEN_FLOAT_OR_IMAG;
         } else {
             lex->tok_kind = MP_TOKEN_INTEGER;
-            if (is_char(lex, '0') && is_following_letter(lex)) {
+            if (is_char(lex, '0') && is_following_base_char(lex)) {
                 forced_integer = true;
             }
         }
diff --git a/tests/float/float1.py b/tests/float/float1.py
@@ -4,6 +4,9 @@
 print(.12)
 print(1.)
 print(1.2)
+print(0e0)
+print(0e+0)
+print(0e-0)
 
 # float construction
 print(float(1.2))

Original file line number	Diff line number	Diff line change
`@@ -105,8 +105,9 @@ STATIC bool is_following_digit(mp_lexer_t *lex) {`
`105`	`105`	`return unichar_isdigit(lex->chr1);`
`106`	`106`	`}`
`107`	`107`
`108`		`-STATIC bool is_following_letter(mp_lexer_t *lex) {`
`109`		`- return unichar_isalpha(lex->chr1);`
	`108`	`+STATIC bool is_following_base_char(mp_lexer_t *lex) {`
	`109`	`+ const unichar chr1 = lex->chr1 \| 0x20;`
	`110`	`+ return chr1 == 'b' \|\| chr1 == 'o' \|\| chr1 == 'x';`
`110`	`111`	`}`
`111`	`112`
`112`	`113`	`STATIC bool is_following_odigit(mp_lexer_t *lex) {`
`@@ -541,7 +542,7 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, bool first_token) {`
`541`	`542`	`lex->tok_kind = MP_TOKEN_FLOAT_OR_IMAG;`
`542`	`543`	`} else {`
`543`	`544`	`lex->tok_kind = MP_TOKEN_INTEGER;`
`544`		`- if (is_char(lex, '0') && is_following_letter(lex)) {`
	`545`	`+ if (is_char(lex, '0') && is_following_base_char(lex)) {`
`545`	`546`	`forced_integer = true;`
`546`	`547`	`}`
`547`	`548`	`}`