From 62615675d6ae0be66fc78d1210c732a88c5abac5 Mon Sep 17 00:00:00 2001
From: Nevada Sanchez <sanchez.nevada@gmail.com>
Date: Tue, 21 Mar 2017 12:59:35 -0400
Subject: [PATCH 1/3] Allow underscores in numeric literals in lib2to3.

---
 Lib/lib2to3/pgen2/tokenize.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index d14db60f7da89ea..f5c132aac311b63 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -54,10 +54,10 @@ def maybe(*choices): return group(*choices) + '?'
 Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
 Name = r'[a-zA-Z_]\w*'
 
-Binnumber = r'0[bB][01]*'
-Hexnumber = r'0[xX][\da-fA-F]*[lL]?'
-Octnumber = r'0[oO]?[0-7]*[lL]?'
-Decnumber = r'[1-9]\d*[lL]?'
+Binnumber = r'0[bB][01_]*'
+Hexnumber = r'0[xX][\da-fA-F_]*[lL]?'
+Octnumber = r'0[oO]?[0-7_]*[lL]?'
+Decnumber = r'[1-9][\d_]*[lL]?'
 Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber)
 Exponent = r'[eE][-+]?\d+'
 Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)

From 2fd46ccb325c20ee22e9ac66699de8adf46c75c4 Mon Sep 17 00:00:00 2001
From: Nevada Sanchez <sanchez.nevada@gmail.com>
Date: Wed, 22 Mar 2017 10:51:34 -0400
Subject: [PATCH 2/3] Stricter literal parsing for Python 3.6 in
 lib2to3.pgen2.tokenize.

---
 Lib/lib2to3/pgen2/tokenize.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index f5c132aac311b63..fba0fa2c00dccd9 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -54,16 +54,16 @@ def maybe(*choices): return group(*choices) + '?'
 Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
 Name = r'[a-zA-Z_]\w*'
 
-Binnumber = r'0[bB][01_]*'
-Hexnumber = r'0[xX][\da-fA-F_]*[lL]?'
-Octnumber = r'0[oO]?[0-7_]*[lL]?'
-Decnumber = r'[1-9][\d_]*[lL]?'
+Binnumber = r'0[bB]_?[01]+(?:_[01]+)*'
+Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?'
+Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?'
+Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?')
 Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber)
-Exponent = r'[eE][-+]?\d+'
-Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
-Expfloat = r'\d+' + Exponent
+Exponent = r'[eE][-+]?\d+(?:_\d+)*'
+Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent)
+Expfloat = r'\d+(?:_\d+)*' + Exponent
 Floatnumber = group(Pointfloat, Expfloat)
-Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]')
+Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]')
 Number = group(Imagnumber, Floatnumber, Intnumber)
 
 # Tail end of ' string.

From b6995a0b469404cc861c1f6aad93a53952bee43b Mon Sep 17 00:00:00 2001
From: Nevada Sanchez <sanchez.nevada@gmail.com>
Date: Thu, 23 Mar 2017 16:19:57 -0400
Subject: [PATCH 3/3] Add test case for underscores in literals in Python 3.

---
 Lib/lib2to3/tests/data/py3_test_grammar.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/Lib/lib2to3/tests/data/py3_test_grammar.py b/Lib/lib2to3/tests/data/py3_test_grammar.py
index cf31a5411a855a7..0b9bee0ab42ea62 100644
--- a/Lib/lib2to3/tests/data/py3_test_grammar.py
+++ b/Lib/lib2to3/tests/data/py3_test_grammar.py
@@ -72,6 +72,28 @@ def testLongIntegers(self):
         x = 0b100000000000000000000000000000000000000000000000000000000000000000000
         x = 0B111111111111111111111111111111111111111111111111111111111111111111111
 
+    def testUnderscoresInNumbers(self):
+        # Integers
+        x = 1_0
+        x = 123_456_7_89
+        x = 0xabc_123_4_5
+        x = 0X_abc_123
+        x = 0B11_01
+        x = 0b_11_01
+        x = 0o45_67
+        x = 0O_45_67
+
+        # Floats
+        x = 3_1.4
+        x = 03_1.4
+        x = 3_1.
+        x = .3_1
+        x = 3.1_4
+        x = 0_3.1_4
+        x = 3e1_4
+        x = 3_1e+4_1
+        x = 3_1E-4_1
+
     def testFloats(self):
         x = 3.14
         x = 314.