From 7f193fa020eae96638569d232242a22873222bbe Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Mon, 3 May 2021 14:40:19 +0300 Subject: [PATCH 01/30] modernize ColorDelegator.recolorize_main() code --- Lib/idlelib/colorizer.py | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index 3c527409731afa..dfcf4fe6999a58 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -231,14 +231,10 @@ def recolorize(self): def recolorize_main(self): "Evaluate text and apply colorizing tags." next = "1.0" - while True: - item = self.tag_nextrange("TODO", next) - if not item: - break - head, tail = item - self.tag_remove("SYNC", head, tail) - item = self.tag_prevrange("SYNC", head) - head = item[1] if item else "1.0" + while todo_tag_range := self.tag_nextrange("TODO", next): + self.tag_remove("SYNC", todo_tag_range[0], todo_tag_range[1]) + sync_tag_range = self.tag_prevrange("SYNC", todo_tag_range[0]) + head = sync_tag_range[1] if sync_tag_range else "1.0" chars = "" next = head @@ -257,22 +253,20 @@ def recolorize_main(self): for tag in self.tagdefs: self.tag_remove(tag, mark, next) chars = chars + line - m = self.prog.search(chars) - while m: + for m in self.prog.finditer(chars): for key, value in m.groupdict().items(): - if value: - a, b = m.span(key) - self.tag_add(key, - head + "+%dc" % a, - head + "+%dc" % b) - if value in ("def", "class"): - m1 = self.idprog.match(chars, b) - if m1: - a, b = m1.span(1) - self.tag_add("DEFINITION", - head + "+%dc" % a, - head + "+%dc" % b) - m = self.prog.search(chars, m.end()) + if not value: + continue + a, b = m.span(key) + self.tag_add(key, + head + "+%dc" % a, + head + "+%dc" % b) + if value in ("def", "class"): + if m1 := self.idprog.match(chars, b): + a, b = m1.span(1) + self.tag_add("DEFINITION", + head + "+%dc" % a, + head + "+%dc" % b) if "SYNC" in self.tag_names(next + "-1c"): head = next chars = "" From 1b4f728efdb9b06cdeca2e2c48c010910be010f2 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Mon, 3 May 2021 14:40:31 +0300 Subject: [PATCH 02/30] add htest source example for pattern-matching --- Lib/idlelib/colorizer.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index dfcf4fe6999a58..aa432c1e327dd0 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -1,6 +1,7 @@ import builtins import keyword import re +import textwrap import time from idlelib.config import idleConf @@ -298,22 +299,27 @@ def _color_delegator(parent): # htest # top = Toplevel(parent) top.title("Test ColorDelegator") x, y = map(int, parent.geometry().split('+')[1:]) - top.geometry("700x250+%d+%d" % (x + 20, y + 175)) - source = ( - "if True: int ('1') # keyword, builtin, string, comment\n" - "elif False: print(0)\n" - "else: float(None)\n" - "if iF + If + IF: 'keyword matching must respect case'\n" - "if'': x or'' # valid keyword-string no-space combinations\n" - "async def f(): await g()\n" - "# All valid prefixes for unicode and byte strings should be colored.\n" - "'x', '''x''', \"x\", \"\"\"x\"\"\"\n" - "r'x', u'x', R'x', U'x', f'x', F'x'\n" - "fr'x', Fr'x', fR'x', FR'x', rf'x', rF'x', Rf'x', RF'x'\n" - "b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x'\n" - "# Invalid combinations of legal characters should be half colored.\n" - "ur'x', ru'x', uf'x', fu'x', UR'x', ufr'x', rfu'x', xf'x', fx'x'\n" - ) + top.geometry("700x350+%d+%d" % (x + 20, y + 175)) + source = textwrap.dedent("""\ + if True: int ('1') # keyword, builtin, string, comment + elif False: print(0) + else: float(None) + if iF + If + IF: 'keyword matching must respect case' + if'': x or'' # valid keyword-string no-space combinations + async def f(): await g() + # All valid prefixes for unicode and byte strings should be colored. + 'x', '''x''', "x", \"""x\""" + r'x', u'x', R'x', U'x', f'x', F'x' + fr'x', Fr'x', fR'x', FR'x', rf'x', rF'x', Rf'x', RF'x' + b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x' + # Invalid combinations of legal characters should be half colored. + ur'x', ru'x', uf'x', fu'x', UR'x', ufr'x', rfu'x', xf'x', fx'x' + match point: + case (x, 0): + print(f"X={x}") + case _: + raise ValueError("Not a point") + """) text = Text(top, background="white") text.pack(expand=1, fill="both") text.insert("insert", source) From f977c5039746a520d5045a72eed63724f52d41fc Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Mon, 3 May 2021 16:04:29 +0300 Subject: [PATCH 03/30] initial implementation of pattern-matching soft-keyword colorization --- Lib/idlelib/colorizer.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index aa432c1e327dd0..26796f0c6066a6 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -17,6 +17,20 @@ def any(name, alternates): def make_pat(): kw = r"\b" + any("KEYWORD", keyword.kwlist) + r"\b" + match_softkw = ( + r"(^|(?<=\n))[ \t]*" + # at beginning of string or after \n + r"(?Pmatch)\b" + + r"(?!([ \t]|\\\n)*[=,])" # not followed by = or , + ) + match_case_default = ( + r"(^|(?<=\n))[ \t]*" + # at beginning of string or after \n + r"(?Pcase)[ \t]+(?P_)[ \t]*:" + ) + match_case_nondefault = ( + r"(^|(?<=\n))[ \t]*" + # at beginning of string or after \n + r"(?Pcase)\b" + + r"(?!([ \t]|\\\n)*[=,])" # not followed by = or , + ) builtinlist = [str(name) for name in dir(builtins) if not name.startswith('_') and name not in keyword.kwlist] @@ -28,12 +42,21 @@ def make_pat(): sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?" dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?' string = any("STRING", [sq3string, dq3string, sqstring, dqstring]) - return (kw + "|" + builtin + "|" + comment + "|" + string + - "|" + any("SYNC", [r"\n"])) + return "|".join([ + builtin, comment, string, any("SYNC", [r"\n"]), + kw, match_softkw, match_case_default, match_case_nondefault, + # kw, match_case_default, + ]) prog = re.compile(make_pat(), re.S) idprog = re.compile(r"\s+(\w+)", re.S) +prog_group_name_to_tag = { + "PM_MATCH": "KEYWORD", + "PM_DEFAULT_CASE": "KEYWORD", + "PM_DEFAULT_UNDERSCORE": "KEYWORD", + "PM_NONDEFAULT_CASE": "KEYWORD", +} def color_config(text): @@ -259,7 +282,8 @@ def recolorize_main(self): if not value: continue a, b = m.span(key) - self.tag_add(key, + tag = prog_group_name_to_tag.get(key, key) + self.tag_add(tag, head + "+%dc" % a, head + "+%dc" % b) if value in ("def", "class"): @@ -314,11 +338,17 @@ async def f(): await g() b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x' # Invalid combinations of legal characters should be half colored. ur'x', ru'x', uf'x', fu'x', UR'x', ufr'x', rfu'x', xf'x', fx'x' + 'case _:' match point: case (x, 0): print(f"X={x}") case _: raise ValueError("Not a point") + # The following statement should all be in the default color for code. + match = ( + case, + _, + ) """) text = Text(top, background="white") text.pack(expand=1, fill="both") From df1c1a5ac53eef22d1afbec90d68df40f532896a Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Mon, 3 May 2021 16:29:13 +0300 Subject: [PATCH 04/30] additional test cases for pattern-matching soft keyword colorization --- Lib/idlelib/colorizer.py | 3 +- Lib/idlelib/idle_test/test_colorizer.py | 38 ++++++++++++++++++------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index 26796f0c6066a6..17d75487d296ea 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -338,7 +338,6 @@ async def f(): await g() b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x' # Invalid combinations of legal characters should be half colored. ur'x', ru'x', uf'x', fu'x', UR'x', ufr'x', rfu'x', xf'x', fx'x' - 'case _:' match point: case (x, 0): print(f"X={x}") @@ -349,6 +348,8 @@ async def f(): await g() case, _, ) + 'case _:' + "match x:" """) text = Text(top, background="white") text.pack(expand=1, fill="both") diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index c31c49236ca0b9..0e0f425eebd116 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -1,11 +1,11 @@ "Test colorizer, coverage 93%." - from idlelib import colorizer from test.support import requires import unittest from unittest import mock from functools import partial +import textwrap from tkinter import Tk, Text from idlelib import config from idlelib.percolator import Percolator @@ -19,15 +19,27 @@ 'extensions': config.IdleUserConfParser(''), } -source = ( - "if True: int ('1') # keyword, builtin, string, comment\n" - "elif False: print(0) # 'string' in comment\n" - "else: float(None) # if in comment\n" - "if iF + If + IF: 'keyword matching must respect case'\n" - "if'': x or'' # valid string-keyword no-space combinations\n" - "async def f(): await g()\n" - "'x', '''x''', \"x\", \"\"\"x\"\"\"\n" +source = textwrap.dedent("""\ + if True: int ('1') # keyword, builtin, string, comment + elif False: print(0) # 'string' in comment + else: float(None) # if in comment + if iF + If + IF: 'keyword matching must respect case' + if'': x or'' # valid string-keyword no-space combinations + async def f(): await g() + 'x', '''x''', "x", \"""x\""" + match point: + case (x, 0): + print(f"X={x}") + case _: + raise ValueError("Not a point") + # The following statement should all be in the default color for code. + match = ( + case, + _, ) + 'case _:' + "match x:" +""") def setUpModule(): @@ -366,6 +378,11 @@ def test_recolorize_main(self, mock_notify): ('6.0', ('KEYWORD',)), ('6.10', ('DEFINITION',)), ('6.11', ()), ('7.0', ('STRING',)), ('7.4', ()), ('7.5', ('STRING',)), ('7.12', ()), ('7.14', ('STRING',)), + ('8.0', ('KEYWORD',)), + ('9.4', ('KEYWORD',)), + ('11.4', ('KEYWORD',)), ('11.9', ('KEYWORD',)), + ('14.0', ()), ('15.4', ()), ('16.4', ()), + ('18.1', ('STRING',)), ('19.1', ('STRING',)), # SYNC at the end of every line. ('1.55', ('SYNC',)), ('2.50', ('SYNC',)), ('3.34', ('SYNC',)), ) @@ -395,7 +412,8 @@ def test_recolorize_main(self, mock_notify): eq(text.tag_nextrange('STRING', '7.3'), ('7.5', '7.12')) eq(text.tag_nextrange('STRING', '7.12'), ('7.14', '7.17')) eq(text.tag_nextrange('STRING', '7.17'), ('7.19', '7.26')) - eq(text.tag_nextrange('SYNC', '7.0'), ('7.26', '9.0')) + eq(text.tag_nextrange('SYNC', '7.0'), ('7.26', '8.0')) + eq(text.tag_nextrange('SYNC', '19.0'), ('19.10', '21.0')) @mock.patch.object(colorizer.ColorDelegator, 'recolorize') @mock.patch.object(colorizer.ColorDelegator, 'notify_range') From 816ea8d69e67ba976318142bb5b9bb1ec57ea8c1 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Mon, 3 May 2021 16:38:07 +0300 Subject: [PATCH 05/30] remove dead code comment --- Lib/idlelib/colorizer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index 17d75487d296ea..b62822493e3b8d 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -45,7 +45,6 @@ def make_pat(): return "|".join([ builtin, comment, string, any("SYNC", [r"\n"]), kw, match_softkw, match_case_default, match_case_nondefault, - # kw, match_case_default, ]) From 146bf70c98912b741e1edfd8874b47edd477f246 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Mon, 3 May 2021 17:17:12 +0300 Subject: [PATCH 06/30] also ignore match/case immediately followed by )]} --- Lib/idlelib/colorizer.py | 9 +++++---- Lib/idlelib/idle_test/test_colorizer.py | 7 ++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index b62822493e3b8d..637100a80cd53b 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -20,7 +20,7 @@ def make_pat(): match_softkw = ( r"(^|(?<=\n))[ \t]*" + # at beginning of string or after \n r"(?Pmatch)\b" + - r"(?!([ \t]|\\\n)*[=,])" # not followed by = or , + r"(?!([ \t]|\\\n)*[=,)\]}])" # not followed by any of =,)]} ) match_case_default = ( r"(^|(?<=\n))[ \t]*" + # at beginning of string or after \n @@ -29,7 +29,7 @@ def make_pat(): match_case_nondefault = ( r"(^|(?<=\n))[ \t]*" + # at beginning of string or after \n r"(?Pcase)\b" + - r"(?!([ \t]|\\\n)*[=,])" # not followed by = or , + r"(?!([ \t]|\\\n)*[=,)\]}])" # not followed by any of =,)]} ) builtinlist = [str(name) for name in dir(builtins) if not name.startswith('_') and @@ -322,7 +322,7 @@ def _color_delegator(parent): # htest # top = Toplevel(parent) top.title("Test ColorDelegator") x, y = map(int, parent.geometry().split('+')[1:]) - top.geometry("700x350+%d+%d" % (x + 20, y + 175)) + top.geometry("700x500+%d+%d" % (x + 20, y + 175)) source = textwrap.dedent("""\ if True: int ('1') # keyword, builtin, string, comment elif False: print(0) @@ -347,7 +347,8 @@ async def f(): await g() case, _, ) - 'case _:' + ''' + case _:''' "match x:" """) text = Text(top, background="white") diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index 0e0f425eebd116..f8c711d145622d 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -37,7 +37,8 @@ async def f(): await g() case, _, ) - 'case _:' + ''' + case _:''' "match x:" """) @@ -382,7 +383,7 @@ def test_recolorize_main(self, mock_notify): ('9.4', ('KEYWORD',)), ('11.4', ('KEYWORD',)), ('11.9', ('KEYWORD',)), ('14.0', ()), ('15.4', ()), ('16.4', ()), - ('18.1', ('STRING',)), ('19.1', ('STRING',)), + ('19.0', ('STRING',)), ('20.1', ('STRING',)), # SYNC at the end of every line. ('1.55', ('SYNC',)), ('2.50', ('SYNC',)), ('3.34', ('SYNC',)), ) @@ -413,7 +414,7 @@ def test_recolorize_main(self, mock_notify): eq(text.tag_nextrange('STRING', '7.12'), ('7.14', '7.17')) eq(text.tag_nextrange('STRING', '7.17'), ('7.19', '7.26')) eq(text.tag_nextrange('SYNC', '7.0'), ('7.26', '8.0')) - eq(text.tag_nextrange('SYNC', '19.0'), ('19.10', '21.0')) + eq(text.tag_nextrange('SYNC', '20.0'), ('20.10', '22.0')) @mock.patch.object(colorizer.ColorDelegator, 'recolorize') @mock.patch.object(colorizer.ColorDelegator, 'notify_range') From ff463a6e03c109a6eb034a1d4ee57a7455d962cc Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Mon, 3 May 2021 19:20:19 +0300 Subject: [PATCH 07/30] simplify regexps using re.MULTILINE + more test cases --- Lib/idlelib/colorizer.py | 16 ++++++++++------ Lib/idlelib/idle_test/test_colorizer.py | 16 ++++++++++------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index 637100a80cd53b..edb09006f31315 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -18,16 +18,16 @@ def any(name, alternates): def make_pat(): kw = r"\b" + any("KEYWORD", keyword.kwlist) + r"\b" match_softkw = ( - r"(^|(?<=\n))[ \t]*" + # at beginning of string or after \n + r"^[ \t]*" + # at beginning of line + possible indentation r"(?Pmatch)\b" + r"(?!([ \t]|\\\n)*[=,)\]}])" # not followed by any of =,)]} ) match_case_default = ( - r"(^|(?<=\n))[ \t]*" + # at beginning of string or after \n + r"^[ \t]*" + # at beginning of line + possible indentation r"(?Pcase)[ \t]+(?P_)[ \t]*:" ) match_case_nondefault = ( - r"(^|(?<=\n))[ \t]*" + # at beginning of string or after \n + r"^[ \t]*" + # at beginning of line + possible indentation r"(?Pcase)\b" + r"(?!([ \t]|\\\n)*[=,)\]}])" # not followed by any of =,)]} ) @@ -48,8 +48,8 @@ def make_pat(): ]) -prog = re.compile(make_pat(), re.S) -idprog = re.compile(r"\s+(\w+)", re.S) +prog = re.compile(make_pat(), re.DOTALL | re.MULTILINE) +idprog = re.compile(r"\s+(\w+)") prog_group_name_to_tag = { "PM_MATCH": "KEYWORD", "PM_DEFAULT_CASE": "KEYWORD", @@ -322,7 +322,7 @@ def _color_delegator(parent): # htest # top = Toplevel(parent) top.title("Test ColorDelegator") x, y = map(int, parent.geometry().split('+')[1:]) - top.geometry("700x500+%d+%d" % (x + 20, y + 175)) + top.geometry("700x550+%d+%d" % (x + 20, y + 175)) source = textwrap.dedent("""\ if True: int ('1') # keyword, builtin, string, comment elif False: print(0) @@ -332,6 +332,10 @@ def _color_delegator(parent): # htest # async def f(): await g() # All valid prefixes for unicode and byte strings should be colored. 'x', '''x''', "x", \"""x\""" + 'abc\\ + def' + '''abc\\ + def''' r'x', u'x', R'x', U'x', f'x', F'x' fr'x', Fr'x', fR'x', FR'x', rf'x', rF'x', Rf'x', RF'x' b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x' diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index f8c711d145622d..36e197234f1711 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -27,6 +27,10 @@ if'': x or'' # valid string-keyword no-space combinations async def f(): await g() 'x', '''x''', "x", \"""x\""" + 'abc\\ + def' + '''abc\\ + def''' match point: case (x, 0): print(f"X={x}") @@ -379,11 +383,11 @@ def test_recolorize_main(self, mock_notify): ('6.0', ('KEYWORD',)), ('6.10', ('DEFINITION',)), ('6.11', ()), ('7.0', ('STRING',)), ('7.4', ()), ('7.5', ('STRING',)), ('7.12', ()), ('7.14', ('STRING',)), - ('8.0', ('KEYWORD',)), - ('9.4', ('KEYWORD',)), - ('11.4', ('KEYWORD',)), ('11.9', ('KEYWORD',)), - ('14.0', ()), ('15.4', ()), ('16.4', ()), - ('19.0', ('STRING',)), ('20.1', ('STRING',)), + ('12.0', ('KEYWORD',)), + ('13.4', ('KEYWORD',)), + ('15.4', ('KEYWORD',)), ('15.9', ('KEYWORD',)), + ('18.0', ()), ('19.4', ()), ('20.4', ()), + ('23.0', ('STRING',)), ('24.1', ('STRING',)), # SYNC at the end of every line. ('1.55', ('SYNC',)), ('2.50', ('SYNC',)), ('3.34', ('SYNC',)), ) @@ -414,7 +418,7 @@ def test_recolorize_main(self, mock_notify): eq(text.tag_nextrange('STRING', '7.12'), ('7.14', '7.17')) eq(text.tag_nextrange('STRING', '7.17'), ('7.19', '7.26')) eq(text.tag_nextrange('SYNC', '7.0'), ('7.26', '8.0')) - eq(text.tag_nextrange('SYNC', '20.0'), ('20.10', '22.0')) + eq(text.tag_nextrange('SYNC', '24.0'), ('24.10', '26.0')) @mock.patch.object(colorizer.ColorDelegator, 'recolorize') @mock.patch.object(colorizer.ColorDelegator, 'notify_range') From 975887020e1305ec45d427ce0f68ae4212090592 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Thu, 6 May 2021 10:00:58 +0300 Subject: [PATCH 08/30] refactor and mark all lone underscores in case patterns as keywords --- Lib/idlelib/colorizer.py | 97 ++++++++++++++----------- Lib/idlelib/idle_test/test_colorizer.py | 25 +++---- 2 files changed, 67 insertions(+), 55 deletions(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index edb09006f31315..abdc2f5a4e3570 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -15,21 +15,18 @@ def any(name, alternates): return "(?P<%s>" % name + "|".join(alternates) + ")" -def make_pat(): +def make_pats(): kw = r"\b" + any("KEYWORD", keyword.kwlist) + r"\b" match_softkw = ( r"^[ \t]*" + # at beginning of line + possible indentation - r"(?Pmatch)\b" + - r"(?!([ \t]|\\\n)*[=,)\]}])" # not followed by any of =,)]} + r"(?Pmatch)\b" + + r"(?!(?:[ \t]|\\\n)*[=,)\]}])" # not followed by any of =,)]} ) - match_case_default = ( + case_softkw_and_pattern = ( r"^[ \t]*" + # at beginning of line + possible indentation - r"(?Pcase)[ \t]+(?P_)[ \t]*:" - ) - match_case_nondefault = ( - r"^[ \t]*" + # at beginning of line + possible indentation - r"(?Pcase)\b" + - r"(?!([ \t]|\\\n)*[=,)\]}])" # not followed by any of =,)]} + r"(?Pcase)\b" + + r"(?!(?:[ \t]|\\\n)*[=,)\]}])" + # not followed by any of =,)]} + r"(?P.*?):[ \t]*$" ) builtinlist = [str(name) for name in dir(builtins) if not name.startswith('_') and @@ -42,22 +39,30 @@ def make_pat(): sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?" dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?' string = any("STRING", [sq3string, dq3string, sqstring, dqstring]) - return "|".join([ - builtin, comment, string, any("SYNC", [r"\n"]), - kw, match_softkw, match_case_default, match_case_nondefault, - ]) - - -prog = re.compile(make_pat(), re.DOTALL | re.MULTILINE) + prog = re.compile("|".join([ + builtin, comment, string, kw, + match_softkw, case_softkw_and_pattern, + any("SYNC", [r"\n"]), + ]), re.DOTALL | re.MULTILINE) + pattern_prog = re.compile("|".join([ + builtin, comment, string, kw, any("UNDERSCORE_SOFTKW", [r"\b_\b"]) + ]), re.DOTALL) + return prog, pattern_prog + + +prog, pattern_prog = make_pats() idprog = re.compile(r"\s+(\w+)") prog_group_name_to_tag = { - "PM_MATCH": "KEYWORD", - "PM_DEFAULT_CASE": "KEYWORD", - "PM_DEFAULT_UNDERSCORE": "KEYWORD", - "PM_NONDEFAULT_CASE": "KEYWORD", + "MATCH_SOFTKW": "KEYWORD", + "CASE_SOFTKW": "KEYWORD", + "UNDERSCORE_SOFTKW": "KEYWORD", } +def matched_named_groups(re_match): + return ((k, v) for (k, v) in re_match.groupdict().items() if v) + + def color_config(text): """Set color options of Text widget. @@ -275,22 +280,8 @@ def recolorize_main(self): return for tag in self.tagdefs: self.tag_remove(tag, mark, next) - chars = chars + line - for m in self.prog.finditer(chars): - for key, value in m.groupdict().items(): - if not value: - continue - a, b = m.span(key) - tag = prog_group_name_to_tag.get(key, key) - self.tag_add(tag, - head + "+%dc" % a, - head + "+%dc" % b) - if value in ("def", "class"): - if m1 := self.idprog.match(chars, b): - a, b = m1.span(1) - self.tag_add("DEFINITION", - head + "+%dc" % a, - head + "+%dc" % b) + chars += line + self._add_tags_in_section(chars, head) if "SYNC" in self.tag_names(next + "-1c"): head = next chars = "" @@ -309,6 +300,30 @@ def recolorize_main(self): if DEBUG: print("colorizing stopped") return + def _add_tag(self, start, end, head, matched_group_name): + tag = prog_group_name_to_tag.get(matched_group_name, + matched_group_name) + self.tag_add(tag, + f"{head}+{start:d}c", + f"{head}+{end:d}c") + + def _add_tags_in_section(self, chars, head): + "Add tags to a given section of code." + for m in self.prog.finditer(chars): + for name, matched_text in matched_named_groups(m): + a, b = m.span(name) + if name == "CASE_PATTERN": + for m1 in pattern_prog.finditer(matched_text): + for name, matched_text in matched_named_groups(m1): + a1, b1 = m1.span() + self._add_tag(a + a1, a + b1, head, name) + else: + self._add_tag(a, b, head, name) + if matched_text in ("def", "class"): + if m1 := self.idprog.match(chars, b): + a, b = m1.span(1) + self._add_tag(a, b, head, "DEFINITION") + def removecolors(self): "Remove all colorizing tags." for tag in self.tagdefs: @@ -344,13 +359,11 @@ async def f(): await g() match point: case (x, 0): print(f"X={x}") + case [_, [_], "_", + _]: + pass case _: raise ValueError("Not a point") - # The following statement should all be in the default color for code. - match = ( - case, - _, - ) ''' case _:''' "match x:" diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index 36e197234f1711..413e5225d96585 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -34,17 +34,15 @@ async def f(): await g() match point: case (x, 0): print(f"X={x}") + case [_, [_], "_", + _]: + pass case _: raise ValueError("Not a point") - # The following statement should all be in the default color for code. - match = ( - case, - _, - ) ''' case _:''' "match x:" -""") + """) def setUpModule(): @@ -63,7 +61,7 @@ def test_any(self): def test_make_pat(self): # Tested in more detail by testing prog. - self.assertTrue(colorizer.make_pat()) + self.assertTrue(colorizer.make_pats()) def test_prog(self): prog = colorizer.prog @@ -124,7 +122,7 @@ def setUpClass(cls): requires('gui') root = cls.root = Tk() root.withdraw() - text = cls.text = Text(root) + cls.text = Text(root) @classmethod def tearDownClass(cls): @@ -169,7 +167,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - cls.percolator.redir.close() + cls.percolator.close() del cls.percolator, cls.text cls.root.update_idletasks() cls.root.destroy() @@ -385,9 +383,10 @@ def test_recolorize_main(self, mock_notify): ('7.12', ()), ('7.14', ('STRING',)), ('12.0', ('KEYWORD',)), ('13.4', ('KEYWORD',)), - ('15.4', ('KEYWORD',)), ('15.9', ('KEYWORD',)), - ('18.0', ()), ('19.4', ()), ('20.4', ()), - ('23.0', ('STRING',)), ('24.1', ('STRING',)), + ('15.4', ('KEYWORD',)), ('15.10', ('KEYWORD',)), ('15.14', ('KEYWORD',)), ('15.19', ('STRING',)), + ('16.12', ('KEYWORD',)), + ('18.4', ('KEYWORD',)), ('18.9', ('KEYWORD',)), + ('21.0', ('STRING',)), ('22.1', ('STRING',)), # SYNC at the end of every line. ('1.55', ('SYNC',)), ('2.50', ('SYNC',)), ('3.34', ('SYNC',)), ) @@ -418,7 +417,7 @@ def test_recolorize_main(self, mock_notify): eq(text.tag_nextrange('STRING', '7.12'), ('7.14', '7.17')) eq(text.tag_nextrange('STRING', '7.17'), ('7.19', '7.26')) eq(text.tag_nextrange('SYNC', '7.0'), ('7.26', '8.0')) - eq(text.tag_nextrange('SYNC', '24.0'), ('24.10', '26.0')) + eq(text.tag_nextrange('SYNC', '22.0'), ('22.10', '24.0')) @mock.patch.object(colorizer.ColorDelegator, 'recolorize') @mock.patch.object(colorizer.ColorDelegator, 'notify_range') From 4adc3189315a3dba2ee5b466458148b53960901d Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Thu, 6 May 2021 10:10:55 +0300 Subject: [PATCH 09/30] fix comments in htest code sample --- Lib/idlelib/colorizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index abdc2f5a4e3570..1d4e01d0bbbe10 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -345,13 +345,13 @@ def _color_delegator(parent): # htest # if iF + If + IF: 'keyword matching must respect case' if'': x or'' # valid keyword-string no-space combinations async def f(): await g() - # All valid prefixes for unicode and byte strings should be colored. + # Strings should be entirely colored, including quotes. 'x', '''x''', "x", \"""x\""" 'abc\\ def' '''abc\\ def''' - r'x', u'x', R'x', U'x', f'x', F'x' + # All valid prefixes for unicode and byte strings should be colored. r'x', u'x', R'x', U'x', f'x', F'x' fr'x', Fr'x', fR'x', FR'x', rf'x', rF'x', Rf'x', RF'x' b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x' # Invalid combinations of legal characters should be half colored. From 1d9ce7f3f296d8d5a234974873f809d8b245ff57 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Thu, 6 May 2021 23:42:18 +0300 Subject: [PATCH 10/30] handle case guard and capture patterns --- Lib/idlelib/colorizer.py | 15 ++++++++++----- Lib/idlelib/idle_test/test_colorizer.py | 14 ++++++++------ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index 1d4e01d0bbbe10..d6223262629b66 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -26,7 +26,10 @@ def make_pats(): r"^[ \t]*" + # at beginning of line + possible indentation r"(?Pcase)\b" + r"(?!(?:[ \t]|\\\n)*[=,)\]}])" + # not followed by any of =,)]} - r"(?P.*?):[ \t]*$" + r"(?P.*?)" + # case pattern + r"(?:(?P\bif\b).*?)?" + # possible guard + r"(?:(?P\bas\b)(?P.*?))?" + # possible capture + r":" ) builtinlist = [str(name) for name in dir(builtins) if not name.startswith('_') and @@ -55,6 +58,8 @@ def make_pats(): prog_group_name_to_tag = { "MATCH_SOFTKW": "KEYWORD", "CASE_SOFTKW": "KEYWORD", + "CASE_AS": "KEYWORD", + "CASE_IF": "KEYWORD", "UNDERSCORE_SOFTKW": "KEYWORD", } @@ -312,7 +317,7 @@ def _add_tags_in_section(self, chars, head): for m in self.prog.finditer(chars): for name, matched_text in matched_named_groups(m): a, b = m.span(name) - if name == "CASE_PATTERN": + if name in {"CASE_PATTERN", "CAPTURE_PATTERN"}: for m1 in pattern_prog.finditer(matched_text): for name, matched_text in matched_named_groups(m1): a1, b1 = m1.span() @@ -357,13 +362,13 @@ async def f(): await g() # Invalid combinations of legal characters should be half colored. ur'x', ru'x', uf'x', fu'x', UR'x', ufr'x', rfu'x', xf'x', fx'x' match point: - case (x, 0): + case (x, 0) as _: print(f"X={x}") case [_, [_], "_", _]: pass - case _: - raise ValueError("Not a point") + case _ if _: + raise ValueError("Not a point _") ''' case _:''' "match x:" diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index 413e5225d96585..7e5517d556d816 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -32,13 +32,13 @@ async def f(): await g() '''abc\\ def''' match point: - case (x, 0): + case (x, 0) as _: print(f"X={x}") case [_, [_], "_", _]: pass - case _: - raise ValueError("Not a point") + case _ if _: + raise ValueError("Not a point _") ''' case _:''' "match x:" @@ -382,11 +382,13 @@ def test_recolorize_main(self, mock_notify): ('7.0', ('STRING',)), ('7.4', ()), ('7.5', ('STRING',)), ('7.12', ()), ('7.14', ('STRING',)), ('12.0', ('KEYWORD',)), - ('13.4', ('KEYWORD',)), + ('13.4', ('KEYWORD',)), ('13.16', ('KEYWORD',)), ('13.19', ('KEYWORD',)), ('15.4', ('KEYWORD',)), ('15.10', ('KEYWORD',)), ('15.14', ('KEYWORD',)), ('15.19', ('STRING',)), ('16.12', ('KEYWORD',)), - ('18.4', ('KEYWORD',)), ('18.9', ('KEYWORD',)), - ('21.0', ('STRING',)), ('22.1', ('STRING',)), + ('18.4', ('KEYWORD',)), ('18.9', ('KEYWORD',)), ('18.11', ('KEYWORD',)), ('18.14', (),), + ('19.25', ('STRING',)), ('19.38', ('STRING',)), + ('21.0', ('STRING',)), + ('22.1', ('STRING',)), # SYNC at the end of every line. ('1.55', ('SYNC',)), ('2.50', ('SYNC',)), ('3.34', ('SYNC',)), ) From 21c2f79b01d301d562bf695de950f43dd3e4c89e Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Fri, 7 May 2021 00:09:48 +0300 Subject: [PATCH 11/30] use single example source in tests --- Lib/idlelib/colorizer.py | 32 ++----------------- Lib/idlelib/idle_test/test_colorizer.py | 41 +++++++++++++++---------- 2 files changed, 26 insertions(+), 47 deletions(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index d6223262629b66..d28b7376ef4503 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -337,42 +337,14 @@ def removecolors(self): def _color_delegator(parent): # htest # from tkinter import Toplevel, Text + from idlelib.idle_test.test_colorizer import source from idlelib.percolator import Percolator top = Toplevel(parent) top.title("Test ColorDelegator") x, y = map(int, parent.geometry().split('+')[1:]) top.geometry("700x550+%d+%d" % (x + 20, y + 175)) - source = textwrap.dedent("""\ - if True: int ('1') # keyword, builtin, string, comment - elif False: print(0) - else: float(None) - if iF + If + IF: 'keyword matching must respect case' - if'': x or'' # valid keyword-string no-space combinations - async def f(): await g() - # Strings should be entirely colored, including quotes. - 'x', '''x''', "x", \"""x\""" - 'abc\\ - def' - '''abc\\ - def''' - # All valid prefixes for unicode and byte strings should be colored. r'x', u'x', R'x', U'x', f'x', F'x' - fr'x', Fr'x', fR'x', FR'x', rf'x', rF'x', Rf'x', RF'x' - b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x' - # Invalid combinations of legal characters should be half colored. - ur'x', ru'x', uf'x', fu'x', UR'x', ufr'x', rfu'x', xf'x', fx'x' - match point: - case (x, 0) as _: - print(f"X={x}") - case [_, [_], "_", - _]: - pass - case _ if _: - raise ValueError("Not a point _") - ''' - case _:''' - "match x:" - """) + text = Text(top, background="white") text.pack(expand=1, fill="both") text.insert("insert", source) diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index 7e5517d556d816..c5c8b8cb563ab7 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -24,13 +24,20 @@ elif False: print(0) # 'string' in comment else: float(None) # if in comment if iF + If + IF: 'keyword matching must respect case' - if'': x or'' # valid string-keyword no-space combinations + if'': x or'' # valid keyword-string no-space combinations async def f(): await g() + # Strings should be entirely colored, including quotes. 'x', '''x''', "x", \"""x\""" 'abc\\ def' '''abc\\ def''' + # All valid prefixes for unicode and byte strings should be colored. + r'x', u'x', R'x', U'x', f'x', F'x' + fr'x', Fr'x', fR'x', FR'x', rf'x', rF'x', Rf'x', RF'x' + b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x' + # Invalid combinations of legal characters should be half colored. + ur'x', ru'x', uf'x', fu'x', UR'x', ufr'x', rfu'x', xf'x', fx'x' match point: case (x, 0) as _: print(f"X={x}") @@ -379,16 +386,16 @@ def test_recolorize_main(self, mock_notify): ('4.0', ('KEYWORD',)), ('4.3', ()), ('4.6', ()), ('5.2', ('STRING',)), ('5.8', ('KEYWORD',)), ('5.10', ('STRING',)), ('6.0', ('KEYWORD',)), ('6.10', ('DEFINITION',)), ('6.11', ()), - ('7.0', ('STRING',)), ('7.4', ()), ('7.5', ('STRING',)), - ('7.12', ()), ('7.14', ('STRING',)), - ('12.0', ('KEYWORD',)), - ('13.4', ('KEYWORD',)), ('13.16', ('KEYWORD',)), ('13.19', ('KEYWORD',)), - ('15.4', ('KEYWORD',)), ('15.10', ('KEYWORD',)), ('15.14', ('KEYWORD',)), ('15.19', ('STRING',)), - ('16.12', ('KEYWORD',)), - ('18.4', ('KEYWORD',)), ('18.9', ('KEYWORD',)), ('18.11', ('KEYWORD',)), ('18.14', (),), - ('19.25', ('STRING',)), ('19.38', ('STRING',)), - ('21.0', ('STRING',)), - ('22.1', ('STRING',)), + ('8.0', ('STRING',)), ('8.4', ()), ('8.5', ('STRING',)), + ('8.12', ()), ('8.14', ('STRING',)), + ('19.0', ('KEYWORD',)), + ('20.4', ('KEYWORD',)), ('20.16', ('KEYWORD',)), ('20.19', ('KEYWORD',)), + ('22.4', ('KEYWORD',)), ('22.10', ('KEYWORD',)), ('22.14', ('KEYWORD',)), ('22.19', ('STRING',)), + ('23.12', ('KEYWORD',)), + ('25.4', ('KEYWORD',)), ('25.9', ('KEYWORD',)), ('25.11', ('KEYWORD',)), ('25.14', (),), + ('26.25', ('STRING',)), ('26.38', ('STRING',)), + ('28.0', ('STRING',)), + ('29.1', ('STRING',)), # SYNC at the end of every line. ('1.55', ('SYNC',)), ('2.50', ('SYNC',)), ('3.34', ('SYNC',)), ) @@ -414,12 +421,12 @@ def test_recolorize_main(self, mock_notify): eq(text.tag_nextrange('COMMENT', '2.0'), ('2.22', '2.43')) eq(text.tag_nextrange('SYNC', '2.0'), ('2.43', '3.0')) eq(text.tag_nextrange('STRING', '2.0'), ('4.17', '4.53')) - eq(text.tag_nextrange('STRING', '7.0'), ('7.0', '7.3')) - eq(text.tag_nextrange('STRING', '7.3'), ('7.5', '7.12')) - eq(text.tag_nextrange('STRING', '7.12'), ('7.14', '7.17')) - eq(text.tag_nextrange('STRING', '7.17'), ('7.19', '7.26')) - eq(text.tag_nextrange('SYNC', '7.0'), ('7.26', '8.0')) - eq(text.tag_nextrange('SYNC', '22.0'), ('22.10', '24.0')) + eq(text.tag_nextrange('STRING', '8.0'), ('8.0', '8.3')) + eq(text.tag_nextrange('STRING', '8.3'), ('8.5', '8.12')) + eq(text.tag_nextrange('STRING', '8.12'), ('8.14', '8.17')) + eq(text.tag_nextrange('STRING', '8.17'), ('8.19', '8.26')) + eq(text.tag_nextrange('SYNC', '8.0'), ('8.26', '9.0')) + eq(text.tag_nextrange('SYNC', '29.0'), ('29.10', '31.0')) @mock.patch.object(colorizer.ColorDelegator, 'recolorize') @mock.patch.object(colorizer.ColorDelegator, 'notify_range') From 288fea305788429caeb5f07013fc894e2626a73b Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Fri, 7 May 2021 00:28:08 +0300 Subject: [PATCH 12/30] fix highlighting in case guard and capture patterns --- Lib/idlelib/colorizer.py | 16 +++++++++++----- Lib/idlelib/idle_test/test_colorizer.py | 10 +++++++--- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index d28b7376ef4503..30496fe3b7ade8 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -27,8 +27,8 @@ def make_pats(): r"(?Pcase)\b" + r"(?!(?:[ \t]|\\\n)*[=,)\]}])" + # not followed by any of =,)]} r"(?P.*?)" + # case pattern - r"(?:(?P\bif\b).*?)?" + # possible guard - r"(?:(?P\bas\b)(?P.*?))?" + # possible capture + r"(?P\bif\b.*?)?" + # possible guard + r"(?P\bas\b.*?)?" + # possible capture r":" ) builtinlist = [str(name) for name in dir(builtins) @@ -47,13 +47,14 @@ def make_pats(): match_softkw, case_softkw_and_pattern, any("SYNC", [r"\n"]), ]), re.DOTALL | re.MULTILINE) + guard_prog = re.compile("|".join([builtin, comment, string, kw])) pattern_prog = re.compile("|".join([ builtin, comment, string, kw, any("UNDERSCORE_SOFTKW", [r"\b_\b"]) ]), re.DOTALL) - return prog, pattern_prog + return prog, guard_prog, pattern_prog -prog, pattern_prog = make_pats() +prog, guard_prog, pattern_prog = make_pats() idprog = re.compile(r"\s+(\w+)") prog_group_name_to_tag = { "MATCH_SOFTKW": "KEYWORD", @@ -317,11 +318,16 @@ def _add_tags_in_section(self, chars, head): for m in self.prog.finditer(chars): for name, matched_text in matched_named_groups(m): a, b = m.span(name) - if name in {"CASE_PATTERN", "CAPTURE_PATTERN"}: + if name in {"CASE_PATTERN", "CASE_AS"}: for m1 in pattern_prog.finditer(matched_text): for name, matched_text in matched_named_groups(m1): a1, b1 = m1.span() self._add_tag(a + a1, a + b1, head, name) + elif name == "CASE_IF": + for m1 in guard_prog.finditer(matched_text): + for name, matched_text in matched_named_groups(m1): + a1, b1 = m1.span() + self._add_tag(a + a1, a + b1, head, name) else: self._add_tag(a, b, head, name) if matched_text in ("def", "class"): diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index c5c8b8cb563ab7..2093208b87a66d 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -46,6 +46,7 @@ async def f(): await g() pass case _ if _: raise ValueError("Not a point _") + case _ if ("a" if _ else set()): pass ''' case _:''' "match x:" @@ -394,8 +395,11 @@ def test_recolorize_main(self, mock_notify): ('23.12', ('KEYWORD',)), ('25.4', ('KEYWORD',)), ('25.9', ('KEYWORD',)), ('25.11', ('KEYWORD',)), ('25.14', (),), ('26.25', ('STRING',)), ('26.38', ('STRING',)), - ('28.0', ('STRING',)), - ('29.1', ('STRING',)), + ('27.4', ('KEYWORD',)), ('27.9', ('KEYWORD',)), ('27.11', ('KEYWORD',)), + ('27.15', ('STRING',)), ('27.19', ('KEYWORD',)), ('27.22', ()), + ('27.24', ('KEYWORD',)), ('27.29', ('BUILTIN',)), ('27.37', ('KEYWORD',)), + ('29.0', ('STRING',)), + ('30.1', ('STRING',)), # SYNC at the end of every line. ('1.55', ('SYNC',)), ('2.50', ('SYNC',)), ('3.34', ('SYNC',)), ) @@ -426,7 +430,7 @@ def test_recolorize_main(self, mock_notify): eq(text.tag_nextrange('STRING', '8.12'), ('8.14', '8.17')) eq(text.tag_nextrange('STRING', '8.17'), ('8.19', '8.26')) eq(text.tag_nextrange('SYNC', '8.0'), ('8.26', '9.0')) - eq(text.tag_nextrange('SYNC', '29.0'), ('29.10', '31.0')) + eq(text.tag_nextrange('SYNC', '30.0'), ('30.10', '32.0')) @mock.patch.object(colorizer.ColorDelegator, 'recolorize') @mock.patch.object(colorizer.ColorDelegator, 'notify_range') From 29886933cfa2e9b998c1ceeed573795a7bcc2c0c Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Sun, 9 May 2021 10:08:26 +0300 Subject: [PATCH 13/30] add a NEWS entry --- Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst diff --git a/Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst b/Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst new file mode 100644 index 00000000000000..bb5f583c1fa503 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst @@ -0,0 +1,2 @@ +Highlight the new pattern-matching "soft keywords": ``match``, +``case``, and ``_``. From c6015dcb6dc1b4fc04b23c98a601bce3e9156f4f Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Sun, 9 May 2021 10:39:16 +0300 Subject: [PATCH 14/30] more tests for function defs --- Lib/idlelib/idle_test/test_colorizer.py | 37 +++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index 2093208b87a66d..aefc9fcf18b06c 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -432,6 +432,43 @@ def test_recolorize_main(self, mock_notify): eq(text.tag_nextrange('SYNC', '8.0'), ('8.26', '9.0')) eq(text.tag_nextrange('SYNC', '30.0'), ('30.10', '32.0')) + @mock.patch.object(colorizer.ColorDelegator, 'notify_range') + def test_def_statement(self, mock_notify): + text = self.text + color = self.color + eq = self.assertEqual + + # empty def + text.insert('insert', 'def') + text.tag_add('TODO', '1.0', 'end-1c') + color.recolorize_main() + eq(text.tag_names('1.0'), ('KEYWORD',)) + text.delete('1.0', 'end-1c') + + # def followed by identifier + text.insert('1.0', 'def foo:') + text.tag_add('TODO', '1.0', 'end-1c') + color.recolorize_main() + eq(text.tag_names('1.0'), ('KEYWORD',)) + eq(text.tag_names('1.4'), ('DEFINITION',)) + text.delete('1.0', 'end-1c') + + # def followed by partial identifier + text.insert('1.0', 'def fo') + text.tag_add('TODO', '1.0', 'end-1c') + color.recolorize_main() + eq(text.tag_names('1.0'), ('KEYWORD',)) + eq(text.tag_names('1.4'), ('DEFINITION',)) + text.delete('1.0', 'end-1c') + + # def followed by non-keyword + text.insert('1.0', 'def ++') + text.tag_add('TODO', '1.0', 'end-1c') + color.recolorize_main() + eq(text.tag_names('1.0'), ('KEYWORD',)) + eq(text.tag_names('1.4'), ()) + text.delete('1.0', 'end-1c') + @mock.patch.object(colorizer.ColorDelegator, 'recolorize') @mock.patch.object(colorizer.ColorDelegator, 'notify_range') def test_removecolors(self, mock_notify, mock_recolorize): From 68d41fe3b5615d2bf9e165a107d6a7bc4b6e5921 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Sun, 9 May 2021 10:40:57 +0300 Subject: [PATCH 15/30] improved doc-strings and indentation as per code review --- Lib/idlelib/colorizer.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index 30496fe3b7ade8..99eaded86bb85d 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -21,7 +21,7 @@ def make_pats(): r"^[ \t]*" + # at beginning of line + possible indentation r"(?Pmatch)\b" + r"(?!(?:[ \t]|\\\n)*[=,)\]}])" # not followed by any of =,)]} - ) + ) case_softkw_and_pattern = ( r"^[ \t]*" + # at beginning of line + possible indentation r"(?Pcase)\b" + @@ -30,7 +30,7 @@ def make_pats(): r"(?P\bif\b.*?)?" + # possible guard r"(?P\bas\b.*?)?" + # possible capture r":" - ) + ) builtinlist = [str(name) for name in dir(builtins) if not name.startswith('_') and name not in keyword.kwlist] @@ -43,10 +43,11 @@ def make_pats(): dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?' string = any("STRING", [sq3string, dq3string, sqstring, dqstring]) prog = re.compile("|".join([ - builtin, comment, string, kw, - match_softkw, case_softkw_and_pattern, - any("SYNC", [r"\n"]), - ]), re.DOTALL | re.MULTILINE) + builtin, comment, string, kw, + match_softkw, case_softkw_and_pattern, + any("SYNC", [r"\n"]), + ]), + re.DOTALL | re.MULTILINE) guard_prog = re.compile("|".join([builtin, comment, string, kw])) pattern_prog = re.compile("|".join([ builtin, comment, string, kw, any("UNDERSCORE_SOFTKW", [r"\b_\b"]) @@ -66,6 +67,7 @@ def make_pats(): def matched_named_groups(re_match): + "Get only the non-empty named groups from an re.Match object." return ((k, v) for (k, v) in re_match.groupdict().items() if v) @@ -307,6 +309,16 @@ def recolorize_main(self): return def _add_tag(self, start, end, head, matched_group_name): + """Add a tag to a given range in the text widget. + + This is a utility function, receiving the range as `start` and + `end` positions, each of which is a number of characters + relative to the given `head` index in the text widget. + + The tag to add is determined by `matched_group_name`, which is + the name of a regular expression "named group" as matched by + by the relevant highlighting regexps. + """ tag = prog_group_name_to_tag.get(matched_group_name, matched_group_name) self.tag_add(tag, @@ -314,7 +326,13 @@ def _add_tag(self, start, end, head, matched_group_name): f"{head}+{end:d}c") def _add_tags_in_section(self, chars, head): - "Add tags to a given section of code." + """Parse and add highlighting tags to a given part of the text. + + `chars` is a string with the text to parse and to which + highlighting is to be applied. + + `head` is the index in the text widget where the text is found. + """ for m in self.prog.finditer(chars): for name, matched_text in matched_named_groups(m): a, b = m.span(name) From 12b7dd557b996dfd35d23103ef13fe23bac9ad35 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Sun, 9 May 2021 10:45:56 +0300 Subject: [PATCH 16/30] add test with long multi-line string at beginning of text This should exercise a branch not previously covered by the tests. --- Lib/idlelib/idle_test/test_colorizer.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index aefc9fcf18b06c..5e54fa484e18b8 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -469,6 +469,23 @@ def test_def_statement(self, mock_notify): eq(text.tag_names('1.4'), ()) text.delete('1.0', 'end-1c') + @mock.patch.object(colorizer.ColorDelegator, 'notify_range') + def test_long_multiline_string(self, notify_range): + text = self.text + color = self.color + eq = self.assertEqual + + text.insert('insert', textwrap.dedent('''\ + """a + b + c + d + e""" + ''')) + text.tag_add('TODO', '1.0', 'end-1c') + color.recolorize_main() + eq(text.tag_nextrange('STRING', '1.0'), ('1.0', '5.4')) + @mock.patch.object(colorizer.ColorDelegator, 'recolorize') @mock.patch.object(colorizer.ColorDelegator, 'notify_range') def test_removecolors(self, mock_notify, mock_recolorize): From 4bdbe2add1f503eb87bb6d7e7c5d3b07c41cb8b4 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Sun, 9 May 2021 11:02:58 +0300 Subject: [PATCH 17/30] remove unused import --- Lib/idlelib/colorizer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index 99eaded86bb85d..a6ca339b7330ab 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -1,7 +1,6 @@ import builtins import keyword import re -import textwrap import time from idlelib.config import idleConf From d550c0fbae633206ffdad5f69c19ba8b66ec1391 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Sun, 9 May 2021 22:05:57 +0300 Subject: [PATCH 18/30] add more reference links in NEWS entry --- .../next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst b/Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst index bb5f583c1fa503..1bd4e65f081865 100644 --- a/Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst +++ b/Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst @@ -1,2 +1,3 @@ -Highlight the new pattern-matching "soft keywords": ``match``, -``case``, and ``_``. +Highlight the new :ref:`match ` statement's +:ref:`soft keywords `: :keyword:`match`, +:keyword:`case `, and :keyword:`_ `. From ee98cf32d1d81aa7a9025a6514d8f9d1585fbcd9 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Mon, 10 May 2021 17:52:24 +0300 Subject: [PATCH 19/30] simplify handling of case softkw, and bring back specific handling of default case This now doesn't mark `case` and `_` in some cases handled before. --- Lib/idlelib/colorizer.py | 21 ++++++++++++--------- Lib/idlelib/idle_test/test_colorizer.py | 22 ++++++++++++---------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index a6ca339b7330ab..a7a83cf6d0fd51 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -21,15 +21,16 @@ def make_pats(): r"(?Pmatch)\b" + r"(?!(?:[ \t]|\\\n)*[=,)\]}])" # not followed by any of =,)]} ) + case_default = ( + r"^[ \t]*" + # at beginning of line + possible indentation + r"(?Pcase)" + + r"[ \t]+(?P_)[ \t]*:" + ) case_softkw_and_pattern = ( r"^[ \t]*" + # at beginning of line + possible indentation - r"(?Pcase)\b" + - r"(?!(?:[ \t]|\\\n)*[=,)\]}])" + # not followed by any of =,)]} - r"(?P.*?)" + # case pattern - r"(?P\bif\b.*?)?" + # possible guard - r"(?P\bas\b.*?)?" + # possible capture - r":" - ) + r"(?Pcase)\b" + + r"(?![ \t]*[=,)\]}])" # not followed by any of =,)]} + ) builtinlist = [str(name) for name in dir(builtins) if not name.startswith('_') and name not in keyword.kwlist] @@ -43,7 +44,8 @@ def make_pats(): string = any("STRING", [sq3string, dq3string, sqstring, dqstring]) prog = re.compile("|".join([ builtin, comment, string, kw, - match_softkw, case_softkw_and_pattern, + match_softkw, case_default, + case_softkw_and_pattern, any("SYNC", [r"\n"]), ]), re.DOTALL | re.MULTILINE) @@ -59,9 +61,10 @@ def make_pats(): prog_group_name_to_tag = { "MATCH_SOFTKW": "KEYWORD", "CASE_SOFTKW": "KEYWORD", + "CASE_DEFAULT_UNDERSCORE": "KEYWORD", + "CASE_SOFTKW2": "KEYWORD", "CASE_AS": "KEYWORD", "CASE_IF": "KEYWORD", - "UNDERSCORE_SOFTKW": "KEYWORD", } diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index 5e54fa484e18b8..8363b44d411a32 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -44,9 +44,9 @@ async def f(): await g() case [_, [_], "_", _]: pass - case _ if _: - raise ValueError("Not a point _") case _ if ("a" if _ else set()): pass + case _: + raise ValueError("Not a point _") ''' case _:''' "match x:" @@ -390,14 +390,16 @@ def test_recolorize_main(self, mock_notify): ('8.0', ('STRING',)), ('8.4', ()), ('8.5', ('STRING',)), ('8.12', ()), ('8.14', ('STRING',)), ('19.0', ('KEYWORD',)), - ('20.4', ('KEYWORD',)), ('20.16', ('KEYWORD',)), ('20.19', ('KEYWORD',)), - ('22.4', ('KEYWORD',)), ('22.10', ('KEYWORD',)), ('22.14', ('KEYWORD',)), ('22.19', ('STRING',)), - ('23.12', ('KEYWORD',)), - ('25.4', ('KEYWORD',)), ('25.9', ('KEYWORD',)), ('25.11', ('KEYWORD',)), ('25.14', (),), - ('26.25', ('STRING',)), ('26.38', ('STRING',)), - ('27.4', ('KEYWORD',)), ('27.9', ('KEYWORD',)), ('27.11', ('KEYWORD',)), - ('27.15', ('STRING',)), ('27.19', ('KEYWORD',)), ('27.22', ()), - ('27.24', ('KEYWORD',)), ('27.29', ('BUILTIN',)), ('27.37', ('KEYWORD',)), + ('20.4', ('KEYWORD',)), ('20.16', ('KEYWORD',)),# ('20.19', ('KEYWORD',)), + #('22.4', ('KEYWORD',)), ('22.10', ('KEYWORD',)), ('22.14', ('KEYWORD',)), ('22.19', ('STRING',)), + #('23.12', ('KEYWORD',)), + ('24.8', ('KEYWORD',)), + ('25.4', ('KEYWORD',)),# ('25.9', ('KEYWORD',)), + ('25.11', ('KEYWORD',)), ('25.15', ('STRING',)), + ('25.19', ('KEYWORD',)), ('25.22', ()), + ('25.24', ('KEYWORD',)), ('25.29', ('BUILTIN',)), ('25.37', ('KEYWORD',)), + ('26.4', ('KEYWORD',)),# ('26.9', ('KEYWORD',)), ('26.11', ('KEYWORD',)), ('26.14', (),), + ('27.25', ('STRING',)), ('27.38', ('STRING',)), ('29.0', ('STRING',)), ('30.1', ('STRING',)), # SYNC at the end of every line. From 9a34c3baab85a0b55c32e4661040e9c1b63f0d07 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Mon, 10 May 2021 21:34:54 +0300 Subject: [PATCH 20/30] add test simulating typing and deleting --- Lib/idlelib/idle_test/test_colorizer.py | 46 +++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index 8363b44d411a32..533b1f3d8b455b 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -3,6 +3,7 @@ from test.support import requires import unittest from unittest import mock +from .tkinter_testing_utils import run_in_tk_mainloop from functools import partial import textwrap @@ -488,6 +489,51 @@ def test_long_multiline_string(self, notify_range): color.recolorize_main() eq(text.tag_nextrange('STRING', '1.0'), ('1.0', '5.4')) + @run_in_tk_mainloop + def test_incremental_editing(self): + text = self.text + eq = self.assertEqual + + # Simulate typing 'inte'. During this, the highlighting should + # change from normal to keyword to builtin to normal. + text.insert('insert', 'i') + yield + eq(text.tag_nextrange('BUILTIN', '1.0'), ()) + eq(text.tag_nextrange('KEYWORD', '1.0'), ()) + + text.insert('insert', 'n') + yield + eq(text.tag_nextrange('BUILTIN', '1.0'), ()) + eq(text.tag_nextrange('KEYWORD', '1.0'), ('1.0', '1.2')) + + text.insert('insert', 't') + yield + eq(text.tag_nextrange('BUILTIN', '1.0'), ('1.0', '1.3')) + eq(text.tag_nextrange('KEYWORD', '1.0'), ()) + + text.insert('insert', 'e') + yield + eq(text.tag_nextrange('BUILTIN', '1.0'), ()) + eq(text.tag_nextrange('KEYWORD', '1.0'), ()) + + # Simulate deleting three characters from the end of 'inte'. + # During this, the highlighting should change from normal to + # builtin to keyword to normal. + text.delete('insert-1c', 'insert') + yield + eq(text.tag_nextrange('BUILTIN', '1.0'), ('1.0', '1.3')) + eq(text.tag_nextrange('KEYWORD', '1.0'), ()) + + text.delete('insert-1c', 'insert') + yield + eq(text.tag_nextrange('BUILTIN', '1.0'), ()) + eq(text.tag_nextrange('KEYWORD', '1.0'), ('1.0', '1.2')) + + text.delete('insert-1c', 'insert') + yield + eq(text.tag_nextrange('BUILTIN', '1.0'), ()) + eq(text.tag_nextrange('KEYWORD', '1.0'), ()) + @mock.patch.object(colorizer.ColorDelegator, 'recolorize') @mock.patch.object(colorizer.ColorDelegator, 'notify_range') def test_removecolors(self, mock_notify, mock_recolorize): From 25dfd1a95a3e1eb9efdf017c50c6053138af0fb9 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Mon, 10 May 2021 21:54:02 +0300 Subject: [PATCH 21/30] fix highlighting of underscore in case, and its tests --- Lib/idlelib/colorizer.py | 4 ++-- Lib/idlelib/idle_test/test_colorizer.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index a7a83cf6d0fd51..d82d47c18502d4 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -24,12 +24,12 @@ def make_pats(): case_default = ( r"^[ \t]*" + # at beginning of line + possible indentation r"(?Pcase)" + - r"[ \t]+(?P_)[ \t]*:" + r"[ \t]+(?P_\b)" ) case_softkw_and_pattern = ( r"^[ \t]*" + # at beginning of line + possible indentation r"(?Pcase)\b" + - r"(?![ \t]*[=,)\]}])" # not followed by any of =,)]} + r"(?![ \t]*(?:[=,)\]}]|_\b))" # not followed by any of =,)]} ) builtinlist = [str(name) for name in dir(builtins) if not name.startswith('_') and diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index 533b1f3d8b455b..b6da477d1a825b 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -395,11 +395,11 @@ def test_recolorize_main(self, mock_notify): #('22.4', ('KEYWORD',)), ('22.10', ('KEYWORD',)), ('22.14', ('KEYWORD',)), ('22.19', ('STRING',)), #('23.12', ('KEYWORD',)), ('24.8', ('KEYWORD',)), - ('25.4', ('KEYWORD',)),# ('25.9', ('KEYWORD',)), + ('25.4', ('KEYWORD',)), ('25.9', ('KEYWORD',)), ('25.11', ('KEYWORD',)), ('25.15', ('STRING',)), ('25.19', ('KEYWORD',)), ('25.22', ()), ('25.24', ('KEYWORD',)), ('25.29', ('BUILTIN',)), ('25.37', ('KEYWORD',)), - ('26.4', ('KEYWORD',)),# ('26.9', ('KEYWORD',)), ('26.11', ('KEYWORD',)), ('26.14', (),), + ('26.4', ('KEYWORD',)), ('26.9', ('KEYWORD',)),# ('26.11', ('KEYWORD',)), ('26.14', (),), ('27.25', ('STRING',)), ('27.38', ('STRING',)), ('29.0', ('STRING',)), ('30.1', ('STRING',)), From 1c87c8affe277766bfca4e6161010d7f753e3035 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Mon, 10 May 2021 22:28:43 +0300 Subject: [PATCH 22/30] avoid highlighting match and case in more scenarios (+ tests) --- Lib/idlelib/colorizer.py | 17 +++- Lib/idlelib/idle_test/test_colorizer.py | 107 +++++++++++++++++++----- 2 files changed, 98 insertions(+), 26 deletions(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index d82d47c18502d4..284534234d4ac3 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -19,8 +19,13 @@ def make_pats(): match_softkw = ( r"^[ \t]*" + # at beginning of line + possible indentation r"(?Pmatch)\b" + - r"(?!(?:[ \t]|\\\n)*[=,)\]}])" # not followed by any of =,)]} - ) + r"(?![ \t]*(?:" + "|".join([ # not followed by ... + r"[:,;=^&|@~)\]}]", # a character which means it can't be a + # pattern-matching statement + r"\b(?:" + r"|".join(keyword.kwlist) + r")\b", # a keyword + ]) + + r"))" + ) case_default = ( r"^[ \t]*" + # at beginning of line + possible indentation r"(?Pcase)" + @@ -29,7 +34,13 @@ def make_pats(): case_softkw_and_pattern = ( r"^[ \t]*" + # at beginning of line + possible indentation r"(?Pcase)\b" + - r"(?![ \t]*(?:[=,)\]}]|_\b))" # not followed by any of =,)]} + r"(?![ \t]*(?:" + "|".join([ # not followed by ... + r"_\b", # a lone underscore + r"[:,;=^&|@~)\]}]", # a character which means it can't be a + # pattern-matching case + r"\b(?:" + r"|".join(keyword.kwlist) + r")\b", # a keyword + ]) + + r"))" ) builtinlist = [str(name) for name in dir(builtins) if not name.startswith('_') and diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index b6da477d1a825b..a6adedf8cc85df 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -435,42 +435,103 @@ def test_recolorize_main(self, mock_notify): eq(text.tag_nextrange('SYNC', '8.0'), ('8.26', '9.0')) eq(text.tag_nextrange('SYNC', '30.0'), ('30.10', '32.0')) - @mock.patch.object(colorizer.ColorDelegator, 'notify_range') - def test_def_statement(self, mock_notify): + def _assert_highlighting(self, source, tags): text = self.text color = self.color eq = self.assertEqual - # empty def - text.insert('insert', 'def') + text.delete('1.0', 'end-1c') + text.insert('insert', source) text.tag_add('TODO', '1.0', 'end-1c') color.recolorize_main() - eq(text.tag_names('1.0'), ('KEYWORD',)) + + text_tags = {} + for tag in set(text.tag_names()) - {'sel', 'TODO', 'SYNC'}: + tag_ranges = [rng.string for rng in text.tag_ranges(tag)] + for pair in zip(tag_ranges[::2], tag_ranges[1::2]): + text_tags.setdefault(tag, []).append(pair) + eq(text_tags, tags) + text.delete('1.0', 'end-1c') + @mock.patch.object(colorizer.ColorDelegator, 'notify_range') + def test_def_statement(self, mock_notify): + # empty def + self._assert_highlighting('def', {'KEYWORD': [('1.0', '1.3')]}) + # def followed by identifier - text.insert('1.0', 'def foo:') - text.tag_add('TODO', '1.0', 'end-1c') - color.recolorize_main() - eq(text.tag_names('1.0'), ('KEYWORD',)) - eq(text.tag_names('1.4'), ('DEFINITION',)) - text.delete('1.0', 'end-1c') + self._assert_highlighting('def foo:', {'KEYWORD': [('1.0', '1.3')], + 'DEFINITION': [('1.4', '1.7')]}) # def followed by partial identifier - text.insert('1.0', 'def fo') - text.tag_add('TODO', '1.0', 'end-1c') - color.recolorize_main() - eq(text.tag_names('1.0'), ('KEYWORD',)) - eq(text.tag_names('1.4'), ('DEFINITION',)) - text.delete('1.0', 'end-1c') + self._assert_highlighting('def fo', {'KEYWORD': [('1.0', '1.3')], + 'DEFINITION': [('1.4', '1.6')]}) # def followed by non-keyword - text.insert('1.0', 'def ++') - text.tag_add('TODO', '1.0', 'end-1c') - color.recolorize_main() - eq(text.tag_names('1.0'), ('KEYWORD',)) - eq(text.tag_names('1.4'), ()) - text.delete('1.0', 'end-1c') + self._assert_highlighting('def ++', {'KEYWORD': [('1.0', '1.3')]}) + + @mock.patch.object(colorizer.ColorDelegator, 'notify_range') + def test_match_soft_keyword(self, mock_notify): + # empty match + self._assert_highlighting('match', {'KEYWORD': [('1.0', '1.5')]}) + + # match followed by partial identifier + self._assert_highlighting('match fo', {'KEYWORD': [('1.0', '1.5')]}) + + # match followed by identifier and colon + self._assert_highlighting('match foo:', {'KEYWORD': [('1.0', '1.5')]}) + + # match followed by keyword + self._assert_highlighting('match and', {'KEYWORD': [('1.6', '1.9')]}) + + # match followed by builtin with keyword prefix + self._assert_highlighting('match int:', {'KEYWORD': [('1.0', '1.5')], + 'BUILTIN': [('1.6', '1.9')]}) + + # match followed by non-text operator + self._assert_highlighting('match^', {}) + self._assert_highlighting('match @', {}) + + # match followed by colon + self._assert_highlighting('match :', {}) + + # match followed by comma + self._assert_highlighting('match\t,', {}) + + # match followed by a lone underscore + self._assert_highlighting('match _:', {'KEYWORD': [('1.0', '1.5')]}) + + @mock.patch.object(colorizer.ColorDelegator, 'notify_range') + def test_case_soft_keyword(self, mock_notify): + # empty case + self._assert_highlighting('case', {'KEYWORD': [('1.0', '1.4')]}) + + # case followed by partial identifier + self._assert_highlighting('case fo', {'KEYWORD': [('1.0', '1.4')]}) + + # case followed by identifier and colon + self._assert_highlighting('case foo:', {'KEYWORD': [('1.0', '1.4')]}) + + # case followed by keyword + self._assert_highlighting('case and', {'KEYWORD': [('1.5', '1.8')]}) + + # case followed by builtin with keyword prefix + self._assert_highlighting('case int:', {'KEYWORD': [('1.0', '1.4')], + 'BUILTIN': [('1.5', '1.8')]}) + + # case followed by non-text operator + self._assert_highlighting('case^', {}) + self._assert_highlighting('case @', {}) + + # case followed by colon + self._assert_highlighting('case :', {}) + + # case followed by comma + self._assert_highlighting('case\t,', {}) + + # case followed by a lone underscore + self._assert_highlighting('case _:', {'KEYWORD': [('1.0', '1.4'), + ('1.5', '1.6')]}) @mock.patch.object(colorizer.ColorDelegator, 'notify_range') def test_long_multiline_string(self, notify_range): From 78980dab8b07c58b20f98b49a9e3b89954527c11 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Mon, 10 May 2021 23:00:23 +0300 Subject: [PATCH 23/30] add info in idle help about soft keyword highlighting --- Doc/library/idle.rst | 5 +++++ Lib/idlelib/help.html | 22 ++++++++++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 3c302115b5f408..f40838649cb063 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -613,6 +613,11 @@ keywords, builtin class and function names, names following ``class`` and ``def``, strings, and comments. For any text window, these are the cursor (when present), found text (when possible), and selected text. +IDLE also highlights the :ref:`soft keywords ` :keyword:`match`, +:keyword:`case `, and :keyword:`_ ` in +pattern-matching statements. However, this highlighting is not perfect and may +be incorrect in some rare cases. + Text coloring is done in the background, so uncolorized text is occasionally visible. To change the color scheme, use the Configure IDLE dialog Highlighting tab. The marking of debugger breakpoint lines in the editor and diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index e80384b7775222..02972ca7ef191e 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -5,7 +5,7 @@ - IDLE — Python 3.10.0a6 documentation + IDLE — Python 3.10.0a7 documentation @@ -18,7 +18,7 @@ @@ -71,7 +71,7 @@

Navigation

  • - 3.10.0a6 Documentation » + 3.10.0a7 Documentation »
  • The Python Standard Library »
    • @@ -581,6 +581,10 @@

    Text colorsclass and def, strings, and comments. For any text window, these are the cursor (when present), found text (when possible), and selected text.

    +

    IDLE also highlights the soft keywords match, +case, and _ in +pattern-matching statements. However, this highlighting is not perfect and may be +incorrect in some rare cases.

    Text coloring is done in the background, so uncolorized text is occasionally visible. To change the color scheme, use the Configure IDLE dialog Highlighting tab. The marking of debugger breakpoint lines in the editor and @@ -685,7 +689,7 @@

    Running user codesys.modules starts with more entries, -and threading.activeCount() returns 2 instead of 1.

    +and threading.active_count() returns 2 instead of 1.

    By default, IDLE runs user code in a separate OS process rather than in the user interface process that runs the shell and editor. In the execution process, it replaces sys.stdin, sys.stdout, and sys.stderr @@ -971,7 +975,7 @@

    Navigation

  • - 3.10.0a6 Documentation » + 3.10.0a7 Documentation »
  • The Python Standard Library »
    • @@ -997,13 +1001,19 @@

    Navigation

    © Copyright 2001-2021, Python Software Foundation.
    + This page is licensed under the Python Software Foundation License Version 2. +
    + Examples, recipes, and other code in the documentation are additionally licensed under the Zero Clause BSD License. +
    + See History and License for more information. +

    The Python Software Foundation is a non-profit corporation. Please donate.

    - Last updated on Mar 29, 2021. + Last updated on May 10, 2021. Found a bug?
    From 574806317ba657b1f4c4b460aa8c8ee968c45bca Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Mon, 10 May 2021 23:13:32 +0300 Subject: [PATCH 24/30] clean up _assert_highlighting and add a doc-string --- Lib/idlelib/idle_test/test_colorizer.py | 27 ++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index a6adedf8cc85df..ad46bffa88d08e 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -435,22 +435,31 @@ def test_recolorize_main(self, mock_notify): eq(text.tag_nextrange('SYNC', '8.0'), ('8.26', '9.0')) eq(text.tag_nextrange('SYNC', '30.0'), ('30.10', '32.0')) - def _assert_highlighting(self, source, tags): + def _assert_highlighting(self, source, tag_ranges): + """Check highlighting of a given piece of code. + + This inserts just this code into the Text widget. It will then + check that the resulting highlighting tag ranges exactly match + those described in the given `tag_ranges` dict. + + Note that the irrelevant tags 'sel', 'TODO' and 'SYNC' are + ignored. + """ text = self.text - color = self.color - eq = self.assertEqual text.delete('1.0', 'end-1c') text.insert('insert', source) text.tag_add('TODO', '1.0', 'end-1c') - color.recolorize_main() + self.color.recolorize_main() - text_tags = {} + # Make a dict with highlighting tag ranges in the Text widget. + text_tag_ranges = {} for tag in set(text.tag_names()) - {'sel', 'TODO', 'SYNC'}: - tag_ranges = [rng.string for rng in text.tag_ranges(tag)] - for pair in zip(tag_ranges[::2], tag_ranges[1::2]): - text_tags.setdefault(tag, []).append(pair) - eq(text_tags, tags) + indexes = [rng.string for rng in text.tag_ranges(tag)] + for index_pair in zip(indexes[::2], indexes[1::2]): + text_tag_ranges.setdefault(tag, []).append(index_pair) + + self.assertEqual(text_tag_ranges, tag_ranges) text.delete('1.0', 'end-1c') From 6c0f5c22c4d2c7c9e037f994689364c664687e55 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Mon, 10 May 2021 23:18:33 +0300 Subject: [PATCH 25/30] refactor mocking of notify_range() with _assert_highlighting --- Lib/idlelib/idle_test/test_colorizer.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index ad46bffa88d08e..93da53e6eb4078 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -447,10 +447,11 @@ def _assert_highlighting(self, source, tag_ranges): """ text = self.text - text.delete('1.0', 'end-1c') - text.insert('insert', source) - text.tag_add('TODO', '1.0', 'end-1c') - self.color.recolorize_main() + with mock.patch.object(colorizer.ColorDelegator, 'notify_range'): + text.delete('1.0', 'end-1c') + text.insert('insert', source) + text.tag_add('TODO', '1.0', 'end-1c') + self.color.recolorize_main() # Make a dict with highlighting tag ranges in the Text widget. text_tag_ranges = {} @@ -461,10 +462,10 @@ def _assert_highlighting(self, source, tag_ranges): self.assertEqual(text_tag_ranges, tag_ranges) - text.delete('1.0', 'end-1c') + with mock.patch.object(colorizer.ColorDelegator, 'notify_range'): + text.delete('1.0', 'end-1c') - @mock.patch.object(colorizer.ColorDelegator, 'notify_range') - def test_def_statement(self, mock_notify): + def test_def_statement(self): # empty def self._assert_highlighting('def', {'KEYWORD': [('1.0', '1.3')]}) @@ -479,8 +480,7 @@ def test_def_statement(self, mock_notify): # def followed by non-keyword self._assert_highlighting('def ++', {'KEYWORD': [('1.0', '1.3')]}) - @mock.patch.object(colorizer.ColorDelegator, 'notify_range') - def test_match_soft_keyword(self, mock_notify): + def test_match_soft_keyword(self): # empty match self._assert_highlighting('match', {'KEYWORD': [('1.0', '1.5')]}) @@ -510,8 +510,7 @@ def test_match_soft_keyword(self, mock_notify): # match followed by a lone underscore self._assert_highlighting('match _:', {'KEYWORD': [('1.0', '1.5')]}) - @mock.patch.object(colorizer.ColorDelegator, 'notify_range') - def test_case_soft_keyword(self, mock_notify): + def test_case_soft_keyword(self): # empty case self._assert_highlighting('case', {'KEYWORD': [('1.0', '1.4')]}) From 4b2b8d7903598d1f4f43029571157bd2eb467e49 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Mon, 10 May 2021 23:20:57 +0300 Subject: [PATCH 26/30] refactor another test to use _assert_highlighting() --- Lib/idlelib/idle_test/test_colorizer.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index 93da53e6eb4078..e782e12d1e87c9 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -541,22 +541,15 @@ def test_case_soft_keyword(self): self._assert_highlighting('case _:', {'KEYWORD': [('1.0', '1.4'), ('1.5', '1.6')]}) - @mock.patch.object(colorizer.ColorDelegator, 'notify_range') - def test_long_multiline_string(self, notify_range): - text = self.text - color = self.color - eq = self.assertEqual - - text.insert('insert', textwrap.dedent('''\ + def test_long_multiline_string(self): + source = textwrap.dedent('''\ """a b c d e""" - ''')) - text.tag_add('TODO', '1.0', 'end-1c') - color.recolorize_main() - eq(text.tag_nextrange('STRING', '1.0'), ('1.0', '5.4')) + ''') + self._assert_highlighting(source, {'STRING': [('1.0', '5.4')]}) @run_in_tk_mainloop def test_incremental_editing(self): From 99a67f6c4ac0bd1e1d43a0afed116f9b1e43b9b8 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Tue, 11 May 2021 10:41:18 +0300 Subject: [PATCH 27/30] update coverage percentage at head of test file --- Lib/idlelib/idle_test/test_colorizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index e782e12d1e87c9..5d7feac2f980d4 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -1,4 +1,4 @@ -"Test colorizer, coverage 93%." +"Test colorizer, coverage 99%." from idlelib import colorizer from test.support import requires import unittest From b991d56da7f3af9516f6a9cbbef837194898521d Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Tue, 11 May 2021 10:46:59 +0300 Subject: [PATCH 28/30] add a What's New entry --- Doc/whatsnew/3.10.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 2c6569fccd624d..f70896ceb32310 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1030,6 +1030,12 @@ Terry Jan Reedy in :issue:`37892`.) We expect to backport these shell changes to a future 3.9 maintenance release. +Highlight the new :ref:`soft keywords ` :keyword:`match`, +:keyword:`case `, and :keyword:`_ ` in +pattern-matching statements. However, this highlighting is not perfect +and may be incorrect in some rare cases. (Contributed by Tal Einat +in bpo-44010.) + importlib.metadata ------------------ From 8b4f201875b2b5c92b24263d943c21cc25d13c2c Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Tue, 11 May 2021 10:55:54 +0300 Subject: [PATCH 29/30] improve wording in NEWS, What's New and docs, and update help.html --- Doc/library/idle.rst | 5 +++-- Doc/whatsnew/3.10.rst | 4 ++-- Lib/idlelib/help.html | 19 ++++++++++--------- .../2021-05-09-09-02-09.bpo-44010.TaLe9x.rst | 2 ++ 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index f40838649cb063..faa34e69ff15d7 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -615,8 +615,9 @@ present), found text (when possible), and selected text. IDLE also highlights the :ref:`soft keywords ` :keyword:`match`, :keyword:`case `, and :keyword:`_ ` in -pattern-matching statements. However, this highlighting is not perfect and may -be incorrect in some rare cases. +pattern-matching statements. However, this highlighting is not perfect and +will be incorrect in some rare cases, including some ``_``-s in ``case`` +patterns. Text coloring is done in the background, so uncolorized text is occasionally visible. To change the color scheme, use the Configure IDLE dialog diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index f70896ceb32310..295a6c1efd730c 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1033,8 +1033,8 @@ release. Highlight the new :ref:`soft keywords ` :keyword:`match`, :keyword:`case `, and :keyword:`_ ` in pattern-matching statements. However, this highlighting is not perfect -and may be incorrect in some rare cases. (Contributed by Tal Einat -in bpo-44010.) +and will be incorrect in some rare cases, including some ``_``-s in +``case`` patterns. (Contributed by Tal Einat in bpo-44010.) importlib.metadata ------------------ diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index 02972ca7ef191e..19041c6054e4cc 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -5,7 +5,7 @@ - IDLE — Python 3.10.0a7 documentation + IDLE — Python 3.11.0a0 documentation @@ -18,7 +18,7 @@ @@ -71,7 +71,7 @@

    Navigation

  • - 3.10.0a7 Documentation » + 3.11.0a0 Documentation »
  • The Python Standard Library »
    • @@ -102,7 +102,7 @@

    Navigation

    IDLE

    -

    Source code: Lib/idlelib/

    +

    Source code: Lib/idlelib/

    IDLE is Python’s Integrated Development and Learning Environment.

    IDLE has the following features:

    @@ -583,8 +583,9 @@

    Text colorssoft keywords match, case, and _ in -pattern-matching statements. However, this highlighting is not perfect and may be -incorrect in some rare cases.

    +pattern-matching statements. However, this highlighting is not perfect and +will be incorrect in some rare cases, including some _-s in case +patterns.

    Text coloring is done in the background, so uncolorized text is occasionally visible. To change the color scheme, use the Configure IDLE dialog Highlighting tab. The marking of debugger breakpoint lines in the editor and @@ -943,7 +944,7 @@

    This Page

    • Report a Bug
    • - Show Source
      • @@ -975,7 +976,7 @@

      Navigation

    • - 3.10.0a7 Documentation » + 3.11.0a0 Documentation »
    • The Python Standard Library »
      • @@ -1013,7 +1014,7 @@

      Navigation



      - Last updated on May 10, 2021. + Last updated on May 11, 2021. Found a bug?
      diff --git a/Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst b/Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst index 1bd4e65f081865..becd331f6d789c 100644 --- a/Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst +++ b/Misc/NEWS.d/next/IDLE/2021-05-09-09-02-09.bpo-44010.TaLe9x.rst @@ -1,3 +1,5 @@ Highlight the new :ref:`match ` statement's :ref:`soft keywords `: :keyword:`match`, :keyword:`case `, and :keyword:`_ `. +However, this highlighting is not perfect and will be incorrect in some +rare cases, including some ``_``-s in ``case`` patterns. From ae3e7e157280d6844f7eea508f758c2533dfc966 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Tue, 11 May 2021 12:07:13 +0300 Subject: [PATCH 30/30] remove dead code, recently added but no longer used --- Lib/idlelib/colorizer.py | 33 ++++++------------------- Lib/idlelib/idle_test/test_colorizer.py | 2 +- 2 files changed, 9 insertions(+), 26 deletions(-) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index 284534234d4ac3..e9f19c145c8673 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -14,7 +14,7 @@ def any(name, alternates): return "(?P<%s>" % name + "|".join(alternates) + ")" -def make_pats(): +def make_pat(): kw = r"\b" + any("KEYWORD", keyword.kwlist) + r"\b" match_softkw = ( r"^[ \t]*" + # at beginning of line + possible indentation @@ -60,22 +60,16 @@ def make_pats(): any("SYNC", [r"\n"]), ]), re.DOTALL | re.MULTILINE) - guard_prog = re.compile("|".join([builtin, comment, string, kw])) - pattern_prog = re.compile("|".join([ - builtin, comment, string, kw, any("UNDERSCORE_SOFTKW", [r"\b_\b"]) - ]), re.DOTALL) - return prog, guard_prog, pattern_prog + return prog -prog, guard_prog, pattern_prog = make_pats() +prog = make_pat() idprog = re.compile(r"\s+(\w+)") prog_group_name_to_tag = { "MATCH_SOFTKW": "KEYWORD", "CASE_SOFTKW": "KEYWORD", "CASE_DEFAULT_UNDERSCORE": "KEYWORD", "CASE_SOFTKW2": "KEYWORD", - "CASE_AS": "KEYWORD", - "CASE_IF": "KEYWORD", } @@ -349,22 +343,11 @@ def _add_tags_in_section(self, chars, head): for m in self.prog.finditer(chars): for name, matched_text in matched_named_groups(m): a, b = m.span(name) - if name in {"CASE_PATTERN", "CASE_AS"}: - for m1 in pattern_prog.finditer(matched_text): - for name, matched_text in matched_named_groups(m1): - a1, b1 = m1.span() - self._add_tag(a + a1, a + b1, head, name) - elif name == "CASE_IF": - for m1 in guard_prog.finditer(matched_text): - for name, matched_text in matched_named_groups(m1): - a1, b1 = m1.span() - self._add_tag(a + a1, a + b1, head, name) - else: - self._add_tag(a, b, head, name) - if matched_text in ("def", "class"): - if m1 := self.idprog.match(chars, b): - a, b = m1.span(1) - self._add_tag(a, b, head, "DEFINITION") + self._add_tag(a, b, head, name) + if matched_text in ("def", "class"): + if m1 := self.idprog.match(chars, b): + a, b = m1.span(1) + self._add_tag(a, b, head, "DEFINITION") def removecolors(self): "Remove all colorizing tags." diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index 5d7feac2f980d4..498480a74e3551 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -70,7 +70,7 @@ def test_any(self): def test_make_pat(self): # Tested in more detail by testing prog. - self.assertTrue(colorizer.make_pats()) + self.assertTrue(colorizer.make_pat()) def test_prog(self): prog = colorizer.prog