From 0e6cca83625ea4aa07ce0c76bebfdfb9c02763d0 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 16 May 2019 16:07:29 -0700 Subject: [PATCH 001/242] autoformat with black --- IPython/terminal/shortcuts.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/IPython/terminal/shortcuts.py b/IPython/terminal/shortcuts.py index 28aa5b4eb1b..11bd4480305 100644 --- a/IPython/terminal/shortcuts.py +++ b/IPython/terminal/shortcuts.py @@ -87,10 +87,23 @@ def create_ipython_shortcuts(shell): def newline_or_execute_outer(shell): + + import black def newline_or_execute(event): """When the user presses return, insert a newline or execute the code.""" b = event.current_buffer d = b.document + def try_reformat(): + try: + tbc = b.delete_before_cursor(len(d.text[:d.cursor_position])) + fmt= black.format_str(tbc, mode=black.FileMode()) + if not tbc.endswith('\n') and fmt.endswith('\n'): + fmt = fmt[:-1] + b.insert_text(fmt) + #print(f'no eexc |{tbc[-1]}|,|{d.text[-1]}|, |{fmt[-3:-1]}|') + except Exception as e: + b.insert_text(tbc) + if b.complete_state: cc = b.complete_state.current_completion @@ -118,9 +131,11 @@ def newline_or_execute(event): return if (status != 'incomplete') and b.accept_handler: + try_reformat() b.validate_and_handle() else: if shell.autoindent: + try_reformat() b.insert_text('\n' + indent) else: b.insert_text('\n') From e962a2b0e1032f5ac3f0c0cd1d40f9f0c177584b Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 6 Jul 2019 21:30:28 -0700 Subject: [PATCH 002/242] some more work --- IPython/terminal/interactiveshell.py | 24 +++++++++++++++++ IPython/terminal/shortcuts.py | 39 +++++++++++++++++----------- 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index be738f8f4de..1a4d5627957 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -87,6 +87,14 @@ def get_default_editor(): _use_simple_prompt = ('IPY_TEST_SIMPLE_PROMPT' in os.environ) or (not _is_tty) +def black_reformat_handler(text_before_cursor): + import black + formatted_text = black.format_str(text_before_cursor, mode=black.FileMode()) + if not text_before_cursor.endswith('\n') and formatted_text.endswith('\n'): + formatted_text = formatted_text[:-1] + return formatted_text + + class TerminalInteractiveShell(InteractiveShell): space_for_menu = Integer(6, help='Number of line at the bottom of the screen ' 'to reserve for the completion menu' @@ -120,6 +128,11 @@ def debugger_cls(self): help="Shortcut style to use at the prompt. 'vi' or 'emacs'.", ).tag(config=True) + autoformatter = Unicode(None, + help="Autoformatter to reformat Terminal code. Can be `'black'` or `None`", + allow_none=True + ).tag(config=True) + mouse_support = Bool(False, help="Enable mouse support in the prompt\n(Note: prevents selecting text with the mouse)" ).tag(config=True) @@ -150,6 +163,16 @@ def _editing_mode(self, change): if self.pt_app: self.pt_app.editing_mode = u_mode + @observe('autoformatter') + def _autoformatter_changed(self, change): + formatter = change.new + if formatter is None: + self.reformat_handler = lambda x:x + elif formatter == 'black': + self.reformat_handler = black_reformat_handler + else: + raise ValueError + @observe('highlighting_style') @observe('colors') def _highlighting_style_changed(self, change): @@ -246,6 +269,7 @@ def init_display_formatter(self): self.display_formatter.ipython_display_formatter.enabled = False def init_prompt_toolkit_cli(self): + self.reformat_handler = lambda x:x if self.simple_prompt: # Fall back to plain non-interactive output for tests. # This is very limited. diff --git a/IPython/terminal/shortcuts.py b/IPython/terminal/shortcuts.py index 11bd4480305..ef0f65c5558 100644 --- a/IPython/terminal/shortcuts.py +++ b/IPython/terminal/shortcuts.py @@ -44,6 +44,15 @@ def create_ipython_shortcuts(shell): & insert_mode ))(return_handler) + def reformat_and_execute(event): + reformat_text_before_cursor(event.current_buffer, event.current_buffer.document, shell) + event.current_buffer.validate_and_handle() + + kb.add('escape', 'enter', filter=(has_focus(DEFAULT_BUFFER) + & ~has_selection + & insert_mode + ))(reformat_and_execute) + kb.add('c-\\')(force_exit) kb.add('c-p', filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER)) @@ -86,24 +95,21 @@ def create_ipython_shortcuts(shell): return kb +def reformat_text_before_cursor(buffer, document, shell): + text = buffer.delete_before_cursor(len(document.text[:document.cursor_position])) + try: + formatted_text = shell.reformat_handler(text) + buffer.insert_text(formatted_text) + except Exception as e: + buffer.insert_text(text) + + def newline_or_execute_outer(shell): - import black def newline_or_execute(event): """When the user presses return, insert a newline or execute the code.""" b = event.current_buffer d = b.document - def try_reformat(): - try: - tbc = b.delete_before_cursor(len(d.text[:d.cursor_position])) - fmt= black.format_str(tbc, mode=black.FileMode()) - if not tbc.endswith('\n') and fmt.endswith('\n'): - fmt = fmt[:-1] - b.insert_text(fmt) - #print(f'no eexc |{tbc[-1]}|,|{d.text[-1]}|, |{fmt[-3:-1]}|') - except Exception as e: - b.insert_text(tbc) - if b.complete_state: cc = b.complete_state.current_completion @@ -120,7 +126,11 @@ def try_reformat(): else: check_text = d.text[:d.cursor_position] status, indent = shell.check_complete(check_text) - + + # if all we have after the cursor is whitespace: reformat current text + # before cursor + if d.text[d.cursor_position:].isspace(): + reformat_text_before_cursor(b, d, shell) if not (d.on_last_line or d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end() ): @@ -131,11 +141,10 @@ def try_reformat(): return if (status != 'incomplete') and b.accept_handler: - try_reformat() + reformat_text_before_cursor(b, d, shell) b.validate_and_handle() else: if shell.autoindent: - try_reformat() b.insert_text('\n' + indent) else: b.insert_text('\n') From cbfdf70d36f0ff2d5ebbedb61163f2d8859c487e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 31 Jul 2019 15:55:13 -0700 Subject: [PATCH 003/242] Provide hooks for arbitrary mimetypes handling. This should allow at some point inline images, an math in the terminal. --- IPython/core/displayhook.py | 2 +- IPython/core/displaypub.py | 19 ++++++-- IPython/core/interactiveshell.py | 2 +- IPython/terminal/interactiveshell.py | 2 + IPython/terminal/prompts.py | 11 +++++ docs/source/whatsnew/pr/mimerenderer.rst | 55 ++++++++++++++++++++++++ 6 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 docs/source/whatsnew/pr/mimerenderer.rst diff --git a/IPython/core/displayhook.py b/IPython/core/displayhook.py index 2128c82d12f..29d41dd5e27 100644 --- a/IPython/core/displayhook.py +++ b/IPython/core/displayhook.py @@ -153,7 +153,7 @@ def compute_format_data(self, result): # This can be set to True by the write_output_prompt method in a subclass prompt_end_newline = False - def write_format_data(self, format_dict, md_dict=None): + def write_format_data(self, format_dict, md_dict=None) -> None: """Write the format data dict to the frontend. This default version of this method simply writes the plain text diff --git a/IPython/core/displaypub.py b/IPython/core/displaypub.py index 9625da2a843..d769692e969 100644 --- a/IPython/core/displaypub.py +++ b/IPython/core/displaypub.py @@ -19,7 +19,7 @@ import sys from traitlets.config.configurable import Configurable -from traitlets import List +from traitlets import List, Dict # This used to be defined here - it is imported for backwards compatibility from .display import publish_display_data @@ -28,6 +28,7 @@ # Main payload class #----------------------------------------------------------------------------- + class DisplayPublisher(Configurable): """A traited class that publishes display data to frontends. @@ -35,6 +36,10 @@ class DisplayPublisher(Configurable): be accessed there. """ + def __init__(self, shell=None, *args, **kwargs): + self.shell = shell + super().__init__(*args, **kwargs) + def _validate_data(self, data, metadata=None): """Validate the display data. @@ -53,7 +58,7 @@ def _validate_data(self, data, metadata=None): raise TypeError('metadata must be a dict, got: %r' % data) # use * to indicate transient, update are keyword-only - def publish(self, data, metadata=None, source=None, *, transient=None, update=False, **kwargs): + def publish(self, data, metadata=None, source=None, *, transient=None, update=False, **kwargs) -> None: """Publish data and metadata to all frontends. See the ``display_data`` message in the messaging documentation for @@ -98,7 +103,15 @@ def publish(self, data, metadata=None, source=None, *, transient=None, update=Fa rather than creating a new output. """ - # The default is to simply write the plain text data using sys.stdout. + handlers = {} + if self.shell is not None: + handlers = self.shell.mime_renderers + + for mime, handler in handlers.items(): + if mime in data: + handler(data[mime], metadata.get(mime, None)) + return + if 'text/plain' in data: print(data['text/plain']) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 6dbf307cdd5..cc7ff1dd36c 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -863,7 +863,7 @@ def init_display_formatter(self): self.configurables.append(self.display_formatter) def init_display_pub(self): - self.display_pub = self.display_pub_class(parent=self) + self.display_pub = self.display_pub_class(parent=self, shell=self) self.configurables.append(self.display_pub) def init_data_pub(self): diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index be738f8f4de..262c574a4f9 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -88,6 +88,8 @@ def get_default_editor(): _use_simple_prompt = ('IPY_TEST_SIMPLE_PROMPT' in os.environ) or (not _is_tty) class TerminalInteractiveShell(InteractiveShell): + mime_renderers = Dict().tag(config=True) + space_for_menu = Integer(6, help='Number of line at the bottom of the screen ' 'to reserve for the completion menu' ).tag(config=True) diff --git a/IPython/terminal/prompts.py b/IPython/terminal/prompts.py index 1a7563bda78..db1b751a026 100644 --- a/IPython/terminal/prompts.py +++ b/IPython/terminal/prompts.py @@ -89,3 +89,14 @@ def write_output_prompt(self): ) else: sys.stdout.write(prompt_txt) + + def write_format_data(self, format_dict, md_dict=None) -> None: + if self.shell.mime_renderers: + + for mime, handler in self.shell.mime_renderers.items(): + if mime in format_dict: + handler(format_dict[mime], None) + return + + super().write_format_data(format_dict, md_dict) + diff --git a/docs/source/whatsnew/pr/mimerenderer.rst b/docs/source/whatsnew/pr/mimerenderer.rst new file mode 100644 index 00000000000..44aeb9d6d3f --- /dev/null +++ b/docs/source/whatsnew/pr/mimerenderer.rst @@ -0,0 +1,55 @@ +Arbitrary Mimetypes Handing in Terminal +======================================= + +When using IPython terminal it is now possible to register function to handle +arbitrary mimetypes (``TerminalInteractiveShell.mime_renderers`` ``Dict`` +configurable). While rendering non-text based representation was possible in +many jupyter frontend; it was not possible in terminal IPython, as usually +terminal are limited to displaying text. As many terminal these days provide +escape sequences to display non-text; bringing this loved feature to IPython CLI +made a lot of sens. This functionality will not only allow inline images; but +allow opening of external program; for example ``fmplayer`` to "display" sound +files. + +Here is a complete IPython tension to display images inline and convert math to +png, before displaying it inline :: + + + from base64 import encodebytes + from IPython.lib.latextools import latex_to_png + + + def mathcat(data, meta): + png = latex_to_png(f'$${data}$$'.replace('\displaystyle', '').replace('$$$', '$$')) + imcat(png, meta) + + IMAGE_CODE = '\033]1337;File=name=name;inline=true;:{}\a' + + def imcat(image_data, metadata): + try: + print(IMAGE_CODE.format(encodebytes(image_data).decode())) + # bug workaround + except: + print(IMAGE_CODE.format(image_data)) + + def register_mimerenderer(ipython, mime, handler): + ipython.display_formatter.active_types.append(mime) + ipython.display_formatter.formatters[mime].enabled = True + ipython.mime_renderers[mime] = handler + + def load_ipython_extension(ipython): + register_mimerenderer(ipython, 'image/png', imcat) + register_mimerenderer(ipython, 'image/jpeg', imcat) + register_mimerenderer(ipython, 'text/latex', mathcat) + +This example only work for iterm2 on mac os and skip error handling for brevity. +One could also invoke an external viewer with ``subporcess.run()`` and a +tempfile, which is left as an exercise. + +So far only the hooks necessary for this are in place, but no default mime +renderers added; so inline images will only be available via extensions. We will +progressively enable these features by default in the next few releases, and +contribution is welcomed. + + + From 72841b001d5502cbdec41fd17fda8e57825124ce Mon Sep 17 00:00:00 2001 From: Tony Fast Date: Wed, 14 Aug 2019 17:20:57 -0400 Subject: [PATCH 004/242] A test for completely builtin names --- IPython/core/completer.py | 52 +++++++++++++--------------- IPython/core/tests/test_completer.py | 14 ++++++++ 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index e7bb0f8bb97..4906ae9d951 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -2014,7 +2014,7 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, # Start with a clean slate of completions matches = [] - custom_res = self.dispatch_custom_completer(text) + # FIXME: we should extend our api to return a dict with completions for # different types of objects. The rlcomplete() method could then # simply collapse the dict into a list for readline, but we'd have @@ -2025,29 +2025,24 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, full_text = line_buffer completions = self._jedi_matches( cursor_pos, cursor_line, full_text) - if custom_res is not None: - # did custom completers produce something? - matches = [(m, 'custom') for m in custom_res] + + if self.merge_completions: + matches = [] + for matcher in self.matchers: + try: + matches.extend([(m, matcher.__qualname__) + for m in matcher(text)]) + except: + # Show the ugly traceback if the matcher causes an + # exception, but do NOT crash the kernel! + sys.excepthook(*sys.exc_info()) else: - # Extend the list of completions with the results of each - # matcher, so we return results to the user from all - # namespaces. - if self.merge_completions: - matches = [] - for matcher in self.matchers: - try: - matches.extend([(m, matcher.__qualname__) - for m in matcher(text)]) - except: - # Show the ugly traceback if the matcher causes an - # exception, but do NOT crash the kernel! - sys.excepthook(*sys.exc_info()) - else: - for matcher in self.matchers: - matches = [(m, matcher.__qualname__) - for m in matcher(text)] - if matches: - break + for matcher in self.matchers: + matches = [(m, matcher.__qualname__) + for m in matcher(text)] + if matches: + break + seen = set() filtered_matches = set() for m in matches: @@ -2056,17 +2051,20 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, filtered_matches.add(m) seen.add(t) - _filtered_matches = sorted( - set(filtered_matches), key=lambda x: completions_sorting_key(x[0]))\ - [:MATCHES_LIMIT] + _filtered_matches = sorted(filtered_matches, key=lambda x: completions_sorting_key(x[0])) + custom_res = [(m, 'custom') for m in self.dispatch_custom_completer(text) or []] + + _filtered_matches = custom_res or _filtered_matches + + _filtered_matches = _filtered_matches[:MATCHES_LIMIT] _matches = [m[0] for m in _filtered_matches] origins = [m[1] for m in _filtered_matches] self.matches = _matches return text, _matches, origins, completions - + def fwd_unicode_match(self, text:str) -> Tuple[str, list]: if self._names is None: self._names = [] diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index d364d1c1a10..5f2210d4d32 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -175,6 +175,20 @@ def complete_A(a, existing_completions): ip.complete("x.") + def test_custom_completion_ordering(self): + """Test that errors from custom attribute completers are silenced.""" + ip = get_ipython() + + _, matches = ip.complete('in') + assert matches.index('input') < matches.index('int') + + def complete_example(a): + return ['example2', 'example1'] + + ip.Completer.custom_completers.add_re('ex*', complete_example) + _, matches = ip.complete('ex') + assert matches.index('example2') < matches.index('example1') + def test_unicode_completions(self): ip = get_ipython() # Some strings that trigger different types of completion. Check them both From b97f9ba51941b6a832abfa3ca9090bf366833876 Mon Sep 17 00:00:00 2001 From: Timo Kaufmann Date: Thu, 22 Aug 2019 23:04:17 +0200 Subject: [PATCH 005/242] Silence pager stderr This is run as a test to determine if a pager is available. If there is no pager, there is a fallback. Therefore, I think it is a bug that in the no-pager case ipython will print sh: less not found To stderr. --- IPython/core/page.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/page.py b/IPython/core/page.py index b5d5b1f56d1..cf9fb21e3de 100644 --- a/IPython/core/page.py +++ b/IPython/core/page.py @@ -210,7 +210,7 @@ def pager_page(strng, start=0, screen_lines=0, pager_cmd=None): retval = None # if I use popen4, things hang. No idea why. #pager,shell_out = os.popen4(pager_cmd) - pager = os.popen(pager_cmd, 'w') + pager = os.popen(pager_cmd + ' 2>/dev/null', 'w') try: pager_encoding = pager.encoding or sys.stdout.encoding pager.write(strng) From 91d36d325aff4990062901556aa9581d2b22c897 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 30 Aug 2019 09:52:21 -0700 Subject: [PATCH 006/242] back to development --- IPython/core/release.py | 4 ++-- tools/release_helper.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 0ad67d00f56..2cb20d7ac20 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 8 +_version_minor = 9 _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] diff --git a/tools/release_helper.sh b/tools/release_helper.sh index d39726ae8b7..67ce086859f 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -78,7 +78,7 @@ echo echo "Let\'s commit : git commit -am \"release $VERSION\" -S" echo $GREEN"Press enter to continue"$NOR read -git commit -am "release $VERSION" # -S +git commit -am "release $VERSION" -S echo echo "git push origin \$BRANCH ?" From f634c37ee523a9ad83343a1ae2079af98d80ad66 Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 3 Sep 2019 18:48:55 +0300 Subject: [PATCH 007/242] Fix for Python 4 --- tools/gen_latex_symbols.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/gen_latex_symbols.py b/tools/gen_latex_symbols.py index 038d8ea2e5a..2f6d339c358 100644 --- a/tools/gen_latex_symbols.py +++ b/tools/gen_latex_symbols.py @@ -11,7 +11,7 @@ import os, sys -if not sys.version_info[0] == 3: +if sys.version_info[0] < 3: print("This script must be run with Python 3, exiting...") sys.exit(1) From 3e5fef7a03182eaf4eb1f8b8676876d59fd4bd10 Mon Sep 17 00:00:00 2001 From: Niclas Date: Thu, 5 Sep 2019 01:50:59 +0200 Subject: [PATCH 008/242] switching to gc.get_referrers() allows for a very simple and (hopefully) efficient update of old class instances --- IPython/extensions/autoreload.py | 58 +++++--------------------------- 1 file changed, 9 insertions(+), 49 deletions(-) diff --git a/IPython/extensions/autoreload.py b/IPython/extensions/autoreload.py index da9cc0e13ef..ada680fcf08 100644 --- a/IPython/extensions/autoreload.py +++ b/IPython/extensions/autoreload.py @@ -115,7 +115,7 @@ import traceback import types import weakref -import inspect +import gc from importlib import import_module from importlib.util import source_from_cache from imp import reload @@ -268,56 +268,16 @@ def update_function(old, new): pass -def update_instances(old, new, objects=None, visited={}): - """Iterate through objects recursively, searching for instances of old and - replace their __class__ reference with new. If no objects are given, start - with the current ipython workspace. - """ - if objects is None: - # make sure visited is cleaned when not called recursively - visited = {} - # find ipython workspace stack frame - frame = next(frame_nfo.frame for frame_nfo in inspect.stack() - if 'trigger' in frame_nfo.function) - # build generator for non-private variable values from workspace - shell = frame.f_locals['self'].shell - user_ns = shell.user_ns - user_ns_hidden = shell.user_ns_hidden - nonmatching = object() - objects = ( value for key, value in user_ns.items() - if not key.startswith('_') - and (value is not user_ns_hidden.get(key, nonmatching)) - and not inspect.ismodule(value)) +def update_instances(old, new): + """Use garbage collector to find all instances that refer to the old + class definition and update their __class__ to point to the new class + definition""" - # use dict values if objects is a dict but don't touch private variables - if hasattr(objects, 'items'): - objects = (value for key, value in objects.items() - if not str(key).startswith('_') - and not inspect.ismodule(value) ) + refs = gc.get_referrers(old) - # try if objects is iterable - try: - for obj in (obj for obj in objects if id(obj) not in visited): - # add current object to visited to avoid revisiting - visited.update({id(obj):obj}) - - # update, if object is instance of old_class (but no subclasses) - if type(obj) is old: - obj.__class__ = new - - - # if object is instance of other class, look for nested instances - if hasattr(obj, '__dict__') and not (inspect.isfunction(obj) - or inspect.ismethod(obj)): - update_instances(old, new, obj.__dict__, visited) - - # if object is a container, search it - if hasattr(obj, 'items') or (hasattr(obj, '__contains__') - and not isinstance(obj, str)): - update_instances(old, new, obj, visited) - - except TypeError: - pass + for ref in refs: + if type(ref) is old: + ref.__class__ = new def update_class(old, new): From ca2480743b1047ecb42325c2fc1f94b0048c2474 Mon Sep 17 00:00:00 2001 From: Mohammad Hossein Sekhavat Date: Thu, 5 Sep 2019 11:24:00 +0000 Subject: [PATCH 009/242] Prevent caching outputs in _, __, ___ when self.cache_size is not positive --- IPython/core/displayhook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/displayhook.py b/IPython/core/displayhook.py index 2128c82d12f..9f160b3ba23 100644 --- a/IPython/core/displayhook.py +++ b/IPython/core/displayhook.py @@ -198,7 +198,7 @@ def update_user_ns(self, result): """Update user_ns with various things like _, __, _1, etc.""" # Avoid recursive reference when displaying _oh/Out - if result is not self.shell.user_ns['_oh']: + if self.cache_size and result is not self.shell.user_ns['_oh']: if len(self.shell.user_ns['_oh']) >= self.cache_size and self.do_full_cache: self.cull_cache() From 6c5d9c3e0ac601ba6c57c13ac1069f760fcb1999 Mon Sep 17 00:00:00 2001 From: mfh92 <48296509+mfh92@users.noreply.github.com> Date: Wed, 18 Sep 2019 10:43:06 -0700 Subject: [PATCH 010/242] typo fix one double-quote symbol was misplaced. --- IPython/core/magics/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/magics/basic.py b/IPython/core/magics/basic.py index 3721cbc4eb0..906d3d5ebdf 100644 --- a/IPython/core/magics/basic.py +++ b/IPython/core/magics/basic.py @@ -124,7 +124,7 @@ def alias_magic(self, line=''): In [6]: %whereami Out[6]: u'/home/testuser' - In [7]: %alias_magic h history -p "-l 30" --line + In [7]: %alias_magic h history "-p -l 30" --line Created `%h` as an alias for `%history -l 30`. """ From b24a72067ebe33c3ed608e85713457fff7ae6749 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 24 Sep 2019 12:24:23 +0200 Subject: [PATCH 011/242] %env: hide likely secrets by default if an environment variable contains 'KEY' or 'TOKEN' or 'SECRET' hide it when printing the whole environment with %env --- IPython/core/magics/osm.py | 8 +++++++- IPython/core/tests/test_magic.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/IPython/core/magics/osm.py b/IPython/core/magics/osm.py index c6d1539bd6d..ddc9b6f5874 100644 --- a/IPython/core/magics/osm.py +++ b/IPython/core/magics/osm.py @@ -446,7 +446,13 @@ def env(self, parameter_s=''): raise UsageError(err) if len(bits) > 1: return self.set_env(parameter_s) - return dict(os.environ) + env = dict(os.environ) + # hide likely secrets when printing the whole environment + for key in list(env): + if any(s in key.lower() for s in ('key', 'token', 'secret')): + env[key] = '' + + return env @line_magic def set_env(self, parameter_s): diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index a845ec7db76..9882d052db1 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -11,6 +11,7 @@ import warnings from textwrap import dedent from unittest import TestCase +from unittest import mock from importlib import invalidate_caches from io import StringIO @@ -733,6 +734,24 @@ def test_env(self): env = _ip.magic("env") self.assertTrue(isinstance(env, dict)) + def test_env_secret(self): + env = _ip.magic("env") + hidden = "" + with mock.patch.dict( + os.environ, + { + "API_KEY": "abc123", + "SECRET_THING": "ssshhh", + "JUPYTER_TOKEN": "", + "VAR": "abc" + } + ): + env = _ip.magic("env") + assert env["API_KEY"] == hidden + assert env["SECRET_THING"] == hidden + assert env["JUPYTER_TOKEN"] == hidden + assert env["VAR"] == "abc" + def test_env_get_set_simple(self): env = _ip.magic("env var val1") self.assertEqual(env, None) From 753ddcfbf7c889aba4a26cc3701f274563d24e75 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Sat, 5 Oct 2019 20:35:38 +0200 Subject: [PATCH 012/242] IPython directive: correctly clear cout --- IPython/sphinxext/ipython_directive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/sphinxext/ipython_directive.py b/IPython/sphinxext/ipython_directive.py index 60c1b51db32..0481624c42c 100644 --- a/IPython/sphinxext/ipython_directive.py +++ b/IPython/sphinxext/ipython_directive.py @@ -585,7 +585,7 @@ def process_input(self, data, input_prompt, lineno): if self.warning_is_error: raise RuntimeError('Non Expected warning in `{}` line {}'.format(filename, lineno)) - self.cout.truncate(0) + self.clear_cout() return (ret, input_lines, processed_output, is_doctest, decorator, image_file, image_directive) From ed126381f35fbda0ed28e403c247539125c390dc Mon Sep 17 00:00:00 2001 From: Vidar Tonaas Fauske Date: Thu, 10 Oct 2019 15:30:33 +0100 Subject: [PATCH 013/242] Easier empty, updatable display --- IPython/core/display.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/IPython/core/display.py b/IPython/core/display.py index d31350cb895..465c000c55a 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -296,6 +296,13 @@ def display(*objs, include=None, exclude=None, metadata=None, transient=None, di if transient: kwargs['transient'] = transient + if not objs and display_id: + # if given no objects, but still a request for a display_id, + # we assume the user wants to insert an empty output that + # can be updated later + objs = [{}] + raw = True + if not raw: format = InteractiveShell.instance().display_formatter.format From 1af9c258bd27cb161cea48b8288af7dfd04bf1c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=93=D0=B5=D0=BE=D1=80=D0=B3=D0=B8=D0=B9=20=D0=A4=D1=80?= =?UTF-8?q?=D0=BE=D0=BB=D0=BE=D0=B2?= Date: Fri, 11 Oct 2019 11:16:38 +0300 Subject: [PATCH 014/242] restore terminal title on exit (xterm) --- IPython/terminal/interactiveshell.py | 9 ++++++++- IPython/utils/terminal.py | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index be738f8f4de..eb49e67d329 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -8,7 +8,7 @@ from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC from IPython.utils import io from IPython.utils.py3compat import input -from IPython.utils.terminal import toggle_set_term_title, set_term_title +from IPython.utils.terminal import toggle_set_term_title, set_term_title, restore_term_title from IPython.utils.process import abbrev_cwd from traitlets import ( Bool, Unicode, Dict, Integer, observe, Instance, Type, default, Enum, Union, @@ -238,6 +238,10 @@ def init_term_title(self, change=None): else: toggle_set_term_title(False) + def restore_term_title(self): + if self.term_title: + restore_term_title() + def init_display_formatter(self): super(TerminalInteractiveShell, self).init_display_formatter() # terminal only supports plain text @@ -507,6 +511,9 @@ def mainloop(self, display_banner=DISPLAY_BANNER_DEPRECATED): if hasattr(self, '_eventloop'): self._eventloop.stop() + self.restore_term_title() + + _inputhook = None def inputhook(self, context): if self._inputhook is not None: diff --git a/IPython/utils/terminal.py b/IPython/utils/terminal.py index 949275e431f..4e1800208c0 100644 --- a/IPython/utils/terminal.py +++ b/IPython/utils/terminal.py @@ -58,14 +58,26 @@ def _set_term_title(*args,**kw): pass +def _restore_term_title(): + pass + + def _set_term_title_xterm(title): """ Change virtual terminal title in xterm-workalikes """ + # save the current title to the xterm "stack" + sys.stdout.write('\033[22;0t') sys.stdout.write('\033]0;%s\007' % title) + +def _restore_term_title_xterm(): + sys.stdout.write('\033[23;0t') + + if os.name == 'posix': TERM = os.environ.get('TERM','') if TERM.startswith('xterm'): _set_term_title = _set_term_title_xterm + _restore_term_title = _restore_term_title_xterm elif sys.platform == 'win32': try: import ctypes @@ -100,6 +112,13 @@ def set_term_title(title): _set_term_title(title) +def restore_term_title(): + """Restore, if possible, terminal title to the original state""" + if ignore_termtitle: + return + _restore_term_title() + + def freeze_term_title(): warnings.warn("This function is deprecated, use toggle_set_term_title()") global ignore_termtitle From b732611ea58cee34d747c286a17db711f401424e Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Thu, 17 Oct 2019 17:11:55 +1100 Subject: [PATCH 015/242] Change matplotlib.pyplot import syntax (for circular import bug) --- IPython/core/pylabtools.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/IPython/core/pylabtools.py b/IPython/core/pylabtools.py index 4423ed5d408..d42c088980c 100644 --- a/IPython/core/pylabtools.py +++ b/IPython/core/pylabtools.py @@ -310,12 +310,12 @@ def activate_matplotlib(backend): # magic of switch_backend(). matplotlib.rcParams['backend'] = backend - import matplotlib.pyplot - matplotlib.pyplot.switch_backend(backend) + # Due to circular imports, pyplot may be only partially initialised + # when this function runs. + # So avoid needing matplotlib attribute-lookup to access pyplot. + from matplotlib import pyplot as plt - # This must be imported last in the matplotlib series, after - # backend/interactivity choices have been made - import matplotlib.pyplot as plt + plt.switch_backend(backend) plt.show._needmain = False # We need to detect at runtime whether show() is called by the user. From efbef07096df261cb1775bffaeb7c99905ce21a6 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Fri, 18 Oct 2019 09:44:20 +1100 Subject: [PATCH 016/242] Build fix: pip should not ignore sub-dependency version requirements. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3369a9695f1..b14f7d7354f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ install: - pip install pip --upgrade - pip install setuptools --upgrade - pip install -e file://$PWD#egg=ipython[test] --upgrade - - pip install trio curio + - pip install trio curio --upgrade --upgrade-strategy eager - pip install 'pytest<4' matplotlib - pip install codecov check-manifest --upgrade From 60e7c6190e28d68aa4e7c7373cb5f5083260d9f5 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 22 Oct 2019 16:26:02 -0700 Subject: [PATCH 017/242] Update trio and curio dependencies eagerly. Otherwise we get an old version of attr; which fails at runtime. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3369a9695f1..b14f7d7354f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ install: - pip install pip --upgrade - pip install setuptools --upgrade - pip install -e file://$PWD#egg=ipython[test] --upgrade - - pip install trio curio + - pip install trio curio --upgrade --upgrade-strategy eager - pip install 'pytest<4' matplotlib - pip install codecov check-manifest --upgrade From 19e61cacb1a241ec4941e504260a3d6e9bbb883a Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 22 Oct 2019 16:15:15 -0700 Subject: [PATCH 018/242] Prepare release notes for 7.9 (long due). I will try to release this Friday. --- docs/source/whatsnew/version7.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 8e78853e4c3..7a912e62888 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,19 @@ 7.x Series ============ +.. _version 790: + +IPython 7.9 is a small release with a couple of improvement and bug fixes. + + - Xterm terminal title should be restored on exit :ghpull:`11910` + - special variables ``_``,``__``, ``___`` are not set anymore when cache size + is 0 or less. :ghpull:`11877` + - Autoreload should have regained some speed by using a new heuristic logic to + find all objects needing reload. This should avoid large objects traversal + like pandas dataframes. :ghpull:`11876` + - Get ready for Python 4. :ghpull:`11874` + + .. _version780: IPython 7.8.0 From 3b26543268f073b0ee48c649ff66096d1b2f0c92 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 25 Oct 2019 11:02:27 -0700 Subject: [PATCH 019/242] update version 7.rst --- docs/source/whatsnew/version7.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 7a912e62888..520ad110f61 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -13,6 +13,12 @@ IPython 7.9 is a small release with a couple of improvement and bug fixes. find all objects needing reload. This should avoid large objects traversal like pandas dataframes. :ghpull:`11876` - Get ready for Python 4. :ghpull:`11874` + - `%env` Magic nonw has euristic to hide potentially sensitive values :ghpull:`11896` + +This is a small release despite a number of Pull Request Pending that need to +be reviewed/worked on. Many of the core developers have been busy outside of +IPython/Jupyter and we thanks all contributor for their patience; we'll work on +these as soon as we have time. .. _version780: From 60b2ea96cc8146857d22ed20081c34a1d6d19327 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 25 Oct 2019 11:08:09 -0700 Subject: [PATCH 020/242] update github stats --- docs/source/whatsnew/github-stats-7.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/source/whatsnew/github-stats-7.rst b/docs/source/whatsnew/github-stats-7.rst index f3eca4fee29..23ff55c83f9 100644 --- a/docs/source/whatsnew/github-stats-7.rst +++ b/docs/source/whatsnew/github-stats-7.rst @@ -2,6 +2,27 @@ Issues closed in the 7.x development cycle ========================================== +Issues closed in 7.9 +-------------------- + +GitHub stats for 2019/08/30 - 2019/10/25 (tag: 7.8.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 1 issues and merged 9 pull requests. +The full list can be seen `on GitHub `__ + +The following 8 authors contributed 20 commits. + +* Benjamin Ragan-Kelley +* Hugo +* Matthias Bussonnier +* mfh92 +* Mohammad Hossein Sekhavat +* Niclas +* Vidar Tonaas Fauske +* Георгий Фролов + Issues closed in 7.8 -------------------- From 2eec187d323fa1d1c1a645a8876a69aa3be7fba1 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 25 Oct 2019 11:10:16 -0700 Subject: [PATCH 021/242] release 7.9.0 --- IPython/core/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 2cb20d7ac20..0cafd66a148 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -24,7 +24,7 @@ _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From f273a59d0a7169acb0293dc6caf962336723f96d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 25 Oct 2019 11:12:18 -0700 Subject: [PATCH 022/242] back to development --- IPython/core/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 0cafd66a148..3961477ed7c 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 9 +_version_minor = 10 _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From a02911aacbdae6598000fdbd5a08139b24f96b43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6kcen=20Eraslan?= Date: Sun, 27 Oct 2019 13:47:20 -0400 Subject: [PATCH 023/242] Fix restoring more than 2 variables and add support for storing multiple variables. --- IPython/extensions/storemagic.py | 91 +++++++++++---------- IPython/extensions/tests/test_storemagic.py | 23 ++++-- 2 files changed, 67 insertions(+), 47 deletions(-) diff --git a/IPython/extensions/storemagic.py b/IPython/extensions/storemagic.py index dae325afe6a..51b79ad314e 100644 --- a/IPython/extensions/storemagic.py +++ b/IPython/extensions/storemagic.py @@ -20,12 +20,15 @@ from traitlets import Bool -def restore_aliases(ip): +def restore_aliases(ip, alias=None): staliases = ip.db.get('stored_aliases', {}) - for k,v in staliases.items(): - #print "restore alias",k,v # dbg - #self.alias_table[k] = v - ip.alias_manager.define_alias(k,v) + if alias is None: + for k,v in staliases.items(): + #print "restore alias",k,v # dbg + #self.alias_table[k] = v + ip.alias_manager.define_alias(k,v) + else: + ip.alias_manager.define_alias(alias, staliases[alias]) def refresh_variables(ip): @@ -58,13 +61,13 @@ class StoreMagics(Magics): """Lightweight persistence for python variables. Provides the %store magic.""" - + autorestore = Bool(False, help= """If True, any %store-d variables will be automatically restored when IPython starts. """ ).tag(config=True) - + def __init__(self, shell): super(StoreMagics, self).__init__(shell=shell) self.shell.configurables.append(self) @@ -94,13 +97,13 @@ def store(self, parameter_s=''): * ``%store`` - Show list of all variables and their current values - * ``%store spam`` - Store the *current* value of the variable spam - to disk + * ``%store spam bar`` - Store the *current* value of the variables spam + and bar to disk * ``%store -d spam`` - Remove the variable and its value from storage * ``%store -z`` - Remove all variables from storage - * ``%store -r`` - Refresh all variables from store (overwrite - current vals) - * ``%store -r spam bar`` - Refresh specified variables from store + * ``%store -r`` - Refresh all variables, aliases and directory history + from store (overwrite current vals) + * ``%store -r spam bar`` - Refresh specified variables and aliases from store (delete current val) * ``%store foo >a.txt`` - Store value of foo to new file a.txt * ``%store foo >>a.txt`` - Append value of foo to file a.txt @@ -116,7 +119,7 @@ def store(self, parameter_s=''): """ opts,argsl = self.parse_options(parameter_s,'drz',mode='string') - args = argsl.split(None,1) + args = argsl.split() ip = self.shell db = ip.db # delete @@ -141,7 +144,10 @@ def store(self, parameter_s=''): try: obj = db['autorestore/' + arg] except KeyError: - print("no stored variable %s" % arg) + try: + restore_aliases(ip, alias=arg) + except KeyError: + print("no stored variable or alias %s" % arg) else: ip.user_ns[arg] = obj else: @@ -189,38 +195,39 @@ def store(self, parameter_s=''): return # %store foo - try: - obj = ip.user_ns[args[0]] - except KeyError: - # it might be an alias - name = args[0] + for arg in args: try: - cmd = ip.alias_manager.retrieve_alias(name) - except ValueError: - raise UsageError("Unknown variable '%s'" % name) - - staliases = db.get('stored_aliases',{}) - staliases[name] = cmd - db['stored_aliases'] = staliases - print("Alias stored: %s (%s)" % (name, cmd)) - return - - else: - modname = getattr(inspect.getmodule(obj), '__name__', '') - if modname == '__main__': - print(textwrap.dedent("""\ - Warning:%s is %s - Proper storage of interactively declared classes (or instances - of those classes) is not possible! Only instances - of classes in real modules on file system can be %%store'd. - """ % (args[0], obj) )) + obj = ip.user_ns[arg] + except KeyError: + # it might be an alias + name = arg + try: + cmd = ip.alias_manager.retrieve_alias(name) + except ValueError: + raise UsageError("Unknown variable '%s'" % name) + + staliases = db.get('stored_aliases',{}) + staliases[name] = cmd + db['stored_aliases'] = staliases + print("Alias stored: %s (%s)" % (name, cmd)) return - #pickled = pickle.dumps(obj) - db[ 'autorestore/' + args[0] ] = obj - print("Stored '%s' (%s)" % (args[0], obj.__class__.__name__)) + + else: + modname = getattr(inspect.getmodule(obj), '__name__', '') + if modname == '__main__': + print(textwrap.dedent("""\ + Warning:%s is %s + Proper storage of interactively declared classes (or instances + of those classes) is not possible! Only instances + of classes in real modules on file system can be %%store'd. + """ % (arg, obj) )) + return + #pickled = pickle.dumps(obj) + db[ 'autorestore/' + arg ] = obj + print("Stored '%s' (%s)" % (arg, obj.__class__.__name__)) def load_ipython_extension(ip): """Load the extension in IPython.""" ip.register_magics(StoreMagics) - + diff --git a/IPython/extensions/tests/test_storemagic.py b/IPython/extensions/tests/test_storemagic.py index a7b41bab533..6f8371d336f 100644 --- a/IPython/extensions/tests/test_storemagic.py +++ b/IPython/extensions/tests/test_storemagic.py @@ -10,29 +10,42 @@ def setup_module(): def test_store_restore(): assert 'bar' not in ip.user_ns, "Error: some other test leaked `bar` in user_ns" assert 'foo' not in ip.user_ns, "Error: some other test leaked `foo` in user_ns" + assert 'foobar' not in ip.user_ns, "Error: some other test leaked `foobar` in user_ns" + assert 'foobaz' not in ip.user_ns, "Error: some other test leaked `foobaz` in user_ns" ip.user_ns['foo'] = 78 ip.magic('alias bar echo "hello"') + ip.user_ns['foobar'] = 79 + ip.user_ns['foobaz'] = '80' tmpd = tempfile.mkdtemp() ip.magic('cd ' + tmpd) ip.magic('store foo') ip.magic('store bar') - + ip.magic('store foobar foobaz') + # Check storing nt.assert_equal(ip.db['autorestore/foo'], 78) nt.assert_in('bar', ip.db['stored_aliases']) - + nt.assert_equal(ip.db['autorestore/foobar'], 79) + nt.assert_equal(ip.db['autorestore/foobaz'], '80') + # Remove those items ip.user_ns.pop('foo', None) + ip.user_ns.pop('foobar', None) + ip.user_ns.pop('foobaz', None) ip.alias_manager.undefine_alias('bar') ip.magic('cd -') ip.user_ns['_dh'][:] = [] - + # Check restoring - ip.magic('store -r') + ip.magic('store -r foo bar foobar foobaz') nt.assert_equal(ip.user_ns['foo'], 78) assert ip.alias_manager.is_alias('bar') + nt.assert_equal(ip.user_ns['foobar'], 79) + nt.assert_equal(ip.user_ns['foobaz'], '80') + + ip.magic('store -r') # restores _dh too nt.assert_in(os.path.realpath(tmpd), ip.user_ns['_dh']) - + os.rmdir(tmpd) def test_autorestore(): From 7411df1d3d1b52349760546e857045363701e5d2 Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Fri, 11 Oct 2019 13:06:10 +0100 Subject: [PATCH 024/242] ENH: Experimental input hook for wxpython phoenix --- IPython/terminal/pt_inputhooks/wx.py | 59 +++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/IPython/terminal/pt_inputhooks/wx.py b/IPython/terminal/pt_inputhooks/wx.py index e6b4f57aafc..aef35d0579f 100644 --- a/IPython/terminal/pt_inputhooks/wx.py +++ b/IPython/terminal/pt_inputhooks/wx.py @@ -137,11 +137,66 @@ def inputhook_wx3(context): pass return 0 + +def inputhook_wxphoenix(context): + """Run the wx event loop by processing pending events only. + + This is equivalent to inputhook_wx3, but has been updated to work with + wxPython Phoenix. + """ + + # See inputhook_wx3 for in-line comments. + try: + app = wx.GetApp() + if app is not None: + assert wx.IsMainThread() + + if not callable(signal.getsignal(signal.SIGINT)): + signal.signal(signal.SIGINT, signal.default_int_handler) + + evtloop = wx.GUIEventLoop() + ea = wx.EventLoopActivator(evtloop) + t = clock() + while not context.input_is_ready(): + while evtloop.Pending(): + t = clock() + evtloop.Dispatch() + + # Not all events will be procesed by Dispatch - + # we have to call ProcessPendingEvents as well + # to ensure that all events get processed. + app.ProcessPendingEvents() + evtloop.ProcessIdle() + + used_time = clock() - t + + if used_time > 10.0: + time.sleep(1.0) + elif used_time > 0.1: + time.sleep(0.05) + else: + time.sleep(0.001) + del ea + except KeyboardInterrupt: + pass + return 0 + + if sys.platform == 'darwin': # On OSX, evtloop.Pending() always returns True, regardless of there being # any events pending. As such we can't use implementations 1 or 3 of the # inputhook as those depend on a pending/dispatch loop. inputhook = inputhook_wx2 else: - # This is our default implementation - inputhook = inputhook_wx3 + + # Get the major wx version number + if hasattr(wx, '__version__'): + major_version = wx.__version__[0] + else: + major_version = '3' + + # Use the phoenix hook for wxpython >= 4 + if int(major_version) >= 4: + inputhook = inputhook_wxphoenix + else: + inputhook = inputhook_wx3 From 0f50c5a5e557b355d6bcf9953ca201524b1687c0 Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Fri, 18 Oct 2019 18:55:39 +0100 Subject: [PATCH 025/242] RF: Make version check a bit more robust --- IPython/terminal/pt_inputhooks/wx.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/IPython/terminal/pt_inputhooks/wx.py b/IPython/terminal/pt_inputhooks/wx.py index aef35d0579f..c4115998a27 100644 --- a/IPython/terminal/pt_inputhooks/wx.py +++ b/IPython/terminal/pt_inputhooks/wx.py @@ -190,13 +190,14 @@ def inputhook_wxphoenix(context): else: # Get the major wx version number - if hasattr(wx, '__version__'): - major_version = wx.__version__[0] - else: - major_version = '3' + major_version = 3 + try: + major_version = int(wx.__version__[0]) + except Exception: + pass # Use the phoenix hook for wxpython >= 4 - if int(major_version) >= 4: + if major_version >= 4: inputhook = inputhook_wxphoenix else: inputhook = inputhook_wx3 From dae0175ce27712307a126712009566f692bc8b77 Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Sat, 19 Oct 2019 13:33:36 +0100 Subject: [PATCH 026/242] RF: Change wx strategy to that used by ipykernel.eventloops.loop_wx. Seems to work fine on both linux and macOS. --- IPython/terminal/pt_inputhooks/wx.py | 113 +++++++++++++-------------- 1 file changed, 56 insertions(+), 57 deletions(-) diff --git a/IPython/terminal/pt_inputhooks/wx.py b/IPython/terminal/pt_inputhooks/wx.py index c4115998a27..ce7ea40addc 100644 --- a/IPython/terminal/pt_inputhooks/wx.py +++ b/IPython/terminal/pt_inputhooks/wx.py @@ -139,65 +139,64 @@ def inputhook_wx3(context): def inputhook_wxphoenix(context): - """Run the wx event loop by processing pending events only. + """Run the wx event loop until the user provides more input. - This is equivalent to inputhook_wx3, but has been updated to work with - wxPython Phoenix. + This function uses the same approach to that used in + ipykernel.eventloops.loop_wx. """ - # See inputhook_wx3 for in-line comments. - try: - app = wx.GetApp() - if app is not None: - assert wx.IsMainThread() - - if not callable(signal.getsignal(signal.SIGINT)): - signal.signal(signal.SIGINT, signal.default_int_handler) - - evtloop = wx.GUIEventLoop() - ea = wx.EventLoopActivator(evtloop) - t = clock() - while not context.input_is_ready(): - while evtloop.Pending(): - t = clock() - evtloop.Dispatch() - - # Not all events will be procesed by Dispatch - - # we have to call ProcessPendingEvents as well - # to ensure that all events get processed. - app.ProcessPendingEvents() - evtloop.ProcessIdle() - - used_time = clock() - t - - if used_time > 10.0: - time.sleep(1.0) - elif used_time > 0.1: - time.sleep(0.05) - else: - time.sleep(0.001) - del ea - except KeyboardInterrupt: - pass - return 0 - - -if sys.platform == 'darwin': - # On OSX, evtloop.Pending() always returns True, regardless of there being - # any events pending. As such we can't use implementations 1 or 3 of the - # inputhook as those depend on a pending/dispatch loop. + app = wx.GetApp() + + if app is None: + return + + # Wx uses milliseconds + poll_interval = 100 + + # This function gets polled periodically; when + # input is ready, the wx main loop is stopped. + def wake(): + if context.input_is_ready(): + app.ExitMainLoop() + + # We have to put the wx.Timer in a wx.Frame for it to fire properly. + # We make the Frame hidden when we create it in the main app below. + class TimerFrame(wx.Frame): + def __init__(self, func): + wx.Frame.__init__(self, None, -1) + self.timer = wx.Timer(self) + self.timer.Start(poll_interval) + self.Bind(wx.EVT_TIMER, self.on_timer) + self.func = func + + def on_timer(self, event): + self.func() + + frame = TimerFrame(wake) + frame.Show(False) + + # The import of wx on Linux sets the handler for signal.SIGINT + # to 0. This is a bug in wx or gtk. We fix by just setting it + # back to the Python default. + if not callable(signal.getsignal(signal.SIGINT)): + signal.signal(signal.SIGINT, signal.default_int_handler) + + app.MainLoop() + +# Get the major wx version number to figure out what input hook we should use. +major_version = 3 +try: + major_version = int(wx.__version__[0]) +except Exception: + pass + +# Use the phoenix hook on all platforms for wxpython >= 4 +if major_version >= 4: + inputhook = inputhook_wxphoenix +# On OSX, evtloop.Pending() always returns True, regardless of there being +# any events pending. As such we can't use implementations 1 or 3 of the +# inputhook as those depend on a pending/dispatch loop. +elif sys.platform == 'darwin': inputhook = inputhook_wx2 else: - - # Get the major wx version number - major_version = 3 - try: - major_version = int(wx.__version__[0]) - except Exception: - pass - - # Use the phoenix hook for wxpython >= 4 - if major_version >= 4: - inputhook = inputhook_wxphoenix - else: - inputhook = inputhook_wx3 + inputhook = inputhook_wx3 From 73517c8eeba9b8c30a8d2affa8c9b9fd51adc121 Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Sat, 19 Oct 2019 13:42:47 +0100 Subject: [PATCH 027/242] RF: Factor out keyboard interrupt catch --- IPython/terminal/pt_inputhooks/wx.py | 154 ++++++++++++++------------- 1 file changed, 82 insertions(+), 72 deletions(-) diff --git a/IPython/terminal/pt_inputhooks/wx.py b/IPython/terminal/pt_inputhooks/wx.py index ce7ea40addc..df41e7e202e 100644 --- a/IPython/terminal/pt_inputhooks/wx.py +++ b/IPython/terminal/pt_inputhooks/wx.py @@ -8,30 +8,41 @@ import wx +def ignore_keyboardinterrupts(func): + """Decorator which causes KeyboardInterrupt exceptions to be + ignored during execution of the decorated function. + """ + def wrapper(*args, **kwargs): + try: + func(*args, **kwargs) + except KeyboardInterrupt: + pass + return wrapper + + +@ignore_keyboardinterrupts def inputhook_wx1(context): """Run the wx event loop by processing pending events only. This approach seems to work, but its performance is not great as it relies on having PyOS_InputHook called regularly. """ - try: - app = wx.GetApp() - if app is not None: - assert wx.Thread_IsMain() - - # Make a temporary event loop and process system events until - # there are no more waiting, then allow idle events (which - # will also deal with pending or posted wx events.) - evtloop = wx.EventLoop() - ea = wx.EventLoopActivator(evtloop) - while evtloop.Pending(): - evtloop.Dispatch() - app.ProcessIdle() - del ea - except KeyboardInterrupt: - pass + app = wx.GetApp() + if app is not None: + assert wx.Thread_IsMain() + + # Make a temporary event loop and process system events until + # there are no more waiting, then allow idle events (which + # will also deal with pending or posted wx events.) + evtloop = wx.EventLoop() + ea = wx.EventLoopActivator(evtloop) + while evtloop.Pending(): + evtloop.Dispatch() + app.ProcessIdle() + del ea return 0 + class EventLoopTimer(wx.Timer): def __init__(self, func): @@ -41,6 +52,7 @@ def __init__(self, func): def Notify(self): self.func() + class EventLoopRunner(object): def Run(self, time, input_is_ready): @@ -55,6 +67,8 @@ def check_stdin(self): self.timer.Stop() self.evtloop.Exit() + +@ignore_keyboardinterrupts def inputhook_wx2(context): """Run the wx event loop, polling for stdin. @@ -69,19 +83,18 @@ def inputhook_wx2(context): but eventually performance would suffer from calling select/kbhit too often. """ - try: - app = wx.GetApp() - if app is not None: - assert wx.Thread_IsMain() - elr = EventLoopRunner() - # As this time is made shorter, keyboard response improves, but idle - # CPU load goes up. 10 ms seems like a good compromise. - elr.Run(time=10, # CHANGE time here to control polling interval - input_is_ready=context.input_is_ready) - except KeyboardInterrupt: - pass + app = wx.GetApp() + if app is not None: + assert wx.Thread_IsMain() + elr = EventLoopRunner() + # As this time is made shorter, keyboard response improves, but idle + # CPU load goes up. 10 ms seems like a good compromise. + elr.Run(time=10, # CHANGE time here to control polling interval + input_is_ready=context.input_is_ready) return 0 + +@ignore_keyboardinterrupts def inputhook_wx3(context): """Run the wx event loop by processing pending events only. @@ -90,54 +103,50 @@ def inputhook_wx3(context): time.sleep is inserted. This is needed, otherwise, CPU usage is at 100%. This sleep time should be tuned though for best performance. """ - # We need to protect against a user pressing Control-C when IPython is - # idle and this is running. We trap KeyboardInterrupt and pass. - try: - app = wx.GetApp() - if app is not None: - assert wx.Thread_IsMain() - - # The import of wx on Linux sets the handler for signal.SIGINT - # to 0. This is a bug in wx or gtk. We fix by just setting it - # back to the Python default. - if not callable(signal.getsignal(signal.SIGINT)): - signal.signal(signal.SIGINT, signal.default_int_handler) - - evtloop = wx.EventLoop() - ea = wx.EventLoopActivator(evtloop) - t = clock() - while not context.input_is_ready(): - while evtloop.Pending(): - t = clock() - evtloop.Dispatch() - app.ProcessIdle() - # We need to sleep at this point to keep the idle CPU load - # low. However, if sleep to long, GUI response is poor. As - # a compromise, we watch how often GUI events are being processed - # and switch between a short and long sleep time. Here are some - # stats useful in helping to tune this. - # time CPU load - # 0.001 13% - # 0.005 3% - # 0.01 1.5% - # 0.05 0.5% - used_time = clock() - t - if used_time > 10.0: - # print 'Sleep for 1 s' # dbg - time.sleep(1.0) - elif used_time > 0.1: - # Few GUI events coming in, so we can sleep longer - # print 'Sleep for 0.05 s' # dbg - time.sleep(0.05) - else: - # Many GUI events coming in, so sleep only very little - time.sleep(0.001) - del ea - except KeyboardInterrupt: - pass + app = wx.GetApp() + if app is not None: + assert wx.Thread_IsMain() + + # The import of wx on Linux sets the handler for signal.SIGINT + # to 0. This is a bug in wx or gtk. We fix by just setting it + # back to the Python default. + if not callable(signal.getsignal(signal.SIGINT)): + signal.signal(signal.SIGINT, signal.default_int_handler) + + evtloop = wx.EventLoop() + ea = wx.EventLoopActivator(evtloop) + t = clock() + while not context.input_is_ready(): + while evtloop.Pending(): + t = clock() + evtloop.Dispatch() + app.ProcessIdle() + # We need to sleep at this point to keep the idle CPU load + # low. However, if sleep to long, GUI response is poor. As + # a compromise, we watch how often GUI events are being processed + # and switch between a short and long sleep time. Here are some + # stats useful in helping to tune this. + # time CPU load + # 0.001 13% + # 0.005 3% + # 0.01 1.5% + # 0.05 0.5% + used_time = clock() - t + if used_time > 10.0: + # print 'Sleep for 1 s' # dbg + time.sleep(1.0) + elif used_time > 0.1: + # Few GUI events coming in, so we can sleep longer + # print 'Sleep for 0.05 s' # dbg + time.sleep(0.05) + else: + # Many GUI events coming in, so sleep only very little + time.sleep(0.001) + del ea return 0 +@ignore_keyboardinterrupts def inputhook_wxphoenix(context): """Run the wx event loop until the user provides more input. @@ -183,6 +192,7 @@ def on_timer(self, event): app.MainLoop() + # Get the major wx version number to figure out what input hook we should use. major_version = 3 try: From 64512e81710717447f59498de6e306c3d72d470f Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Sun, 20 Oct 2019 13:44:32 +0100 Subject: [PATCH 028/242] RF: Clean up wxphoenix input hook. --- IPython/terminal/pt_inputhooks/wx.py | 55 +++++++++++++++------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/IPython/terminal/pt_inputhooks/wx.py b/IPython/terminal/pt_inputhooks/wx.py index df41e7e202e..ebe3b1c384e 100644 --- a/IPython/terminal/pt_inputhooks/wx.py +++ b/IPython/terminal/pt_inputhooks/wx.py @@ -9,8 +9,12 @@ def ignore_keyboardinterrupts(func): - """Decorator which causes KeyboardInterrupt exceptions to be - ignored during execution of the decorated function. + """Decorator which causes KeyboardInterrupt exceptions to be ignored during + execution of the decorated function. + + This is used by the inputhook functions to handle the event where the user + presses CTRL+C while IPython is idle, and the inputhook loop is running. In + this case, we want to ignore interrupts. """ def wrapper(*args, **kwargs): try: @@ -159,34 +163,34 @@ def inputhook_wxphoenix(context): if app is None: return - # Wx uses milliseconds + if context.input_is_ready(): + return + + assert wx.IsMainThread() + + # Wx uses milliseconds poll_interval = 100 - # This function gets polled periodically; when - # input is ready, the wx main loop is stopped. - def wake(): + # We have to create a dummy wx.Frame, otherwise wx.App.MainLoop will know + # that it has nothing to do, and will return immediately. + frame = getattr(inputhook_wxphoenix, '_frame', None) + if frame is None: + inputhook_wxphoenix._frame = frame = wx.Frame(None) + frame.Show(False) + + # Use a wx.Timer to periodically check whether input is ready - as soon as + # it is, we exit the main loop + def poll(ev): if context.input_is_ready(): app.ExitMainLoop() - # We have to put the wx.Timer in a wx.Frame for it to fire properly. - # We make the Frame hidden when we create it in the main app below. - class TimerFrame(wx.Frame): - def __init__(self, func): - wx.Frame.__init__(self, None, -1) - self.timer = wx.Timer(self) - self.timer.Start(poll_interval) - self.Bind(wx.EVT_TIMER, self.on_timer) - self.func = func - - def on_timer(self, event): - self.func() - - frame = TimerFrame(wake) - frame.Show(False) - - # The import of wx on Linux sets the handler for signal.SIGINT - # to 0. This is a bug in wx or gtk. We fix by just setting it - # back to the Python default. + timer = wx.Timer() + timer.Start(poll_interval) + timer.Bind(wx.EVT_TIMER, poll) + + # The import of wx on Linux sets the handler for signal.SIGINT to 0. This + # is a bug in wx or gtk. We fix by just setting it back to the Python + # default. if not callable(signal.getsignal(signal.SIGINT)): signal.signal(signal.SIGINT, signal.default_int_handler) @@ -195,6 +199,7 @@ def on_timer(self, event): # Get the major wx version number to figure out what input hook we should use. major_version = 3 + try: major_version = int(wx.__version__[0]) except Exception: From 879050a347776f9818d0003fb0e8ea09b434e42a Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Sun, 20 Oct 2019 13:51:03 +0100 Subject: [PATCH 029/242] DOC: Clarify method in docstring --- IPython/terminal/pt_inputhooks/wx.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/IPython/terminal/pt_inputhooks/wx.py b/IPython/terminal/pt_inputhooks/wx.py index ebe3b1c384e..1ec92ca22b0 100644 --- a/IPython/terminal/pt_inputhooks/wx.py +++ b/IPython/terminal/pt_inputhooks/wx.py @@ -155,7 +155,9 @@ def inputhook_wxphoenix(context): """Run the wx event loop until the user provides more input. This function uses the same approach to that used in - ipykernel.eventloops.loop_wx. + ipykernel.eventloops.loop_wx. The wx.MainLoop is executed, and a wx.Timer + is used to periodically poll the context for input. As soon as input is + ready, the wx.MainLoop is stopped. """ app = wx.GetApp() From 96ed7a3d20543c70b300728b1eecb18acabda13e Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Sun, 20 Oct 2019 14:05:53 +0100 Subject: [PATCH 030/242] DOC: More info in docstring. --- IPython/terminal/pt_inputhooks/wx.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/IPython/terminal/pt_inputhooks/wx.py b/IPython/terminal/pt_inputhooks/wx.py index 1ec92ca22b0..d918e67c330 100644 --- a/IPython/terminal/pt_inputhooks/wx.py +++ b/IPython/terminal/pt_inputhooks/wx.py @@ -154,7 +154,9 @@ def inputhook_wx3(context): def inputhook_wxphoenix(context): """Run the wx event loop until the user provides more input. - This function uses the same approach to that used in + This input hook is suitable for use with wxPython >= 4 (a.k.a. Phoenix). + + It uses the same approach to that used in ipykernel.eventloops.loop_wx. The wx.MainLoop is executed, and a wx.Timer is used to periodically poll the context for input. As soon as input is ready, the wx.MainLoop is stopped. From c3461c7abe1b1d0de05b88dcfb64d601f4f8a1af Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Wed, 30 Oct 2019 02:40:51 +0000 Subject: [PATCH 031/242] RF: Use SetExitOnFrameDelete instead of creating a frame --- IPython/terminal/pt_inputhooks/wx.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/IPython/terminal/pt_inputhooks/wx.py b/IPython/terminal/pt_inputhooks/wx.py index d918e67c330..618f092b4f9 100644 --- a/IPython/terminal/pt_inputhooks/wx.py +++ b/IPython/terminal/pt_inputhooks/wx.py @@ -175,13 +175,6 @@ def inputhook_wxphoenix(context): # Wx uses milliseconds poll_interval = 100 - # We have to create a dummy wx.Frame, otherwise wx.App.MainLoop will know - # that it has nothing to do, and will return immediately. - frame = getattr(inputhook_wxphoenix, '_frame', None) - if frame is None: - inputhook_wxphoenix._frame = frame = wx.Frame(None) - frame.Show(False) - # Use a wx.Timer to periodically check whether input is ready - as soon as # it is, we exit the main loop def poll(ev): @@ -198,6 +191,9 @@ def poll(ev): if not callable(signal.getsignal(signal.SIGINT)): signal.signal(signal.SIGINT, signal.default_int_handler) + # The SetExitOnFrameDelete call allows us to run the wx mainloop without + # having a frame open. + app.SetExitOnFrameDelete(False) app.MainLoop() From e33e98fe8de68c8af80b9158a8458a9b626a9f5b Mon Sep 17 00:00:00 2001 From: Gerrit Buss Date: Fri, 1 Nov 2019 10:00:06 +0100 Subject: [PATCH 032/242] static message instead of function call to prevent shell slowdown --- IPython/terminal/interactiveshell.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index eb49e67d329..533c7cc5897 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -377,14 +377,12 @@ def _extra_prompt_options(self): """ Return the current layout option for the current Terminal InteractiveShell """ - def get_message(): - return PygmentsTokens(self.prompts.in_prompt_tokens()) return { 'complete_in_thread': False, 'lexer':IPythonPTLexer(), 'reserve_space_for_menu':self.space_for_menu, - 'message': get_message, + 'message': PygmentsTokens(self.prompts.in_prompt_tokens()), 'prompt_continuation': ( lambda width, lineno, is_soft_wrap: PygmentsTokens(self.prompts.continuation_prompt_tokens(width))), From 467667270d4d0e4ac3885d229b8f51cef72171b5 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 2 Nov 2019 20:51:25 -0700 Subject: [PATCH 033/242] Do no precompute when using vi mode. --- IPython/terminal/interactiveshell.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 533c7cc5897..0dda4284a8e 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -377,12 +377,24 @@ def _extra_prompt_options(self): """ Return the current layout option for the current Terminal InteractiveShell """ + def get_message(): + return PygmentsTokens(self.prompts.in_prompt_tokens()) + + if self.editing_mode == 'emacs': + # with emacs mode the prompt is (usually) static, so we call only + # the function once. With VI mode it can toggle between [ins] and + # [nor] so we can't precompute. + # here I'm going to favor the default keybinding which almost + # everybody uses to decrease CPU usage. + # if we have issues with users with custom Prompts we can see how to + # work around this. + get_message = get_message() return { 'complete_in_thread': False, 'lexer':IPythonPTLexer(), 'reserve_space_for_menu':self.space_for_menu, - 'message': PygmentsTokens(self.prompts.in_prompt_tokens()), + 'message': get_message, 'prompt_continuation': ( lambda width, lineno, is_soft_wrap: PygmentsTokens(self.prompts.continuation_prompt_tokens(width))), From a4fabd4b48882f109c982884a492ef2665f10f15 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Mon, 4 Nov 2019 20:47:23 -0500 Subject: [PATCH 034/242] add py37/38 for appveyor --- appveyor.yml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 7fbecff4318..d7406b41ac5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -12,10 +12,35 @@ environment: PYTHON_VERSION: "3.6.x" PYTHON_ARCH: "64" + - PYTHON: "C:\\Python37" + PYTHON_VERSION: "3.7.x" + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python37-x64" + PYTHON_VERSION: "3.7.x" + PYTHON_ARCH: "64" + + - PYTHON: "C:\\Python38" + PYTHON_VERSION: "3.8.x" + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python38-x64" + PYTHON_VERSION: "3.8.x" + PYTHON_ARCH: "64" + init: - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%" install: + - ps: | + Set-PSDebug -Trace 1 + if ($Env:PYTHON_VERSION.StartsWith("3.8")) { + if ($Env:PYTHON_ARCH -eq "64") { + iex "choco install python3 --version=3.8 --no-progress --params /InstallDir:$Env:PYTHON_HOME" + } else { + iex "choco install python3 --version=3.8 --no-progress --params /InstallDir:$Env:PYTHON_HOME --x86" + } + } - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - "%CMD_IN_ENV% python -m pip install --upgrade setuptools pip" - "%CMD_IN_ENV% pip install nose coverage" @@ -24,4 +49,3 @@ install: - "%CMD_IN_ENV% cd results" test_script: - "%CMD_IN_ENV% iptest --coverage xml" - From 63b0de49b0e86ef79479087de7f324d0cd180714 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Mon, 4 Nov 2019 20:48:41 -0500 Subject: [PATCH 035/242] fix env var for py38 install --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d7406b41ac5..992c7b0743a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -36,9 +36,9 @@ install: Set-PSDebug -Trace 1 if ($Env:PYTHON_VERSION.StartsWith("3.8")) { if ($Env:PYTHON_ARCH -eq "64") { - iex "choco install python3 --version=3.8 --no-progress --params /InstallDir:$Env:PYTHON_HOME" + iex "choco install python3 --version=3.8 --no-progress --params /InstallDir:$Env:PYTHON" } else { - iex "choco install python3 --version=3.8 --no-progress --params /InstallDir:$Env:PYTHON_HOME --x86" + iex "choco install python3 --version=3.8 --no-progress --params /InstallDir:$Env:PYTHON --x86" } } - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" From de62a081d1d8f4f7a301026fcfa7cfb7317f96b2 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Mon, 11 Nov 2019 07:59:38 -0500 Subject: [PATCH 036/242] skip %HOME% 3.8 test on windows, update docs for get_home_dir --- IPython/utils/path.py | 3 ++- IPython/utils/tests/test_path.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/IPython/utils/path.py b/IPython/utils/path.py index f8665975a1a..f677f1c7cd0 100644 --- a/IPython/utils/path.py +++ b/IPython/utils/path.py @@ -175,7 +175,8 @@ def get_home_dir(require_writable=False): Uses os.path.expanduser('~'), and checks for writability. See stdlib docs for how this is determined. - $HOME is first priority on *ALL* platforms. + For Python <3.8, $HOME is first priority on *ALL* platforms. + For Python >=3.8 on Windows, %HOME% is no longer considered. Parameters ---------- diff --git a/IPython/utils/tests/test_path.py b/IPython/utils/tests/test_path.py index a4c1fc814b8..26b7d321ff3 100644 --- a/IPython/utils/tests/test_path.py +++ b/IPython/utils/tests/test_path.py @@ -28,7 +28,7 @@ # Platform-dependent imports try: - import winreg as wreg + import winreg as wreg except ImportError: #Fake _winreg module on non-windows platforms import types @@ -128,6 +128,7 @@ def test_get_home_dir_2(): nt.assert_equal(home_dir, unfrozen) +@skipif(sys.version_info > (3,8) and os.name == 'nt') @with_environment def test_get_home_dir_3(): """get_home_dir() uses $HOME if set""" @@ -247,7 +248,7 @@ def test_filefind(): def test_get_long_path_name_win32(): with TemporaryDirectory() as tmpdir: - # Make a long path. Expands the path of tmpdir prematurely as it may already have a long + # Make a long path. Expands the path of tmpdir prematurely as it may already have a long # path component, so ensure we include the long form of it long_path = os.path.join(path.get_long_path_name(tmpdir), 'this is my long path name') os.makedirs(long_path) @@ -311,7 +312,7 @@ def test_get_py_filename(): def test_unicode_in_filename(): """When a file doesn't exist, the exception raised should be safe to call str() on - i.e. in Python 2 it must only have ASCII characters. - + https://github.com/ipython/ipython/issues/875 """ try: From 93500dbe00103146a977c66d8c8afac3120c49b0 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Mon, 11 Nov 2019 08:13:14 -0500 Subject: [PATCH 037/242] add skip_win32_py38 decorator, skip another test --- IPython/testing/decorators.py | 11 ++++++++--- IPython/utils/tests/test_path.py | 7 +++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/IPython/testing/decorators.py b/IPython/testing/decorators.py index 91e7193cbe6..4539a72a8ca 100644 --- a/IPython/testing/decorators.py +++ b/IPython/testing/decorators.py @@ -108,9 +108,9 @@ def make_label_dec(label, ds=None): >>> slow = make_label_dec('slow') >>> slow.__doc__ "Labels a test as 'slow'." - + And one that uses multiple labels and a custom docstring: - + >>> rare = make_label_dec(['slow','hard'], ... "Mix labels 'slow' and 'hard' for rare tests.") >>> rare.__doc__ @@ -280,7 +280,7 @@ def module_not_available(module): def decorated_dummy(dec, name): """Return a dummy function decorated with dec, with the given name. - + Examples -------- import IPython.testing.decorators as dec @@ -318,6 +318,11 @@ def decorated_dummy(dec, name): skip_if_no_x11 = skipif(_x11_skip_cond, _x11_skip_msg) + +# Decorators to skip certain tests on specific platform/python combinations +skip_win32_py38 = skipif(sys.version_info > (3,8) and os.name == 'nt') + + # not a decorator itself, returns a dummy function to be used as setup def skip_file_no_x11(name): warnings.warn("The function `skip_file_no_x11` is deprecated since IPython 4.0", diff --git a/IPython/utils/tests/test_path.py b/IPython/utils/tests/test_path.py index 26b7d321ff3..b34f9ca926c 100644 --- a/IPython/utils/tests/test_path.py +++ b/IPython/utils/tests/test_path.py @@ -21,11 +21,13 @@ from IPython import paths from IPython.testing import decorators as dec from IPython.testing.decorators import (skip_if_not_win32, skip_win32, - onlyif_unicode_paths,) + onlyif_unicode_paths, skipif, + skip_win32_py38,) from IPython.testing.tools import make_tempfile, AssertPrints from IPython.utils import path from IPython.utils.tempdir import TemporaryDirectory + # Platform-dependent imports try: import winreg as wreg @@ -128,7 +130,7 @@ def test_get_home_dir_2(): nt.assert_equal(home_dir, unfrozen) -@skipif(sys.version_info > (3,8) and os.name == 'nt') +@skip_win32_py38 @with_environment def test_get_home_dir_3(): """get_home_dir() uses $HOME if set""" @@ -146,6 +148,7 @@ def test_get_home_dir_4(): # this should still succeed, but we don't care what the answer is home = path.get_home_dir(False) +@skip_win32_py38 @with_environment def test_get_home_dir_5(): """raise HomeDirError if $HOME is specified, but not a writable dir""" From 0f6d2f42f2a10eeb5f78c57314b2f8cc026aed19 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 11 Nov 2019 12:41:43 -0800 Subject: [PATCH 038/242] Adopt NEP 29, Drop Python 3.5 and Numpy <1.14 This lays out the foundation of adopting NEP29 in IPython, And drop support for older versions of Python in the metadata, setup.py and __init__. This does not yet remove any codepath from within IPython. --- .travis.yml | 1 - IPython/__init__.py | 5 +++-- README.rst | 8 ++++++++ setup.py | 11 +++++------ 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index b14f7d7354f..f0d4dc6f126 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,6 @@ addons: python: - 3.6 - - 3.5 sudo: false diff --git a/IPython/__init__.py b/IPython/__init__.py index 043a946ab26..4fb77107680 100644 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -27,12 +27,13 @@ #----------------------------------------------------------------------------- # Don't forget to also update setup.py when this changes! -if sys.version_info < (3, 5): +if sys.version_info < (3, 6): raise ImportError( """ -IPython 7.0+ supports Python 3.5 and above. +IPython 7.10+ supports Python 3.6 and above. When using Python 2.7, please install IPython 5.x LTS Long Term Support version. Python 3.3 and 3.4 were supported up to IPython 6.x. +Python 3.5 was supported with IPython 7.0 to 7.9. See IPython `README.rst` file for more information: diff --git a/README.rst b/README.rst index f708ca0ecef..9899a538544 100644 --- a/README.rst +++ b/README.rst @@ -10,6 +10,10 @@ .. image:: https://www.codetriage.com/ipython/ipython/badges/users.svg :target: https://www.codetriage.com/ipython/ipython/ +.. image:: https://raster.shields.io/badge/Follows-NEP29-brightgreen.png + :target: https://numpy.org/neps/nep-0029-deprecation_policy.html + + =========================================== IPython: Productive Interactive Computing =========================================== @@ -23,6 +27,10 @@ contribute to the project. **IPython versions and Python Support** +Starting with IPython 7.10, IPython follows `NEP 29 `_ + +**IPython 7.10+** requires Python version 3.6 and above. + **IPython 7.0** requires Python version 3.5 and above. **IPython 6.x** requires Python version 3.3 and above. diff --git a/setup.py b/setup.py index 47c9af24307..638446de9e0 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ # # This check is also made in IPython/__init__, don't forget to update both when # changing Python version requirements. -if sys.version_info < (3, 5): +if sys.version_info < (3, 6): pip_message = 'This may be due to an out of date pip. Make sure you have pip >= 9.0.1.' try: import pip @@ -42,9 +42,10 @@ error = """ -IPython 7.0+ supports Python 3.5 and above. +IPython 7.10+ supports Python 3.6 and above, following NEP 29. When using Python 2.7, please install IPython 5.x LTS Long Term Support version. Python 3.3 and 3.4 were supported up to IPython 6.x. +Python 3.5 was supported with IPython 7.0 to 7.9. See IPython `README.rst` file for more information: @@ -175,7 +176,7 @@ parallel = ['ipyparallel'], qtconsole = ['qtconsole'], doc = ['Sphinx>=1.3'], - test = ['nose>=0.10.1', 'requests', 'testpath', 'pygments', 'nbformat', 'ipykernel', 'numpy'], + test = ['nose>=0.10.1', 'requests', 'testpath', 'pygments', 'nbformat', 'ipykernel', 'numpy>=1.14'], terminal = [], kernel = ['ipykernel'], nbformat = ['nbformat'], @@ -199,11 +200,9 @@ # but requires pip >= 6. pip < 6 ignores these. extras_require.update({ - ':python_version == "3.4"': ['typing'], ':sys_platform != "win32"': ['pexpect'], ':sys_platform == "darwin"': ['appnope'], ':sys_platform == "win32"': ['colorama'], - ':sys_platform == "win32" and python_version < "3.6"': ['win_unicode_console>=0.5'], }) # FIXME: re-specify above platform dependencies for pip < 6 # These would result in non-portable bdists. @@ -229,7 +228,7 @@ extras_require['all'] = everything if 'setuptools' in sys.modules: - setuptools_extra_args['python_requires'] = '>=3.5' + setuptools_extra_args['python_requires'] = '>=3.6' setuptools_extra_args['zip_safe'] = False setuptools_extra_args['entry_points'] = { 'console_scripts': find_entry_points(), From 390d0400cfafb2270b35bc5634f19e40b2a2e249 Mon Sep 17 00:00:00 2001 From: Srinivas Reddy Thatiparthy Date: Tue, 12 Nov 2019 14:25:44 +0530 Subject: [PATCH 039/242] Change absolute imports to relative imports to facilitate processes embedding kernel or debugger --- IPython/config.py | 2 +- IPython/conftest.py | 6 +++--- IPython/core/alias.py | 2 +- IPython/core/completerlib.py | 6 +++--- IPython/core/formatters.py | 10 +++++----- IPython/core/historyapp.py | 4 ++-- IPython/core/hooks.py | 6 +++--- IPython/core/magic.py | 12 ++++++------ IPython/core/prefilter.py | 8 ++++---- IPython/core/profiledir.py | 4 ++-- 10 files changed, 30 insertions(+), 30 deletions(-) diff --git a/IPython/config.py b/IPython/config.py index cf2bacafad1..964f46f10ac 100644 --- a/IPython/config.py +++ b/IPython/config.py @@ -7,7 +7,7 @@ import sys from warnings import warn -from IPython.utils.shimmodule import ShimModule, ShimWarning +from .utils.shimmodule import ShimModule, ShimWarning warn("The `IPython.config` package has been deprecated since IPython 4.0. " "You should import from traitlets.config instead.", ShimWarning) diff --git a/IPython/conftest.py b/IPython/conftest.py index b9d1f065353..8b2af8c020a 100644 --- a/IPython/conftest.py +++ b/IPython/conftest.py @@ -6,11 +6,11 @@ import pathlib import shutil -from IPython.testing import tools +from .testing import tools def get_ipython(): - from IPython.terminal.interactiveshell import TerminalInteractiveShell + from .terminal.interactiveshell import TerminalInteractiveShell if TerminalInteractiveShell._instance: return TerminalInteractiveShell.instance() @@ -60,7 +60,7 @@ def inject(): builtins.ip = get_ipython() builtins.ip.system = types.MethodType(xsys, ip) builtins.ip.builtin_trap.activate() - from IPython.core import page + from .core import page page.pager_page = nopage # yield diff --git a/IPython/core/alias.py b/IPython/core/alias.py index 4577becf7f6..2ad990231a0 100644 --- a/IPython/core/alias.py +++ b/IPython/core/alias.py @@ -25,7 +25,7 @@ import sys from traitlets.config.configurable import Configurable -from IPython.core.error import UsageError +from .error import UsageError from traitlets import List, Instance from logging import error diff --git a/IPython/core/completerlib.py b/IPython/core/completerlib.py index 9b14bf7c715..9e592b0817e 100644 --- a/IPython/core/completerlib.py +++ b/IPython/core/completerlib.py @@ -30,9 +30,9 @@ from zipimport import zipimporter # Our own imports -from IPython.core.completer import expand_user, compress_user -from IPython.core.error import TryNext -from IPython.utils._process_common import arg_split +from .completer import expand_user, compress_user +from .error import TryNext +from ..utils._process_common import arg_split # FIXME: this should be pulled in with the right call via the component system from IPython import get_ipython diff --git a/IPython/core/formatters.py b/IPython/core/formatters.py index fd93d3747b5..237b959b9a6 100644 --- a/IPython/core/formatters.py +++ b/IPython/core/formatters.py @@ -20,10 +20,10 @@ from decorator import decorator from traitlets.config.configurable import Configurable -from IPython.core.getipython import get_ipython -from IPython.utils.sentinel import Sentinel -from IPython.utils.dir2 import get_real_method -from IPython.lib import pretty +from .getipython import get_ipython +from ..utils.sentinel import Sentinel +from ..utils.dir2 import get_real_method +from ..lib import pretty from traitlets import ( Bool, Dict, Integer, Unicode, CUnicode, ObjectName, List, ForwardDeclaredInstance, @@ -1015,7 +1015,7 @@ def format_display_data(obj, include=None, exclude=None): data dict. If this is set all format types will be computed, except for those included in this argument. """ - from IPython.core.interactiveshell import InteractiveShell + from .interactiveshell import InteractiveShell return InteractiveShell.instance().display_formatter.format( obj, diff --git a/IPython/core/historyapp.py b/IPython/core/historyapp.py index 3bcc697a20c..a6437eff26e 100644 --- a/IPython/core/historyapp.py +++ b/IPython/core/historyapp.py @@ -9,9 +9,9 @@ import sqlite3 from traitlets.config.application import Application -from IPython.core.application import BaseIPythonApplication +from .application import BaseIPythonApplication from traitlets import Bool, Int, Dict -from IPython.utils.io import ask_yes_no +from ..utils.io import ask_yes_no trim_hist_help = """Trim the IPython history database to the last 1000 entries. diff --git a/IPython/core/hooks.py b/IPython/core/hooks.py index 66a544d7d8c..b0637d8c154 100644 --- a/IPython/core/hooks.py +++ b/IPython/core/hooks.py @@ -40,7 +40,7 @@ def load_ipython_extension(ip): import warnings import sys -from IPython.core.error import TryNext +from .error import TryNext # List here all the default hooks. For now it's just the editor functions # but over time we'll move here all the public API for user-accessible things. @@ -83,7 +83,7 @@ def editor(self, filename, linenum=None, wait=True): raise TryNext() import tempfile -from IPython.utils.decorators import undoc +from ..utils.decorators import undoc @undoc def fix_error_editor(self,filename,linenum,column,msg): @@ -212,7 +212,7 @@ def pre_run_code_hook(self): def clipboard_get(self): """ Get text from the clipboard. """ - from IPython.lib.clipboard import ( + from ..lib.clipboard import ( osx_clipboard_get, tkinter_clipboard_get, win32_clipboard_get ) diff --git a/IPython/core/magic.py b/IPython/core/magic.py index 2b41617bed7..bc51677f083 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -17,13 +17,13 @@ from getopt import getopt, GetoptError from traitlets.config.configurable import Configurable -from IPython.core import oinspect -from IPython.core.error import UsageError -from IPython.core.inputtransformer2 import ESC_MAGIC, ESC_MAGIC2 +from . import oinspect +from .error import UsageError +from .inputtransformer2 import ESC_MAGIC, ESC_MAGIC2 from decorator import decorator -from IPython.utils.ipstruct import Struct -from IPython.utils.process import arg_split -from IPython.utils.text import dedent +from ..utils.ipstruct import Struct +from ..utils.process import arg_split +from ..utils.text import dedent from traitlets import Bool, Dict, Instance, observe from logging import error diff --git a/IPython/core/prefilter.py b/IPython/core/prefilter.py index 0262a29bf3b..dbf185e6a42 100644 --- a/IPython/core/prefilter.py +++ b/IPython/core/prefilter.py @@ -12,16 +12,16 @@ from keyword import iskeyword import re -from IPython.core.autocall import IPyAutocall +from .autocall import IPyAutocall from traitlets.config.configurable import Configurable -from IPython.core.inputtransformer2 import ( +from .inputtransformer2 import ( ESC_MAGIC, ESC_QUOTE, ESC_QUOTE2, ESC_PAREN, ) -from IPython.core.macro import Macro -from IPython.core.splitinput import LineInfo +from .macro import Macro +from .splitinput import LineInfo from traitlets import ( List, Integer, Unicode, Bool, Instance, CRegExp diff --git a/IPython/core/profiledir.py b/IPython/core/profiledir.py index 6ab600f3004..3199dfd5d64 100644 --- a/IPython/core/profiledir.py +++ b/IPython/core/profiledir.py @@ -9,8 +9,8 @@ import errno from traitlets.config.configurable import LoggingConfigurable -from IPython.paths import get_ipython_package_dir -from IPython.utils.path import expand_path, ensure_dir_exists +from ..paths import get_ipython_package_dir +from ..utils.path import expand_path, ensure_dir_exists from traitlets import Unicode, Bool, observe #----------------------------------------------------------------------------- From 595fa2512d0ddadcfc71736d8de6023cdf9d40fc Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Tue, 12 Nov 2019 08:19:20 -0500 Subject: [PATCH 040/242] remove choco appveyor --- appveyor.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 992c7b0743a..5ed18735ff0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -32,15 +32,6 @@ init: - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%" install: - - ps: | - Set-PSDebug -Trace 1 - if ($Env:PYTHON_VERSION.StartsWith("3.8")) { - if ($Env:PYTHON_ARCH -eq "64") { - iex "choco install python3 --version=3.8 --no-progress --params /InstallDir:$Env:PYTHON" - } else { - iex "choco install python3 --version=3.8 --no-progress --params /InstallDir:$Env:PYTHON --x86" - } - } - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - "%CMD_IN_ENV% python -m pip install --upgrade setuptools pip" - "%CMD_IN_ENV% pip install nose coverage" From 2a6002333514150368366b10e184062e550dbda3 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 12 Nov 2019 14:25:01 -0800 Subject: [PATCH 041/242] Add whats new --- .../whatsnew/pr/prompt-update-modifications.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 docs/source/whatsnew/pr/prompt-update-modifications.rst diff --git a/docs/source/whatsnew/pr/prompt-update-modifications.rst b/docs/source/whatsnew/pr/prompt-update-modifications.rst new file mode 100644 index 00000000000..77d557b8936 --- /dev/null +++ b/docs/source/whatsnew/pr/prompt-update-modifications.rst @@ -0,0 +1,14 @@ +Prompt Rendering Performance improvements +========================================= + +Pull Request :ghpull:`11933` introduced an optimisation in the prompt rendering +logic that should decrease the resource usage of IPython when using the +_default_ configuration but could potentially introduce a regression of +functionalities if you are using a custom prompt. + +We know assume if you haven't changed the default keybindings that the prompt +**will not change** during the duration of your input – which is for example +not true when using vi insert mode that switches between `[ins]` and `[nor]` +for the current mode. + +If you are experiencing any issue let us know. From fd1a027a7f133d9adf9dd46bfc581e2afb079482 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 13 Nov 2019 20:15:57 -0800 Subject: [PATCH 042/242] Let anyone add "good first issue" --- .meeseeksdev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.meeseeksdev.yml b/.meeseeksdev.yml index 924dc8eeaeb..b52022dde07 100644 --- a/.meeseeksdev.yml +++ b/.meeseeksdev.yml @@ -12,6 +12,7 @@ special: config: tag: only: + - good first issue - async/await - backported - help wanted From 802df205b23bec291959ef495328637a1a786f2e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 17 Nov 2019 09:46:02 -0800 Subject: [PATCH 043/242] Use proper subprocess commands instead of modifying the command line. This should in particular work on windows; as 2>/dev/null might not have been proper windows syntax --- IPython/core/page.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/IPython/core/page.py b/IPython/core/page.py index cf9fb21e3de..afdded29b1a 100644 --- a/IPython/core/page.py +++ b/IPython/core/page.py @@ -15,9 +15,11 @@ import os +import io import re import sys import tempfile +import subprocess from io import UnsupportedOperation @@ -208,9 +210,13 @@ def pager_page(strng, start=0, screen_lines=0, pager_cmd=None): else: try: retval = None - # if I use popen4, things hang. No idea why. - #pager,shell_out = os.popen4(pager_cmd) - pager = os.popen(pager_cmd + ' 2>/dev/null', 'w') + # Emulate os.popen, but redirect stderr + proc = subprocess.Popen(pager_cmd, + shell=True, + stdin=subprocess.PIPE, + stderr=subprocess.DEVNULL + ) + pager = os._wrap_close(io.TextIOWrapper(proc.stdin), proc) try: pager_encoding = pager.encoding or sys.stdout.encoding pager.write(strng) From da7bac9111c66c1a8b9aa5bccf464c3d8dfa93b3 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 17 Nov 2019 10:10:12 -0800 Subject: [PATCH 044/242] Update documentation, and prepare inine matplotlib. --- IPython/terminal/interactiveshell.py | 2 +- IPython/terminal/pt_inputhooks/__init__.py | 2 +- docs/source/config/index.rst | 1 + docs/source/config/shell_mimerenderer.rst | 60 ++++++++++++++++++++++ docs/source/whatsnew/pr/mimerenderer.rst | 44 ++-------------- 5 files changed, 67 insertions(+), 42 deletions(-) create mode 100644 docs/source/config/shell_mimerenderer.rst diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 262c574a4f9..cdc55987ced 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -516,7 +516,7 @@ def inputhook(self, context): active_eventloop = None def enable_gui(self, gui=None): - if gui: + if gui and (gui != 'inline') : self.active_eventloop, self._inputhook =\ get_inputhook_name_and_func(gui) else: diff --git a/IPython/terminal/pt_inputhooks/__init__.py b/IPython/terminal/pt_inputhooks/__init__.py index 3766973e826..284293ddde4 100644 --- a/IPython/terminal/pt_inputhooks/__init__.py +++ b/IPython/terminal/pt_inputhooks/__init__.py @@ -12,7 +12,7 @@ 'tk', 'wx', 'pyglet', 'glut', - 'osx', + 'osx' ] registered = {} diff --git a/docs/source/config/index.rst b/docs/source/config/index.rst index 0fe4f20f668..28e6994cc21 100644 --- a/docs/source/config/index.rst +++ b/docs/source/config/index.rst @@ -29,6 +29,7 @@ Extending and integrating with IPython extensions/index integrating custommagics + shell_mimerenderer inputtransforms callbacks eventloops diff --git a/docs/source/config/shell_mimerenderer.rst b/docs/source/config/shell_mimerenderer.rst new file mode 100644 index 00000000000..eda16142674 --- /dev/null +++ b/docs/source/config/shell_mimerenderer.rst @@ -0,0 +1,60 @@ + +.. _shell_mimerenderer: + + +Mime Renderer Extensions +======================== + +Like it's cousins, Jupyter Notebooks and JupyterLab, Terminal IPython can be +thought to render a number of mimetypes in the shell. This can be used to either +display inline images if your terminal emulator supports it; or open some +display results with external file viewers. + +Registering new mimetype handlers can so far only be done my extensions and +requires 4 steps: + + - Define a callable that takes 2 parameters:``data`` and ``metadata``; return + value of the callable is so far ignored. This callable is responsible for + "displaying" the given mimetype. Which can be sending the right escape + sequences and bytes to the current terminal; or open an external program. - + - Appending the right mimetype to ``ipython.display_formatter.active_types`` + for IPython to know it should not ignore those mimetypes. + - Enabling the given mimetype: ``ipython.display_formatter.formatters[mime].enabled = True`` + - Registering above callable with mimetype handler: + ``ipython.mime_renderers[mime] = handler`` + + +Here is a complete IPython extension to display images inline and convert math +to png, before displaying it inline for iterm2 on macOS :: + + + from base64 import encodebytes + from IPython.lib.latextools import latex_to_png + + + def mathcat(data, meta): + png = latex_to_png(f'$${data}$$'.replace('\displaystyle', '').replace('$$$', '$$')) + imcat(png, meta) + + IMAGE_CODE = '\033]1337;File=name=name;inline=true;:{}\a' + + def imcat(image_data, metadata): + try: + print(IMAGE_CODE.format(encodebytes(image_data).decode())) + # bug workaround + except: + print(IMAGE_CODE.format(image_data)) + + def register_mimerenderer(ipython, mime, handler): + ipython.display_formatter.active_types.append(mime) + ipython.display_formatter.formatters[mime].enabled = True + ipython.mime_renderers[mime] = handler + + def load_ipython_extension(ipython): + register_mimerenderer(ipython, 'image/png', imcat) + register_mimerenderer(ipython, 'image/jpeg', imcat) + register_mimerenderer(ipython, 'text/latex', mathcat) + +This example only work for iterm2 on macOS and skip error handling for brevity. +One could also invoke an external viewer with ``subporcess.run()`` and a +temporary file, which is left as an exercise. diff --git a/docs/source/whatsnew/pr/mimerenderer.rst b/docs/source/whatsnew/pr/mimerenderer.rst index 44aeb9d6d3f..58d474a1515 100644 --- a/docs/source/whatsnew/pr/mimerenderer.rst +++ b/docs/source/whatsnew/pr/mimerenderer.rst @@ -2,54 +2,18 @@ Arbitrary Mimetypes Handing in Terminal ======================================= When using IPython terminal it is now possible to register function to handle -arbitrary mimetypes (``TerminalInteractiveShell.mime_renderers`` ``Dict`` -configurable). While rendering non-text based representation was possible in +arbitrary mimetypes. While rendering non-text based representation was possible in many jupyter frontend; it was not possible in terminal IPython, as usually terminal are limited to displaying text. As many terminal these days provide escape sequences to display non-text; bringing this loved feature to IPython CLI made a lot of sens. This functionality will not only allow inline images; but -allow opening of external program; for example ``fmplayer`` to "display" sound +allow opening of external program; for example ``mplayer`` to "display" sound files. -Here is a complete IPython tension to display images inline and convert math to -png, before displaying it inline :: - - - from base64 import encodebytes - from IPython.lib.latextools import latex_to_png - - - def mathcat(data, meta): - png = latex_to_png(f'$${data}$$'.replace('\displaystyle', '').replace('$$$', '$$')) - imcat(png, meta) - - IMAGE_CODE = '\033]1337;File=name=name;inline=true;:{}\a' - - def imcat(image_data, metadata): - try: - print(IMAGE_CODE.format(encodebytes(image_data).decode())) - # bug workaround - except: - print(IMAGE_CODE.format(image_data)) - - def register_mimerenderer(ipython, mime, handler): - ipython.display_formatter.active_types.append(mime) - ipython.display_formatter.formatters[mime].enabled = True - ipython.mime_renderers[mime] = handler - - def load_ipython_extension(ipython): - register_mimerenderer(ipython, 'image/png', imcat) - register_mimerenderer(ipython, 'image/jpeg', imcat) - register_mimerenderer(ipython, 'text/latex', mathcat) - -This example only work for iterm2 on mac os and skip error handling for brevity. -One could also invoke an external viewer with ``subporcess.run()`` and a -tempfile, which is left as an exercise. - So far only the hooks necessary for this are in place, but no default mime renderers added; so inline images will only be available via extensions. We will progressively enable these features by default in the next few releases, and contribution is welcomed. - - +We welcome any feedback on the API. See :ref:`shell_mimerenderer` for more +informations. From bf42a49afb349be18957d7b115e98f451c9c3af5 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 17 Nov 2019 11:09:48 -0800 Subject: [PATCH 045/242] update code to work with black more recent versions --- IPython/terminal/interactiveshell.py | 1 - IPython/terminal/shortcuts.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 1a4d5627957..e02cf26a56b 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -269,7 +269,6 @@ def init_display_formatter(self): self.display_formatter.ipython_display_formatter.enabled = False def init_prompt_toolkit_cli(self): - self.reformat_handler = lambda x:x if self.simple_prompt: # Fall back to plain non-interactive output for tests. # This is very limited. diff --git a/IPython/terminal/shortcuts.py b/IPython/terminal/shortcuts.py index ef0f65c5558..e44e34277e2 100644 --- a/IPython/terminal/shortcuts.py +++ b/IPython/terminal/shortcuts.py @@ -129,7 +129,8 @@ def newline_or_execute(event): # if all we have after the cursor is whitespace: reformat current text # before cursor - if d.text[d.cursor_position:].isspace(): + after_cursor = d.text[d.cursor_position:] + if not after_cursor.strip(): reformat_text_before_cursor(b, d, shell) if not (d.on_last_line or d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end() From e498a4308f88cabc0e2cb264597dea73c91cc49f Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 17 Nov 2019 12:14:48 -0800 Subject: [PATCH 046/242] add whats new --- docs/source/whatsnew/pr/autoformatting.rst | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 docs/source/whatsnew/pr/autoformatting.rst diff --git a/docs/source/whatsnew/pr/autoformatting.rst b/docs/source/whatsnew/pr/autoformatting.rst new file mode 100644 index 00000000000..dff810ec766 --- /dev/null +++ b/docs/source/whatsnew/pr/autoformatting.rst @@ -0,0 +1,24 @@ +Code autoformatting +=================== + +The IPython terminal can now auto format your code just before entering a new +line or executing a command. To do so use the +``--TerminalInteractiveShell.autoformatter`` option and set it to ``'black'``; +if black is installed IPython will use black to format your code when possible. + +IPython cannot always properly format your code; in particular it will +auto formatting with *black* will only work if: + + - Your code does not contains magics or special python syntax. + + - There is no code after your cursor. + +The Black API is also still in motion; so this may not work with all versions of +black. + +It should be possible to register custom reformatter, though the API is till in +flux. + + + + From 17611ad6ef7a41a6df9f0f655c92b670f01ff14b Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 17 Nov 2019 12:35:22 -0800 Subject: [PATCH 047/242] Deprecate some methods the should not be used anymore And cleanup some code that had no effect/should be unused. --- IPython/core/oinspect.py | 46 ++++++++++++++-------------------------- 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index 749f97959e6..0ad709405de 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -76,7 +76,7 @@ def pylight(code): 'call_def', 'call_docstring', # These won't be printed but will be used to determine how to # format the object - 'ismagic', 'isalias', 'isclass', 'argspec', 'found', 'name' + 'ismagic', 'isalias', 'isclass', 'found', 'name' ] @@ -200,26 +200,39 @@ def is_simple_callable(obj): return (inspect.isfunction(obj) or inspect.ismethod(obj) or \ isinstance(obj, _builtin_func_type) or isinstance(obj, _builtin_meth_type)) - +@undoc def getargspec(obj): """Wrapper around :func:`inspect.getfullargspec` on Python 3, and :func:inspect.getargspec` on Python 2. In addition to functions and methods, this can also handle objects with a ``__call__`` attribute. + + DEPRECATED: Deprecated since 7.10. Do not use, will be removed. """ + + warnings.warn('`getargspec` function is deprecated as of IPython 7.10' + 'and will be removed in future versions.', DeprecationWarning, stacklevel=2) + if safe_hasattr(obj, '__call__') and not is_simple_callable(obj): obj = obj.__call__ return inspect.getfullargspec(obj) - +@undoc def format_argspec(argspec): """Format argspect, convenience wrapper around inspect's. This takes a dict instead of ordered arguments and calls inspect.format_argspec with the arguments in the necessary order. + + DEPRECATED: Do not use; will be removed in future versions. """ + + warnings.warn('`format_argspec` function is deprecated as of IPython 7.10' + 'and will be removed in future versions.', DeprecationWarning, stacklevel=2) + + return inspect.formatargspec(argspec['args'], argspec['varargs'], argspec['varkw'], argspec['defaults']) @@ -916,33 +929,6 @@ def _info(self, obj, oname='', info=None, detail_level=0) -> dict: if call_ds: out['call_docstring'] = call_ds - # Compute the object's argspec as a callable. The key is to decide - # whether to pull it from the object itself, from its __init__ or - # from its __call__ method. - - if inspect.isclass(obj): - # Old-style classes need not have an __init__ - callable_obj = getattr(obj, "__init__", None) - elif callable(obj): - callable_obj = obj - else: - callable_obj = None - - if callable_obj is not None: - try: - argspec = getargspec(callable_obj) - except Exception: - # For extensions/builtins we can't retrieve the argspec - pass - else: - # named tuples' _asdict() method returns an OrderedDict, but we - # we want a normal - out['argspec'] = argspec_dict = dict(argspec._asdict()) - # We called this varkw before argspec became a named tuple. - # With getfullargspec it's also called varkw. - if 'varkw' not in argspec_dict: - argspec_dict['varkw'] = argspec_dict.pop('keywords') - return object_info(**out) @staticmethod From 39b518189261f3b31d59cc9740db502785c3365e Mon Sep 17 00:00:00 2001 From: kousik Date: Mon, 18 Nov 2019 21:07:43 +0530 Subject: [PATCH 048/242] Removed codepath for Python < 3.6 #11949 --- IPython/core/interactiveshell.py | 11 ++--------- IPython/lib/pretty.py | 2 +- IPython/terminal/interactiveshell.py | 11 ----------- tools/gen_latex_symbols.py | 4 ---- 4 files changed, 3 insertions(+), 25 deletions(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index d1b59d76c0e..3eddd312ea8 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -166,13 +166,7 @@ def removed_co_newlocals(function:types.FunctionType) -> types.FunctionType: # we still need to run things using the asyncio eventloop, but there is no # async integration from .async_helpers import (_asyncio_runner, _asyncify, _pseudo_sync_runner) -if sys.version_info > (3, 5): - from .async_helpers import _curio_runner, _trio_runner, _should_be_async -else : - _curio_runner = _trio_runner = None - - def _should_be_async(cell:str)->bool: - return False +from .async_helpers import _curio_runner, _trio_runner, _should_be_async def _ast_asyncify(cell:str, wrapper_name:str) -> ast.Module: @@ -2245,8 +2239,7 @@ def init_magics(self): m.NamespaceMagics, m.OSMagics, m.PackagingMagics, m.PylabMagics, m.ScriptMagics, ) - if sys.version_info >(3,5): - self.register_magics(m.AsyncMagics) + self.register_magics(m.AsyncMagics) # Register Magic Aliases mman = self.magics_manager diff --git a/IPython/lib/pretty.py b/IPython/lib/pretty.py index 39d23ddcd8a..9d8935e9ee4 100644 --- a/IPython/lib/pretty.py +++ b/IPython/lib/pretty.py @@ -99,7 +99,7 @@ def _repr_pretty_(self, p, cycle): MAX_SEQ_LENGTH = 1000 # The language spec says that dicts preserve order from 3.7, but CPython # does so from 3.6, so it seems likely that people will expect that. -DICT_IS_ORDERED = sys.version_info >= (3, 6) +DICT_IS_ORDERED = True _re_pattern_type = type(re.compile('')) def _safe_getattr(obj, attr, default=None): diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 0dda4284a8e..9b0ee2bef8b 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -424,21 +424,10 @@ def prompt_for_code(self): **self._extra_prompt_options()) return text - def enable_win_unicode_console(self): - if sys.version_info >= (3, 6): - # Since PEP 528, Python uses the unicode APIs for the Windows - # console by default, so WUC shouldn't be needed. - return - - import win_unicode_console - win_unicode_console.enable() - def init_io(self): if sys.platform not in {'win32', 'cli'}: return - self.enable_win_unicode_console() - import colorama colorama.init() diff --git a/tools/gen_latex_symbols.py b/tools/gen_latex_symbols.py index 2f6d339c358..c3fed125109 100644 --- a/tools/gen_latex_symbols.py +++ b/tools/gen_latex_symbols.py @@ -11,10 +11,6 @@ import os, sys -if sys.version_info[0] < 3: - print("This script must be run with Python 3, exiting...") - sys.exit(1) - # Import the Julia LaTeX symbols print('Importing latex_symbols.js from Julia...') import requests From aa324b5c104f090d0b24c4293f7488943faec66a Mon Sep 17 00:00:00 2001 From: kousik Date: Mon, 18 Nov 2019 22:50:23 +0530 Subject: [PATCH 049/242] Removed codepath for Python < 3.6 from test functions #11949 --- IPython/core/tests/test_async_helpers.py | 531 ++++++++++---------- IPython/core/tests/test_completer.py | 7 +- IPython/extensions/tests/test_autoreload.py | 1 - IPython/lib/tests/test_display.py | 3 +- 4 files changed, 269 insertions(+), 273 deletions(-) diff --git a/IPython/core/tests/test_async_helpers.py b/IPython/core/tests/test_async_helpers.py index f23eae06acc..2432d8b5e4e 100644 --- a/IPython/core/tests/test_async_helpers.py +++ b/IPython/core/tests/test_async_helpers.py @@ -14,298 +14,297 @@ iprc = lambda x: ip.run_cell(dedent(x)).raise_error() iprc_nr = lambda x: ip.run_cell(dedent(x)) -if sys.version_info > (3, 5): - from IPython.core.async_helpers import _should_be_async - - class AsyncTest(TestCase): - def test_should_be_async(self): - nt.assert_false(_should_be_async("False")) - nt.assert_true(_should_be_async("await bar()")) - nt.assert_true(_should_be_async("x = await bar()")) - nt.assert_false( - _should_be_async( - dedent( - """ - async def awaitable(): - pass - """ - ) +from IPython.core.async_helpers import _should_be_async + +class AsyncTest(TestCase): + def test_should_be_async(self): + nt.assert_false(_should_be_async("False")) + nt.assert_true(_should_be_async("await bar()")) + nt.assert_true(_should_be_async("x = await bar()")) + nt.assert_false( + _should_be_async( + dedent( + """ + async def awaitable(): + pass + """ ) ) - - def _get_top_level_cases(self): - # These are test cases that should be valid in a function - # but invalid outside of a function. - test_cases = [] - test_cases.append(('basic', "{val}")) - - # Note, in all conditional cases, I use True instead of - # False so that the peephole optimizer won't optimize away - # the return, so CPython will see this as a syntax error: - # - # while True: - # break - # return - # - # But not this: - # - # while False: - # return - # - # See https://bugs.python.org/issue1875 - - test_cases.append(('if', dedent(""" - if True: + ) + + def _get_top_level_cases(self): + # These are test cases that should be valid in a function + # but invalid outside of a function. + test_cases = [] + test_cases.append(('basic', "{val}")) + + # Note, in all conditional cases, I use True instead of + # False so that the peephole optimizer won't optimize away + # the return, so CPython will see this as a syntax error: + # + # while True: + # break + # return + # + # But not this: + # + # while False: + # return + # + # See https://bugs.python.org/issue1875 + + test_cases.append(('if', dedent(""" + if True: + {val} + """))) + + test_cases.append(('while', dedent(""" + while True: + {val} + break + """))) + + test_cases.append(('try', dedent(""" + try: + {val} + except: + pass + """))) + + test_cases.append(('except', dedent(""" + try: + pass + except: + {val} + """))) + + test_cases.append(('finally', dedent(""" + try: + pass + except: + pass + finally: + {val} + """))) + + test_cases.append(('for', dedent(""" + for _ in range(4): + {val} + """))) + + + test_cases.append(('nested', dedent(""" + if True: + while True: {val} - """))) + break + """))) - test_cases.append(('while', dedent(""" + test_cases.append(('deep-nested', dedent(""" + if True: while True: - {val} break - """))) + for x in range(3): + if True: + while True: + for x in range(3): + {val} + """))) - test_cases.append(('try', dedent(""" - try: - {val} - except: - pass - """))) + return test_cases - test_cases.append(('except', dedent(""" - try: - pass - except: - {val} - """))) + def _get_ry_syntax_errors(self): + # This is a mix of tests that should be a syntax error if + # return or yield whether or not they are in a function - test_cases.append(('finally', dedent(""" - try: - pass - except: - pass - finally: - {val} - """))) + test_cases = [] - test_cases.append(('for', dedent(""" - for _ in range(4): + test_cases.append(('class', dedent(""" + class V: + {val} + """))) + + test_cases.append(('nested-class', dedent(""" + class V: + class C: {val} - """))) + """))) + return test_cases - test_cases.append(('nested', dedent(""" - if True: - while True: - {val} - break - """))) - test_cases.append(('deep-nested', dedent(""" - if True: - while True: + def test_top_level_return_error(self): + tl_err_test_cases = self._get_top_level_cases() + tl_err_test_cases.extend(self._get_ry_syntax_errors()) + + vals = ('return', 'yield', 'yield from (_ for _ in range(3))', + dedent(''' + def f(): + pass + return + '''), + ) + + for test_name, test_case in tl_err_test_cases: + # This example should work if 'pass' is used as the value + with self.subTest((test_name, 'pass')): + iprc(test_case.format(val='pass')) + + # It should fail with all the values + for val in vals: + with self.subTest((test_name, val)): + msg = "Syntax error not raised for %s, %s" % (test_name, val) + with self.assertRaises(SyntaxError, msg=msg): + iprc(test_case.format(val=val)) + + def test_in_func_no_error(self): + # Test that the implementation of top-level return/yield + # detection isn't *too* aggressive, and works inside a function + func_contexts = [] + + func_contexts.append(('func', False, dedent(""" + def f():"""))) + + func_contexts.append(('method', False, dedent(""" + class MyClass: + def __init__(self): + """))) + + func_contexts.append(('async-func', True, dedent(""" + async def f():"""))) + + func_contexts.append(('async-method', True, dedent(""" + class MyClass: + async def f(self):"""))) + + func_contexts.append(('closure', False, dedent(""" + def f(): + def g(): + """))) + + def nest_case(context, case): + # Detect indentation + lines = context.strip().splitlines() + prefix_len = 0 + for c in lines[-1]: + if c != ' ': break - for x in range(3): - if True: - while True: - for x in range(3): - {val} - """))) + prefix_len += 1 - return test_cases + indented_case = indent(case, ' ' * (prefix_len + 4)) + return context + '\n' + indented_case - def _get_ry_syntax_errors(self): - # This is a mix of tests that should be a syntax error if - # return or yield whether or not they are in a function + # Gather and run the tests - test_cases = [] + # yield is allowed in async functions, starting in Python 3.6, + # and yield from is not allowed in any version + vals = ('return', 'yield', 'yield from (_ for _ in range(3))') + async_safe = (True, + True, + False) + vals = tuple(zip(vals, async_safe)) - test_cases.append(('class', dedent(""" - class V: - {val} - """))) - - test_cases.append(('nested-class', dedent(""" - class V: - class C: - {val} - """))) - - return test_cases - - - def test_top_level_return_error(self): - tl_err_test_cases = self._get_top_level_cases() - tl_err_test_cases.extend(self._get_ry_syntax_errors()) - - vals = ('return', 'yield', 'yield from (_ for _ in range(3))', - dedent(''' - def f(): - pass - return - '''), - ) - - for test_name, test_case in tl_err_test_cases: - # This example should work if 'pass' is used as the value - with self.subTest((test_name, 'pass')): - iprc(test_case.format(val='pass')) - - # It should fail with all the values - for val in vals: - with self.subTest((test_name, val)): - msg = "Syntax error not raised for %s, %s" % (test_name, val) - with self.assertRaises(SyntaxError, msg=msg): - iprc(test_case.format(val=val)) - - def test_in_func_no_error(self): - # Test that the implementation of top-level return/yield - # detection isn't *too* aggressive, and works inside a function - func_contexts = [] - - func_contexts.append(('func', False, dedent(""" - def f():"""))) - - func_contexts.append(('method', False, dedent(""" - class MyClass: - def __init__(self): - """))) - - func_contexts.append(('async-func', True, dedent(""" - async def f():"""))) - - func_contexts.append(('async-method', True, dedent(""" - class MyClass: - async def f(self):"""))) - - func_contexts.append(('closure', False, dedent(""" - def f(): - def g(): - """))) - - def nest_case(context, case): - # Detect indentation - lines = context.strip().splitlines() - prefix_len = 0 - for c in lines[-1]: - if c != ' ': - break - prefix_len += 1 - - indented_case = indent(case, ' ' * (prefix_len + 4)) - return context + '\n' + indented_case - - # Gather and run the tests - - # yield is allowed in async functions, starting in Python 3.6, - # and yield from is not allowed in any version - vals = ('return', 'yield', 'yield from (_ for _ in range(3))') - async_safe = (True, - sys.version_info >= (3, 6), - False) - vals = tuple(zip(vals, async_safe)) - - success_tests = zip(self._get_top_level_cases(), repeat(False)) - failure_tests = zip(self._get_ry_syntax_errors(), repeat(True)) - - tests = chain(success_tests, failure_tests) - - for context_name, async_func, context in func_contexts: - for (test_name, test_case), should_fail in tests: - nested_case = nest_case(context, test_case) - - for val, async_safe in vals: - val_should_fail = (should_fail or - (async_func and not async_safe)) - - test_id = (context_name, test_name, val) - cell = nested_case.format(val=val) - - with self.subTest(test_id): - if val_should_fail: - msg = ("SyntaxError not raised for %s" % - str(test_id)) - with self.assertRaises(SyntaxError, msg=msg): - iprc(cell) - - print(cell) - else: - iprc(cell) + success_tests = zip(self._get_top_level_cases(), repeat(False)) + failure_tests = zip(self._get_ry_syntax_errors(), repeat(True)) - def test_nonlocal(self): - # fails if outer scope is not a function scope or if var not defined - with self.assertRaises(SyntaxError): - iprc("nonlocal x") - iprc(""" - x = 1 - def f(): - nonlocal x - x = 10000 - yield x - """) - iprc(""" - def f(): - def g(): - nonlocal x - x = 10000 - yield x - """) - - # works if outer scope is a function scope and var exists - iprc(""" - def f(): - x = 20 - def g(): - nonlocal x - x = 10000 - yield x - """) + tests = chain(success_tests, failure_tests) + for context_name, async_func, context in func_contexts: + for (test_name, test_case), should_fail in tests: + nested_case = nest_case(context, test_case) - def test_execute(self): - iprc(""" - import asyncio - await asyncio.sleep(0.001) - """ - ) + for val, async_safe in vals: + val_should_fail = (should_fail or + (async_func and not async_safe)) - def test_autoawait(self): - iprc("%autoawait False") - iprc("%autoawait True") - iprc(""" - from asyncio import sleep - await sleep(0.1) - """ - ) + test_id = (context_name, test_name, val) + cell = nested_case.format(val=val) - @skip_without('curio') - def test_autoawait_curio(self): - iprc("%autoawait curio") + with self.subTest(test_id): + if val_should_fail: + msg = ("SyntaxError not raised for %s" % + str(test_id)) + with self.assertRaises(SyntaxError, msg=msg): + iprc(cell) - @skip_without('trio') - def test_autoawait_trio(self): - iprc("%autoawait trio") + print(cell) + else: + iprc(cell) - @skip_without('trio') - def test_autoawait_trio_wrong_sleep(self): - iprc("%autoawait trio") - res = iprc_nr(""" - import asyncio - await asyncio.sleep(0) + def test_nonlocal(self): + # fails if outer scope is not a function scope or if var not defined + with self.assertRaises(SyntaxError): + iprc("nonlocal x") + iprc(""" + x = 1 + def f(): + nonlocal x + x = 10000 + yield x """) - with nt.assert_raises(TypeError): - res.raise_error() - - @skip_without('trio') - def test_autoawait_asyncio_wrong_sleep(self): - iprc("%autoawait asyncio") - res = iprc_nr(""" - import trio - await trio.sleep(0) + iprc(""" + def f(): + def g(): + nonlocal x + x = 10000 + yield x """) - with nt.assert_raises(RuntimeError): - res.raise_error() - - def tearDown(self): - ip.loop_runner = "asyncio" + # works if outer scope is a function scope and var exists + iprc(""" + def f(): + x = 20 + def g(): + nonlocal x + x = 10000 + yield x + """) + + + def test_execute(self): + iprc(""" + import asyncio + await asyncio.sleep(0.001) + """ + ) + + def test_autoawait(self): + iprc("%autoawait False") + iprc("%autoawait True") + iprc(""" + from asyncio import sleep + await sleep(0.1) + """ + ) + + @skip_without('curio') + def test_autoawait_curio(self): + iprc("%autoawait curio") + + @skip_without('trio') + def test_autoawait_trio(self): + iprc("%autoawait trio") + + @skip_without('trio') + def test_autoawait_trio_wrong_sleep(self): + iprc("%autoawait trio") + res = iprc_nr(""" + import asyncio + await asyncio.sleep(0) + """) + with nt.assert_raises(TypeError): + res.raise_error() + + @skip_without('trio') + def test_autoawait_asyncio_wrong_sleep(self): + iprc("%autoawait asyncio") + res = iprc_nr(""" + import trio + await trio.sleep(0) + """) + with nt.assert_raises(RuntimeError): + res.raise_error() + + + def tearDown(self): + ip.loop_runner = "asyncio" diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index 5f2210d4d32..2920d453936 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -485,10 +485,9 @@ def _(line, cursor_pos, expect, message, completion): 5, 6, "real" ) - if sys.version_info > (3, 4): - yield _, "a[0].from_", 10, "a[0].from_bytes", "Should have completed on a[0].from_: %s", Completion( - 5, 10, "from_bytes" - ) + yield _, "a[0].from_", 10, "a[0].from_bytes", "Should have completed on a[0].from_: %s", Completion( + 5, 10, "from_bytes" + ) def test_omit__names(self): # also happens to test IPCompleter as a configurable diff --git a/IPython/extensions/tests/test_autoreload.py b/IPython/extensions/tests/test_autoreload.py index 6300b1b67fe..fe62db0c353 100644 --- a/IPython/extensions/tests/test_autoreload.py +++ b/IPython/extensions/tests/test_autoreload.py @@ -140,7 +140,6 @@ def pickle_get_current_class(obj): class TestAutoreload(Fixture): - @skipif(sys.version_info < (3, 6)) def test_reload_enums(self): import enum mod_name, mod_fn = self.new_module(textwrap.dedent(""" diff --git a/IPython/lib/tests/test_display.py b/IPython/lib/tests/test_display.py index f56aca4b43b..533bb46d059 100644 --- a/IPython/lib/tests/test_display.py +++ b/IPython/lib/tests/test_display.py @@ -49,8 +49,7 @@ def test_instantiation_FileLink(): """FileLink: Test class can be instantiated""" fl = display.FileLink('example.txt') # TODO: remove if when only Python >= 3.6 is supported - if sys.version_info >= (3, 6): - fl = display.FileLink(pathlib.PurePath('example.txt')) + fl = display.FileLink(pathlib.PurePath('example.txt')) def test_warning_on_non_existent_path_FileLink(): """FileLink: Calling _repr_html_ on non-existent files returns a warning From 747cdeb276ec8c88ad646b809882c11bd63543e9 Mon Sep 17 00:00:00 2001 From: kousik Date: Wed, 20 Nov 2019 10:34:15 +0530 Subject: [PATCH 050/242] Added deprecation warning for enable_win_unicode_console #11953 --- IPython/terminal/interactiveshell.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 9b0ee2bef8b..5a0aac8db20 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -424,6 +424,14 @@ def prompt_for_code(self): **self._extra_prompt_options()) return text + def enable_win_unicode_console(self): + # Since IPython 7.10 doesn't support python < 3.6 and PEP 528, Python uses the unicode APIs for the Windows + # console by default, so WUC shouldn't be needed. + from warnings import warn + warn("`enable_win_unicode_console` is deprecated since IPython 7.10, does not do anything and will be removed in the future", + DeprecationWarning, + stacklevel=2) + def init_io(self): if sys.platform not in {'win32', 'cli'}: return From f612098348e7b6f8dddbc8eca767f681db458a23 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 21 Nov 2019 11:14:00 -0800 Subject: [PATCH 051/242] typo --- docs/source/config/shell_mimerenderer.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/config/shell_mimerenderer.rst b/docs/source/config/shell_mimerenderer.rst index eda16142674..0fb2ffd9862 100644 --- a/docs/source/config/shell_mimerenderer.rst +++ b/docs/source/config/shell_mimerenderer.rst @@ -56,5 +56,5 @@ to png, before displaying it inline for iterm2 on macOS :: register_mimerenderer(ipython, 'text/latex', mathcat) This example only work for iterm2 on macOS and skip error handling for brevity. -One could also invoke an external viewer with ``subporcess.run()`` and a +One could also invoke an external viewer with ``subprocess.run()`` and a temporary file, which is left as an exercise. From 172dc6b7786c612786c130af3eaeb92e60ad7ebb Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 21 Nov 2019 11:29:08 -0800 Subject: [PATCH 052/242] Thanks original work --- docs/source/whatsnew/pr/mimerenderer.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/source/whatsnew/pr/mimerenderer.rst b/docs/source/whatsnew/pr/mimerenderer.rst index 58d474a1515..339905f58e1 100644 --- a/docs/source/whatsnew/pr/mimerenderer.rst +++ b/docs/source/whatsnew/pr/mimerenderer.rst @@ -17,3 +17,6 @@ contribution is welcomed. We welcome any feedback on the API. See :ref:`shell_mimerenderer` for more informations. + +This is originally based on work form in :ghpull:`10610` from stephanh42 +started over two years ago, and still a lot need to be done. From dffa2c33dea3ecc09256a66a6ddbc876ce0629a9 Mon Sep 17 00:00:00 2001 From: grey275 Date: Thu, 21 Nov 2019 12:18:13 -0800 Subject: [PATCH 053/242] remove unchanging boolean DICT_IS_ORDERED and dependent unreachable code --- IPython/lib/pretty.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/IPython/lib/pretty.py b/IPython/lib/pretty.py index 9d8935e9ee4..49c76ad902e 100644 --- a/IPython/lib/pretty.py +++ b/IPython/lib/pretty.py @@ -97,9 +97,6 @@ def _repr_pretty_(self, p, cycle): MAX_SEQ_LENGTH = 1000 -# The language spec says that dicts preserve order from 3.7, but CPython -# does so from 3.6, so it seems likely that people will expect that. -DICT_IS_ORDERED = True _re_pattern_type = type(re.compile('')) def _safe_getattr(obj, attr, default=None): @@ -606,11 +603,6 @@ def inner(obj, p, cycle): step = len(start) p.begin_group(step, start) keys = obj.keys() - # if dict isn't large enough to be truncated, sort keys before displaying - # From Python 3.7, dicts preserve order by definition, so we don't sort. - if not DICT_IS_ORDERED \ - and not (p.max_seq_length and len(obj) >= p.max_seq_length): - keys = _sorted_for_pprint(keys) for idx, key in p._enumerate(keys): if idx: p.text(',') From 0c371883a01eafe804e9c4b6b0971b069111fd1e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 21 Nov 2019 15:26:52 -0800 Subject: [PATCH 054/242] Update readme with links to (som) similar projects. --- README.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.rst b/README.rst index 9899a538544..3e13610a443 100644 --- a/README.rst +++ b/README.rst @@ -117,3 +117,15 @@ version:: ... install_requires=install_req ) + +Alternatives to IPython +======================= + +IPython may not be to your taste; if that's the case there might be similar +project that you might want to use: + +- the classic Python REPL. +- `bpython `_ +- `mypython `_ +- `ptpython and ptipython ` +- `xonsh ` From 13e486ebe73887527d34fea748233d9fce1268cc Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 22 Nov 2019 09:43:14 -0800 Subject: [PATCH 055/242] Add a failing test if we %run something with multiprocessing --- IPython/core/tests/test_run.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/IPython/core/tests/test_run.py b/IPython/core/tests/test_run.py index 42f2e5c422e..38d71b31740 100644 --- a/IPython/core/tests/test_run.py +++ b/IPython/core/tests/test_run.py @@ -539,6 +539,35 @@ def test_run_tb(): del ip.user_ns['bar'] del ip.user_ns['foo'] + +def test_multiprocessing_run(): + """Set we can run mutiprocesgin without messing up up main namespace + + Note that import `nose.tools as nt` mdify the value s + sys.module['__mp_main__'] so wee need to temporarily set it to None to test + the issue. + """ + with TemporaryDirectory() as td: + mpm = sys.modules.get('__mp_main__') + assert mpm is not None + sys.modules['__mp_main__'] = None + try: + path = pjoin(td, 'test.py') + with open(path, 'w') as f: + f.write("import multiprocessing\nprint('hoy')") + with capture_output() as io: + _ip.run_line_magic('run', path) + _ip.run_cell("i_m_undefined") + out = io.stdout + nt.assert_in("hoy", out) + nt.assert_not_in("AttributeError", out) + nt.assert_in("NameError", out) + nt.assert_equal(out.count("---->"), 1) + except: + raise + finally: + sys.modules['__mp_main__'] = mpm + @dec.knownfailureif(sys.platform == 'win32', "writes to io.stdout aren't captured on Windows") def test_script_tb(): """Test traceback offset in `ipython script.py`""" From 13441028e1d087c9ecf96d9fe4349a0e82350afc Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 22 Nov 2019 09:45:02 -0800 Subject: [PATCH 056/242] Fix the issue with %run-ing something with multiprocessing This should fix gh-9978 and gh-11941 --- IPython/core/magics/execution.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/IPython/core/magics/execution.py b/IPython/core/magics/execution.py index d9e66b545a7..438d0b5f0ad 100644 --- a/IPython/core/magics/execution.py +++ b/IPython/core/magics/execution.py @@ -853,6 +853,8 @@ def run(): sys.argv = save_argv if restore_main: sys.modules['__main__'] = restore_main + if '__mp_main__' in sys.modules: + sys.modules['__mp_main__'] = restore_main else: # Remove from sys.modules the reference to main_mod we'd # added. Otherwise it will trap references to objects From c16c0a7be7b43423001e7b9b0ef8c97d26d69667 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 24 Nov 2019 12:20:45 -0800 Subject: [PATCH 057/242] Update the what's new. --- docs/source/whatsnew/development.rst | 61 ++++++++++++++++++- docs/source/whatsnew/pr/autoformatting.rst | 24 -------- docs/source/whatsnew/pr/mimerenderer.rst | 22 ------- .../pr/prompt-update-modifications.rst | 14 ----- 4 files changed, 60 insertions(+), 61 deletions(-) delete mode 100644 docs/source/whatsnew/pr/autoformatting.rst delete mode 100644 docs/source/whatsnew/pr/mimerenderer.rst delete mode 100644 docs/source/whatsnew/pr/prompt-update-modifications.rst diff --git a/docs/source/whatsnew/development.rst b/docs/source/whatsnew/development.rst index 27afd6f5d0e..e2d90affc74 100644 --- a/docs/source/whatsnew/development.rst +++ b/docs/source/whatsnew/development.rst @@ -24,10 +24,69 @@ Need to be updated: +Prompt Rendering Performance improvements +========================================= + +Pull Request :ghpull:`11933` introduced an optimisation in the prompt rendering +logic that should decrease the resource usage of IPython when using the +_default_ configuration but could potentially introduce a regression of +functionalities if you are using a custom prompt. + +We know assume if you haven't changed the default keybindings that the prompt +**will not change** during the duration of your input – which is for example +not true when using vi insert mode that switches between `[ins]` and `[nor]` +for the current mode. + +If you are experiencing any issue let us know. + +Code autoformatting +=================== + +The IPython terminal can now auto format your code just before entering a new +line or executing a command. To do so use the +``--TerminalInteractiveShell.autoformatter`` option and set it to ``'black'``; +if black is installed IPython will use black to format your code when possible. + +IPython cannot always properly format your code; in particular it will +auto formatting with *black* will only work if: + + - Your code does not contains magics or special python syntax. + + - There is no code after your cursor. + +The Black API is also still in motion; so this may not work with all versions of +black. + +It should be possible to register custom reformatter, though the API is till in +flux. + +Arbitrary Mimetypes Handing in Terminal +======================================= + +When using IPython terminal it is now possible to register function to handle +arbitrary mimetypes. While rendering non-text based representation was possible in +many jupyter frontend; it was not possible in terminal IPython, as usually +terminal are limited to displaying text. As many terminal these days provide +escape sequences to display non-text; bringing this loved feature to IPython CLI +made a lot of sens. This functionality will not only allow inline images; but +allow opening of external program; for example ``mplayer`` to "display" sound +files. + +So far only the hooks necessary for this are in place, but no default mime +renderers added; so inline images will only be available via extensions. We will +progressively enable these features by default in the next few releases, and +contribution is welcomed. + +We welcome any feedback on the API. See :ref:`shell_mimerenderer` for more +informations. + +This is originally based on work form in :ghpull:`10610` from stephanh42 +started over two years ago, and still a lot need to be done. + .. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT. Backwards incompatible changes ------------------------------ -.. DO NOT EDIT THIS LINE BEFORE RELEASE. INCOMPAT INSERTION POINT. +.. DO NOT EDIT THIS LINE BEFORE RELEASE. INCOMPAT INSERTION POINT. \ No newline at end of file diff --git a/docs/source/whatsnew/pr/autoformatting.rst b/docs/source/whatsnew/pr/autoformatting.rst deleted file mode 100644 index dff810ec766..00000000000 --- a/docs/source/whatsnew/pr/autoformatting.rst +++ /dev/null @@ -1,24 +0,0 @@ -Code autoformatting -=================== - -The IPython terminal can now auto format your code just before entering a new -line or executing a command. To do so use the -``--TerminalInteractiveShell.autoformatter`` option and set it to ``'black'``; -if black is installed IPython will use black to format your code when possible. - -IPython cannot always properly format your code; in particular it will -auto formatting with *black* will only work if: - - - Your code does not contains magics or special python syntax. - - - There is no code after your cursor. - -The Black API is also still in motion; so this may not work with all versions of -black. - -It should be possible to register custom reformatter, though the API is till in -flux. - - - - diff --git a/docs/source/whatsnew/pr/mimerenderer.rst b/docs/source/whatsnew/pr/mimerenderer.rst deleted file mode 100644 index 339905f58e1..00000000000 --- a/docs/source/whatsnew/pr/mimerenderer.rst +++ /dev/null @@ -1,22 +0,0 @@ -Arbitrary Mimetypes Handing in Terminal -======================================= - -When using IPython terminal it is now possible to register function to handle -arbitrary mimetypes. While rendering non-text based representation was possible in -many jupyter frontend; it was not possible in terminal IPython, as usually -terminal are limited to displaying text. As many terminal these days provide -escape sequences to display non-text; bringing this loved feature to IPython CLI -made a lot of sens. This functionality will not only allow inline images; but -allow opening of external program; for example ``mplayer`` to "display" sound -files. - -So far only the hooks necessary for this are in place, but no default mime -renderers added; so inline images will only be available via extensions. We will -progressively enable these features by default in the next few releases, and -contribution is welcomed. - -We welcome any feedback on the API. See :ref:`shell_mimerenderer` for more -informations. - -This is originally based on work form in :ghpull:`10610` from stephanh42 -started over two years ago, and still a lot need to be done. diff --git a/docs/source/whatsnew/pr/prompt-update-modifications.rst b/docs/source/whatsnew/pr/prompt-update-modifications.rst deleted file mode 100644 index 77d557b8936..00000000000 --- a/docs/source/whatsnew/pr/prompt-update-modifications.rst +++ /dev/null @@ -1,14 +0,0 @@ -Prompt Rendering Performance improvements -========================================= - -Pull Request :ghpull:`11933` introduced an optimisation in the prompt rendering -logic that should decrease the resource usage of IPython when using the -_default_ configuration but could potentially introduce a regression of -functionalities if you are using a custom prompt. - -We know assume if you haven't changed the default keybindings that the prompt -**will not change** during the duration of your input – which is for example -not true when using vi insert mode that switches between `[ins]` and `[nor]` -for the current mode. - -If you are experiencing any issue let us know. From aaa90f97ebd2db14b1f25b37a0e5af22c643b01d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 24 Nov 2019 13:27:11 -0800 Subject: [PATCH 058/242] some more update to wn and migrate to the right file --- docs/source/whatsnew/development.rst | 65 +------------------ docs/source/whatsnew/version7.rst | 96 ++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 63 deletions(-) diff --git a/docs/source/whatsnew/development.rst b/docs/source/whatsnew/development.rst index e2d90affc74..9d4493a3a53 100644 --- a/docs/source/whatsnew/development.rst +++ b/docs/source/whatsnew/development.rst @@ -11,7 +11,7 @@ This document describes in-flight development work. `docs/source/whatsnew/pr` folder -Released .... ...., 2017 +Released .... ...., 2019 Need to be updated: @@ -22,71 +22,10 @@ Need to be updated: pr/* - - -Prompt Rendering Performance improvements -========================================= - -Pull Request :ghpull:`11933` introduced an optimisation in the prompt rendering -logic that should decrease the resource usage of IPython when using the -_default_ configuration but could potentially introduce a regression of -functionalities if you are using a custom prompt. - -We know assume if you haven't changed the default keybindings that the prompt -**will not change** during the duration of your input – which is for example -not true when using vi insert mode that switches between `[ins]` and `[nor]` -for the current mode. - -If you are experiencing any issue let us know. - -Code autoformatting -=================== - -The IPython terminal can now auto format your code just before entering a new -line or executing a command. To do so use the -``--TerminalInteractiveShell.autoformatter`` option and set it to ``'black'``; -if black is installed IPython will use black to format your code when possible. - -IPython cannot always properly format your code; in particular it will -auto formatting with *black* will only work if: - - - Your code does not contains magics or special python syntax. - - - There is no code after your cursor. - -The Black API is also still in motion; so this may not work with all versions of -black. - -It should be possible to register custom reformatter, though the API is till in -flux. - -Arbitrary Mimetypes Handing in Terminal -======================================= - -When using IPython terminal it is now possible to register function to handle -arbitrary mimetypes. While rendering non-text based representation was possible in -many jupyter frontend; it was not possible in terminal IPython, as usually -terminal are limited to displaying text. As many terminal these days provide -escape sequences to display non-text; bringing this loved feature to IPython CLI -made a lot of sens. This functionality will not only allow inline images; but -allow opening of external program; for example ``mplayer`` to "display" sound -files. - -So far only the hooks necessary for this are in place, but no default mime -renderers added; so inline images will only be available via extensions. We will -progressively enable these features by default in the next few releases, and -contribution is welcomed. - -We welcome any feedback on the API. See :ref:`shell_mimerenderer` for more -informations. - -This is originally based on work form in :ghpull:`10610` from stephanh42 -started over two years ago, and still a lot need to be done. - .. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT. Backwards incompatible changes ------------------------------ -.. DO NOT EDIT THIS LINE BEFORE RELEASE. INCOMPAT INSERTION POINT. \ No newline at end of file +.. DO NOT EDIT THIS LINE BEFORE RELEASE. INCOMPAT INSERTION POINT. diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 520ad110f61..7cf432004f2 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,8 +2,104 @@ 7.x Series ============ +IPython 7.10.0 +============== + +IPython 7.10 is the first double digit minor release in the last decade, and +first since the release of IPython 1.0, previous double digit minor release was +in August 2009. + +We've been trying to give you regular release on the last Friday of every month +for a guaranty of rapid access to bug fixes and new features. + +Unlike the previous first few releases that have seen only a couple of code +changes, 7.10 bring a number of changes, new features and bugfixes. + +Stop Support for Python 3.5 – Adopt NEP 29 +------------------------------------------ + +IPython has decided to follow the informational `NEP 29 +`_ which layout a clear +policy as to which version of (C)Python and NumPy are supported. + +We thus dropped support for Python 3.5, and cleaned up a number of code path +that were Python-version dependant. If you are on 3.5 or earlier pip should +automatically give you the latest compatible version of IPython so you do not +need to pin to a given version. + + +Prompt Rendering Performance improvements +----------------------------------------- + +Pull Request :ghpull:`11933` introduced an optimisation in the prompt rendering +logic that should decrease the resource usage of IPython when using the +_default_ configuration but could potentially introduce a regression of +functionalities if you are using a custom prompt. + +We know assume if you haven't changed the default keybindings that the prompt +**will not change** during the duration of your input – which is for example +not true when using vi insert mode that switches between `[ins]` and `[nor]` +for the current mode. + +If you are experiencing any issue let us know. + +Code autoformatting +------------------- + +The IPython terminal can now auto format your code just before entering a new +line or executing a command. To do so use the +``--TerminalInteractiveShell.autoformatter`` option and set it to ``'black'``; +if black is installed IPython will use black to format your code when possible. + +IPython cannot always properly format your code; in particular it will +auto formatting with *black* will only work if: + + - Your code does not contains magics or special python syntax. + + - There is no code after your cursor. + +The Black API is also still in motion; so this may not work with all versions of +black. + +It should be possible to register custom reformatter, though the API is till in +flux. + +Arbitrary Mimetypes Handing in Terminal +--------------------------------------- + +When using IPython terminal it is now possible to register function to handle +arbitrary mimetypes. While rendering non-text based representation was possible in +many jupyter frontend; it was not possible in terminal IPython, as usually +terminal are limited to displaying text. As many terminal these days provide +escape sequences to display non-text; bringing this loved feature to IPython CLI +made a lot of sens. This functionality will not only allow inline images; but +allow opening of external program; for example ``mplayer`` to "display" sound +files. + +So far only the hooks necessary for this are in place, but no default mime +renderers added; so inline images will only be available via extensions. We will +progressively enable these features by default in the next few releases, and +contribution is welcomed. + +We welcome any feedback on the API. See :ref:`shell_mimerenderer` for more +informations. + +This is originally based on work form in :ghpull:`10610` from stephanh42 +started over two years ago, and still a lot need to be done. + +MISC +---- + + - Completions can define their own ordering :ghpull:`11855` + - Enable Plotting in the same cell than the one that import matplotlib + :ghpull:`11916` + - Allow to store and restore multiple variables at once :ghpull:`11930` + .. _version 790: +IPython 7.9.0 +============= + IPython 7.9 is a small release with a couple of improvement and bug fixes. - Xterm terminal title should be restored on exit :ghpull:`11910` From ff71f9f50d7ab15e40672e1663f9db6884e12c0c Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Mon, 25 Nov 2019 20:35:21 +0300 Subject: [PATCH 059/242] Expand docs on how cells are executed --- docs/source/development/execution.rst | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/source/development/execution.rst b/docs/source/development/execution.rst index 3d5351bd0d3..817bfd5d8d5 100644 --- a/docs/source/development/execution.rst +++ b/docs/source/development/execution.rst @@ -1,13 +1,14 @@ .. _execution_semantics: -Execution semantics in the IPython kernel -========================================= +Execution of cells in the IPython kernel +======================================== -The execution of user code consists of the following phases: +When IPython kernel receives `execute_request `_ +with user code, it processes the message in the following phases: 1. Fire the ``pre_execute`` event. 2. Fire the ``pre_run_cell`` event unless silent is ``True``. -3. Execute the ``code`` field, see below for details. +3. Execute ``run_cell`` method to preprocess ``code``, compile and run it, see below for details. 4. If execution succeeds, expressions in ``user_expressions`` are computed. This ensures that any error in the expressions don't affect the main code execution. 5. Fire the ``post_execute`` event. @@ -18,9 +19,15 @@ The execution of user code consists of the following phases: :doc:`/config/callbacks` -To understand how the ``code`` field is executed, one must know that Python -code can be compiled in one of three modes (controlled by the ``mode`` argument -to the :func:`compile` builtin): +Running user ``code`` +===================== + +First, the ``code`` cell is transformed to expand ``%magic`` and ``!system`` +by ``IPython.core.inputtransformer2``. Then is being compiled using standard +Python :func:`compile` function and executed. + +Not specific to IPython, all Python code can be compiled in one of three modes +(see ``mode`` argument to the standard :func:`compile` function): *single* Valid for a single interactive statement (though the source can contain @@ -50,16 +57,16 @@ execution in 'single' mode, and then: - If there is more than one block: - * if the last one is a single line long, run all but the last in 'exec' mode + * if the last block is a single line long, run all but the last in 'exec' mode and the very last one in 'single' mode. This makes it easy to type simple expressions at the end to see computed values. - * if the last one is no more than two lines long, run all but the last in + * if the last block is no more than two lines long, run all but the last in 'exec' mode and the very last one in 'single' mode. This makes it easy to type simple expressions at the end to see computed values. - otherwise (last one is also multiline), run all in 'exec' mode - * otherwise (last one is also multiline), run all in 'exec' mode as a single + * otherwise (last block is also multiline), run all in 'exec' mode as a single unit. From 7382f0c50e144bc0d9dee77336a7edbefcb0afbd Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Mon, 25 Nov 2019 20:40:52 +0300 Subject: [PATCH 060/242] Fill in some missing words --- docs/source/development/execution.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/development/execution.rst b/docs/source/development/execution.rst index 817bfd5d8d5..73e386d5c8e 100644 --- a/docs/source/development/execution.rst +++ b/docs/source/development/execution.rst @@ -23,11 +23,11 @@ Running user ``code`` ===================== First, the ``code`` cell is transformed to expand ``%magic`` and ``!system`` -by ``IPython.core.inputtransformer2``. Then is being compiled using standard -Python :func:`compile` function and executed. +commands by ``IPython.core.inputtransformer2``. Then expanded cell is compiled +using standard Python :func:`compile` function and executed. -Not specific to IPython, all Python code can be compiled in one of three modes -(see ``mode`` argument to the standard :func:`compile` function): +Python :func:`compile` function provides ``mode`` argument to select one +of three ways of compiling code: *single* Valid for a single interactive statement (though the source can contain From c2a5384f364ecc5dded4ad892f13527880ae8517 Mon Sep 17 00:00:00 2001 From: Jonathan Slenders Date: Tue, 14 May 2019 22:45:27 +0200 Subject: [PATCH 061/242] Support prompt_toolkit 3.0 as well. --- IPython/terminal/interactiveshell.py | 19 +++++++++++++++++-- setup.py | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 7970ced9041..a185915f387 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -25,6 +25,7 @@ from prompt_toolkit.shortcuts import PromptSession, CompleteStyle, print_formatted_text from prompt_toolkit.styles import DynamicStyle, merge_styles from prompt_toolkit.styles.pygments import style_from_pygments_cls, style_from_pygments_dict +from prompt_toolkit import __version__ as ptk_version from pygments.styles import get_style_by_name from pygments.style import Style @@ -38,6 +39,7 @@ from .shortcuts import create_ipython_shortcuts DISPLAY_BANNER_DEPRECATED = object() +PTK3 = ptk_version.startswith('3.') class _NoStyle(Style): pass @@ -415,7 +417,7 @@ def get_message(): # work around this. get_message = get_message() - return { + options = { 'complete_in_thread': False, 'lexer':IPythonPTLexer(), 'reserve_space_for_menu':self.space_for_menu, @@ -432,8 +434,11 @@ def get_message(): processor=HighlightMatchingBracketProcessor(chars='[](){}'), filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() & Condition(lambda: self.highlight_matching_brackets))], - 'inputhook': self.inputhook, } + if not PTK3: + options['inputhook'] = self.inputhook + + return options def prompt_for_code(self): if self.rl_next_input: @@ -559,6 +564,16 @@ def enable_gui(self, gui=None): else: self.active_eventloop = self._inputhook = None + # For prompt_toolkit 3.0. We have to create an asyncio event loop with + # this inputhook. + if PTK3: + if self._inputhook: + from prompt_toolkit.eventloop import set_eventloop_with_inputhook + set_eventloop_with_inputhook(self._inputhook) + else: + import asyncio + asyncio.set_event_loop(asyncio.new_event_loop()) + # Run !system commands directly, not through pipes, so terminal programs # work correctly. system = InteractiveShell.system_raw diff --git a/setup.py b/setup.py index 638446de9e0..022520db442 100755 --- a/setup.py +++ b/setup.py @@ -190,7 +190,7 @@ 'decorator', 'pickleshare', 'traitlets>=4.2', - 'prompt_toolkit>=2.0.0,<2.1.0', + 'prompt_toolkit>=2.0.0,<3.1.0', 'pygments', 'backcall', ] From 6ba2738be374b724bce99df8538297e441ab2c28 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 26 Nov 2019 09:48:37 -0800 Subject: [PATCH 062/242] update whats new --- docs/source/whatsnew/version7.rst | 47 +++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 7cf432004f2..af17dec8e06 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -27,6 +27,14 @@ that were Python-version dependant. If you are on 3.5 or earlier pip should automatically give you the latest compatible version of IPython so you do not need to pin to a given version. +Support for Prompt Toolkit 3.0 +------------------------------ + +Prompt Toolkit 3.0 was release a week before IPython 7.10 and introduces a few +breaking changes. We believe IPython 7.10 should be compatible with both Prompt +Toolkit 2.x and 3.x, though it has not been extensively tested with 3.x so +please report any issues. + Prompt Rendering Performance improvements ----------------------------------------- @@ -61,11 +69,11 @@ auto formatting with *black* will only work if: The Black API is also still in motion; so this may not work with all versions of black. -It should be possible to register custom reformatter, though the API is till in +It should be possible to register custom formatter, though the API is till in flux. -Arbitrary Mimetypes Handing in Terminal ---------------------------------------- +Arbitrary Mimetypes Handing in Terminal (Aka inline images in terminal) +----------------------------------------------------------------------- When using IPython terminal it is now possible to register function to handle arbitrary mimetypes. While rendering non-text based representation was possible in @@ -84,7 +92,7 @@ contribution is welcomed. We welcome any feedback on the API. See :ref:`shell_mimerenderer` for more informations. -This is originally based on work form in :ghpull:`10610` from stephanh42 +This is originally based on work form in :ghpull:`10610` from @stephanh42 started over two years ago, and still a lot need to be done. MISC @@ -95,6 +103,33 @@ MISC :ghpull:`11916` - Allow to store and restore multiple variables at once :ghpull:`11930` +You can see `all pull-requests `_ for this release. + +API Changes +----------- + +Change of API and exposed objects automatically detected using `frappuccino `_ : + +The following items are new in IPython 7.10: + + IPython.terminal.shortcuts.reformat_text_before_cursor + + IPython.terminal.prompts.RichPromptDisplayHook.write_format_data + + IPython.terminal.interactiveshell.black_reformat_handler + +The following signatures differ between 7.9 and 7.10: + + - IPython.extensions.storemagic.restore_aliases(ip) + + IPython.extensions.storemagic.restore_aliases(ip, alias='None') + +Special Thanks +-------------- + + - @stephanh42 who started the work on inline images in terminal 2 years ago + - @augustogoulart who spent a lot of time triaging issues and responding to + users. + - @con-f-use who is my (@Carreau) first sponsor on GitHub, as a reminder if you + like IPython, Jupyter and many other library of the SciPy stack you can + donate to numfocus.org non profit + .. _version 790: IPython 7.9.0 @@ -109,12 +144,12 @@ IPython 7.9 is a small release with a couple of improvement and bug fixes. find all objects needing reload. This should avoid large objects traversal like pandas dataframes. :ghpull:`11876` - Get ready for Python 4. :ghpull:`11874` - - `%env` Magic nonw has euristic to hide potentially sensitive values :ghpull:`11896` + - `%env` Magic now has heuristic to hide potentially sensitive values :ghpull:`11896` This is a small release despite a number of Pull Request Pending that need to be reviewed/worked on. Many of the core developers have been busy outside of IPython/Jupyter and we thanks all contributor for their patience; we'll work on -these as soon as we have time. +these as soon as we have time. .. _version780: From c671abf59674b09c92ff570dadbd3c97565286f7 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 26 Nov 2019 13:40:22 -0800 Subject: [PATCH 063/242] update changes in API --- docs/source/whatsnew/version7.rst | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index af17dec8e06..6c3dbab387a 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -108,14 +108,20 @@ You can see `all pull-requests `_ : +Change of API and exposed objects automatically detected using `frappuccino `_ (still in beta): -The following items are new in IPython 7.10: - + IPython.terminal.shortcuts.reformat_text_before_cursor - + IPython.terminal.prompts.RichPromptDisplayHook.write_format_data - + IPython.terminal.interactiveshell.black_reformat_handler +The following items are new in IPython 7.10:: -The following signatures differ between 7.9 and 7.10: + + IPython.terminal.shortcuts.reformat_text_before_cursor(buffer, document, shell) + + IPython.terminal.interactiveshell.PTK3 + + IPython.terminal.interactiveshell.black_reformat_handler(text_before_cursor) + + IPython.terminal.prompts.RichPromptDisplayHook.write_format_data(self, format_dict, md_dict='None') + +The following items have been removed in 7.10:: + + - IPython.lib.pretty.DICT_IS_ORDERED + +The following signatures differ between versions:: - IPython.extensions.storemagic.restore_aliases(ip) + IPython.extensions.storemagic.restore_aliases(ip, alias='None') From d5e408bdc738e3320fc6110f32c08fcb2fe83d93 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 27 Nov 2019 15:16:03 -0800 Subject: [PATCH 064/242] update github stats --- docs/source/whatsnew/github-stats-7.rst | 27 +++++++++++++++++++++++++ tools/release_helper.sh | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/source/whatsnew/github-stats-7.rst b/docs/source/whatsnew/github-stats-7.rst index 23ff55c83f9..0d8b28bcc4a 100644 --- a/docs/source/whatsnew/github-stats-7.rst +++ b/docs/source/whatsnew/github-stats-7.rst @@ -1,6 +1,33 @@ Issues closed in the 7.x development cycle ========================================== +Issues closed in 7.10 +--------------------- + +GitHub stats for 2019/10/25 - 2019/11/27 (tag: 7.9.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 4 issues and merged 22 pull requests. +The full list can be seen `on GitHub `__ + +The following 15 authors contributed 101 commits. + +* anatoly techtonik +* Ben Lewis +* Benjamin Ragan-Kelley +* Gerrit Buss +* grey275 +* Gökcen Eraslan +* Jonathan Slenders +* Joris Van den Bossche +* kousik +* Matthias Bussonnier +* Nicholas Bollweg +* Paul McCarthy +* Srinivas Reddy Thatiparthy +* Timo Kaufmann +* Tony Fast Issues closed in 7.9 -------------------- diff --git a/tools/release_helper.sh b/tools/release_helper.sh index 67ce086859f..92e59ae532f 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -42,7 +42,7 @@ echo $BLUE"here are all the authors that contributed to this release:"$NOR git log --format="%aN <%aE>" $PREV_RELEASE... | sort -u -f echo -echo $BLUE"If you see any duplicates cancel (Ctrl-C), then edit .mailmap."$GREEN"Press enter to continue:"$NOR +echo $BLUE"If you see any duplicates cancel (Ctrl-C), then edit .mailmap.\n"$GREEN"Press enter to continue:"$NOR read echo $BLUE"generating stats"$NOR From cd9f5228dcee55d170f22e55ecea807ee79b3bbd Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 27 Nov 2019 15:18:18 -0800 Subject: [PATCH 065/242] release 7.10.0 --- IPython/core/release.py | 2 +- tools/release_helper.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 3961477ed7c..af592917a69 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -24,7 +24,7 @@ _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] diff --git a/tools/release_helper.sh b/tools/release_helper.sh index 92e59ae532f..8935ef0e91e 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -75,8 +75,8 @@ echo $BLUE"Attempting to build package..."$NOR tools/build_release echo -echo "Let\'s commit : git commit -am \"release $VERSION\" -S" -echo $GREEN"Press enter to continue"$NOR +echo "Let's commit : git commit -am \"release $VERSION\" -S" +echo $GREEN"Press enter to commit"$NOR read git commit -am "release $VERSION" -S From 5bac48502c470f66db3629468dfecb52639ff357 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 27 Nov 2019 16:15:43 -0800 Subject: [PATCH 066/242] back to dev --- IPython/core/release.py | 4 ++-- tools/release_helper.sh | 26 ++++++++++++++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index af592917a69..5cf2110adf0 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 10 +_version_minor = 11 _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] diff --git a/tools/release_helper.sh b/tools/release_helper.sh index 8935ef0e91e..d7040cc742b 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -81,10 +81,28 @@ read git commit -am "release $VERSION" -S echo -echo "git push origin \$BRANCH ?" -echo "Press enter to continue" +echo $BLUE"git push origin \$BRANCH ($BRANCH)?"$NOR +echo $GREEN"Make sure you can push"$NOR +echo $GREEN"Press enter to continue"$NOR read git push origin $BRANCH -# git tag -am "release $VERSION" "$VERSION" -s -# git push origin $VERSION + +echo +echo "Let's tag : git tag -am \"release $VERSION\" \"$VERSION\" -s" +echo $GREEN"Press enter to wtagcommit"$NOR +read +git tag -am "release $VERSION" "$VERSION" -s + +echo +echo $BLUE"And push the tag: git push origin \$VERSION ?"$NOR +echo $GREEN"Press enter to continue"$NOR +read +git push origin $VERSION + +echo +echo $BLUE"let's : git checkout $VERSION"$NOR +echo $GREEN"Press enter to continue"$NOR +read +git checkout $VERSION + From 7995e64e9d85d4e8ef8dd236d5fff97524508700 Mon Sep 17 00:00:00 2001 From: Jonathan Slenders Date: Wed, 27 Nov 2019 23:20:37 +0100 Subject: [PATCH 067/242] Run the prompt in a separate asyncio loop. This prevents crashes from situations where the user starts scheduling new coroutines that don't terminate. Like import asyncio async def func(): while True: pass asyncio.ensure_future(func()) This would freeze the event loop and can't even be cancelled by pressing control-c, because prompt_toolkit will never gain control from the event loop to process key bindings. --- IPython/terminal/interactiveshell.py | 27 ++++++++++++++++++++------- setup.py | 2 +- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index a185915f387..26188aeba8a 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -1,5 +1,6 @@ """IPython terminal interface using prompt_toolkit""" +import asyncio import os import sys import warnings @@ -309,6 +310,7 @@ def prompt(): editing_mode = getattr(EditingMode, self.editing_mode.upper()) + self.pt_loop = asyncio.new_event_loop() self.pt_app = PromptSession( editing_mode=editing_mode, key_bindings=key_bindings, @@ -448,10 +450,21 @@ def prompt_for_code(self): default = '' with patch_stdout(raw=True): - text = self.pt_app.prompt( - default=default, -# pre_run=self.pre_prompt,# reset_current_buffer=True, - **self._extra_prompt_options()) + # In order to make sure that asyncio code written in the + # interactive shell doesn't interfere with the prompt, we run the + # prompt in a different event loop. + # If we don't do this, people could spawn coroutine with a + # while/true inside which will freeze the prompt. + + old_loop = asyncio.get_event_loop() + asyncio.set_event_loop(self.pt_loop) + try: + text = self.pt_app.prompt( + default=default, + **self._extra_prompt_options()) + finally: + # Restore the original event loop. + asyncio.set_event_loop(old_loop) return text def enable_win_unicode_console(self): @@ -568,11 +581,11 @@ def enable_gui(self, gui=None): # this inputhook. if PTK3: if self._inputhook: - from prompt_toolkit.eventloop import set_eventloop_with_inputhook - set_eventloop_with_inputhook(self._inputhook) + from prompt_toolkit.eventloop import new_eventloop_with_inputhook + self.pt_loop = new_eventloop_with_inputhook(self._inputhook) else: import asyncio - asyncio.set_event_loop(asyncio.new_event_loop()) + self.pt_loop = asyncio.new_event_loop() # Run !system commands directly, not through pipes, so terminal programs # work correctly. diff --git a/setup.py b/setup.py index 022520db442..593e3a6a0af 100755 --- a/setup.py +++ b/setup.py @@ -190,7 +190,7 @@ 'decorator', 'pickleshare', 'traitlets>=4.2', - 'prompt_toolkit>=2.0.0,<3.1.0', + 'prompt_toolkit>=2.0.0,<3.1.0,!=3.0.0,!=3.0.1', 'pygments', 'backcall', ] From a4255f1db13453f0fe7398218f3c0470400b0df5 Mon Sep 17 00:00:00 2001 From: Jonathan Slenders Date: Sat, 30 Nov 2019 15:12:53 +0100 Subject: [PATCH 068/242] Add asyncio input hook for event loop integration. --- IPython/terminal/pt_inputhooks/__init__.py | 3 +- IPython/terminal/pt_inputhooks/asyncio.py | 43 ++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 IPython/terminal/pt_inputhooks/asyncio.py diff --git a/IPython/terminal/pt_inputhooks/__init__.py b/IPython/terminal/pt_inputhooks/__init__.py index 284293ddde4..c7ba58dc4be 100644 --- a/IPython/terminal/pt_inputhooks/__init__.py +++ b/IPython/terminal/pt_inputhooks/__init__.py @@ -12,7 +12,8 @@ 'tk', 'wx', 'pyglet', 'glut', - 'osx' + 'osx', + 'asyncio' ] registered = {} diff --git a/IPython/terminal/pt_inputhooks/asyncio.py b/IPython/terminal/pt_inputhooks/asyncio.py new file mode 100644 index 00000000000..53d49741b40 --- /dev/null +++ b/IPython/terminal/pt_inputhooks/asyncio.py @@ -0,0 +1,43 @@ +""" +Inputhook for running the original asyncio event loop while we're waiting for +input. + +By default, in IPython, we run the prompt with a different asyncio event loop, +because otherwise we risk that people are freezing the prompt by scheduling bad +coroutines. E.g., a coroutine that does a while/true and never yield back +control to the loop. We can't cancel that. + +However, sometimes we want the asyncio loop to keep running while waiting for +a prompt. + +The following example will print the numbers from 1 to 10 above the prompt, +while we are waiting for input. (This works also because we use +prompt_toolkit`s `patch_stdout`):: + + In [1]: import asyncio + + In [2]: %gui asyncio + + In [3]: async def f(): + ...: for i in range(10): + ...: await asyncio.sleep(1) + ...: print(i) + + + In [4]: asyncio.ensure_future(f()) + +""" +import asyncio + +# Keep reference to the original asyncio loop, because getting the event loop +# within the input hook would return the other loop. +loop = asyncio.get_event_loop() + + +def inputhook(context): + def stop(): + loop.stop() + + loop.add_reader(context.fileno(), stop) + context.fileno() + loop.run_forever() From 8768212e2f10ebffd0118d79eebfcc6f8debb6b2 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 30 Nov 2019 08:48:09 -0800 Subject: [PATCH 069/242] Handle case where the shell does not have arbitrary mimerenderer handler. That can be the case in test-suites; for example bokeh Closes #11976 --- IPython/core/displaypub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/displaypub.py b/IPython/core/displaypub.py index d769692e969..f651a2a0cf6 100644 --- a/IPython/core/displaypub.py +++ b/IPython/core/displaypub.py @@ -105,7 +105,7 @@ def publish(self, data, metadata=None, source=None, *, transient=None, update=Fa handlers = {} if self.shell is not None: - handlers = self.shell.mime_renderers + handlers = getattr(self.shell, 'mime_renderers', {}) for mime, handler in handlers.items(): if mime in data: From f52ca01cad84187a87ee0159778886b3764dbb18 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 30 Nov 2019 09:00:14 -0800 Subject: [PATCH 070/242] Add prompt toolkit 3 compatibility to ipdb closes #11962 closes #11975 --- IPython/terminal/debugger.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/IPython/terminal/debugger.py b/IPython/terminal/debugger.py index 696603f61e9..ba5ce1bada8 100644 --- a/IPython/terminal/debugger.py +++ b/IPython/terminal/debugger.py @@ -17,6 +17,9 @@ from prompt_toolkit.enums import EditingMode from prompt_toolkit.formatted_text import PygmentsTokens +from prompt_toolkit import __version__ as ptk_version +PTK3 = ptk_version.startswith('3.') + class TerminalPdb(Pdb): """Standalone IPython debugger.""" @@ -49,20 +52,23 @@ def get_prompt_tokens(): & ~cursor_in_leading_ws ))(display_completions_like_readline) - self.pt_app = PromptSession( - message=(lambda: PygmentsTokens(get_prompt_tokens())), - editing_mode=getattr(EditingMode, self.shell.editing_mode.upper()), - key_bindings=kb, - history=self.shell.debugger_history, - completer=self._ptcomp, - enable_history_search=True, - mouse_support=self.shell.mouse_support, - complete_style=self.shell.pt_complete_style, - style=self.shell.style, - inputhook=self.shell.inputhook, - color_depth=self.shell.color_depth, + options = dict( + message=(lambda: PygmentsTokens(get_prompt_tokens())), + editing_mode=getattr(EditingMode, self.shell.editing_mode.upper()), + key_bindings=kb, + history=self.shell.debugger_history, + completer=self._ptcomp, + enable_history_search=True, + mouse_support=self.shell.mouse_support, + complete_style=self.shell.pt_complete_style, + style=self.shell.style, + color_depth=self.shell.color_depth, ) + if not PTK3: + options['inputhook'] = self.inputhook + self.pt_app = PromptSession(**options) + def cmdloop(self, intro=None): """Repeatedly issue a prompt, accept input, parse an initial prefix off the received input, and dispatch to action methods, passing them From 5465c56db8e58789edb6c137c219d4cdbd9029c9 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 30 Nov 2019 09:57:07 -0800 Subject: [PATCH 071/242] some updates to the release scripts --- tools/release_helper.sh | 79 +++++++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 26 deletions(-) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index d7040cc742b..82a7bc0208e 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -9,14 +9,6 @@ python -c 'import sphinx' python -c 'import sphinx_rtd_theme' python -c 'import nose' -echo -n 'PREV_RELEASE (X.y.z):' -read PREV_RELEASE -echo -n 'MILESTONE (X.y):' -read MILESTONE -echo -n 'VERSION (X.y.z):' -read VERSION -echo -n 'branch (master|X.y):' -read branch BLACK=$(tput setaf 1) RED=$(tput setaf 1) @@ -28,30 +20,65 @@ CYAN=$(tput setaf 6) WHITE=$(tput setaf 7) NOR=$(tput sgr0) -echo -echo $BLUE"Updating what's new with informations from docs/source/whatsnew/pr"$NOR -python tools/update_whatsnew.py -echo -echo $BLUE"please move the contents of "docs/source/whatsnew/development.rst" to version-X.rst"$NOR -echo $GREEN"Press enter to continue"$NOR -read +echo -n "PREV_RELEASE (X.y.z) [$PREV_RELEASE]: " +read input +PREV_RELEASE=${input:-$PREV_RELEASE} +echo -n "MILESTONE (X.y) [$MILESTONE]: " +read input +MILESTONE=${input:-$MILESTONE} +echo -n "VERSION (X.y.z) [$VERSION]:" +read input +VERSION=${input:-$VERSION} +echo -n "branch (master|X.y) [$branch]:" +read input +branch=${input:-$branch} + +ask_section(){ + echo + echo $BLUE"$1"$NOR + echo -n $GREEN"Press Enter to continue, S to skip: "$GREEN + read -n1 value + echo + if [ -z $value ] || [ $value = 'y' ] ; then + return 0 + fi + return 1 +} + -echo -echo $BLUE"here are all the authors that contributed to this release:"$NOR -git log --format="%aN <%aE>" $PREV_RELEASE... | sort -u -f echo -echo $BLUE"If you see any duplicates cancel (Ctrl-C), then edit .mailmap.\n"$GREEN"Press enter to continue:"$NOR -read +if ask_section "Updating what's new with informations from docs/source/whatsnew/pr" +then + python tools/update_whatsnew.py -echo $BLUE"generating stats"$NOR -python tools/github_stats.py --milestone $MILESTONE > stats.rst + echo + echo $BLUE"please move the contents of "docs/source/whatsnew/development.rst" to version-X.rst"$NOR + echo $GREEN"Press enter to continue"$NOR + read +fi -echo $BLUE"stats.rst files generated."$NOR -echo $GREEN"Please merge it with the right file (github-stats-X.rst) and commit."$NOR -echo $GREEN"press enter to continue."$NOR -read +if ask_section "Gen Stats, and authors" +then + + echo + echo $BLUE"here are all the authors that contributed to this release:"$NOR + git log --format="%aN <%aE>" $PREV_RELEASE... | sort -u -f + + echo + echo $BLUE"If you see any duplicates cancel (Ctrl-C), then edit .mailmap.\n"$GREEN"Press enter to continue:"$NOR + read + + echo $BLUE"generating stats"$NOR + python tools/github_stats.py --milestone $MILESTONE > stats.rst + + echo $BLUE"stats.rst files generated."$NOR + echo $GREEN"Please merge it with the right file (github-stats-X.rst) and commit."$NOR + echo $GREEN"press enter to continue."$NOR + read + +fi echo "Cleaning repository" git clean -xfdi From 33775656cb4137d918fe38a7d38a4206cceba934 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 30 Nov 2019 09:23:06 -0800 Subject: [PATCH 072/242] print the version on crash --- IPython/core/crashhandler.py | 6 ++++-- tools/release_helper.sh | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/IPython/core/crashhandler.py b/IPython/core/crashhandler.py index 2117edb5c0b..1e0b429d09a 100644 --- a/IPython/core/crashhandler.py +++ b/IPython/core/crashhandler.py @@ -29,6 +29,8 @@ from IPython.utils.sysinfo import sys_info from IPython.utils.py3compat import input +from IPython.core.release import __version__ as version + #----------------------------------------------------------------------------- # Code #----------------------------------------------------------------------------- @@ -68,7 +70,7 @@ """ _lite_message_template = """ -If you suspect this is an IPython bug, please report it at: +If you suspect this is an IPython {version} bug, please report it at: https://github.com/ipython/ipython/issues or send an email to the mailing list at {email} @@ -222,5 +224,5 @@ def crash_handler_lite(etype, evalue, tb): else: # we are not in a shell, show generic config config = "c." - print(_lite_message_template.format(email=author_email, config=config), file=sys.stderr) + print(_lite_message_template.format(email=author_email, config=config, version=version), file=sys.stderr) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index d7040cc742b..7251ec1d752 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -105,4 +105,8 @@ echo $GREEN"Press enter to continue"$NOR read git checkout $VERSION +# ./tools/release +# ls ./dist +# shasum -a 256 dist/* + From ac224067d1e05be54fcc91a3036dbf07a8a4ce68 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 30 Nov 2019 11:48:01 -0800 Subject: [PATCH 073/242] Remove useless call to u_format and redundant list comprehension One step toward #11987 --- IPython/core/tests/test_handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/tests/test_handlers.py b/IPython/core/tests/test_handlers.py index e87e29dbd31..02860c8b4af 100644 --- a/IPython/core/tests/test_handlers.py +++ b/IPython/core/tests/test_handlers.py @@ -49,11 +49,11 @@ def test_handlers(): # For many of the below, we're also checking that leading whitespace # turns off the esc char, which it should unless there is a continuation # line. - run([(i,py3compat.u_format(o)) for i,o in \ + run( [('"no change"', '"no change"'), # normal (u"lsmagic", "get_ipython().run_line_magic('lsmagic', '')"), # magic #("a = b # PYTHON-MODE", '_i'), # emacs -- avoids _in cache - ]]) + ]) # Objects which are instances of IPyAutocall are *always* autocalled autocallable = Autocallable() From a771dadfde60a612739274c6fbc73db783621963 Mon Sep 17 00:00:00 2001 From: Jonathan Slenders Date: Sat, 30 Nov 2019 21:42:13 +0100 Subject: [PATCH 074/242] Improve cleanup of asyncio input hook. --- IPython/terminal/pt_inputhooks/asyncio.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/IPython/terminal/pt_inputhooks/asyncio.py b/IPython/terminal/pt_inputhooks/asyncio.py index 53d49741b40..e28603864f9 100644 --- a/IPython/terminal/pt_inputhooks/asyncio.py +++ b/IPython/terminal/pt_inputhooks/asyncio.py @@ -38,6 +38,9 @@ def inputhook(context): def stop(): loop.stop() - loop.add_reader(context.fileno(), stop) - context.fileno() - loop.run_forever() + fileno = context.fileno() + loop.add_reader(fileno, stop) + try: + loop.run_forever() + finally: + loop.remove_reader(fileno) From c812a49785e55339d131017718e594f4a7d45154 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 30 Nov 2019 17:29:33 -0800 Subject: [PATCH 075/242] Be more explicit on how we handle osx clipboard. The return values can only be bytes as we do bytes manipulations before. So if we need to return text; we always need to decode. We can now also leave the default encoding out; as py3compat.decode takecare of that for us --- IPython/lib/clipboard.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/IPython/lib/clipboard.py b/IPython/lib/clipboard.py index 1b8e756b5c6..316a8ab1f8a 100644 --- a/IPython/lib/clipboard.py +++ b/IPython/lib/clipboard.py @@ -32,15 +32,15 @@ def win32_clipboard_get(): win32clipboard.CloseClipboard() return text -def osx_clipboard_get(): +def osx_clipboard_get() -> str: """ Get the clipboard's text on OS X. """ p = subprocess.Popen(['pbpaste', '-Prefer', 'ascii'], stdout=subprocess.PIPE) - text, stderr = p.communicate() + bytes_, stderr = p.communicate() # Text comes in with old Mac \r line endings. Change them to \n. - text = text.replace(b'\r', b'\n') - text = py3compat.cast_unicode(text, py3compat.DEFAULT_ENCODING) + bytes_ = bytes_.replace(b'\r', b'\n') + text = py3compat.decode(bytes_) return text def tkinter_clipboard_get(): From b70b111b3f5a6cab3a89847eb98ce6b36f9f287d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 30 Nov 2019 17:51:06 -0800 Subject: [PATCH 076/242] Enforce some more types since we are Py3only --- IPython/paths.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/IPython/paths.py b/IPython/paths.py index 82fba5ab45f..73631b58288 100644 --- a/IPython/paths.py +++ b/IPython/paths.py @@ -12,7 +12,7 @@ ensure_dir_exists, fs_encoding) from IPython.utils import py3compat -def get_ipython_dir(): +def get_ipython_dir() -> str: """Get the IPython directory for this platform and user. This uses the logic in `get_home_dir` to find the home directory @@ -28,10 +28,9 @@ def get_ipython_dir(): home_dir = get_home_dir() xdg_dir = get_xdg_dir() - # import pdb; pdb.set_trace() # dbg if 'IPYTHON_DIR' in env: - warn('The environment variable IPYTHON_DIR is deprecated. ' - 'Please use IPYTHONDIR instead.') + warn('The environment variable IPYTHON_DIR is deprecated since IPython 3.0. ' + 'Please use IPYTHONDIR instead.', DeprecationWarning) ipdir = env.get('IPYTHONDIR', env.get('IPYTHON_DIR', None)) if ipdir is None: # not set explicitly, use ~/.ipython @@ -67,8 +66,8 @@ def get_ipython_dir(): warn("IPython parent '{0}' is not a writable location," " using a temp directory.".format(parent)) ipdir = tempfile.mkdtemp() - - return py3compat.cast_unicode(ipdir, fs_encoding) + assert isinstance(ipdir, str), "all path manipulation should be str(unicode), but are not." + return ipdir def get_ipython_cache_dir(): From 4a42866eee1d79b2d07c3fc5472d67f19929d077 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 30 Nov 2019 18:03:12 -0800 Subject: [PATCH 077/242] More cleaning and enforcing of types. Also drop some reminder python 2 compatibility imports. And use some context manager when appropriate instead of manual closing --- IPython/paths.py | 9 +++++---- IPython/utils/path.py | 17 +++++++---------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/IPython/paths.py b/IPython/paths.py index 73631b58288..bbe3d5c7cad 100644 --- a/IPython/paths.py +++ b/IPython/paths.py @@ -70,7 +70,7 @@ def get_ipython_dir() -> str: return ipdir -def get_ipython_cache_dir(): +def get_ipython_cache_dir() -> str: """Get the cache directory it is created if it does not exist.""" xdgdir = get_xdg_cache_dir() if xdgdir is None: @@ -81,13 +81,14 @@ def get_ipython_cache_dir(): elif not _writable_dir(xdgdir): return get_ipython_dir() - return py3compat.cast_unicode(ipdir, fs_encoding) + return ipdir -def get_ipython_package_dir(): +def get_ipython_package_dir() -> str: """Get the base directory where IPython itself is installed.""" ipdir = os.path.dirname(IPython.__file__) - return py3compat.cast_unicode(ipdir, fs_encoding) + assert isinstance(ipdir, str) + return ipdir def get_ipython_module_path(module_str): diff --git a/IPython/utils/path.py b/IPython/utils/path.py index f677f1c7cd0..5276f8617fe 100644 --- a/IPython/utils/path.py +++ b/IPython/utils/path.py @@ -169,7 +169,7 @@ class HomeDirError(Exception): pass -def get_home_dir(require_writable=False): +def get_home_dir(require_writable=False) -> str: """Return the 'home' directory, as a unicode string. Uses os.path.expanduser('~'), and checks for writability. @@ -197,21 +197,18 @@ def get_home_dir(require_writable=False): if not _writable_dir(homedir) and os.name == 'nt': # expanduser failed, use the registry to get the 'My Documents' folder. try: - try: - import winreg as wreg # Py 3 - except ImportError: - import _winreg as wreg # Py 2 - key = wreg.OpenKey( + import winreg as wreg + with wreg.OpenKey( wreg.HKEY_CURRENT_USER, r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders" - ) - homedir = wreg.QueryValueEx(key,'Personal')[0] - key.Close() + ) as key: + homedir = wreg.QueryValueEx(key,'Personal')[0] except: pass if (not require_writable) or _writable_dir(homedir): - return py3compat.cast_unicode(homedir, fs_encoding) + assert isinstance(homedir, str), "Homedir shoudl be unicode not bytes" + return homedir else: raise HomeDirError('%s is not a writable dir, ' 'set $HOME environment variable to override' % homedir) From 364457bf98c4335115906209ef7066ea03e142bc Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 30 Nov 2019 18:06:24 -0800 Subject: [PATCH 078/242] Add some missing deprecation warnings --- IPython/utils/path.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/IPython/utils/path.py b/IPython/utils/path.py index 5276f8617fe..f2f1ea06f5a 100644 --- a/IPython/utils/path.py +++ b/IPython/utils/path.py @@ -251,31 +251,31 @@ def get_xdg_cache_dir(): @undoc def get_ipython_dir(): - warn("get_ipython_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2) + warn("get_ipython_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2) from IPython.paths import get_ipython_dir return get_ipython_dir() @undoc def get_ipython_cache_dir(): - warn("get_ipython_cache_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2) + warn("get_ipython_cache_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2) from IPython.paths import get_ipython_cache_dir return get_ipython_cache_dir() @undoc def get_ipython_package_dir(): - warn("get_ipython_package_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2) + warn("get_ipython_package_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2) from IPython.paths import get_ipython_package_dir return get_ipython_package_dir() @undoc def get_ipython_module_path(module_str): - warn("get_ipython_module_path has moved to the IPython.paths module since IPython 4.0.", stacklevel=2) + warn("get_ipython_module_path has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2) from IPython.paths import get_ipython_module_path return get_ipython_module_path(module_str) @undoc def locate_profile(profile='default'): - warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", stacklevel=2) + warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2) from IPython.paths import locate_profile return locate_profile(profile=profile) From 8c4e8b5da7d0caa0ee88e3a11410e9b3f33551f6 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 30 Nov 2019 19:15:17 -0800 Subject: [PATCH 079/242] Strange wondows regression --- IPython/utils/path.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/IPython/utils/path.py b/IPython/utils/path.py index f2f1ea06f5a..d52cad0a2e5 100644 --- a/IPython/utils/path.py +++ b/IPython/utils/path.py @@ -198,11 +198,12 @@ def get_home_dir(require_writable=False) -> str: # expanduser failed, use the registry to get the 'My Documents' folder. try: import winreg as wreg - with wreg.OpenKey( + key = wreg.OpenKey( wreg.HKEY_CURRENT_USER, r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders" - ) as key: - homedir = wreg.QueryValueEx(key,'Personal')[0] + ) + homedir = wreg.QueryValueEx(key,'Personal')[0] + key.Close() except: pass From 595f4426a61a25c8146f9a022ef2ca66eab28aa4 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 30 Nov 2019 19:24:31 -0800 Subject: [PATCH 080/242] correct patching ? --- IPython/utils/path.py | 7 +++---- IPython/utils/tests/test_path.py | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/IPython/utils/path.py b/IPython/utils/path.py index d52cad0a2e5..f2f1ea06f5a 100644 --- a/IPython/utils/path.py +++ b/IPython/utils/path.py @@ -198,12 +198,11 @@ def get_home_dir(require_writable=False) -> str: # expanduser failed, use the registry to get the 'My Documents' folder. try: import winreg as wreg - key = wreg.OpenKey( + with wreg.OpenKey( wreg.HKEY_CURRENT_USER, r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders" - ) - homedir = wreg.QueryValueEx(key,'Personal')[0] - key.Close() + ) as key: + homedir = wreg.QueryValueEx(key,'Personal')[0] except: pass diff --git a/IPython/utils/tests/test_path.py b/IPython/utils/tests/test_path.py index b34f9ca926c..987177bc101 100644 --- a/IPython/utils/tests/test_path.py +++ b/IPython/utils/tests/test_path.py @@ -171,8 +171,12 @@ def test_get_home_dir_8(): env.pop(key, None) class key: + def __enter__(self): + pass def Close(self): pass + def __exit__(*args, **kwargs): + pass with patch.object(wreg, 'OpenKey', return_value=key()), \ patch.object(wreg, 'QueryValueEx', return_value=[abspath(HOME_TEST_DIR)]): From 0aac4ecaafaf327dc34f2d6790d429c7a23e2cbb Mon Sep 17 00:00:00 2001 From: Chemss Eddine Ben Hassine Date: Sun, 1 Dec 2019 18:35:09 +0100 Subject: [PATCH 081/242] Fix #11984 Removed unused py3compat import --- IPython/core/tests/test_autocall.py | 1 - 1 file changed, 1 deletion(-) diff --git a/IPython/core/tests/test_autocall.py b/IPython/core/tests/test_autocall.py index 10a4e0d477d..ded9f78858a 100644 --- a/IPython/core/tests/test_autocall.py +++ b/IPython/core/tests/test_autocall.py @@ -7,7 +7,6 @@ """ from IPython.core.splitinput import LineInfo from IPython.core.prefilter import AutocallChecker -from IPython.utils import py3compat def doctest_autocall(): """ From ac70fcde4501b916215e67670ab0bbd50dc1ab55 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 1 Dec 2019 09:39:51 -0800 Subject: [PATCH 082/242] update whats new with 7.10.1 releases --- docs/source/whatsnew/version7.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 6c3dbab387a..c4ba18635b2 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,17 @@ 7.x Series ============ +.. _version 7101: + +IPython 7.10.1 +============== + +IPython 7.10.1 fix a couple of incompatibilities with Prompt toolkit 3 (please +update Prompt toolkit to 3.0.2 at least), and fixes some interaction with +headless IPython. + +.. _version 7100: + IPython 7.10.0 ============== From e508b6e4dc46c2b425ec7144da590eb2c3a0cbd7 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 1 Dec 2019 09:44:51 -0800 Subject: [PATCH 083/242] update v7 stats --- docs/source/whatsnew/github-stats-7.rst | 15 +++++++++++++++ tools/release_helper.sh | 3 ++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/source/whatsnew/github-stats-7.rst b/docs/source/whatsnew/github-stats-7.rst index 0d8b28bcc4a..958954ebf5d 100644 --- a/docs/source/whatsnew/github-stats-7.rst +++ b/docs/source/whatsnew/github-stats-7.rst @@ -1,6 +1,21 @@ Issues closed in the 7.x development cycle ========================================== +Issues closed in 7.10.1 +----------------------- + +GitHub stats for 2019/11/27 - 2019/12/01 (tag: 7.10.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 5 issues and merged 7 pull requests. +The full list can be seen `on GitHub `__ + +The following 2 authors contributed 14 commits. + +* Jonathan Slenders +* Matthias Bussonnier + Issues closed in 7.10 --------------------- diff --git a/tools/release_helper.sh b/tools/release_helper.sh index 82a7bc0208e..bed8ed69253 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -67,7 +67,8 @@ then git log --format="%aN <%aE>" $PREV_RELEASE... | sort -u -f echo - echo $BLUE"If you see any duplicates cancel (Ctrl-C), then edit .mailmap.\n"$GREEN"Press enter to continue:"$NOR + echo $BLUE"If you see any duplicates cancel (Ctrl-C), then edit .mailmap." + echo $GREEN"Press enter to continue:"$NOR read echo $BLUE"generating stats"$NOR From b93782e5359bff127a2da1d3cc0019ee8d75099e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 1 Dec 2019 09:49:46 -0800 Subject: [PATCH 084/242] release 7.10.1 --- IPython/core/release.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 5cf2110adf0..a928a6b8825 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 11 -_version_patch = 0 +_version_minor = 10 +_version_patch = 1 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 24145ee1ff606ed7cf470bea13ef2dea77baf604 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 1 Dec 2019 09:54:11 -0800 Subject: [PATCH 085/242] back to dev, and tools updates --- IPython/core/release.py | 6 +++--- tools/release_helper.sh | 22 +++++++++++++++++++--- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index a928a6b8825..5cf2110adf0 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 10 -_version_patch = 1 +_version_minor = 11 +_version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] diff --git a/tools/release_helper.sh b/tools/release_helper.sh index bed8ed69253..7f99a8ac31d 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -30,9 +30,9 @@ MILESTONE=${input:-$MILESTONE} echo -n "VERSION (X.y.z) [$VERSION]:" read input VERSION=${input:-$VERSION} -echo -n "branch (master|X.y) [$branch]:" +echo -n "BRANCH (master|X.y) [$BRANCH]:" read input -branch=${input:-$branch} +BRANCH=${input:-$BRANCH} ask_section(){ echo @@ -117,7 +117,7 @@ git push origin $BRANCH echo echo "Let's tag : git tag -am \"release $VERSION\" \"$VERSION\" -s" -echo $GREEN"Press enter to wtagcommit"$NOR +echo $GREEN"Press enter to tag commit"$NOR read git tag -am "release $VERSION" "$VERSION" -s @@ -127,6 +127,22 @@ echo $GREEN"Press enter to continue"$NOR read git push origin $VERSION + +echo $GREEN"please update version number and back to .dev in ${RED}IPython/core/release.py" +echo ${BLUE}"Do not commit yet – we'll do it later."$NOR + +echo $GREEN"Press enter to continue"$NOR +read + +echo +echo "Let's commit : git commit -am \"back to dev\" -S" +echo $GREEN"Press enter to commit"$NOR +read +git commit -am "back to dev" -S + + + + echo echo $BLUE"let's : git checkout $VERSION"$NOR echo $GREEN"Press enter to continue"$NOR From efc78ade7060282fd6cecd846091f28c6a7fe1eb Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 1 Dec 2019 19:04:13 +0000 Subject: [PATCH 086/242] Add chained exception to 'Plain' mode --- IPython/core/ultratb.py | 85 +++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 33 deletions(-) diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index 0864ba456ef..c05571827a3 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -530,6 +530,30 @@ def _set_ostream(self, val): ostream = property(_get_ostream, _set_ostream) + def get_parts_of_chained_exception(self, evalue): + def get_chained_exception(exception_value): + cause = getattr(exception_value, '__cause__', None) + if cause: + return cause + if getattr(exception_value, '__suppress_context__', False): + return None + return getattr(exception_value, '__context__', None) + + chained_evalue = get_chained_exception(evalue) + + if chained_evalue: + return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__ + + def prepare_chained_exception_message(self, cause): + direct_cause = "\nThe above exception was the direct cause of the following exception:\n" + exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n" + + if cause: + message = [[direct_cause]] + else: + message = [[exception_during_handling]] + return message + def set_colors(self, *args, **kw): """Shorthand access to the color table scheme selector method.""" @@ -603,7 +627,7 @@ def __call__(self, etype, value, elist): self.ostream.write(self.text(etype, value, elist)) self.ostream.write('\n') - def structured_traceback(self, etype, value, elist, tb_offset=None, + def structured_traceback(self, etype, evalue, etb=None, tb_offset=None, context=5): """Return a color formatted string with the traceback info. @@ -612,15 +636,15 @@ def structured_traceback(self, etype, value, elist, tb_offset=None, etype : exception type Type of the exception raised. - value : object + evalue : object Data stored in the exception - elist : list - List of frames, see class docstring for details. + etb : traceback + Traceback of the exception. tb_offset : int, optional Number of frames in the traceback to skip. If not given, the - instance value is used (set in constructor). + instance evalue is used (set in constructor). context : int, optional Number of lines of context information to print. @@ -629,6 +653,14 @@ def structured_traceback(self, etype, value, elist, tb_offset=None, ------- String with formatted exception. """ + # if chained_exc_ids is None: + chained_exc_ids = set() + if isinstance(etb, list): + elist = etb + elif etb is not None: + elist = self._extract_tb(etb) + else: + elist = [] tb_offset = self.tb_offset if tb_offset is None else tb_offset Colors = self.Colors out_list = [] @@ -641,9 +673,21 @@ def structured_traceback(self, etype, value, elist, tb_offset=None, (Colors.normalEm, Colors.Normal) + '\n') out_list.extend(self._format_list(elist)) # The exception info should be a single entry in the list. - lines = ''.join(self._format_exception_only(etype, value)) + lines = ''.join(self._format_exception_only(etype, evalue)) out_list.append(lines) + exception = self.get_parts_of_chained_exception(evalue) + + if exception and not id(exception[1]) in chained_exc_ids: + etype, evalue, etb = exception + chained_exc_ids.add(id(exception[1])) # trace exception to avoid infinite 'cause' loop + chained_exceptions_tb_offset = 0 + out_list = (self.structured_traceback( + etype, evalue, etb, chained_exceptions_tb_offset, context) + + self.prepare_chained_exception_message( + evalue.__cause__)[0] + + out_list) + return out_list def _format_list(self, extracted_list): @@ -763,7 +807,7 @@ def get_exception_only(self, etype, value): etype : exception type value : exception value """ - return ListTB.structured_traceback(self, etype, value, []) + return ListTB.structured_traceback(self, etype, value) def show_exception_only(self, etype, evalue): """Only print the exception type and message, without a traceback. @@ -1013,16 +1057,6 @@ def linereader(file=file, lnum=[lnum], getline=linecache.getline): _format_traceback_lines(lnum, index, lines, Colors, lvals, _line_format))) - def prepare_chained_exception_message(self, cause): - direct_cause = "\nThe above exception was the direct cause of the following exception:\n" - exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n" - - if cause: - message = [[direct_cause]] - else: - message = [[exception_during_handling]] - return message - def prepare_header(self, etype, long_version=False): colors = self.Colors # just a shorthand + quicker name lookup colorsnormal = colors.Normal # used a lot @@ -1117,20 +1151,6 @@ def get_records(self, etb, number_of_lines_of_context, tb_offset): info('\nUnfortunately, your original traceback can not be constructed.\n') return None - def get_parts_of_chained_exception(self, evalue): - def get_chained_exception(exception_value): - cause = getattr(exception_value, '__cause__', None) - if cause: - return cause - if getattr(exception_value, '__suppress_context__', False): - return None - return getattr(exception_value, '__context__', None) - - chained_evalue = get_chained_exception(evalue) - - if chained_evalue: - return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__ - def structured_traceback(self, etype, evalue, etb, tb_offset=None, number_of_lines_of_context=5): """Return a nice text document describing the traceback.""" @@ -1294,9 +1314,8 @@ def structured_traceback(self, etype, value, tb, tb_offset=None, number_of_lines # out-of-date source code. self.check_cache() # Now we can extract and format the exception - elist = self._extract_tb(tb) return ListTB.structured_traceback( - self, etype, value, elist, tb_offset, number_of_lines_of_context + self, etype, value, tb, tb_offset, number_of_lines_of_context ) def stb2text(self, stb): From 695206f27d652531276b73c43c581c1eea71947a Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 1 Dec 2019 19:15:51 +0000 Subject: [PATCH 087/242] Add test --- IPython/core/tests/test_ultratb.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/IPython/core/tests/test_ultratb.py b/IPython/core/tests/test_ultratb.py index e77f44416c3..9190f5eb7ee 100644 --- a/IPython/core/tests/test_ultratb.py +++ b/IPython/core/tests/test_ultratb.py @@ -296,6 +296,11 @@ def test_suppress_exception_chaining(self): tt.AssertPrints("ValueError", suppress=False): ip.run_cell(self.SUPPRESS_CHAINING_CODE) + def test_plain_exception(self): + with tt.AssertPrints(["KeyError", "NameError", "During handling"]): + ip.run_cell("%xmode Plain") + ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE) + class RecursionTest(unittest.TestCase): DEFINITIONS = """ From c273724b5b98650388e4de7e17f488dec886e585 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 1 Dec 2019 21:27:59 +0000 Subject: [PATCH 088/242] Avoid recursive loop --- IPython/core/ultratb.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index c05571827a3..a56b738cae4 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -653,8 +653,10 @@ def structured_traceback(self, etype, evalue, etb=None, tb_offset=None, ------- String with formatted exception. """ - # if chained_exc_ids is None: - chained_exc_ids = set() + if isinstance(etb, tuple): + etb, chained_exc_ids = etb + else: + chained_exc_ids = set() if isinstance(etb, list): elist = etb elif etb is not None: @@ -683,7 +685,8 @@ def structured_traceback(self, etype, evalue, etb=None, tb_offset=None, chained_exc_ids.add(id(exception[1])) # trace exception to avoid infinite 'cause' loop chained_exceptions_tb_offset = 0 out_list = (self.structured_traceback( - etype, evalue, etb, chained_exceptions_tb_offset, context) + etype, evalue, (etb, chained_exc_ids), + chained_exceptions_tb_offset, context) + self.prepare_chained_exception_message( evalue.__cause__)[0] + out_list) From 42d108534cbea8a150e41cbe4de7ff9a12a28191 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 1 Dec 2019 21:44:11 +0000 Subject: [PATCH 089/242] Reset xmode to Verbose --- IPython/core/tests/test_ultratb.py | 1 + 1 file changed, 1 insertion(+) diff --git a/IPython/core/tests/test_ultratb.py b/IPython/core/tests/test_ultratb.py index 9190f5eb7ee..a8763df368f 100644 --- a/IPython/core/tests/test_ultratb.py +++ b/IPython/core/tests/test_ultratb.py @@ -300,6 +300,7 @@ def test_plain_exception(self): with tt.AssertPrints(["KeyError", "NameError", "During handling"]): ip.run_cell("%xmode Plain") ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE) + ip.run_cell("%xmode Verbose") class RecursionTest(unittest.TestCase): From 2705621ca111194e778372390defdc478c58a21d Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 2 Dec 2019 08:51:03 +0000 Subject: [PATCH 090/242] Add more tests --- IPython/core/tests/test_ultratb.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/IPython/core/tests/test_ultratb.py b/IPython/core/tests/test_ultratb.py index a8763df368f..c00b4d30afe 100644 --- a/IPython/core/tests/test_ultratb.py +++ b/IPython/core/tests/test_ultratb.py @@ -296,12 +296,25 @@ def test_suppress_exception_chaining(self): tt.AssertPrints("ValueError", suppress=False): ip.run_cell(self.SUPPRESS_CHAINING_CODE) - def test_plain_exception(self): + def test_plain_direct_cause_error(self): + with tt.AssertPrints(["KeyError", "NameError", "direct cause"]): + ip.run_cell("%xmode Plain") + ip.run_cell(self.DIRECT_CAUSE_ERROR_CODE) + ip.run_cell("%xmode Verbose") + + def test_plain_exception_during_handling_error(self): with tt.AssertPrints(["KeyError", "NameError", "During handling"]): ip.run_cell("%xmode Plain") ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE) ip.run_cell("%xmode Verbose") + def test_plain_suppress_exception_chaining(self): + with tt.AssertNotPrints("ZeroDivisionError"), \ + tt.AssertPrints("ValueError", suppress=False): + ip.run_cell("%xmode Plain") + ip.run_cell(self.SUPPRESS_CHAINING_CODE) + ip.run_cell("%xmode Verbose") + class RecursionTest(unittest.TestCase): DEFINITIONS = """ From 7b234536d859baa939ee3a19004a2d96f86ce6cb Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 2 Dec 2019 09:00:00 +0000 Subject: [PATCH 091/242] fix 'direct cause' message --- IPython/core/ultratb.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index a56b738cae4..3612bc4afde 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -681,15 +681,18 @@ def structured_traceback(self, etype, evalue, etb=None, tb_offset=None, exception = self.get_parts_of_chained_exception(evalue) if exception and not id(exception[1]) in chained_exc_ids: + chained_exception_message = self.prepare_chained_exception_message( + evalue.__cause__)[0] etype, evalue, etb = exception - chained_exc_ids.add(id(exception[1])) # trace exception to avoid infinite 'cause' loop + # Trace exception to avoid infinite 'cause' loop + chained_exc_ids.add(id(exception[1])) chained_exceptions_tb_offset = 0 - out_list = (self.structured_traceback( - etype, evalue, (etb, chained_exc_ids), - chained_exceptions_tb_offset, context) - + self.prepare_chained_exception_message( - evalue.__cause__)[0] - + out_list) + out_list = ( + self.structured_traceback( + etype, evalue, (etb, chained_exc_ids), + chained_exceptions_tb_offset, context) + + chained_exception_message + + out_list) return out_list From 2d3a743daaeef7b18e18ae803951f32ddd6db20c Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 2 Dec 2019 09:55:18 +0000 Subject: [PATCH 092/242] Update docstring --- IPython/core/ultratb.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index 3612bc4afde..eef16cc650a 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -639,8 +639,9 @@ def structured_traceback(self, etype, evalue, etb=None, tb_offset=None, evalue : object Data stored in the exception - etb : traceback - Traceback of the exception. + etb : object + If list: List of frames, see class docstring for details. + If Traceback: Traceback of the exception. tb_offset : int, optional Number of frames in the traceback to skip. If not given, the @@ -653,10 +654,13 @@ def structured_traceback(self, etype, evalue, etb=None, tb_offset=None, ------- String with formatted exception. """ + # This is a workaround to get chained_exc_ids in recursive calls + # etb should not be a tuple if structured_traceback is not recursive if isinstance(etb, tuple): etb, chained_exc_ids = etb else: chained_exc_ids = set() + if isinstance(etb, list): elist = etb elif etb is not None: From 74439e405d372fb90eaa6410eadc9327620f3c9f Mon Sep 17 00:00:00 2001 From: Jonathan Slenders Date: Mon, 2 Dec 2019 18:31:46 +0100 Subject: [PATCH 093/242] Don't nest asyncio input hooks. This fixes the "event loop already running" error. For prompt_toolkit 2.0: this will still nest the event loops, but that's fine, because we have the asyncio event loop running an prompt_toolkit's custom loop. They don't interfere with each other. For prompt_toolkit 3.0: we only use one asyncio event loop when "%gui asyncio" is used. The terminal prompt will now use the active asyncio event loop. --- IPython/terminal/interactiveshell.py | 17 ++++++++++++++--- IPython/terminal/pt_inputhooks/asyncio.py | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 26188aeba8a..aff48d36c5e 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -580,11 +580,22 @@ def enable_gui(self, gui=None): # For prompt_toolkit 3.0. We have to create an asyncio event loop with # this inputhook. if PTK3: - if self._inputhook: - from prompt_toolkit.eventloop import new_eventloop_with_inputhook + import asyncio + from prompt_toolkit.eventloop import new_eventloop_with_inputhook + + if gui == 'asyncio': + # When we integrate the asyncio event loop, run the UI in the + # same event loop as the rest of the code. don't use an actual + # input hook. (Asyncio is not made for nesting event loops.) + self.pt_loop = asyncio.get_event_loop() + + elif self._inputhook: + # If an inputhook was set, create a new asyncio event loop with + # this inputhook for the prompt. self.pt_loop = new_eventloop_with_inputhook(self._inputhook) else: - import asyncio + # When there's no inputhook, run the prompt in a separate + # asyncio event loop. self.pt_loop = asyncio.new_event_loop() # Run !system commands directly, not through pipes, so terminal programs diff --git a/IPython/terminal/pt_inputhooks/asyncio.py b/IPython/terminal/pt_inputhooks/asyncio.py index e28603864f9..95cf194f866 100644 --- a/IPython/terminal/pt_inputhooks/asyncio.py +++ b/IPython/terminal/pt_inputhooks/asyncio.py @@ -28,6 +28,10 @@ """ import asyncio +from prompt_toolkit import __version__ as ptk_version + +PTK3 = ptk_version.startswith('3.') + # Keep reference to the original asyncio loop, because getting the event loop # within the input hook would return the other loop. @@ -35,6 +39,19 @@ def inputhook(context): + """ + Inputhook for asyncio event loop integration. + """ + # For prompt_toolkit 3.0, this input hook literally doesn't do anything. + # The event loop integration here is implemented in `interactiveshell.py` + # by running the prompt itself in the current asyncio loop. The main reason + # for this is that nesting asyncio event loops is unreliable. + if PTK3: + return + + # For prompt_toolkit 2.0, we can run the current asyncio event loop, + # because prompt_toolkit 2.0 uses a different event loop internally. + def stop(): loop.stop() @@ -44,3 +61,4 @@ def stop(): loop.run_forever() finally: loop.remove_reader(fileno) + From 2c41244b453a729067b3a85f99b03261f0a3dd76 Mon Sep 17 00:00:00 2001 From: Jonathan Slenders Date: Mon, 2 Dec 2019 21:07:20 +0100 Subject: [PATCH 094/242] Handle situation where the user used 'asyncio.run' and clears the event loop. --- IPython/terminal/interactiveshell.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index aff48d36c5e..9f7d335ead8 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -449,22 +449,28 @@ def prompt_for_code(self): else: default = '' - with patch_stdout(raw=True): - # In order to make sure that asyncio code written in the - # interactive shell doesn't interfere with the prompt, we run the - # prompt in a different event loop. - # If we don't do this, people could spawn coroutine with a - # while/true inside which will freeze the prompt. + # In order to make sure that asyncio code written in the + # interactive shell doesn't interfere with the prompt, we run the + # prompt in a different event loop. + # If we don't do this, people could spawn coroutine with a + # while/true inside which will freeze the prompt. + try: old_loop = asyncio.get_event_loop() - asyncio.set_event_loop(self.pt_loop) - try: + except RuntimeError: + # This happens when the user used `asyncio.run()`. + old_loop = None + + asyncio.set_event_loop(self.pt_loop) + try: + with patch_stdout(raw=True): text = self.pt_app.prompt( default=default, **self._extra_prompt_options()) - finally: - # Restore the original event loop. - asyncio.set_event_loop(old_loop) + finally: + # Restore the original event loop. + asyncio.set_event_loop(old_loop) + return text def enable_win_unicode_console(self): From 1507a55d532894b51d724be83bcac218c7e8270d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 1 Dec 2019 10:17:52 -0800 Subject: [PATCH 095/242] some sections --- tools/release_helper.sh | 119 ++++++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 53 deletions(-) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index a50bf0edffe..46e00d47789 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -89,68 +89,81 @@ echo $GREEN"please update version number in ${RED}IPython/core/release.py${NOR} echo $GREEN"Press enter to continue"$NOR read -echo -echo "Attempting to build the docs.." -make html -C docs +if ask_section "Build the documentation ?" +then + make html -C docs + echo + echo $GREEN"Check the docs, press enter to continue"$NOR + read -echo -echo $GREEN"Check the docs, press enter to continue"$NOR -read +fi echo echo $BLUE"Attempting to build package..."$NOR tools/build_release +rm dist/* -echo -echo "Let's commit : git commit -am \"release $VERSION\" -S" -echo $GREEN"Press enter to commit"$NOR -read -git commit -am "release $VERSION" -S - -echo -echo $BLUE"git push origin \$BRANCH ($BRANCH)?"$NOR -echo $GREEN"Make sure you can push"$NOR -echo $GREEN"Press enter to continue"$NOR -read -git push origin $BRANCH - -echo -echo "Let's tag : git tag -am \"release $VERSION\" \"$VERSION\" -s" -echo $GREEN"Press enter to tag commit"$NOR -read -git tag -am "release $VERSION" "$VERSION" -s - -echo -echo $BLUE"And push the tag: git push origin \$VERSION ?"$NOR -echo $GREEN"Press enter to continue"$NOR -read -git push origin $VERSION - - -echo $GREEN"please update version number and back to .dev in ${RED}IPython/core/release.py" -echo ${BLUE}"Do not commit yet – we'll do it later."$NOR - -echo $GREEN"Press enter to continue"$NOR -read - -echo -echo "Let's commit : git commit -am \"back to dev\" -S" -echo $GREEN"Press enter to commit"$NOR -read -git commit -am "back to dev" -S - - +if ask_section "Shoudl we commit, tag, push... etc ? " +then + echo + echo "Let's commit : git commit -am \"release $VERSION\" -S" + echo $GREEN"Press enter to commit"$NOR + read + git commit -am "release $VERSION" -S + + echo + echo $BLUE"git push origin \$BRANCH ($BRANCH)?"$NOR + echo $GREEN"Make sure you can push"$NOR + echo $GREEN"Press enter to continue"$NOR + read + git push origin $BRANCH + + echo + echo "Let's tag : git tag -am \"release $VERSION\" \"$VERSION\" -s" + echo $GREEN"Press enter to tag commit"$NOR + read + git tag -am "release $VERSION" "$VERSION" -s + + echo + echo $BLUE"And push the tag: git push origin \$VERSION ?"$NOR + echo $GREEN"Press enter to continue"$NOR + read + git push origin $VERSION + + + echo $GREEN"please update version number and back to .dev in ${RED}IPython/core/release.py" + echo ${BLUE}"Do not commit yet – we'll do it later."$NOR + + echo $GREEN"Press enter to continue"$NOR + read + + echo + echo "Let's commit : git commit -am \"back to dev\" -S" + echo $GREEN"Press enter to commit"$NOR + read + git commit -am "back to dev" -S + + echo + echo $BLUE"let's : git checkout $VERSION"$NOR + echo $GREEN"Press enter to continue"$NOR + read + git checkout $VERSION +fi +if ask_section "Should we build and release ?" +then -echo -echo $BLUE"let's : git checkout $VERSION"$NOR -echo $GREEN"Press enter to continue"$NOR -read -git checkout $VERSION + echo + echo $BLUE"Attempting to build package..."$NOR -# ./tools/release -# ls ./dist -# shasum -a 256 dist/* + tools/build_release + echo '$ shasum -a 256 dist/*' + shasum -a 256 dist/* + if ask_section "upload packages ?" + then + tools/build_release upload + fi +fi From 4286ee62532ee84eb8bcbdb1400fd25610867e13 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 1 Dec 2019 10:20:19 -0800 Subject: [PATCH 096/242] a few typos --- tools/release_helper.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index 46e00d47789..c82285ecd44 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -1,5 +1,5 @@ # Simple tool to help for release -# when releasing with bash, simplei source it to get asked questions. +# when releasing with bash, simple source it to get asked questions. # misc check before starting @@ -104,7 +104,7 @@ echo $BLUE"Attempting to build package..."$NOR tools/build_release rm dist/* -if ask_section "Shoudl we commit, tag, push... etc ? " +if ask_section "Should we commit, tag, push... etc ? " then echo echo "Let's commit : git commit -am \"release $VERSION\" -S" From 565504a7f2a9c4bfbea2b6441f8f222a2a325f6e Mon Sep 17 00:00:00 2001 From: Joseph Kahn Date: Wed, 4 Dec 2019 11:22:45 -0500 Subject: [PATCH 097/242] revert inputhook param location --- IPython/terminal/interactiveshell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 26188aeba8a..e9ba812b310 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -438,7 +438,7 @@ def get_message(): Condition(lambda: self.highlight_matching_brackets))], } if not PTK3: - options['inputhook'] = self.inputhook + options['inputhook'] = self.shell.inputhook return options From bcc84fb01a46f660c08f004842cbd7be3b6c8441 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 4 Dec 2019 11:35:04 -0800 Subject: [PATCH 098/242] Fix the right inputhook Interactive shell and debugger have similar code. The fix was applied to the wrong one, not sure how test were passing. --- IPython/terminal/debugger.py | 2 +- IPython/terminal/interactiveshell.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/terminal/debugger.py b/IPython/terminal/debugger.py index ba5ce1bada8..715f7c19926 100644 --- a/IPython/terminal/debugger.py +++ b/IPython/terminal/debugger.py @@ -66,7 +66,7 @@ def get_prompt_tokens(): ) if not PTK3: - options['inputhook'] = self.inputhook + options['inputhook'] = self.shell.inputhook self.pt_app = PromptSession(**options) def cmdloop(self, intro=None): diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 312b691ace7..9f7d335ead8 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -438,7 +438,7 @@ def get_message(): Condition(lambda: self.highlight_matching_brackets))], } if not PTK3: - options['inputhook'] = self.shell.inputhook + options['inputhook'] = self.inputhook return options From a089b78315ef964f8552ffa52db569a8a924bfc2 Mon Sep 17 00:00:00 2001 From: kousik Date: Thu, 5 Dec 2019 23:08:14 +0530 Subject: [PATCH 099/242] Code Cleanup: Remove unused unused utility functions from 'IPython/utils/py3compat.py' #11985 --- IPython/utils/py3compat.py | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/IPython/utils/py3compat.py b/IPython/utils/py3compat.py index e5716650392..9b58c6feeec 100644 --- a/IPython/utils/py3compat.py +++ b/IPython/utils/py3compat.py @@ -13,8 +13,6 @@ from .encoding import DEFAULT_ENCODING -def no_code(x, encoding=None): - return x def decode(s, encoding=None): encoding = encoding or DEFAULT_ENCODING @@ -44,7 +42,7 @@ def buffer_to_bytes(buf): def _modify_str_or_docstring(str_change_func): @functools.wraps(str_change_func) def wrapper(func_or_str): - if isinstance(func_or_str, string_types): + if isinstance(func_or_str, (str,)): func = None doc = func_or_str else: @@ -66,17 +64,12 @@ def safe_unicode(e): safe to call unicode() on. """ try: - return unicode_type(e) + return str(e) except UnicodeError: pass try: - return str_to_unicode(str(e)) - except UnicodeError: - pass - - try: - return str_to_unicode(repr(e)) + return repr(e) except UnicodeError: pass @@ -156,16 +149,6 @@ def input(prompt=''): builtin_mod_name = "builtins" import builtins as builtin_mod -str_to_unicode = no_code -unicode_to_str = no_code -str_to_bytes = encode -bytes_to_str = decode -cast_bytes_py2 = no_code -cast_unicode_py2 = no_code -buffer_to_bytes_py2 = no_code - -string_types = (str,) -unicode_type = str which = shutil.which From f984edc39a6e247c9deecafb8752464aa8c7235e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 5 Dec 2019 15:54:21 -0800 Subject: [PATCH 100/242] Remove some of the cast_unicode as we can prove they are not bytes. 1) inspect getdoc impementation return None if docs is not str. 2) if a value is split, joined, or concatenated with str, it must be str, or we woudl get a type error. --- IPython/core/oinspect.py | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index 0ad709405de..c376c22d3ae 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -23,6 +23,7 @@ import types import io as stdlib_io from itertools import zip_longest +from typing import Union # IPython's own from IPython.core import page @@ -110,7 +111,7 @@ def get_encoding(obj): encoding, lines = openpy.detect_encoding(buffer.readline) return encoding -def getdoc(obj): +def getdoc(obj) -> Union[str,None]: """Stable wrapper around inspect.getdoc. This can't crash because of attribute problems. @@ -128,11 +129,10 @@ def getdoc(obj): if isinstance(ds, str): return inspect.cleandoc(ds) docstr = inspect.getdoc(obj) - encoding = get_encoding(obj) - return py3compat.cast_unicode(docstr, encoding=encoding) + return docstr -def getsource(obj, oname=''): +def getsource(obj, oname='') -> Union[str,None]: """Wrapper around inspect.getsource. This can be modified by other projects to provide customized source @@ -158,18 +158,15 @@ def getsource(obj, oname=''): if fn is not None: encoding = get_encoding(fn) oname_prefix = ('%s.' % oname) if oname else '' - sources.append(cast_unicode( - ''.join(('# ', oname_prefix, attrname)), - encoding=encoding)) + sources.append(''.join(('# ', oname_prefix, attrname))) if inspect.isfunction(fn): sources.append(dedent(getsource(fn))) else: # Default str/repr only prints function name, # pretty.pretty prints module name too. - sources.append(cast_unicode( - '%s%s = %s\n' % ( - oname_prefix, attrname, pretty(fn)), - encoding=encoding)) + sources.append( + '%s%s = %s\n' % (oname_prefix, attrname, pretty(fn)) + ) if sources: return '\n'.join(sources) else: @@ -191,8 +188,7 @@ def getsource(obj, oname=''): except TypeError: return None - encoding = get_encoding(obj) - return cast_unicode(src, encoding=encoding) + return src def is_simple_callable(obj): @@ -289,7 +285,7 @@ def _get_wrapped(obj): return orig_obj return obj -def find_file(obj): +def find_file(obj) -> str: """Find the absolute path to the file where an object was defined. This is essentially a robust wrapper around `inspect.getabsfile`. @@ -370,18 +366,17 @@ def __init__(self, color_table=InspectColors, self.str_detail_level = str_detail_level self.set_active_scheme(scheme) - def _getdef(self,obj,oname=''): + def _getdef(self,obj,oname='') -> Union[str,None]: """Return the call signature for any callable object. If any exception is generated, None is returned instead and the exception is suppressed.""" try: - hdef = _render_signature(signature(obj), oname) - return cast_unicode(hdef) + return _render_signature(signature(obj), oname) except: return None - def __head(self,h): + def __head(self,h) -> str: """Return a header string with proper colors.""" return '%s%s%s' % (self.color_table.active_colors.header,h, self.color_table.active_colors.normal) @@ -517,7 +512,7 @@ def pfile(self, obj, oname=''): # 0-offset, so we must adjust. page.page(self.format(openpy.read_py_file(ofile, skip_encoding_cookie=False)), lineno - 1) - def _format_fields(self, fields, title_width=0): + def _format_fields(self, fields, title_width=0) -> str: """Formats a list of fields for display. Parameters @@ -536,7 +531,7 @@ def _format_fields(self, fields, title_width=0): title = header(title + ':') + '\n' else: title = header((title + ':').ljust(title_width)) - out.append(cast_unicode(title) + cast_unicode(content)) + out.append(title + cast_unicode(content)) return "\n".join(out) def _mime_format(self, text, formatter=None): @@ -1015,7 +1010,7 @@ def psearch(self,pattern,ns_table,ns_search=[], page.page('\n'.join(sorted(search_result))) -def _render_signature(obj_signature, obj_name): +def _render_signature(obj_signature, obj_name) -> str: """ This was mostly taken from inspect.Signature.__str__. Look there for the comments. From 537d3ccf4c280b39a029956e1d36c5c1e2da880c Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 5 Dec 2019 15:57:41 -0800 Subject: [PATCH 101/242] _format_fields is unused, removing. --- IPython/core/oinspect.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index c376c22d3ae..27513e1fe3c 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -512,28 +512,6 @@ def pfile(self, obj, oname=''): # 0-offset, so we must adjust. page.page(self.format(openpy.read_py_file(ofile, skip_encoding_cookie=False)), lineno - 1) - def _format_fields(self, fields, title_width=0) -> str: - """Formats a list of fields for display. - - Parameters - ---------- - fields : list - A list of 2-tuples: (field_title, field_content) - title_width : int - How many characters to pad titles to. Default to longest title. - """ - out = [] - header = self.__head - if title_width == 0: - title_width = max(len(title) + 2 for title, _ in fields) - for title, content in fields: - if len(content.splitlines()) > 1: - title = header(title + ':') + '\n' - else: - title = header((title + ':').ljust(title_width)) - out.append(title + cast_unicode(content)) - return "\n".join(out) - def _mime_format(self, text, formatter=None): """Return a mime bundle representation of the input text. From 5a8935c7ba75e3dedd18db7008609cb39b690f25 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 5 Dec 2019 14:40:58 -0800 Subject: [PATCH 102/242] Cleanup unused imports. --- IPython/core/completer.py | 2 +- IPython/core/displaypub.py | 2 +- IPython/core/interactiveshell.py | 2 +- IPython/core/magics/basic.py | 1 - IPython/core/magics/packaging.py | 3 +-- IPython/core/tests/refbug.py | 1 - IPython/core/tests/test_async_helpers.py | 1 - IPython/core/tests/test_display.py | 2 +- IPython/core/tests/test_events.py | 1 - IPython/core/tests/test_handlers.py | 1 - IPython/extensions/tests/test_autoreload.py | 3 --- IPython/utils/daemonize.py | 2 +- IPython/utils/frame.py | 1 - IPython/utils/tests/test_io.py | 5 ----- IPython/utils/tests/test_path.py | 4 ++-- IPython/utils/tests/test_process.py | 1 - 16 files changed, 8 insertions(+), 24 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 4906ae9d951..0942798f3b3 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -126,7 +126,7 @@ from contextlib import contextmanager from importlib import import_module -from typing import Iterator, List, Tuple, Iterable, Union +from typing import Iterator, List, Tuple, Iterable from types import SimpleNamespace from traitlets.config.configurable import Configurable diff --git a/IPython/core/displaypub.py b/IPython/core/displaypub.py index f651a2a0cf6..1da0458cf08 100644 --- a/IPython/core/displaypub.py +++ b/IPython/core/displaypub.py @@ -19,7 +19,7 @@ import sys from traitlets.config.configurable import Configurable -from traitlets import List, Dict +from traitlets import List # This used to be defined here - it is imported for backwards compatibility from .display import publish_display_data diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index dc8bd236d3b..d8bb3e59ec9 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -13,7 +13,6 @@ import abc import ast -import asyncio import atexit import builtins as builtin_mod import functools @@ -87,6 +86,7 @@ # NoOpContext is deprecated, but ipykernel imports it from here. # See https://github.com/ipython/ipykernel/issues/157 +# (2016, let's try to remove than in IPython 8.0) from IPython.utils.contexts import NoOpContext try: diff --git a/IPython/core/magics/basic.py b/IPython/core/magics/basic.py index 906d3d5ebdf..5c8649b5408 100644 --- a/IPython/core/magics/basic.py +++ b/IPython/core/magics/basic.py @@ -5,7 +5,6 @@ from logging import error import io from pprint import pformat -import textwrap import sys from warnings import warn diff --git a/IPython/core/magics/packaging.py b/IPython/core/magics/packaging.py index 6477c7defc7..cfee7865f5d 100644 --- a/IPython/core/magics/packaging.py +++ b/IPython/core/magics/packaging.py @@ -12,7 +12,6 @@ import re import shlex import sys -from subprocess import Popen, PIPE from IPython.core.magic import Magics, magics_class, line_magic @@ -101,4 +100,4 @@ def conda(self, line): extra_args.extend(["--prefix", sys.prefix]) self.shell.system(' '.join([conda, command] + extra_args + args)) - print("\nNote: you may need to restart the kernel to use updated packages.") \ No newline at end of file + print("\nNote: you may need to restart the kernel to use updated packages.") diff --git a/IPython/core/tests/refbug.py b/IPython/core/tests/refbug.py index b8de4c81078..92e2eead347 100644 --- a/IPython/core/tests/refbug.py +++ b/IPython/core/tests/refbug.py @@ -16,7 +16,6 @@ #----------------------------------------------------------------------------- # Module imports #----------------------------------------------------------------------------- -import sys from IPython import get_ipython diff --git a/IPython/core/tests/test_async_helpers.py b/IPython/core/tests/test_async_helpers.py index 2432d8b5e4e..2932189c63a 100644 --- a/IPython/core/tests/test_async_helpers.py +++ b/IPython/core/tests/test_async_helpers.py @@ -3,7 +3,6 @@ Should only trigger on python 3.5+ or will have syntax errors. """ -import sys from itertools import chain, repeat import nose.tools as nt from textwrap import dedent, indent diff --git a/IPython/core/tests/test_display.py b/IPython/core/tests/test_display.py index 1fed51127a1..0ce4ad2b429 100644 --- a/IPython/core/tests/test_display.py +++ b/IPython/core/tests/test_display.py @@ -14,7 +14,7 @@ from IPython.utils.io import capture_output from IPython.utils.tempdir import NamedFileInTemporaryDirectory from IPython import paths as ipath -from IPython.testing.tools import AssertPrints, AssertNotPrints +from IPython.testing.tools import AssertNotPrints import IPython.testing.decorators as dec diff --git a/IPython/core/tests/test_events.py b/IPython/core/tests/test_events.py index db79b4ffc8e..a4211ecea4c 100644 --- a/IPython/core/tests/test_events.py +++ b/IPython/core/tests/test_events.py @@ -1,4 +1,3 @@ -from backcall import callback_prototype import unittest from unittest.mock import Mock import nose.tools as nt diff --git a/IPython/core/tests/test_handlers.py b/IPython/core/tests/test_handlers.py index 02860c8b4af..19248177295 100644 --- a/IPython/core/tests/test_handlers.py +++ b/IPython/core/tests/test_handlers.py @@ -10,7 +10,6 @@ # our own packages from IPython.core import autocall from IPython.testing import tools as tt -from IPython.utils import py3compat #----------------------------------------------------------------------------- # Globals diff --git a/IPython/extensions/tests/test_autoreload.py b/IPython/extensions/tests/test_autoreload.py index fe62db0c353..e81bf221515 100644 --- a/IPython/extensions/tests/test_autoreload.py +++ b/IPython/extensions/tests/test_autoreload.py @@ -26,8 +26,6 @@ from unittest import TestCase -from IPython.testing.decorators import skipif - from IPython.extensions.autoreload import AutoreloadMagics from IPython.core.events import EventManager, pre_run_cell @@ -141,7 +139,6 @@ def pickle_get_current_class(obj): class TestAutoreload(Fixture): def test_reload_enums(self): - import enum mod_name, mod_fn = self.new_module(textwrap.dedent(""" from enum import Enum class MyEnum(Enum): diff --git a/IPython/utils/daemonize.py b/IPython/utils/daemonize.py index 0de3852c78c..44b4a2832e0 100644 --- a/IPython/utils/daemonize.py +++ b/IPython/utils/daemonize.py @@ -1,4 +1,4 @@ from warnings import warn -warn("IPython.utils.daemonize has moved to ipyparallel.apps.daemonize", stacklevel=2) +warn("IPython.utils.daemonize has moved to ipyparallel.apps.daemonize since IPython 4.0", DeprecationWarning, stacklevel=2) from ipyparallel.apps.daemonize import daemonize diff --git a/IPython/utils/frame.py b/IPython/utils/frame.py index 11dab31f8d0..74c6d4197f4 100644 --- a/IPython/utils/frame.py +++ b/IPython/utils/frame.py @@ -15,7 +15,6 @@ #----------------------------------------------------------------------------- import sys -from IPython.utils import py3compat #----------------------------------------------------------------------------- # Code diff --git a/IPython/utils/tests/test_io.py b/IPython/utils/tests/test_io.py index e6af8c25a43..83367fa9e9f 100644 --- a/IPython/utils/tests/test_io.py +++ b/IPython/utils/tests/test_io.py @@ -5,9 +5,6 @@ # Distributed under the terms of the Modified BSD License. -import io as stdlib_io -import os.path -import stat import sys from io import StringIO @@ -16,9 +13,7 @@ import nose.tools as nt -from IPython.testing.decorators import skipif, skip_win32 from IPython.utils.io import IOStream, Tee, capture_output -from IPython.utils.tempdir import TemporaryDirectory def test_tee_simple(): diff --git a/IPython/utils/tests/test_path.py b/IPython/utils/tests/test_path.py index 987177bc101..74f21c3b50a 100644 --- a/IPython/utils/tests/test_path.py +++ b/IPython/utils/tests/test_path.py @@ -21,9 +21,9 @@ from IPython import paths from IPython.testing import decorators as dec from IPython.testing.decorators import (skip_if_not_win32, skip_win32, - onlyif_unicode_paths, skipif, + onlyif_unicode_paths, skip_win32_py38,) -from IPython.testing.tools import make_tempfile, AssertPrints +from IPython.testing.tools import make_tempfile from IPython.utils import path from IPython.utils.tempdir import TemporaryDirectory diff --git a/IPython/utils/tests/test_process.py b/IPython/utils/tests/test_process.py index 956b2abbae2..0c81138cd19 100644 --- a/IPython/utils/tests/test_process.py +++ b/IPython/utils/tests/test_process.py @@ -16,7 +16,6 @@ import sys import os -from unittest import TestCase import nose.tools as nt From d525e643e56b2698f6859830c687bf9a0f5571e5 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 5 Dec 2019 16:27:41 -0800 Subject: [PATCH 103/242] cast_unicode code simplification. Also simplify also the creation of a dictionary with dict comprehension. --- IPython/core/oinspect.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index 27513e1fe3c..c27d8b31d8c 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -22,7 +22,7 @@ from textwrap import dedent import types import io as stdlib_io -from itertools import zip_longest + from typing import Union # IPython's own @@ -83,7 +83,7 @@ def pylight(code): def object_info(**kw): """Make an object info dict with all fields present.""" - infodict = dict(zip_longest(info_fields, [None])) + infodict = {k:None for k in info_fields} infodict.update(kw) return infodict @@ -512,7 +512,8 @@ def pfile(self, obj, oname=''): # 0-offset, so we must adjust. page.page(self.format(openpy.read_py_file(ofile, skip_encoding_cookie=False)), lineno - 1) - def _mime_format(self, text, formatter=None): + + def _mime_format(self, text:str, formatter=None) -> dict: """Return a mime bundle representation of the input text. - if `formatter` is None, the returned mime bundle has @@ -528,7 +529,6 @@ def _mime_format(self, text, formatter=None): Formatters returning strings are supported but this behavior is deprecated. """ - text = cast_unicode(text) defaults = { 'text/plain': text, 'text/html': '
' + text + '
' @@ -591,7 +591,7 @@ def _get_info(self, obj, oname='', formatter=None, info=None, detail_level=0): 'text/html': '', } - def append_field(bundle, title, key, formatter=None): + def append_field(bundle, title:str, key:str, formatter=None): field = info[key] if field is not None: formatted_field = self._mime_format(field, formatter) @@ -711,7 +711,8 @@ def _info(self, obj, oname='', info=None, detail_level=0) -> dict: Returns ======= - An object info dict with known fields from `info_fields`. + An object info dict with known fields from `info_fields`. Keys are + strings, values are string or None. """ if info is None: From 788aabe73859ce041a0e3d05533eaa94fff1966c Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 10 Dec 2019 08:58:36 -0500 Subject: [PATCH 104/242] Remove one unnecessary shell=True option in subprocess. See #12023 --- IPython/utils/sysinfo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/utils/sysinfo.py b/IPython/utils/sysinfo.py index 7c29ed028d5..07d14fd8a48 100644 --- a/IPython/utils/sysinfo.py +++ b/IPython/utils/sysinfo.py @@ -55,10 +55,10 @@ def pkg_commit_hash(pkg_path): return "installation", _sysinfo.commit # maybe we are in a repository - proc = subprocess.Popen('git rev-parse --short HEAD', + proc = subprocess.Popen('git rev-parse --short HEAD'.split(' '), stdout=subprocess.PIPE, stderr=subprocess.PIPE, - cwd=pkg_path, shell=True) + cwd=pkg_path) repo_commit, _ = proc.communicate() if repo_commit: return 'repository', repo_commit.strip().decode('ascii') From ee13e3c65f38a87cefffcf618b3387181b4883be Mon Sep 17 00:00:00 2001 From: Jonathan Slenders Date: Wed, 11 Dec 2019 21:03:51 +0100 Subject: [PATCH 105/242] Use separate asyncio event loop for user input in debugger. --- IPython/terminal/debugger.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/IPython/terminal/debugger.py b/IPython/terminal/debugger.py index 715f7c19926..475c6b8ca94 100644 --- a/IPython/terminal/debugger.py +++ b/IPython/terminal/debugger.py @@ -1,3 +1,4 @@ +import asyncio import signal import sys @@ -67,6 +68,7 @@ def get_prompt_tokens(): if not PTK3: options['inputhook'] = self.shell.inputhook + self.pt_loop = asyncio.new_event_loop() self.pt_app = PromptSession(**options) def cmdloop(self, intro=None): @@ -79,6 +81,19 @@ def cmdloop(self, intro=None): if not self.use_rawinput: raise ValueError('Sorry ipdb does not support use_rawinput=False') + # In order to make sure that asyncio code written in the + # interactive shell doesn't interfere with the prompt, we run the + # prompt in a different event loop. + # If we don't do this, people could spawn coroutine with a + # while/true inside which will freeze the prompt. + + try: + old_loop = asyncio.get_event_loop() + except RuntimeError: + # This happens when the user used `asyncio.run()`. + old_loop = None + + self.preloop() try: @@ -93,10 +108,16 @@ def cmdloop(self, intro=None): else: self._ptcomp.ipy_completer.namespace = self.curframe_locals self._ptcomp.ipy_completer.global_namespace = self.curframe.f_globals + + asyncio.set_event_loop(self.pt_loop) try: - line = self.pt_app.prompt() # reset_current_buffer=True) + line = self.pt_app.prompt() except EOFError: line = 'EOF' + finally: + # Restore the original event loop. + asyncio.set_event_loop(old_loop) + line = self.precmd(line) stop = self.onecmd(line) stop = self.postcmd(stop, line) From c2c3b03e6ba8ff3980946be547e6e10a8d4f63dd Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 11 Dec 2019 15:04:26 -0800 Subject: [PATCH 106/242] remove cast_unicode and add some typings --- IPython/core/inputsplitter.py | 8 ++++---- IPython/utils/text.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py index 84aa0a71f0f..0256278acb4 100644 --- a/IPython/core/inputsplitter.py +++ b/IPython/core/inputsplitter.py @@ -386,7 +386,7 @@ def check_complete(self, source): finally: self.reset() - def push(self, lines): + def push(self, lines:str) -> bool: """Push one or more lines of input. This stores the given lines and returns a status code indicating @@ -408,6 +408,7 @@ def push(self, lines): this value is also stored as a private attribute (``_is_complete``), so it can be queried at any time. """ + assert isinstance(lines, str) self._store(lines) source = self.source @@ -677,7 +678,7 @@ def transform_cell(self, cell): finally: self.reset() - def push(self, lines): + def push(self, lines:str) -> bool: """Push one or more lines of IPython input. This stores the given lines and returns a status code indicating @@ -700,9 +701,8 @@ def push(self, lines): this value is also stored as a private attribute (_is_complete), so it can be queried at any time. """ - + assert isinstance(lines, str) # We must ensure all input is pure unicode - lines = cast_unicode(lines, self.encoding) # ''.splitlines() --> [], but we need to push the empty line to transformers lines_list = lines.splitlines() if not lines_list: diff --git a/IPython/utils/text.py b/IPython/utils/text.py index e844203ca0e..14405ede1f7 100644 --- a/IPython/utils/text.py +++ b/IPython/utils/text.py @@ -538,7 +538,7 @@ class FullEvalFormatter(Formatter): """ # copied from Formatter._vformat with minor changes to allow eval # and replace the format_spec code with slicing - def vformat(self, format_string, args, kwargs): + def vformat(self, format_string:str, args, kwargs)->str: result = [] for literal_text, field_name, format_spec, conversion in \ self.parse(format_string): @@ -566,7 +566,7 @@ def vformat(self, format_string, args, kwargs): # format the object and append to the result result.append(self.format_field(obj, '')) - return ''.join(py3compat.cast_unicode(s) for s in result) + return ''.join(result) class DollarFormatter(FullEvalFormatter): From 23e618db87df269d82f828cba5b2f729e5eb5d9d Mon Sep 17 00:00:00 2001 From: kousik Date: Thu, 12 Dec 2019 10:42:41 +0530 Subject: [PATCH 107/242] Removed unused py2 compact functions #12014 --- IPython/utils/py3compat.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/IPython/utils/py3compat.py b/IPython/utils/py3compat.py index 9b58c6feeec..50be2f872b8 100644 --- a/IPython/utils/py3compat.py +++ b/IPython/utils/py3compat.py @@ -157,9 +157,6 @@ def isidentifier(s, dotted=False): return all(isidentifier(a) for a in s.split(".")) return s.isidentifier() -xrange = range -def iteritems(d): return iter(d.items()) -def itervalues(d): return iter(d.values()) getcwd = os.getcwd MethodType = types.MethodType From 0606868436f21c920b7aaff94c8526c8e3550ce7 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 12 Dec 2019 10:18:16 -0800 Subject: [PATCH 108/242] Remove one appveyor test on win32. Appveyor is often the limiting CI. Keep both 64 and 32 on both max/min version of Python supported --- appveyor.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 5ed18735ff0..b79c7f26ff4 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -12,10 +12,6 @@ environment: PYTHON_VERSION: "3.6.x" PYTHON_ARCH: "64" - - PYTHON: "C:\\Python37" - PYTHON_VERSION: "3.7.x" - PYTHON_ARCH: "32" - - PYTHON: "C:\\Python37-x64" PYTHON_VERSION: "3.7.x" PYTHON_ARCH: "64" From b24c2e57e99aba62f8627083e9ed676fbaa4a736 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 12 Dec 2019 15:51:05 -0800 Subject: [PATCH 109/242] Remove Py3compat from the save magic code. closes #12018 --- IPython/core/magics/code.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/IPython/core/magics/code.py b/IPython/core/magics/code.py index 41aa37ca7cf..a1841384651 100644 --- a/IPython/core/magics/code.py +++ b/IPython/core/magics/code.py @@ -29,7 +29,6 @@ from IPython.core.magic import Magics, magics_class, line_magic from IPython.core.oinspect import find_file, find_source_lines from IPython.testing.skipdoctest import skip_doctest -from IPython.utils import py3compat from IPython.utils.contexts import preserve_keys from IPython.utils.path import get_py_filename from warnings import warn @@ -214,9 +213,9 @@ def save(self, parameter_s=''): force = 'f' in opts append = 'a' in opts mode = 'a' if append else 'w' - ext = u'.ipy' if raw else u'.py' + ext = '.ipy' if raw else '.py' fname, codefrom = args[0], " ".join(args[1:]) - if not fname.endswith((u'.py',u'.ipy')): + if not fname.endswith(('.py','.ipy')): fname += ext file_exists = os.path.isfile(fname) if file_exists and not force and not append: @@ -233,14 +232,13 @@ def save(self, parameter_s=''): except (TypeError, ValueError) as e: print(e.args[0]) return - out = py3compat.cast_unicode(cmds) with io.open(fname, mode, encoding="utf-8") as f: if not file_exists or not append: - f.write(u"# coding: utf-8\n") - f.write(out) + f.write("# coding: utf-8\n") + f.write(cmds) # make sure we end on a newline - if not out.endswith(u'\n'): - f.write(u'\n') + if not cmds.endswith('\n'): + f.write('\n') print('The following commands were written to file `%s`:' % fname) print(cmds) From e4b383a79d89b13083477517f06c606ebe98878f Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 12 Dec 2019 15:59:10 -0800 Subject: [PATCH 110/242] Remove py3compat cast_unicode from utils/path. in Python3 os.environ will always be strings, if you need bytes you manipulate os.environb --- IPython/utils/path.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/IPython/utils/path.py b/IPython/utils/path.py index f2f1ea06f5a..0fb6144e19f 100644 --- a/IPython/utils/path.py +++ b/IPython/utils/path.py @@ -15,13 +15,11 @@ from warnings import warn from IPython.utils.process import system -from IPython.utils import py3compat from IPython.utils.decorators import undoc #----------------------------------------------------------------------------- # Code #----------------------------------------------------------------------------- - fs_encoding = sys.getfilesystemencoding() def _writable_dir(path): @@ -226,7 +224,8 @@ def get_xdg_dir(): # use ~/.config if empty OR not set xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config') if xdg and _writable_dir(xdg): - return py3compat.cast_unicode(xdg, fs_encoding) + assert isinstance(xdg, str) + return xdg return None @@ -244,7 +243,8 @@ def get_xdg_cache_dir(): # use ~/.cache if empty OR not set xdg = env.get("XDG_CACHE_HOME", None) or os.path.join(get_home_dir(), '.cache') if xdg and _writable_dir(xdg): - return py3compat.cast_unicode(xdg, fs_encoding) + assert isinstance(xdg, str) + return xdg return None From 2df0cf135595664b4d595d6bf48f748ba03ac1cf Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 12 Dec 2019 16:10:23 -0800 Subject: [PATCH 111/242] remove unused import (cast_unicode) --- IPython/core/inputsplitter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py index 0256278acb4..e7bc6e7f5a3 100644 --- a/IPython/core/inputsplitter.py +++ b/IPython/core/inputsplitter.py @@ -31,7 +31,6 @@ import tokenize import warnings -from IPython.utils.py3compat import cast_unicode from IPython.core.inputtransformer import (leading_indent, classic_prompt, ipy_prompt, From cec52bd225a7f4d9dd41d27f110ddfb165edbb10 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 12 Dec 2019 16:41:15 -0800 Subject: [PATCH 112/242] Remove more references to py3compat. `input` is unnecessary; and so is `u_format` --- IPython/core/tests/test_inputsplitter.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py index 0b3e47d35b5..a39943aed80 100644 --- a/IPython/core/tests/test_inputsplitter.py +++ b/IPython/core/tests/test_inputsplitter.py @@ -14,8 +14,6 @@ from IPython.core.inputtransformer import InputTransformer from IPython.core.tests.test_inputtransformer import syntax, syntax_ml from IPython.testing import tools as tt -from IPython.utils import py3compat -from IPython.utils.py3compat import input #----------------------------------------------------------------------------- # Semi-complete examples (also used as tests) @@ -568,8 +566,8 @@ class CellMagicsCommon(object): def test_whole_cell(self): src = "%%cellm line\nbody\n" out = self.sp.transform_cell(src) - ref = u"get_ipython().run_cell_magic('cellm', 'line', 'body')\n" - nt.assert_equal(out, py3compat.u_format(ref)) + ref = "get_ipython().run_cell_magic('cellm', 'line', 'body')\n" + nt.assert_equal(out, ref) def test_cellmagic_help(self): self.sp.push('%%cellm?') From 939ead039136510adcf4d34c312dc9b6209c1f2b Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 13 Dec 2019 19:59:57 -0800 Subject: [PATCH 113/242] 7.10.2 release notes --- docs/source/whatsnew/version7.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index c4ba18635b2..87c312b8862 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,14 @@ 7.x Series ============ +.. _version 7102: + +IPython 7.10.2 +============== + +IPython 7.10.2 fix a couple of extra incompatibility between IPython, ipdb, +asyncio and Prompt Toolkit 3. + .. _version 7101: IPython 7.10.1 From 6dc6b7250c0b9cba85eed7aea6e728e95c96754c Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 14 Dec 2019 10:23:22 -0800 Subject: [PATCH 114/242] 7.10.2 stats (forward port from 7.10.x branch) --- docs/source/whatsnew/github-stats-7.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/source/whatsnew/github-stats-7.rst b/docs/source/whatsnew/github-stats-7.rst index 958954ebf5d..121c35a4f47 100644 --- a/docs/source/whatsnew/github-stats-7.rst +++ b/docs/source/whatsnew/github-stats-7.rst @@ -1,6 +1,23 @@ Issues closed in the 7.x development cycle ========================================== +Issues closed in 7.10.2 +----------------------- + + +GitHub stats for 2019/12/01 - 2019/12/14 (tag: 7.10.1) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 3 issues and merged 10 pull requests. +The full list can be seen `on GitHub `__ + +The following 3 authors contributed 11 commits. + +* Jonathan Slenders +* Joseph Kahn +* Matthias Bussonnier + Issues closed in 7.10.1 ----------------------- From b87c2b69042bfa48f688e07343934e0d4fee759a Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 14 Dec 2019 14:42:11 -0800 Subject: [PATCH 115/242] Remove try/except import for Py2 compat --- IPython/core/debugger.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py index 9ce896ec634..ebb8dcac0d8 100644 --- a/IPython/core/debugger.py +++ b/IPython/core/debugger.py @@ -153,10 +153,7 @@ def __init__(self, colors=None): # at least raise that limit to 80 chars, which should be enough for # most interactive uses. try: - try: - from reprlib import aRepr # Py 3 - except ImportError: - from repr import aRepr # Py 2 + from reprlib import aRepr aRepr.maxstring = 80 except: # This is only a user-facing convenience, so any error we encounter From d670c431a5e85a930c575d3f69152dc21c38b887 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 15 Dec 2019 05:26:59 +0000 Subject: [PATCH 116/242] fix syntax error --- IPython/core/ultratb.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index eef16cc650a..a770d9028cb 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -627,6 +627,12 @@ def __call__(self, etype, value, elist): self.ostream.write(self.text(etype, value, elist)) self.ostream.write('\n') + def _extract_tb(self, tb): + if tb: + return traceback.extract_tb(tb) + else: + return None + def structured_traceback(self, etype, evalue, etb=None, tb_offset=None, context=5): """Return a color formatted string with the traceback info. @@ -1303,12 +1309,6 @@ def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False, # set_mode also sets the tb_join_char attribute self.set_mode(mode) - def _extract_tb(self, tb): - if tb: - return traceback.extract_tb(tb) - else: - return None - def structured_traceback(self, etype, value, tb, tb_offset=None, number_of_lines_of_context=5): tb_offset = self.tb_offset if tb_offset is None else tb_offset mode = self.mode @@ -1409,7 +1409,10 @@ def structured_traceback(self, etype=None, value=None, tb=None, tb_offset=None, number_of_lines_of_context=5): if etype is None: etype, value, tb = sys.exc_info() - self.tb = tb + if not isinstance(tb, tuple): + # tb is a tuple if this is a chained exception. + # We keep first traceback. + self.tb = tb return FormattedTB.structured_traceback( self, etype, value, tb, tb_offset, number_of_lines_of_context) From c9fc4159fb78cfd97ed42a4acf0df259a93cd4a6 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 15 Dec 2019 13:02:08 +0000 Subject: [PATCH 117/242] save traceback for chained exception --- IPython/core/ultratb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index a770d9028cb..e1f805738a0 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -1409,9 +1409,10 @@ def structured_traceback(self, etype=None, value=None, tb=None, tb_offset=None, number_of_lines_of_context=5): if etype is None: etype, value, tb = sys.exc_info() - if not isinstance(tb, tuple): + if isinstance(tb, tuple): # tb is a tuple if this is a chained exception. - # We keep first traceback. + self.tb = tb[0] + else: self.tb = tb return FormattedTB.structured_traceback( self, etype, value, tb, tb_offset, number_of_lines_of_context) From a05409fd4543127086bb048d6d6015de86e31520 Mon Sep 17 00:00:00 2001 From: Naveen Honest Raj K Date: Mon, 16 Dec 2019 10:49:28 +0530 Subject: [PATCH 118/242] removed snip_print fn because unused Signed-off-by: Naveen Honest Raj K --- IPython/core/page.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/IPython/core/page.py b/IPython/core/page.py index afdded29b1a..ed16b617812 100644 --- a/IPython/core/page.py +++ b/IPython/core/page.py @@ -341,32 +341,3 @@ def page_more(): return False else: return True - - -def snip_print(str,width = 75,print_full = 0,header = ''): - """Print a string snipping the midsection to fit in width. - - print_full: mode control: - - - 0: only snip long strings - - 1: send to page() directly. - - 2: snip long strings and ask for full length viewing with page() - - Return 1 if snipping was necessary, 0 otherwise.""" - - if print_full == 1: - page(header+str) - return 0 - - print(header, end=' ') - if len(str) < width: - print(str) - snip = 0 - else: - whalf = int((width -5)/2) - print(str[:whalf] + ' <...> ' + str[-whalf:]) - snip = 1 - if snip and print_full == 2: - if py3compat.input(header+' Snipped. View (y/n)? [N]').lower() == 'y': - page(str) - return snip From 0546e3994ca0b297788646d3ca28e16d3b7dfe83 Mon Sep 17 00:00:00 2001 From: kousik Date: Mon, 16 Dec 2019 11:47:16 +0530 Subject: [PATCH 119/242] Removed IPython.core.page:snip_print #12038 --- IPython/core/page.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/IPython/core/page.py b/IPython/core/page.py index afdded29b1a..ed16b617812 100644 --- a/IPython/core/page.py +++ b/IPython/core/page.py @@ -341,32 +341,3 @@ def page_more(): return False else: return True - - -def snip_print(str,width = 75,print_full = 0,header = ''): - """Print a string snipping the midsection to fit in width. - - print_full: mode control: - - - 0: only snip long strings - - 1: send to page() directly. - - 2: snip long strings and ask for full length viewing with page() - - Return 1 if snipping was necessary, 0 otherwise.""" - - if print_full == 1: - page(header+str) - return 0 - - print(header, end=' ') - if len(str) < width: - print(str) - snip = 0 - else: - whalf = int((width -5)/2) - print(str[:whalf] + ' <...> ' + str[-whalf:]) - snip = 1 - if snip and print_full == 2: - if py3compat.input(header+' Snipped. View (y/n)? [N]').lower() == 'y': - page(str) - return snip From 9444858e55e7d3de5433e4642f64cc9363fb2381 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 27 Dec 2019 14:12:17 +0100 Subject: [PATCH 120/242] Use update brew tap to fix test failures. This will need to be reverted at some points in the future once https://github.com/minrk/homebrew-python-frameworks/pull/2 is merged --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f0d4dc6f126..c0aad50b4ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ before_install: if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then env | sort if ! which python$TRAVIS_PYTHON_VERSION; then - HOMEBREW_NO_AUTO_UPDATE=1 brew tap minrk/homebrew-python-frameworks + HOMEBREW_NO_AUTO_UPDATE=1 brew tap carreau/homebrew-python-frameworks HOMEBREW_NO_AUTO_UPDATE=1 brew cask install python-framework-${TRAVIS_PYTHON_VERSION/./} fi python3 -m pip install virtualenv From 3697d322d153a470d45de676eb224e3cdc311d8b Mon Sep 17 00:00:00 2001 From: takuya fujiwara Date: Sat, 21 Dec 2019 01:45:26 +0900 Subject: [PATCH 121/242] Remove unused annotate function in py3compat #12011 --- IPython/utils/py3compat.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/IPython/utils/py3compat.py b/IPython/utils/py3compat.py index 50be2f872b8..7a57514a340 100644 --- a/IPython/utils/py3compat.py +++ b/IPython/utils/py3compat.py @@ -190,18 +190,6 @@ def get_closure(f): PYPY = platform.python_implementation() == "PyPy" -def annotate(**kwargs): - """Python 3 compatible function annotation for Python 2.""" - if not kwargs: - raise ValueError('annotations must be provided as keyword arguments') - def dec(f): - if hasattr(f, '__annotations__'): - for k, v in kwargs.items(): - f.__annotations__[k] = v - else: - f.__annotations__ = kwargs - return f - return dec # Parts below taken from six: From d6b29e2567c6d952d9c31f4e37d04c03c7278119 Mon Sep 17 00:00:00 2001 From: Marc Hernandez Cabot Date: Thu, 19 Dec 2019 21:56:17 +0100 Subject: [PATCH 122/242] remove duplicated entry in gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4471ae7e307..1fc0e22a320 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ docs/source/api/generated docs/source/config/options docs/source/config/shortcuts/*.csv docs/source/interactive/magics-generated.txt -docs/source/config/shortcuts/*.csv docs/gh-pages jupyter_notebook/notebook/static/mathjax jupyter_notebook/static/style/*.map From 8d5912e8df4fd1d643bc214c3b6b777a3755410a Mon Sep 17 00:00:00 2001 From: Marc Hernandez Cabot Date: Thu, 19 Dec 2019 22:00:23 +0100 Subject: [PATCH 123/242] fix ipyparallel duplicated entry --- docs/source/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 8a7a6098c3f..bd78b98ae4a 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -225,7 +225,6 @@ def is_stable(extra): intersphinx_mapping = {'python': ('https://docs.python.org/3/', None), 'rpy2': ('https://rpy2.readthedocs.io/en/version_2.8.x/', None), 'jupyterclient': ('https://jupyter-client.readthedocs.io/en/latest/', None), - 'ipyparallel': ('https://ipyparallel.readthedocs.io/en/latest/', None), 'jupyter': ('https://jupyter.readthedocs.io/en/latest/', None), 'jedi': ('https://jedi.readthedocs.io/en/latest/', None), 'traitlets': ('https://traitlets.readthedocs.io/en/latest/', None), From a67a3d70b8204f42e9bc4d570587a32450e4e9b2 Mon Sep 17 00:00:00 2001 From: Marc Hernandez Cabot Date: Thu, 19 Dec 2019 22:02:36 +0100 Subject: [PATCH 124/242] fix argument in format string --- IPython/testing/ipunittest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/testing/ipunittest.py b/IPython/testing/ipunittest.py index 56146d1de26..5a940a5fe91 100644 --- a/IPython/testing/ipunittest.py +++ b/IPython/testing/ipunittest.py @@ -155,7 +155,7 @@ def test(self): # failed should contain at most one item. More than that # is a case we can't handle and should error out on if len(failed) > 1: - err = "Invalid number of test results:" % failed + err = "Invalid number of test results: %s" % failed raise ValueError(err) # Report a normal failure. self.fail('failed doctests: %s' % str(failed[0])) From 003ab938319421376e3d573e27a27833a8e85d93 Mon Sep 17 00:00:00 2001 From: Jonathan Feinberg Date: Sat, 21 Dec 2019 15:39:04 +0100 Subject: [PATCH 125/242] Adding VI-mode in prompt From the code it is apparent that if vi-mode is enabled, it should display in the prompt. However, as the compare done is done against an Enum class, it will always return False. The solution is either to access the Enum's `.value` attribute, or to compare it to the declared Enum value directly. The latter is likely more readable. --- IPython/terminal/prompts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/IPython/terminal/prompts.py b/IPython/terminal/prompts.py index db1b751a026..ee2c5d7052e 100644 --- a/IPython/terminal/prompts.py +++ b/IPython/terminal/prompts.py @@ -7,6 +7,7 @@ from prompt_toolkit.formatted_text import fragment_list_width, PygmentsTokens from prompt_toolkit.shortcuts import print_formatted_text +from prompt_toolkit.enums import EditingMode class Prompts(object): @@ -14,7 +15,7 @@ def __init__(self, shell): self.shell = shell def vi_mode(self): - if (getattr(self.shell.pt_app, 'editing_mode', None) == 'VI' + if (getattr(self.shell.pt_app, 'editing_mode', None) == EditingMode.VI and self.shell.prompt_includes_vi_mode): return '['+str(self.shell.pt_app.app.vi_state.input_mode)[3:6]+'] ' return '' From ee3eebcc0768f845f00a516296959438fdb32384 Mon Sep 17 00:00:00 2001 From: Jonathan Feinberg Date: Sat, 21 Dec 2019 16:07:17 +0100 Subject: [PATCH 126/242] Support vi-mode 'InputFormat' prompt Prompt defining what vi-mode is in, is either 'vi-insert' or 'vi-navigate' (or something 'vi-...' I imagine). From this `[3:6]` is extracted to get either 'ins' or 'nav' in the prompt. This is consistent with the behavior I experience when manipulating a custom prompt from the most updated pip installable package. However, in the case of installing the current master from source, the prompt is on the format 'InputFormat.INSERT' and 'InputFormat.NAVIGATION'. --- IPython/terminal/prompts.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/IPython/terminal/prompts.py b/IPython/terminal/prompts.py index ee2c5d7052e..3f5c07b980e 100644 --- a/IPython/terminal/prompts.py +++ b/IPython/terminal/prompts.py @@ -17,7 +17,12 @@ def __init__(self, shell): def vi_mode(self): if (getattr(self.shell.pt_app, 'editing_mode', None) == EditingMode.VI and self.shell.prompt_includes_vi_mode): - return '['+str(self.shell.pt_app.app.vi_state.input_mode)[3:6]+'] ' + mode = str(self.shell.pt_app.app.vi_state.input_mode) + if mode.startswith('InputMode.'): + mode = mode[10:13].lower() + elif mode.startswith('vi-'): + mode = mode[3:6] + return '['+mode+'] ' return '' From 50afc8844d13a6192a0d00dd49b395787ba162ee Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Thu, 26 Dec 2019 22:06:51 -0600 Subject: [PATCH 127/242] Fix small typos: an -> and and function -> functions --- docs/source/interactive/magics.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/interactive/magics.rst b/docs/source/interactive/magics.rst index bb562f44f20..b61bcdc7f85 100644 --- a/docs/source/interactive/magics.rst +++ b/docs/source/interactive/magics.rst @@ -13,9 +13,9 @@ Built-in magic commands other languages. Here is the help auto-generated from the docstrings of all the available Magics -function that IPython ships with. +functions that IPython ships with. -You can create an register your own Magics with IPython. You can find many user +You can create and register your own Magics with IPython. You can find many user defined Magics on `PyPI `_. Feel free to publish your own and use the ``Framework :: IPython`` trove classifier. From 6a8e15d9f21966ffcb5d020f2d4099489d0c6d3e Mon Sep 17 00:00:00 2001 From: Augusto Date: Sun, 22 Dec 2019 14:17:56 -0300 Subject: [PATCH 128/242] Remove unused with_metaclass function. --- IPython/utils/py3compat.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/IPython/utils/py3compat.py b/IPython/utils/py3compat.py index 7a57514a340..0d34735bb47 100644 --- a/IPython/utils/py3compat.py +++ b/IPython/utils/py3compat.py @@ -189,30 +189,3 @@ def get_closure(f): PY2 = not PY3 PYPY = platform.python_implementation() == "PyPy" - - - -# Parts below taken from six: -# Copyright (c) 2010-2013 Benjamin Peterson -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -def with_metaclass(meta, *bases): - """Create a base class with a metaclass.""" - return meta("_NewBase", bases, {}) From fbeedc0e5e4a5195359adb2209f67b855f54c163 Mon Sep 17 00:00:00 2001 From: kousik Date: Wed, 25 Dec 2019 16:39:24 +0530 Subject: [PATCH 129/242] Removed IPython/utils/py3compat.py unused get_closure function. #12012 --- IPython/utils/py3compat.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/IPython/utils/py3compat.py b/IPython/utils/py3compat.py index 7a57514a340..5395a1be654 100644 --- a/IPython/utils/py3compat.py +++ b/IPython/utils/py3compat.py @@ -181,10 +181,6 @@ def u_format(s): Accepts a string or a function, so it can be used as a decorator.""" return s.format(u='') -def get_closure(f): - """Get a function's closure attribute""" - return f.__closure__ - PY2 = not PY3 PYPY = platform.python_implementation() == "PyPy" From 0c5d00ea919d258742d1d83cc00a93422be5fd0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Miedzi=C5=84ski?= Date: Sun, 22 Dec 2019 22:00:50 +0100 Subject: [PATCH 130/242] Fix handling of repeated NEWLINE tokens --- IPython/core/inputtransformer2.py | 4 ++-- IPython/core/tests/test_inputtransformer2.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/IPython/core/inputtransformer2.py b/IPython/core/inputtransformer2.py index 04f423274da..4562fe01d2c 100644 --- a/IPython/core/inputtransformer2.py +++ b/IPython/core/inputtransformer2.py @@ -674,8 +674,8 @@ def check_complete(self, cell: str): while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types: tokens_by_line[-1].pop() - if len(tokens_by_line) == 1 and not tokens_by_line[-1]: - return 'incomplete', 0 + if not tokens_by_line[-1]: + return 'incomplete', find_last_indent(lines) if tokens_by_line[-1][-1].string == ':': # The last line starts a block (e.g. 'if foo:') diff --git a/IPython/core/tests/test_inputtransformer2.py b/IPython/core/tests/test_inputtransformer2.py index 8d6efc1e221..cde9eca0d06 100644 --- a/IPython/core/tests/test_inputtransformer2.py +++ b/IPython/core/tests/test_inputtransformer2.py @@ -265,6 +265,8 @@ def test_check_complete(): for k in short: cc(c+k) + nt.assert_equal(cc("def f():\n x=0\n \\\n "), ('incomplete', 2)) + def test_check_complete_II(): """ Test that multiple line strings are properly handled. From fee94b46db24987b8918d26f4a146582c2dfafc6 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 27 Dec 2019 19:08:52 +0100 Subject: [PATCH 131/242] update what's new for 7.11 --- docs/source/whatsnew/version7.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 87c312b8862..97c9a8e2445 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,36 @@ 7.x Series ============ +.. _version 711: + +IPython 7.11 +============ + +IPython 7.11 received a couple of compatibility fixes and code cleanup. + +A number of function in the ``py3compat`` have been removed; a number of types +in the IPython code base are now non-ambiguous and now always ``unicode`` +instead of ``Union[Unicode,bytes]``; many of the relevant code path have thus +been simplified/cleaned and types annotation added. + +IPython support several verbosity level from exceptions. ``xmode plain`` now +support chained exceptions. :ghpull:`11999` + +We are starting to remove ``shell=True`` in some usages of subprocess. While not directly +a security issue (as IPython is made to run arbitrary code anyway) it is not good +practice and we'd like to show the example. :ghissue:`12023`. This discussion +was started by ``@mschwager`` thanks to a new auditing tool they are working on +with duo-labs (`dlint `_). + +Work around some bugs in Python 3.9 tokenizer :ghpull:`12057` + +IPython will now print its version after a crash. :ghpull:`11986` + +This is likely the last release from the 7.x series that will see new feature. +The master branch will soon accept large code changes and thrilling new +features; the 7.x branch will only start to accept critical bug fixes, and +update dependencies. + .. _version 7102: IPython 7.10.2 From d3376c96d31f77d13dd34d3d465de698d9b72057 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 27 Dec 2019 19:37:43 +0100 Subject: [PATCH 132/242] update stats-7.11 --- docs/source/whatsnew/github-stats-7.rst | 30 +++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/source/whatsnew/github-stats-7.rst b/docs/source/whatsnew/github-stats-7.rst index 121c35a4f47..dd366d8ad9c 100644 --- a/docs/source/whatsnew/github-stats-7.rst +++ b/docs/source/whatsnew/github-stats-7.rst @@ -1,6 +1,36 @@ Issues closed in the 7.x development cycle ========================================== +Issues closed in 7.11 +--------------------- + +GitHub stats for 2019/12/01 - 2019/12/27 (tag: 7.10.1) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 4 issues and merged 36 pull requests. +The full list can be seen `on GitHub `__ + +The following 16 authors contributed 114 commits. + +* Augusto +* Benjamin Ragan-Kelley +* Chemss Eddine Ben Hassine +* Danny Hermes +* Dominik Miedziński +* Jonathan Feinberg +* Jonathan Slenders +* Joseph Kahn +* kousik +* Kousik Mitra +* Marc Hernandez Cabot +* Matthias Bussonnier +* Naveen Honest Raj K +* Pratyay Pandey +* Quentin Peter +* takuya fujiwara + + Issues closed in 7.10.2 ----------------------- From c6839ecd8f63381014dd37ff17d6979b8d9c1426 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 27 Dec 2019 19:38:24 +0100 Subject: [PATCH 133/242] release 7.11.0 --- IPython/core/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 5cf2110adf0..f47141b3312 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -24,7 +24,7 @@ _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 54057a78f9c1100ec8ad8b06803ad27120fa7889 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 27 Dec 2019 19:40:08 +0100 Subject: [PATCH 134/242] back to dev --- IPython/core/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index f47141b3312..0fe4a422092 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 11 +_version_minor = 12 _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 9a8d1a345f48b7aa85e6a6da5841b65ee1f8de63 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 28 Dec 2019 10:54:13 +0100 Subject: [PATCH 135/242] Reintroduce compatibility functions for a few release; Leaving time for Cython to clean that up. Closes #12068 --- IPython/utils/py3compat.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/IPython/utils/py3compat.py b/IPython/utils/py3compat.py index b8ad4e62fd2..5e1a81a9d57 100644 --- a/IPython/utils/py3compat.py +++ b/IPython/utils/py3compat.py @@ -185,3 +185,10 @@ def u_format(s): PY2 = not PY3 PYPY = platform.python_implementation() == "PyPy" +# Cython still rely on that as a Dec 28 2019 +# See https://github.com/cython/cython/pull/3291 and +# https://github.com/ipython/ipython/issues/12068 +def no_code(x, encoding=None): + return x +unicode_to_str = cast_bytes_py2 = no_code + From 2b346b1a3644aa59606305bef26601d66c149465 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 1 Jan 2020 18:27:56 +0100 Subject: [PATCH 136/242] some updates to release tool --- tools/release_helper.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index c82285ecd44..49474477f11 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -139,10 +139,10 @@ then read echo - echo "Let's commit : git commit -am \"back to dev\" -S" + echo "Let's commit : "$BLUE"git commit -am \"back to dev\""$NOR echo $GREEN"Press enter to commit"$NOR read - git commit -am "back to dev" -S + git commit -am "back to dev" echo echo $BLUE"let's : git checkout $VERSION"$NOR @@ -157,13 +157,15 @@ then echo echo $BLUE"Attempting to build package..."$NOR - tools/build_release + tools/release echo '$ shasum -a 256 dist/*' + echo $RED shasum -a 256 dist/* + echo $NOR if ask_section "upload packages ?" then - tools/build_release upload + tools/release upload fi fi From d96f387391d71e03d872b077e46f408eb21a0c7d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 1 Jan 2020 18:30:00 +0100 Subject: [PATCH 137/242] whatsnew --- docs/source/whatsnew/version7.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 97c9a8e2445..7a77cac5750 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,14 @@ 7.x Series ============ +.. _version 7111: + +IPython 7.11.1 +============== + +A couple of deprecated functions (no-op) have been reintroduces in py3compat as +Cython was still relying on them, and will be removed in a couple of versions. + .. _version 711: IPython 7.11 From 3544c180b971afa87022e3c6b467819713f57311 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 1 Jan 2020 18:31:34 +0100 Subject: [PATCH 138/242] release 7.11.1 --- IPython/core/release.py | 6 +++--- tools/release_helper.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 0fe4a422092..a07fa6e27a9 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 12 -_version_patch = 0 +_version_minor = 11 +_version_patch = 1 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] diff --git a/tools/release_helper.sh b/tools/release_helper.sh index 49474477f11..fe0828ca323 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -107,7 +107,7 @@ rm dist/* if ask_section "Should we commit, tag, push... etc ? " then echo - echo "Let's commit : git commit -am \"release $VERSION\" -S" + echo $BLUE"Let's commit : git commit -am \"release $VERSION\" -S" echo $GREEN"Press enter to commit"$NOR read git commit -am "release $VERSION" -S From 10f852d25397921b5223ec065cb82abfed5d6ace Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 1 Jan 2020 18:33:12 +0100 Subject: [PATCH 139/242] back to dev --- IPython/core/release.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index a07fa6e27a9..0fe4a422092 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 11 -_version_patch = 1 +_version_minor = 12 +_version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From ba155123097ad0b5c5586bb54883399e85139503 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 1 Jan 2020 19:22:29 +0100 Subject: [PATCH 140/242] Update instesphinx for rpy2 as object.inv is gone. --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index bd78b98ae4a..afdff72b5c4 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -223,7 +223,7 @@ def is_stable(extra): htmlhelp_basename = 'ipythondoc' intersphinx_mapping = {'python': ('https://docs.python.org/3/', None), - 'rpy2': ('https://rpy2.readthedocs.io/en/version_2.8.x/', None), + 'rpy2': ('https://rpy2.readthedocs.io/en/latest/', None), 'jupyterclient': ('https://jupyter-client.readthedocs.io/en/latest/', None), 'jupyter': ('https://jupyter.readthedocs.io/en/latest/', None), 'jedi': ('https://jedi.readthedocs.io/en/latest/', None), From 225be320674e2bb5ba2a0ee15056d39d58aeecbe Mon Sep 17 00:00:00 2001 From: takuya fujiwara Date: Tue, 7 Jan 2020 17:46:42 +0900 Subject: [PATCH 141/242] Remove unused _print_statement_sub function in py3compat #12010 --- IPython/utils/py3compat.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/IPython/utils/py3compat.py b/IPython/utils/py3compat.py index 5e1a81a9d57..c7587873005 100644 --- a/IPython/utils/py3compat.py +++ b/IPython/utils/py3compat.py @@ -169,9 +169,6 @@ def execfile(fname, glob, loc=None, compiler=None): # Refactor print statements in doctests. _print_statement_re = re.compile(r"\bprint (?P.*)$", re.MULTILINE) -def _print_statement_sub(match): - expr = match.groups('expr') - return "print(%s)" % expr # Abstract u'abc' syntax: @_modify_str_or_docstring From 0bd2747531438878d16ddff670a325f15df1b858 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 1 Jan 2020 18:42:38 +0100 Subject: [PATCH 142/242] reset color after asking or it stays green --- tools/release_helper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index fe0828ca323..d87c281f454 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -37,7 +37,7 @@ BRANCH=${input:-$BRANCH} ask_section(){ echo echo $BLUE"$1"$NOR - echo -n $GREEN"Press Enter to continue, S to skip: "$GREEN + echo -n $GREEN"Press Enter to continue, S to skip: "$NOR read -n1 value echo if [ -z $value ] || [ $value = 'y' ] ; then From 0e3251db2c48a6327f80cd3fe94fdda641a584a7 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 1 Jan 2020 18:46:47 +0100 Subject: [PATCH 143/242] some updates to release scripts --- tools/release_helper.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index d87c281f454..7489c16d756 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -143,6 +143,13 @@ then echo $GREEN"Press enter to commit"$NOR read git commit -am "back to dev" + + echo + echo $BLUE"git push origin \$BRANCH ($BRANCH)?"$NOR + echo $GREEN"Press enter to continue"$NOR + read + git push origin $BRANCH + echo echo $BLUE"let's : git checkout $VERSION"$NOR @@ -159,8 +166,8 @@ then tools/release - echo '$ shasum -a 256 dist/*' echo $RED + echo '$ shasum -a 256 dist/*' shasum -a 256 dist/* echo $NOR From 39d3e8668ef8f77a691f81fba451824bea8f842a Mon Sep 17 00:00:00 2001 From: ossdev07 Date: Thu, 2 Jan 2020 06:32:46 +0000 Subject: [PATCH 144/242] Added arm64 jobs in Travis-CI Signed-off-by: ossdev07 --- .travis.yml | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index c0aad50b4ea..38da38e850c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,18 +65,38 @@ after_success: matrix: include: - - python: "3.7" + - arch: amd64 + python: "3.7" dist: xenial sudo: true - - python: "3.8-dev" + - arch: arm64 + python: "3.7" dist: xenial sudo: true - - python: "3.7-dev" + - arch: amd64 + python: "3.8-dev" dist: xenial sudo: true - - python: "nightly" + - arch: arm64 + python: "3.8-dev" dist: xenial sudo: true + - arch: amd64 + python: "3.7-dev" + dist: xenial + sudo: true + - arch: arm64 + python: "3.7-dev" + dist: xenial + sudo: true + - arch: amd64 + python: "nightly" + dist: xenial + sudo: true + - arch: arm64 + python: "nightly" + dist: bionic + sudo: true - os: osx language: generic python: 3.6 From 1e6139c81e9acd5d2c771b0718f6f4b8592892e3 Mon Sep 17 00:00:00 2001 From: Jeff Potter Date: Sat, 4 Jan 2020 11:13:08 -0500 Subject: [PATCH 145/242] Exit non-zero when ipython is given a file path to run that doesn't exist --- IPython/core/shellapp.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/IPython/core/shellapp.py b/IPython/core/shellapp.py index a09192f255c..3147c59f9f6 100644 --- a/IPython/core/shellapp.py +++ b/IPython/core/shellapp.py @@ -404,6 +404,9 @@ def _run_cmd_line_code(self): fname = self.file_to_run if os.path.isdir(fname): fname = os.path.join(fname, "__main__.py") + if not os.path.exists(fname): + self.log.warning("File '%s' doesn't exist", fname) + self.exit(2) try: self._exec_file(fname, shell_futures=True) except: From 773930b856c4c051c67ffeb4d7dc6e2820d59985 Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Sun, 12 Jan 2020 00:09:43 -0500 Subject: [PATCH 146/242] MNT: ensure we can show un-pickled figures For Matplotlib < 3.2 inline figures maybe re-hydrated with the canvas attribute as None. In that case `print_figure` will fail, this ensure that a minimal canvas will exist. --- IPython/core/pylabtools.py | 8 ++++++-- IPython/core/tests/test_pylabtools.py | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/IPython/core/pylabtools.py b/IPython/core/pylabtools.py index d42c088980c..cb1ce811984 100644 --- a/IPython/core/pylabtools.py +++ b/IPython/core/pylabtools.py @@ -123,14 +123,18 @@ def print_figure(fig, fmt='png', bbox_inches='tight', **kwargs): } # **kwargs get higher priority kw.update(kwargs) - + bytes_io = BytesIO() + if fig.canvas is None: + from matplotlib.backend_bases import FigureCanvasBase + FigureCanvasBase(fig) + fig.canvas.print_figure(bytes_io, **kw) data = bytes_io.getvalue() if fmt == 'svg': data = data.decode('utf-8') return data - + def retina_figure(fig, **kwargs): """format a figure as a pixel-doubled (retina) PNG""" pngdata = print_figure(fig, fmt='retina', **kwargs) diff --git a/IPython/core/tests/test_pylabtools.py b/IPython/core/tests/test_pylabtools.py index 181e99f9b84..7b724000f58 100644 --- a/IPython/core/tests/test_pylabtools.py +++ b/IPython/core/tests/test_pylabtools.py @@ -248,3 +248,9 @@ def test_qt_gtk(self): def test_no_gui_backends(): for k in ['agg', 'svg', 'pdf', 'ps']: assert k not in pt.backend2gui + + +def test_figure_no_canvas(): + fig = Figure() + fig.canvas = None + pt.print_figure(fig) From b1231d6020323a1680b072c992270a62c4de84b0 Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Sun, 12 Jan 2020 01:35:19 -0500 Subject: [PATCH 147/242] MNT: update pytest.ini to work with pytest > 3 --- .travis.yml | 2 +- pytest.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38da38e850c..1099db1942c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ install: - pip install setuptools --upgrade - pip install -e file://$PWD#egg=ipython[test] --upgrade - pip install trio curio --upgrade --upgrade-strategy eager - - pip install 'pytest<4' matplotlib + - pip install pytest matplotlib - pip install codecov check-manifest --upgrade script: diff --git a/pytest.ini b/pytest.ini index 537ebeb2674..9da365334c6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -addopts = --duration=10 +addopts = --durations=10 From 9ef08404eb19da18d55188cd163d8e51101b3c53 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 13 Jan 2020 13:54:48 -0800 Subject: [PATCH 148/242] Remove some failing ARM 64, and add an environment variable Env variable for better report of travis failure in the UI or the builds are otherwise impossible to distinguish. --- .travis.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38da38e850c..626196e3b12 100644 --- a/.travis.yml +++ b/.travis.yml @@ -72,23 +72,16 @@ matrix: - arch: arm64 python: "3.7" dist: xenial + env: ARM64=True sudo: true - arch: amd64 python: "3.8-dev" dist: xenial sudo: true - - arch: arm64 - python: "3.8-dev" - dist: xenial - sudo: true - arch: amd64 python: "3.7-dev" dist: xenial sudo: true - - arch: arm64 - python: "3.7-dev" - dist: xenial - sudo: true - arch: amd64 python: "nightly" dist: xenial @@ -96,6 +89,7 @@ matrix: - arch: arm64 python: "nightly" dist: bionic + env: ARM64=True sudo: true - os: osx language: generic From f724bc8245c475324acd11568955df67c6670071 Mon Sep 17 00:00:00 2001 From: "Mark E. Haase" Date: Tue, 14 Jan 2020 16:51:52 -0500 Subject: [PATCH 149/242] Experimental patch to keep Trio event loop alive between cells Run Trio in a background thread and use trio.from_thread.run() to schedule cells in that thread. This exports a nursery object named GLOBAL_NURSERY that client code can use to start background tasks without blocking the completion of the current cell. Co-authored-by: Brian Mackintosh --- IPython/core/async_helpers.py | 37 +++++++++++++++++++++++++++++++- IPython/core/interactiveshell.py | 2 +- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/IPython/core/async_helpers.py b/IPython/core/async_helpers.py index 1a7d88991db..fbaae707bb2 100644 --- a/IPython/core/async_helpers.py +++ b/IPython/core/async_helpers.py @@ -42,9 +42,44 @@ def _curio_runner(coroutine): return curio.run(coroutine) +_TRIO_TOKEN = None +_TRIO_NURSERY = None + + +def _init_trio(trio): + global _TRIO_TOKEN + import traceback + import builtins + import threading + + async def trio_entry(): + global _TRIO_TOKEN, _TRIO_NURSERY + _TRIO_TOKEN = trio.hazmat.current_trio_token() + async with trio.open_nursery() as nursery: + _TRIO_NURSERY = nursery + builtins.GLOBAL_NURSERY = nursery + await trio.sleep_forever() + + def trio_entry_sync(): + while True: + try: + trio.run(trio_entry) + except Exception: + print("Exception in trio event loop:", traceback.format_exc()) + + threading.Thread(target=trio_entry_sync).start() + #TODO fix this race condition + import time + while not _TRIO_TOKEN: + time.sleep(0.1) + + def _trio_runner(async_fn): import trio + if not _TRIO_TOKEN: + _init_trio(trio) + async def loc(coro): """ We need the dummy no-op async def to protect from @@ -52,7 +87,7 @@ async def loc(coro): """ return await coro - return trio.run(loc, async_fn) + return trio.from_thread.run(loc, async_fn, trio_token=_TRIO_TOKEN) def _pseudo_sync_runner(coro): diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index d8bb3e59ec9..6a4fc43efb9 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2865,7 +2865,7 @@ def _run_cell(self, raw_cell:str, store_history:bool, silent:bool, shell_futures # when this is the case, we want to run it using the pseudo_sync_runner # so that code can invoke eventloops (for example via the %run , and # `%paste` magic. - if self.should_run_async(raw_cell): + if self.should_run_async(raw_cell) or self.loop_runner is _trio_runner: runner = self.loop_runner else: runner = _pseudo_sync_runner From 018538a05b3db3e47b7c7eb4f025480e6d783a75 Mon Sep 17 00:00:00 2001 From: "Mark E. Haase" Date: Wed, 15 Jan 2020 15:13:18 -0500 Subject: [PATCH 150/242] More work on long-lived Trio loop -Better handling for race condition on loop startup -Better handling of errors for tasks in the global nursery: it doesn't crash the global nursery, it just logs the exception -And some other code cleanup Co-authored-by: Brian Mackintosh --- IPython/core/async_helpers.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/IPython/core/async_helpers.py b/IPython/core/async_helpers.py index fbaae707bb2..6c0f9bacdbe 100644 --- a/IPython/core/async_helpers.py +++ b/IPython/core/async_helpers.py @@ -43,7 +43,6 @@ def _curio_runner(coroutine): _TRIO_TOKEN = None -_TRIO_NURSERY = None def _init_trio(trio): @@ -51,30 +50,36 @@ def _init_trio(trio): import traceback import builtins import threading + # We use an Event to avoid a race condition between starting the Trio thread + # and running Trio code. + thread_start = threading.Event() + + def log_nursery_exc(exc): + import logging + import traceback + exc = '\n'.join(traceback.format_exception(type(exc), exc, + exc.__traceback__)) + logging.error('An exception occurred in a global nursery task.\n%s', + exc) async def trio_entry(): global _TRIO_TOKEN, _TRIO_NURSERY _TRIO_TOKEN = trio.hazmat.current_trio_token() async with trio.open_nursery() as nursery: - _TRIO_NURSERY = nursery + # TODO This hack prevents the nursery from cancelling all child + # tasks when but it's ugly. + nursery._add_exc = log_nursery_exc builtins.GLOBAL_NURSERY = nursery + thread_start.set() await trio.sleep_forever() - def trio_entry_sync(): - while True: - try: - trio.run(trio_entry) - except Exception: - print("Exception in trio event loop:", traceback.format_exc()) - - threading.Thread(target=trio_entry_sync).start() - #TODO fix this race condition - import time - while not _TRIO_TOKEN: - time.sleep(0.1) + threading.Thread(target=trio.run, args=(trio_entry,)).start() + thread_start.wait() def _trio_runner(async_fn): + global _TRIO_TOKEN + import logging import trio if not _TRIO_TOKEN: From 0d29159e6d8a731ae44e9ce0dfeb4d76ae05c81d Mon Sep 17 00:00:00 2001 From: Terry Davis <16829776+terrdavis@users.noreply.github.com> Date: Mon, 20 Jan 2020 21:10:12 -0800 Subject: [PATCH 151/242] Adds support for blame ignore. Windows and non-Windows scripts are provided for initial setup. Commit hashes to ignore are entered in the file .git-blame-ignore-revs --- .git-blame-ignore-revs | 8 ++++++++ tools/configure-git-blame-ignore-revs.bat | 10 ++++++++++ tools/configure-git-blame-ignore-revs.sh | 10 ++++++++++ 3 files changed, 28 insertions(+) create mode 100644 .git-blame-ignore-revs create mode 100644 tools/configure-git-blame-ignore-revs.bat create mode 100644 tools/configure-git-blame-ignore-revs.sh diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000000..70971c53b5a --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,8 @@ +# When making commits that are strictly formatting/style changes, add the +# commit hash here, so git blame can ignore the change. +# See docs for more details: +# https://git-scm.com/docs/git-config#Documentation/git-config.txt-blameignoreRevsFile + +# Example entries: +# # initial black-format +# # rename something internal diff --git a/tools/configure-git-blame-ignore-revs.bat b/tools/configure-git-blame-ignore-revs.bat new file mode 100644 index 00000000000..3fc3c01dcf8 --- /dev/null +++ b/tools/configure-git-blame-ignore-revs.bat @@ -0,0 +1,10 @@ +rem Other config options for blame are markUnblamables and markIgnoredLines. +rem See docs for more details: +rem https://git-scm.com/docs/git-config#Documentation/git-config.txt-blameignoreRevsFile + +rem Uncomment below and rerun script to enable an option. +rem git config blame.markIgnoredLines +rem git config blame.markUnblamables + +git config blame.ignoreRevsFile .git-blame-ignore-revs +git config --get blame.ignoreRevsFile diff --git a/tools/configure-git-blame-ignore-revs.sh b/tools/configure-git-blame-ignore-revs.sh new file mode 100644 index 00000000000..e1892ef4d05 --- /dev/null +++ b/tools/configure-git-blame-ignore-revs.sh @@ -0,0 +1,10 @@ +# Other config options for blame are markUnblamables and markIgnoredLines. +# See docs for more details: +# https://git-scm.com/docs/git-config#Documentation/git-config.txt-blameignoreRevsFile + +# Uncomment below and rerun script to enable an option. +# git config blame.markIgnoredLines +# git config blame.markUnblamables + +git config blame.ignoreRevsFile .git-blame-ignore-revs +git config --get blame.ignoreRevsFile From 2cfa2568e26b50179befe20c23e00be5b5bab2ae Mon Sep 17 00:00:00 2001 From: Terry Davis <16829776+terrdavis@users.noreply.github.com> Date: Mon, 20 Jan 2020 21:11:18 -0800 Subject: [PATCH 152/242] Add section to README about enabling blame ignore revs. --- README.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.rst b/README.rst index 3e13610a443..9c0c89ec8e0 100644 --- a/README.rst +++ b/README.rst @@ -129,3 +129,18 @@ project that you might want to use: - `mypython `_ - `ptpython and ptipython ` - `xonsh ` + +Ignoring commits with git blame.ignoreRevsFile +============================================== + +As of git 2.23, it is possible to make formatting changes without breaking +``git blame``. See the `git documentation +`_ +for more details. + +To use this feature you must: + +- Install git >= 2.23 +- Configure your local git repo by running: + - POSIX: ``tools\configure-git-blame-ignore-revs.sh`` + - Windows: ``tools\configure-git-blame-ignore-revs.bat`` From 24cb49b3c9e6190f56330b7b0967932f16e3d336 Mon Sep 17 00:00:00 2001 From: yangyang Date: Tue, 21 Jan 2020 16:02:21 +0800 Subject: [PATCH 153/242] catch `MemoryError` in compiling --- IPython/core/async_helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/IPython/core/async_helpers.py b/IPython/core/async_helpers.py index 1a7d88991db..fb4cc193250 100644 --- a/IPython/core/async_helpers.py +++ b/IPython/core/async_helpers.py @@ -146,20 +146,20 @@ def _should_be_async(cell: str) -> bool: If it works, assume it should be async. Otherwise Return False. - Not handled yet: If the block of code has a return statement as the top + Not handled yet: If the block of code has a return statement as the top level, it will be seen as async. This is a know limitation. """ if sys.version_info > (3, 8): try: code = compile(cell, "<>", "exec", flags=getattr(ast,'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0)) return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE - except SyntaxError: + except (SyntaxError, MemoryError): return False try: # we can't limit ourself to ast.parse, as it __accepts__ to parse on # 3.7+, but just does not _compile_ code = compile(cell, "<>", "exec") - except SyntaxError: + except (SyntaxError, MemoryError): try: parse_tree = _async_parse_cell(cell) @@ -167,7 +167,7 @@ def _should_be_async(cell: str) -> bool: v = _AsyncSyntaxErrorVisitor() v.visit(parse_tree) - except SyntaxError: + except (SyntaxError, MemoryError): return False return True return False From 4527b9ca0edb17e6c3d454f566cadbc78e91a7fc Mon Sep 17 00:00:00 2001 From: yangyang Date: Tue, 21 Jan 2020 16:30:14 +0800 Subject: [PATCH 154/242] add test --- IPython/core/tests/test_async_helpers.py | 4 ++++ IPython/core/tests/test_ultratb.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/IPython/core/tests/test_async_helpers.py b/IPython/core/tests/test_async_helpers.py index 2932189c63a..86a516ccc1f 100644 --- a/IPython/core/tests/test_async_helpers.py +++ b/IPython/core/tests/test_async_helpers.py @@ -276,6 +276,10 @@ def test_autoawait(self): """ ) + def test_memory_error(self): + with self.assertRaises(MemoryError): + iprc("(" * 200 + ")" * 200) + @skip_without('curio') def test_autoawait_curio(self): iprc("%autoawait curio") diff --git a/IPython/core/tests/test_ultratb.py b/IPython/core/tests/test_ultratb.py index c00b4d30afe..3ab0ce3cf00 100644 --- a/IPython/core/tests/test_ultratb.py +++ b/IPython/core/tests/test_ultratb.py @@ -253,6 +253,13 @@ def test_non_syntaxerror(self): ip.showsyntaxerror() +class MemoryErrorTest(unittest.TestCase): + def test_memoryerror(self): + memoryerror_code = "(" * 200 + ")" * 200 + with tt.AssertPrints("MemoryError"): + ip.run_cell(memoryerror_code) + + class Python3ChainedExceptionsTest(unittest.TestCase): DIRECT_CAUSE_ERROR_CODE = """ try: From 47d58f4213b8d1a898e1508d966f77b5b58b6e71 Mon Sep 17 00:00:00 2001 From: "Mark E. Haase" Date: Fri, 24 Jan 2020 15:36:27 -0500 Subject: [PATCH 155/242] More work on Trio background loop Revert changes to _async_helpers.py. Add a new _trio_runner property to the `InteractiveShell` class for use with a PR on the `ipykernel` repo that populates this property. Co-authored-by: Brian Mackintosh --- IPython/core/async_helpers.py | 42 +------------------------------- IPython/core/interactiveshell.py | 14 ++++++++++- 2 files changed, 14 insertions(+), 42 deletions(-) diff --git a/IPython/core/async_helpers.py b/IPython/core/async_helpers.py index 6c0f9bacdbe..1a7d88991db 100644 --- a/IPython/core/async_helpers.py +++ b/IPython/core/async_helpers.py @@ -42,49 +42,9 @@ def _curio_runner(coroutine): return curio.run(coroutine) -_TRIO_TOKEN = None - - -def _init_trio(trio): - global _TRIO_TOKEN - import traceback - import builtins - import threading - # We use an Event to avoid a race condition between starting the Trio thread - # and running Trio code. - thread_start = threading.Event() - - def log_nursery_exc(exc): - import logging - import traceback - exc = '\n'.join(traceback.format_exception(type(exc), exc, - exc.__traceback__)) - logging.error('An exception occurred in a global nursery task.\n%s', - exc) - - async def trio_entry(): - global _TRIO_TOKEN, _TRIO_NURSERY - _TRIO_TOKEN = trio.hazmat.current_trio_token() - async with trio.open_nursery() as nursery: - # TODO This hack prevents the nursery from cancelling all child - # tasks when but it's ugly. - nursery._add_exc = log_nursery_exc - builtins.GLOBAL_NURSERY = nursery - thread_start.set() - await trio.sleep_forever() - - threading.Thread(target=trio.run, args=(trio_entry,)).start() - thread_start.wait() - - def _trio_runner(async_fn): - global _TRIO_TOKEN - import logging import trio - if not _TRIO_TOKEN: - _init_trio(trio) - async def loc(coro): """ We need the dummy no-op async def to protect from @@ -92,7 +52,7 @@ async def loc(coro): """ return await coro - return trio.from_thread.run(loc, async_fn, trio_token=_TRIO_TOKEN) + return trio.run(loc, async_fn) def _pseudo_sync_runner(coro): diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 6a4fc43efb9..4cbc885cb29 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -695,6 +695,13 @@ def __init__(self, ipython_dir=None, profile_dir=None, self.events.trigger('shell_initialized', self) atexit.register(self.atexit_operations) + # The trio runner is used for running Trio in the foreground thread. It + # is different from `_trio_runner(async_fn)` in `async_helpers.py` + # which calls `trio.run()` for every cell. This runner runs all cells + # inside a single Trio event loop. If used, it is set from + # `ipykernel.kernelapp`. + self.trio_runner = None + def get_ipython(self): """Return the currently running IPython instance.""" return self @@ -715,6 +722,9 @@ def set_autoindent(self,value=None): else: self.autoindent = value + def set_trio_runner(self, tr): + self.trio_runner = tr + #------------------------------------------------------------------------- # init_* methods called by __init__ #------------------------------------------------------------------------- @@ -2865,7 +2875,9 @@ def _run_cell(self, raw_cell:str, store_history:bool, silent:bool, shell_futures # when this is the case, we want to run it using the pseudo_sync_runner # so that code can invoke eventloops (for example via the %run , and # `%paste` magic. - if self.should_run_async(raw_cell) or self.loop_runner is _trio_runner: + if self.trio_runner: + runner = self.trio_runner + elif self.should_run_async(raw_cell): runner = self.loop_runner else: runner = _pseudo_sync_runner From a4e12b3d6861b726fd7a2869b1dac5f9b85e1fa6 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 27 Jan 2020 15:07:59 +0000 Subject: [PATCH 156/242] Start multi-line __repr__s on their own line Fixes gh-12098 --- IPython/lib/pretty.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/IPython/lib/pretty.py b/IPython/lib/pretty.py index 49c76ad902e..d58779cd2f7 100644 --- a/IPython/lib/pretty.py +++ b/IPython/lib/pretty.py @@ -687,7 +687,11 @@ def _repr_pprint(obj, p, cycle): """A pprint that just redirects to the normal repr function.""" # Find newlines and replace them with p.break_() output = repr(obj) - for idx,output_line in enumerate(output.splitlines()): + lines = output.splitlines() + # insert a leading newline for multi-line objects that are indented + if len(lines) > 1 and p.indentation != p.output_width: + p.break_() + for idx, output_line in enumerate(lines): if idx: p.break_() p.text(output_line) From 16865e0fe7123de7db84122246a5a005043a8774 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 27 Jan 2020 15:22:49 +0000 Subject: [PATCH 157/242] Simplify and extend the test for multiline repr `BreakingReprParent` is not really necessary here, testing using lists is easier to understand, and lets us test multiple lengths of list. --- IPython/lib/tests/test_pretty.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/IPython/lib/tests/test_pretty.py b/IPython/lib/tests/test_pretty.py index cd9fb3662f2..695012d6fc9 100644 --- a/IPython/lib/tests/test_pretty.py +++ b/IPython/lib/tests/test_pretty.py @@ -68,11 +68,6 @@ class BreakingRepr(object): def __repr__(self): return "Breaking(\n)" -class BreakingReprParent(object): - def _repr_pretty_(self, p, cycle): - with p.group(4,"TG: ",":"): - p.pretty(BreakingRepr()) - class BadRepr(object): def __repr__(self): @@ -151,8 +146,12 @@ def test_pprint_break_repr(): """ Test that p.break_ is used in repr """ - output = pretty.pretty(BreakingReprParent()) - expected = "TG: Breaking(\n ):" + output = pretty.pretty([[BreakingRepr()]]) + expected = "[[Breaking(\n )]]" + nt.assert_equal(output, expected) + + output = pretty.pretty([[BreakingRepr()]*2]) + expected = "[[Breaking(\n ),\n Breaking(\n )]]" nt.assert_equal(output, expected) def test_bad_repr(): From 5741513a995863bd5677407d4571d24d07fbe9d7 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 27 Jan 2020 17:10:43 +0000 Subject: [PATCH 158/242] Try and use groups instead --- IPython/lib/pretty.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/IPython/lib/pretty.py b/IPython/lib/pretty.py index d58779cd2f7..7c3bffe9cf8 100644 --- a/IPython/lib/pretty.py +++ b/IPython/lib/pretty.py @@ -196,19 +196,22 @@ def __init__(self, output, max_width=79, newline='\n', max_seq_length=MAX_SEQ_LE self.group_queue = GroupQueue(root_group) self.indentation = 0 - def _break_outer_groups(self): + def _break_one_group(self, group): + while group.breakables: + x = self.buffer.popleft() + self.output_width = x.output(self.output, self.output_width) + self.buffer_width -= x.width + while self.buffer and isinstance(self.buffer[0], Text): + x = self.buffer.popleft() + self.output_width = x.output(self.output, self.output_width) + self.buffer_width -= x.width + + def _break_outer_groups(self, force=): while self.max_width < self.output_width + self.buffer_width: group = self.group_queue.deq() if not group: return - while group.breakables: - x = self.buffer.popleft() - self.output_width = x.output(self.output, self.output_width) - self.buffer_width -= x.width - while self.buffer and isinstance(self.buffer[0], Text): - x = self.buffer.popleft() - self.output_width = x.output(self.output, self.output_width) - self.buffer_width -= x.width + self._break_one_group(group) def text(self, obj): """Add literal text to the output.""" @@ -248,6 +251,9 @@ def break_(self): """ Explicitly insert a newline into the output, maintaining correct indentation. """ + group = self.group_queue.deq() + if group: + self._break_one_group(group) self.flush() self.output.write(self.newline) self.output.write(' ' * self.indentation) @@ -688,13 +694,11 @@ def _repr_pprint(obj, p, cycle): # Find newlines and replace them with p.break_() output = repr(obj) lines = output.splitlines() - # insert a leading newline for multi-line objects that are indented - if len(lines) > 1 and p.indentation != p.output_width: - p.break_() - for idx, output_line in enumerate(lines): - if idx: - p.break_() - p.text(output_line) + with p.group(): + for idx, output_line in enumerate(lines): + if idx: + p.break_() + p.text(output_line) def _function_pprint(obj, p, cycle): From deb53640dc4239b70ef95601e03224df0de28917 Mon Sep 17 00:00:00 2001 From: Augusto Date: Wed, 29 Jan 2020 23:24:35 -0300 Subject: [PATCH 159/242] The fix_error_editor hook was deprecated in version 7.0, and no complaints have been raised as of today: https://github.com/ipython/ipython/issues/9649 --- IPython/core/hooks.py | 39 --------------------------------------- 1 file changed, 39 deletions(-) diff --git a/IPython/core/hooks.py b/IPython/core/hooks.py index b0637d8c154..fa732f7ba82 100644 --- a/IPython/core/hooks.py +++ b/IPython/core/hooks.py @@ -37,7 +37,6 @@ def load_ipython_extension(ip): import os import subprocess -import warnings import sys from .error import TryNext @@ -82,44 +81,6 @@ def editor(self, filename, linenum=None, wait=True): if wait and proc.wait() != 0: raise TryNext() -import tempfile -from ..utils.decorators import undoc - -@undoc -def fix_error_editor(self,filename,linenum,column,msg): - """DEPRECATED - - Open the editor at the given filename, linenumber, column and - show an error message. This is used for correcting syntax errors. - The current implementation only has special support for the VIM editor, - and falls back on the 'editor' hook if VIM is not used. - - Call ip.set_hook('fix_error_editor',yourfunc) to use your own function, - """ - - warnings.warn(""" -`fix_error_editor` is deprecated as of IPython 6.0 and will be removed -in future versions. It appears to be used only for automatically fixing syntax -error that has been broken for a few years and has thus been removed. If you -happened to use this function and still need it please make your voice heard on -the mailing list ipython-dev@python.org , or on the GitHub Issue tracker: -https://github.com/ipython/ipython/issues/9649 """, UserWarning) - - def vim_quickfix_file(): - t = tempfile.NamedTemporaryFile() - t.write('%s:%d:%d:%s\n' % (filename,linenum,column,msg)) - t.flush() - return t - if os.path.basename(self.editor) != 'vim': - self.hooks.editor(filename,linenum) - return - t = vim_quickfix_file() - try: - if os.system('vim --cmd "set errorformat=%f:%l:%c:%m" -q ' + t.name): - raise TryNext() - finally: - t.close() - def synchronize_with_editor(self, filename, linenum, column): pass From 42b7b7c21ca0aef8ef1e7d6aa136ae51ecfbefa0 Mon Sep 17 00:00:00 2001 From: ras44 <9282281+ras44@users.noreply.github.com> Date: Fri, 31 Jan 2020 09:34:52 +0100 Subject: [PATCH 160/242] Escape character now displays on readthedocs --- docs/source/interactive/reference.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/interactive/reference.rst b/docs/source/interactive/reference.rst index 08ac1867437..ccd37733252 100644 --- a/docs/source/interactive/reference.rst +++ b/docs/source/interactive/reference.rst @@ -77,8 +77,8 @@ prompt. What follows is a list of these. Caution for Windows users ------------------------- -Windows, unfortunately, uses the '\\' character as a path separator. This is a -terrible choice, because '\\' also represents the escape character in most +Windows, unfortunately, uses the ``\`` character as a path separator. This is a +terrible choice, because ``\`` also represents the escape character in most modern programming languages, including Python. For this reason, using '/' character is recommended if you have problems with ``\``. However, in Windows commands '/' flags options, so you can not use it for the root directory. This From cddef04ae82729b23297bc499f4a68e61eb1ac53 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Fri, 31 Jan 2020 20:49:03 +0000 Subject: [PATCH 161/242] Update IPython/lib/pretty.py --- IPython/lib/pretty.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/lib/pretty.py b/IPython/lib/pretty.py index 7c3bffe9cf8..50cec7006b1 100644 --- a/IPython/lib/pretty.py +++ b/IPython/lib/pretty.py @@ -206,7 +206,7 @@ def _break_one_group(self, group): self.output_width = x.output(self.output, self.output_width) self.buffer_width -= x.width - def _break_outer_groups(self, force=): + def _break_outer_groups(self): while self.max_width < self.output_width + self.buffer_width: group = self.group_queue.deq() if not group: From 7d815833908d778d5d9fc1ac342dc69ff3c4aab4 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 31 Jan 2020 13:43:30 -0800 Subject: [PATCH 162/242] Exclude .git-blame-ignore-revs to fix CI .git-blame-ignore-revs is added so that we can ignore revisions in .git-blame, mostly code reformatting; but CI check that all files are explicitly included (or excluded) from manifest; so we need to list this one explicitly for travis to succeed. --- MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index dee45a2c408..d47a16d42f8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -24,6 +24,8 @@ graft docs exclude docs/\#* exclude docs/man/*.1.gz +exclude .git-blame-ignore-revs + # Examples graft examples From 53f23d4d5fe3f39405d43e9d1c52ba16ffafe349 Mon Sep 17 00:00:00 2001 From: Terry Davis <16829776+terrdavis@users.noreply.github.com> Date: Mon, 20 Jan 2020 22:06:26 -0800 Subject: [PATCH 163/242] Remove unused sys & fallback pathlib (3.5+) imports. --- IPython/lib/tests/test_display.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/IPython/lib/tests/test_display.py b/IPython/lib/tests/test_display.py index 533bb46d059..7e98a1858f4 100644 --- a/IPython/lib/tests/test_display.py +++ b/IPython/lib/tests/test_display.py @@ -14,11 +14,7 @@ #----------------------------------------------------------------------------- from tempfile import NamedTemporaryFile, mkdtemp from os.path import split, join as pjoin, dirname -import sys -try: - import pathlib -except ImportError: - pass +import pathlib from unittest import TestCase, mock import struct import wave From 5935424486087b165ba795ee7cdb28a5bef139e3 Mon Sep 17 00:00:00 2001 From: Terry Davis <16829776+terrdavis@users.noreply.github.com> Date: Mon, 20 Jan 2020 22:25:14 -0800 Subject: [PATCH 164/242] Remove python 2 only import in IPython.core.ultratb --- IPython/core/ultratb.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index e1f805738a0..0caf7ec9758 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -101,10 +101,7 @@ import tokenize import traceback -try: # Python 2 - generate_tokens = tokenize.generate_tokens -except AttributeError: # Python 3 - generate_tokens = tokenize.tokenize +generate_tokens = tokenize.tokenize # For purposes of monkeypatching inspect to fix a bug in it. from inspect import getsourcefile, getfile, getmodule, \ From 38119d16744abe88f8ccac7f38d7bf5898db1388 Mon Sep 17 00:00:00 2001 From: Terry Davis <16829776+terrdavis@users.noreply.github.com> Date: Mon, 20 Jan 2020 22:25:44 -0800 Subject: [PATCH 165/242] Remove python 2 specific line. --- examples/IPython Kernel/ipython-get-history.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/IPython Kernel/ipython-get-history.py b/examples/IPython Kernel/ipython-get-history.py index 561584253a9..5e68bf5c041 100755 --- a/examples/IPython Kernel/ipython-get-history.py +++ b/examples/IPython Kernel/ipython-get-history.py @@ -35,5 +35,4 @@ hist = HistoryAccessor() for session, lineno, cell in hist.get_range(session=session_number, raw=raw): - cell = cell.encode('utf-8') # This line is only needed on Python 2. dest.write(cell + '\n') From 5618e599241e06ee4d731d9368ddafbedc9adc34 Mon Sep 17 00:00:00 2001 From: Terry Davis <16829776+terrdavis@users.noreply.github.com> Date: Mon, 20 Jan 2020 22:29:18 -0800 Subject: [PATCH 166/242] Remove python 2 references from docstrings, where appropriate. --- IPython/core/oinspect.py | 3 +-- IPython/lib/deepreload.py | 8 +------- IPython/lib/lexers.py | 2 +- IPython/lib/pretty.py | 13 +------------ IPython/utils/encoding.py | 2 +- IPython/utils/openpy.py | 6 ++---- 6 files changed, 7 insertions(+), 27 deletions(-) diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index c27d8b31d8c..ab25eeeffca 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -198,8 +198,7 @@ def is_simple_callable(obj): @undoc def getargspec(obj): - """Wrapper around :func:`inspect.getfullargspec` on Python 3, and - :func:inspect.getargspec` on Python 2. + """Wrapper around :func:`inspect.getfullargspec` In addition to functions and methods, this can also handle objects with a ``__call__`` attribute. diff --git a/IPython/lib/deepreload.py b/IPython/lib/deepreload.py index 37dd21abb90..bd8c01b2a75 100644 --- a/IPython/lib/deepreload.py +++ b/IPython/lib/deepreload.py @@ -7,13 +7,7 @@ imported from that module, which is useful when you're changing files deep inside a package. -To use this as your default reload function, type this for Python 2:: - - import __builtin__ - from IPython.lib import deepreload - __builtin__.reload = deepreload.reload - -Or this for Python 3:: +To use this as your default reload function, type this:: import builtins from IPython.lib import deepreload diff --git a/IPython/lib/lexers.py b/IPython/lib/lexers.py index 1677054f78e..4494da56571 100644 --- a/IPython/lib/lexers.py +++ b/IPython/lib/lexers.py @@ -129,7 +129,7 @@ class IPythonPartialTracebackLexer(RegexLexer): """ Partial lexer for IPython tracebacks. - Handles all the non-python output. This works for both Python 2.x and 3.x. + Handles all the non-python output. """ name = 'IPython Partial Traceback' diff --git a/IPython/lib/pretty.py b/IPython/lib/pretty.py index 50cec7006b1..cc68781ba16 100644 --- a/IPython/lib/pretty.py +++ b/IPython/lib/pretty.py @@ -263,18 +263,7 @@ def break_(self): def begin_group(self, indent=0, open=''): """ - Begin a group. If you want support for python < 2.5 which doesn't has - the with statement this is the preferred way: - - p.begin_group(1, '{') - ... - p.end_group(1, '}') - - The python 2.5 expression would be this: - - with p.group(1, '{', '}'): - ... - + Begin a group. The first parameter specifies the indentation for the next line (usually the width of the opening text), the second the opening text. All parameters are optional. diff --git a/IPython/utils/encoding.py b/IPython/utils/encoding.py index 387a24700cf..69a319ef0ef 100644 --- a/IPython/utils/encoding.py +++ b/IPython/utils/encoding.py @@ -44,7 +44,7 @@ def getdefaultencoding(prefer_stream=True): Then fall back on locale.getpreferredencoding(), which should be a sensible platform default (that respects LANG environment), and finally to sys.getdefaultencoding() which is the most conservative option, - and usually ASCII on Python 2 or UTF8 on Python 3. + and usually UTF8 as of Python 3. """ enc = None if prefer_stream: diff --git a/IPython/utils/openpy.py b/IPython/utils/openpy.py index 3046c31587d..c90d2b53a30 100644 --- a/IPython/utils/openpy.py +++ b/IPython/utils/openpy.py @@ -66,8 +66,7 @@ def read_py_file(filename, skip_encoding_cookie=True): The path to the file to read. skip_encoding_cookie : bool If True (the default), and the encoding declaration is found in the first - two lines, that line will be excluded from the output - compiling a - unicode string with an encoding declaration is a SyntaxError in Python 2. + two lines, that line will be excluded from the output. Returns ------- @@ -91,8 +90,7 @@ def read_py_url(url, errors='replace', skip_encoding_cookie=True): bytes.decode(), but here 'replace' is the default. skip_encoding_cookie : bool If True (the default), and the encoding declaration is found in the first - two lines, that line will be excluded from the output - compiling a - unicode string with an encoding declaration is a SyntaxError in Python 2. + two lines, that line will be excluded from the output. Returns ------- From 51cd68a4315a740602b13fefe70aefca6d09f2ec Mon Sep 17 00:00:00 2001 From: Terry Davis <16829776+terrdavis@users.noreply.github.com> Date: Mon, 20 Jan 2020 22:38:33 -0800 Subject: [PATCH 167/242] Remove python 2 only long and unicode type references. --- IPython/lib/pretty.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/IPython/lib/pretty.py b/IPython/lib/pretty.py index cc68781ba16..3115d3f6dbf 100644 --- a/IPython/lib/pretty.py +++ b/IPython/lib/pretty.py @@ -764,12 +764,8 @@ def _exception_pprint(obj, p, cycle): _dict_pprinter_factory('mappingproxy({', '})') _type_pprinters[slice] = _repr_pprint -try: - _type_pprinters[long] = _repr_pprint - _type_pprinters[unicode] = _repr_pprint -except NameError: - _type_pprinters[range] = _repr_pprint - _type_pprinters[bytes] = _repr_pprint +_type_pprinters[range] = _repr_pprint +_type_pprinters[bytes] = _repr_pprint #: printers for types specified by name _deferred_type_pprinters = { From 73e3738540026fd4549b3571cdafa3bcd062ec54 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 31 Jan 2020 17:25:07 -0800 Subject: [PATCH 168/242] update what's new for 7.12 --- docs/source/whatsnew/version7.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 7a77cac5750..e9d4ef3ce66 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,23 @@ 7.x Series ============ +.. _version 712: + +IPython 7.12 +============ + +IPython 7.12 is a minor update that mostly brings code cleanup, removal of +longtime deprecated function and a couple update to documentation cleanup as well. + +Notable changes are the following: + + - Exit non-zero when ipython is given a file path to run that doesn't exist :ghpull:`12074` + - Test PR on ARM64 with Travis-CI :ghpull:`12073` + - Update CI to work with latest Pytest :ghpull:`12086` + - Add infrastructure to run ipykernel eventloop via trio :ghpull:`12097` + - Support git blame ignore revs :ghpull:`12091` + - Start multi-line ``__repr__`` s on their own line :ghpull:`12099` + .. _version 7111: IPython 7.11.1 From b6c74215118fef50f0cd251bd37b0118568604af Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 31 Jan 2020 17:35:55 -0800 Subject: [PATCH 169/242] update stats --- docs/source/whatsnew/github-stats-7.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/source/whatsnew/github-stats-7.rst b/docs/source/whatsnew/github-stats-7.rst index dd366d8ad9c..c2abed72017 100644 --- a/docs/source/whatsnew/github-stats-7.rst +++ b/docs/source/whatsnew/github-stats-7.rst @@ -1,6 +1,30 @@ Issues closed in the 7.x development cycle ========================================== +Issues closed in 7.12 +--------------------- + +GitHub stats for 2020/01/01 - 2020/01/31 (tag: 7.11.1) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 2 issues and merged 14 pull requests. +The full list can be seen `on GitHub `__ + +The following 11 authors contributed 48 commits. + +* Augusto +* Eric Wieser +* Jeff Potter +* Mark E. Haase +* Matthias Bussonnier +* ossdev07 +* ras44 +* takuya fujiwara +* Terry Davis +* Thomas A Caswell +* yangyang + Issues closed in 7.11 --------------------- From 8bda98619aba4bf0aa75d92991c3f7d240d1c781 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 31 Jan 2020 17:37:48 -0800 Subject: [PATCH 170/242] release 7.12.0 --- IPython/core/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 0fe4a422092..b5b9c28227f 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -24,7 +24,7 @@ _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From e40c4012fa1ed4087dd3b55a4403b87828b02f23 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 31 Jan 2020 17:38:53 -0800 Subject: [PATCH 171/242] back to dev --- IPython/core/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index b5b9c28227f..feb09c0e335 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 12 +_version_minor = 13 _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 21073ffabed5ce2f676dbbcd5dde41c2933e9be4 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 1 Feb 2020 20:05:11 +0100 Subject: [PATCH 172/242] Speedup _make_wav When numpy is installed, ~30x speedup; otherwise, ~5x speedup --- IPython/lib/display.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/IPython/lib/display.py b/IPython/lib/display.py index e7c427cbe91..de31788ab97 100644 --- a/IPython/lib/display.py +++ b/IPython/lib/display.py @@ -130,7 +130,6 @@ def reload(self): @staticmethod def _make_wav(data, rate, normalize): """ Transform a numpy array to a PCM bytestring """ - import struct from io import BytesIO import wave @@ -145,7 +144,7 @@ def _make_wav(data, rate, normalize): waveobj.setframerate(rate) waveobj.setsampwidth(2) waveobj.setcomptype('NONE','NONE') - waveobj.writeframes(b''.join([struct.pack(' Date: Sat, 1 Feb 2020 14:02:36 -0800 Subject: [PATCH 173/242] Attempt to fix test failure with generate_tokens It looks like the try/except comments were wrong and generate_tokens is available on Py3 --- IPython/core/ultratb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index 0caf7ec9758..9e7e8fb390d 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -101,7 +101,7 @@ import tokenize import traceback -generate_tokens = tokenize.tokenize +from tokenize import generate_tokens # For purposes of monkeypatching inspect to fix a bug in it. from inspect import getsourcefile, getfile, getmodule, \ From 108fec78d74c91b7271fd791985816a2127032a6 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 6 Feb 2020 18:10:25 -0800 Subject: [PATCH 174/242] Add tool to automatically upgrade a file. Push syntax to 36+, apply black; do the commits and add those comits to git-ignore-revs. --- tools/autoformat_file | 46 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100755 tools/autoformat_file diff --git a/tools/autoformat_file b/tools/autoformat_file new file mode 100755 index 00000000000..3b6b71d3007 --- /dev/null +++ b/tools/autoformat_file @@ -0,0 +1,46 @@ +#!/bin/bash + +set -ueo pipefail +FILE=$1 + +echo "will update $FILE" + +echo $LINENO $? +pyupgrade --py36-plus --exit-zero-even-if-changed $FILE + +echo $LINENO $? +git commit -am"Apply pyupgrade to $FILE + + pyupgrade --py36-plus $FILE + +To ignore those changes when using git blame see the content of +.git-blame-ignore-revs" + +HASH=$(git rev-parse HEAD) + +echo "$HASH # apply pyupgrade to $FILE" >> .git-blame-ignore-revs + +git commit -am'Update .git-blame-ignore-revs with previous commit' + +##### + +black --target-version py36 $FILE + + +git commit -am"Apply black to $FILE + + black --target-version py36 $FILE + +To ignore those changes when using git blame see the content of +.git-blame-ignore-revs" + +HASH=$(git rev-parse HEAD) + +echo "$HASH # apply black to $FILE" >> .git-blame-ignore-revs + +git commit -am'Update .git-blame-ignore-revs with previous commit' + +echo +echo "Updating, reformatting and adding to .git-blame-ignore-revs successful" + + From c69c6b4a89be29df0a047bfd47309c9b1619a398 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 12 Feb 2020 14:52:27 -0800 Subject: [PATCH 175/242] Backport PR #12125: Revert "Use update brew tap to fix test failures." --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cccdfb3db14..a70f00a228a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ before_install: if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then env | sort if ! which python$TRAVIS_PYTHON_VERSION; then - HOMEBREW_NO_AUTO_UPDATE=1 brew tap carreau/homebrew-python-frameworks + HOMEBREW_NO_AUTO_UPDATE=1 brew tap minrk/homebrew-python-frameworks HOMEBREW_NO_AUTO_UPDATE=1 brew cask install python-framework-${TRAVIS_PYTHON_VERSION/./} fi python3 -m pip install virtualenv From b29d90be695c3a565e1df92c7bae4d89e2adcb00 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 13 Feb 2020 09:39:03 -0800 Subject: [PATCH 176/242] Backport PR #12122: Pop the last if path ends with slash --- IPython/terminal/ptutils.py | 2 ++ IPython/terminal/tests/test_interactivshell.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/IPython/terminal/ptutils.py b/IPython/terminal/ptutils.py index 4f21cb04e35..47fd6f40612 100644 --- a/IPython/terminal/ptutils.py +++ b/IPython/terminal/ptutils.py @@ -42,6 +42,8 @@ def _elide(string, *, min_elide=30): object_parts = string.split('.') file_parts = string.split(os.sep) + if file_parts[-1] == '': + file_parts.pop() if len(object_parts) > 3: return '{}.{}\N{HORIZONTAL ELLIPSIS}{}.{}'.format(object_parts[0], object_parts[1][0], object_parts[-2][-1], object_parts[-1]) diff --git a/IPython/terminal/tests/test_interactivshell.py b/IPython/terminal/tests/test_interactivshell.py index 9651a224dad..6bacc8ea48a 100644 --- a/IPython/terminal/tests/test_interactivshell.py +++ b/IPython/terminal/tests/test_interactivshell.py @@ -5,6 +5,7 @@ import sys import unittest +import os from IPython.core.inputtransformer import InputTransformer from IPython.testing import tools as tt @@ -19,6 +20,10 @@ def test_elide(self): _elide('concatenate((a1, a2, ...), axis') # do not raise _elide('concatenate((a1, a2, ..), . axis') # do not raise nt.assert_equal(_elide('aaaa.bbbb.ccccc.dddddd.eeeee.fffff.gggggg.hhhhhh'), 'aaaa.b…g.hhhhhh') + + test_string = os.sep.join(['', 10*'a', 10*'b', 10*'c', '']) + expect_stirng = os.sep + 'a' + '\N{HORIZONTAL ELLIPSIS}' + 'b' + os.sep + 10*'c' + nt.assert_equal(_elide(test_string), expect_stirng) class TestContextAwareCompletion(unittest.TestCase): From 7f97e7bbc218dd6a931461efa09fc35636074e63 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 27 Feb 2020 08:17:18 -0800 Subject: [PATCH 177/242] Backport PR #12130: Ensure set_custom_completer is working. Fixes #11272 --- .travis.yml | 2 +- IPython/core/completer.py | 4 ++++ IPython/core/interactiveshell.py | 2 +- IPython/core/tests/test_interactiveshell.py | 18 ++++++++++++++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a70f00a228a..9bb3a9bb0a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -72,7 +72,7 @@ matrix: - arch: arm64 python: "3.7" dist: xenial - env: ARM64=True + env: ARM64=True IPYTHON_TESTING_TIMEOUT_SCALE=2 sudo: true - arch: amd64 python: "3.8-dev" diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 0942798f3b3..9a81d637f15 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -626,6 +626,8 @@ def __init__(self, namespace=None, global_namespace=None, **kwargs): else: self.global_namespace = global_namespace + self.custom_matchers = [] + super(Completer, self).__init__(**kwargs) def complete(self, text, state): @@ -1122,12 +1124,14 @@ def matchers(self): if self.use_jedi: return [ + *self.custom_matchers, self.file_matches, self.magic_matches, self.dict_key_matches, ] else: return [ + *self.custom_matchers, self.python_matches, self.file_matches, self.magic_matches, diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 4cbc885cb29..848a14a481c 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2218,7 +2218,7 @@ def set_custom_completer(self, completer, pos=0): list where you want the completer to be inserted.""" newcomp = types.MethodType(completer,self.Completer) - self.Completer.matchers.insert(pos,newcomp) + self.Completer.custom_matchers.insert(pos,newcomp) def set_completer_frame(self, frame=None): """Set the frame of the completer.""" diff --git a/IPython/core/tests/test_interactiveshell.py b/IPython/core/tests/test_interactiveshell.py index 578ef27d36f..496e3bd02bc 100644 --- a/IPython/core/tests/test_interactiveshell.py +++ b/IPython/core/tests/test_interactiveshell.py @@ -998,3 +998,21 @@ def test_should_run_async(): assert not ip.should_run_async("a = 5") assert ip.should_run_async("await x") assert ip.should_run_async("import asyncio; await asyncio.sleep(1)") + + +def test_set_custom_completer(): + num_completers = len(ip.Completer.matchers) + + def foo(*args, **kwargs): + return "I'm a completer!" + + ip.set_custom_completer(foo, 0) + + # check that we've really added a new completer + assert len(ip.Completer.matchers) == num_completers + 1 + + # check that the first completer is the function we defined + assert ip.Completer.matchers[0]() == "I'm a completer!" + + # clean up + ip.Completer.custom_matchers.pop() From b0a77c49c76da5ec3646b5f1cb39913cc5f8055d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 27 Feb 2020 08:20:27 -0800 Subject: [PATCH 178/242] Backport PR #12132: Use default shortcuts with terminal debugger --- IPython/terminal/debugger.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/IPython/terminal/debugger.py b/IPython/terminal/debugger.py index 475c6b8ca94..030e4ae8689 100644 --- a/IPython/terminal/debugger.py +++ b/IPython/terminal/debugger.py @@ -6,7 +6,7 @@ from IPython.core.completer import IPCompleter from .ptutils import IPythonPTCompleter -from .shortcuts import suspend_to_bg, cursor_in_leading_ws +from .shortcuts import create_ipython_shortcuts, suspend_to_bg, cursor_in_leading_ws from prompt_toolkit.enums import DEFAULT_BUFFER from prompt_toolkit.filters import (Condition, has_focus, has_selection, @@ -42,21 +42,10 @@ def get_prompt_tokens(): ) self._ptcomp = IPythonPTCompleter(compl) - kb = KeyBindings() - supports_suspend = Condition(lambda: hasattr(signal, 'SIGTSTP')) - kb.add('c-z', filter=supports_suspend)(suspend_to_bg) - - if self.shell.display_completions == 'readlinelike': - kb.add('tab', filter=(has_focus(DEFAULT_BUFFER) - & ~has_selection - & vi_insert_mode | emacs_insert_mode - & ~cursor_in_leading_ws - ))(display_completions_like_readline) - options = dict( message=(lambda: PygmentsTokens(get_prompt_tokens())), editing_mode=getattr(EditingMode, self.shell.editing_mode.upper()), - key_bindings=kb, + key_bindings=create_ipython_shortcuts(self.shell), history=self.shell.debugger_history, completer=self._ptcomp, enable_history_search=True, From a4b22e27f9f07545f97940dc67a58efaa187f8cb Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 27 Feb 2020 08:20:44 -0800 Subject: [PATCH 179/242] Backport PR #12135: add configuration controlling whether the CWD gets added to sys.path --- IPython/core/shellapp.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/IPython/core/shellapp.py b/IPython/core/shellapp.py index 3147c59f9f6..33052c44d15 100644 --- a/IPython/core/shellapp.py +++ b/IPython/core/shellapp.py @@ -60,6 +60,10 @@ colours.""", "Disable using colors for info related things." ) +addflag('ignore-cwd', 'InteractiveShellApp.ignore_cwd', + "Exclude the current working directory from sys.path", + "Include the current working directory in sys.path", +) nosep_config = Config() nosep_config.InteractiveShell.separate_in = '' nosep_config.InteractiveShell.separate_out = '' @@ -168,6 +172,12 @@ class InteractiveShellApp(Configurable): When False, pylab mode should not import any names into the user namespace. """ ).tag(config=True) + ignore_cwd = Bool( + False, + help="""If True, IPython will not add the current working directory to sys.path. + When False, the current working directory is added to sys.path, allowing imports + of modules defined in the current directory.""" + ).tag(config=True) shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True) # whether interact-loop should start @@ -189,8 +199,10 @@ def init_path(self): .. versionchanged:: 7.2 Try to insert after the standard library, instead of first. + .. versionchanged:: 8.0 + Allow optionally not including the current directory in sys.path """ - if '' in sys.path: + if '' in sys.path or self.ignore_cwd: return for idx, path in enumerate(sys.path): parent, last_part = os.path.split(path) From 1c4899a271f404825675be43866adb915051a497 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 27 Feb 2020 08:24:25 -0800 Subject: [PATCH 180/242] Backport PR #12137: Fix inability to interrupt processes on Windows --- IPython/utils/_process_win32.py | 38 ++++++++++++++------- IPython/utils/tests/test_process.py | 53 ++++++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 13 deletions(-) diff --git a/IPython/utils/_process_win32.py b/IPython/utils/_process_win32.py index fb17e4c5d37..6d05bdaa12e 100644 --- a/IPython/utils/_process_win32.py +++ b/IPython/utils/_process_win32.py @@ -18,10 +18,12 @@ import os import sys import ctypes +import time from ctypes import c_int, POINTER from ctypes.wintypes import LPCWSTR, HLOCAL -from subprocess import STDOUT +from subprocess import STDOUT, TimeoutExpired +from threading import Thread # our own imports from ._process_common import read_no_interrupt, process_handler, arg_split as py_arg_split @@ -93,15 +95,29 @@ def _find_cmd(cmd): def _system_body(p): """Callback for _system.""" enc = DEFAULT_ENCODING - for line in read_no_interrupt(p.stdout).splitlines(): - line = line.decode(enc, 'replace') - print(line, file=sys.stdout) - for line in read_no_interrupt(p.stderr).splitlines(): - line = line.decode(enc, 'replace') - print(line, file=sys.stderr) - # Wait to finish for returncode - return p.wait() + def stdout_read(): + for line in read_no_interrupt(p.stdout).splitlines(): + line = line.decode(enc, 'replace') + print(line, file=sys.stdout) + + def stderr_read(): + for line in read_no_interrupt(p.stderr).splitlines(): + line = line.decode(enc, 'replace') + print(line, file=sys.stderr) + + Thread(target=stdout_read).start() + Thread(target=stderr_read).start() + + # Wait to finish for returncode. Unfortunately, Python has a bug where + # wait() isn't interruptible (https://bugs.python.org/issue28168) so poll in + # a loop instead of just doing `return p.wait()`. + while True: + result = p.poll() + if result is None: + time.sleep(0.01) + else: + return result def system(cmd): @@ -116,9 +132,7 @@ def system(cmd): Returns ------- - None : we explicitly do NOT return the subprocess status code, as this - utility is meant to be used extensively in IPython, where any return value - would trigger :func:`sys.displayhook` calls. + int : child process' exit code. """ # The controller provides interactivity with both # stdin and stdout diff --git a/IPython/utils/tests/test_process.py b/IPython/utils/tests/test_process.py index 0c81138cd19..fbc000c3eeb 100644 --- a/IPython/utils/tests/test_process.py +++ b/IPython/utils/tests/test_process.py @@ -15,13 +15,19 @@ #----------------------------------------------------------------------------- import sys +import signal import os +import time +from _thread import interrupt_main # Py 3 +import threading +from unittest import SkipTest import nose.tools as nt from IPython.utils.process import (find_cmd, FindCmdError, arg_split, system, getoutput, getoutputerror, get_output_error_code) +from IPython.utils.capture import capture_output from IPython.testing import decorators as dec from IPython.testing import tools as tt @@ -107,6 +113,49 @@ def test_system_quotes(self): status = system('%s -c "import sys"' % python) self.assertEqual(status, 0) + def assert_interrupts(self, command): + """ + Interrupt a subprocess after a second. + """ + if threading.main_thread() != threading.current_thread(): + raise nt.SkipTest("Can't run this test if not in main thread.") + + # Some tests can overwrite SIGINT handler (by using pdb for example), + # which then breaks this test, so just make sure it's operating + # normally. + signal.signal(signal.SIGINT, signal.default_int_handler) + + def interrupt(): + # Wait for subprocess to start: + time.sleep(0.5) + interrupt_main() + + threading.Thread(target=interrupt).start() + start = time.time() + try: + result = command() + except KeyboardInterrupt: + # Success! + pass + end = time.time() + self.assertTrue( + end - start < 2, "Process didn't die quickly: %s" % (end - start) + ) + return result + + def test_system_interrupt(self): + """ + When interrupted in the way ipykernel interrupts IPython, the + subprocess is interrupted. + """ + def command(): + return system('%s -c "import time; time.sleep(5)"' % python) + + status = self.assert_interrupts(command) + self.assertNotEqual( + status, 0, "The process wasn't interrupted. Status: %s" % (status,) + ) + def test_getoutput(self): out = getoutput('%s "%s"' % (python, self.fname)) # we can't rely on the order the line buffered streams are flushed @@ -131,7 +180,7 @@ def test_getoutput_error(self): out, err = getoutputerror('%s "%s"' % (python, self.fname)) self.assertEqual(out, 'on stdout') self.assertEqual(err, 'on stderr') - + def test_get_output_error_code(self): quiet_exit = '%s -c "import sys; sys.exit(1)"' % python out, err, code = get_output_error_code(quiet_exit) @@ -142,3 +191,5 @@ def test_get_output_error_code(self): self.assertEqual(out, 'on stdout') self.assertEqual(err, 'on stderr') self.assertEqual(code, 0) + + From 48f691cf26e4cd58bd79d96a34667dd5263639f4 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 27 Feb 2020 08:25:51 -0800 Subject: [PATCH 181/242] Backport PR #12141: Run ipdb in separate thread (required when using within asyncio applications). --- IPython/terminal/debugger.py | 45 ++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/IPython/terminal/debugger.py b/IPython/terminal/debugger.py index 475c6b8ca94..96fb33b1440 100644 --- a/IPython/terminal/debugger.py +++ b/IPython/terminal/debugger.py @@ -1,6 +1,7 @@ import asyncio import signal import sys +import threading from IPython.core.debugger import Pdb @@ -81,18 +82,11 @@ def cmdloop(self, intro=None): if not self.use_rawinput: raise ValueError('Sorry ipdb does not support use_rawinput=False') - # In order to make sure that asyncio code written in the - # interactive shell doesn't interfere with the prompt, we run the - # prompt in a different event loop. - # If we don't do this, people could spawn coroutine with a - # while/true inside which will freeze the prompt. - - try: - old_loop = asyncio.get_event_loop() - except RuntimeError: - # This happens when the user used `asyncio.run()`. - old_loop = None - + # In order to make sure that prompt, which uses asyncio doesn't + # interfere with applications in which it's used, we always run the + # prompt itself in a different thread (we can't start an event loop + # within an event loop). This new thread won't have any event loop + # running, and here we run our prompt-loop. self.preloop() @@ -109,14 +103,25 @@ def cmdloop(self, intro=None): self._ptcomp.ipy_completer.namespace = self.curframe_locals self._ptcomp.ipy_completer.global_namespace = self.curframe.f_globals - asyncio.set_event_loop(self.pt_loop) - try: - line = self.pt_app.prompt() - except EOFError: - line = 'EOF' - finally: - # Restore the original event loop. - asyncio.set_event_loop(old_loop) + # Run the prompt in a different thread. + line = '' + keyboard_interrupt = False + + def in_thread(): + nonlocal line, keyboard_interrupt + try: + line = self.pt_app.prompt() + except EOFError: + line = 'EOF' + except KeyboardInterrupt: + keyboard_interrupt = True + + th = threading.Thread(target=in_thread) + th.start() + th.join() + + if keyboard_interrupt: + raise KeyboardInterrupt line = self.precmd(line) stop = self.onecmd(line) From 7af64217dd26c147a8f8149f67eea5a4b649abb4 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 28 Feb 2020 13:36:40 -0800 Subject: [PATCH 182/242] Backport PR #12128: Improves detection of whether tab-completion is in a string and suppresses Jedi --- IPython/core/completer.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 9a81d637f15..985ac5ab9eb 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -1375,18 +1375,18 @@ def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str): try_jedi = True try: - # should we check the type of the node is Error ? + # find the first token in the current tree -- if it is a ' or " then we are in a string + completing_string = False try: - # jedi < 0.11 - from jedi.parser.tree import ErrorLeaf - except ImportError: - # jedi >= 0.11 - from parso.tree import ErrorLeaf + first_child = next(c for c in interpreter._get_module().tree_node.children if hasattr(c, 'value')) + except StopIteration: + pass + else: + # note the value may be ', ", or it may also be ''' or """, or + # in some cases, """what/you/typed..., but all of these are + # strings. + completing_string = len(first_child.value) > 0 and first_child.value[0] in {"'", '"'} - next_to_last_tree = interpreter._get_module().tree_node.children[-2] - completing_string = False - if isinstance(next_to_last_tree, ErrorLeaf): - completing_string = next_to_last_tree.value.lstrip()[0] in {'"', "'"} # if we are in a string jedi is likely not the right candidate for # now. Skip it. try_jedi = not completing_string From 1db16b6ed04f66adcab36ccb834440ba1a6be8a3 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 28 Feb 2020 15:54:59 -0800 Subject: [PATCH 183/242] Backport PR #12158: Whats new in 7.13 --- docs/source/whatsnew/version7.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index e9d4ef3ce66..f54f0e5ef0c 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,28 @@ 7.x Series ============ +.. _version 713: + +IPython 7.13 +============ + +IPython 7.13 is the first release of the 7.x branch since master is diverging +toward an 8.0. Exiting new features have already been merged in 8.0 and will +not be available on the 7.x branch. All the changes bellow have been backported +from the master branch. + + + - Fix inability to run PDB when inside an event loop :ghpull:`12141` + - Fix ability to interrupt some processes on windows :ghpull:`12137` + - Fix debugger shortcuts :ghpull:`12132` + - improve tab completion when inside a string by removing irrelevant elements :ghpull:`12128` + - Fix display of filename tab completion when the path is long :ghpull:`12122` + - Many removal of Python 2 specific code path :ghpull:`12110` + - displaying wav files do not require NumPy anymore, and is 5x to 30x faster :ghpull:`12113` + +See the list of all closed issues and pull request on `github +`_. + .. _version 712: IPython 7.12 From a2ff9368894619ed2d02284862d9d4f47ff939a0 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 28 Feb 2020 16:37:14 -0800 Subject: [PATCH 184/242] Backport PR #12160: update stats --- docs/source/whatsnew/github-stats-7.rst | 26 +++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/source/whatsnew/github-stats-7.rst b/docs/source/whatsnew/github-stats-7.rst index c2abed72017..91955bb2afa 100644 --- a/docs/source/whatsnew/github-stats-7.rst +++ b/docs/source/whatsnew/github-stats-7.rst @@ -1,6 +1,32 @@ Issues closed in the 7.x development cycle ========================================== +Issues closed in 8.12 +--------------------- + + +GitHub stats for 2020/02/01 - 2020/02/28 (tag: 7.12.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 1 issues and merged 24 pull requests. +The full list can be seen `on GitHub `__ + +The following 12 authors contributed 108 commits. + +* Alex Hall +* Augusto +* Coon, Ethan T +* Daniel Hahler +* Inception95 +* Itamar Turner-Trauring +* Jonas Haag +* Jonathan Slenders +* linar-jether +* Matthias Bussonnier +* Nathan Goldbaum +* Terry Davis + Issues closed in 7.12 --------------------- From d0b255042e339b213930fa7fb76cdab81f23af0a Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 28 Feb 2020 16:45:41 -0800 Subject: [PATCH 185/242] release 7.13.0 --- IPython/core/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index feb09c0e335..d818b071dc5 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -24,7 +24,7 @@ _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 98b1e108aab70a7574ba2a861024db75821f5834 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 28 Feb 2020 16:48:25 -0800 Subject: [PATCH 186/242] back to dev --- IPython/core/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index d818b071dc5..feb09c0e335 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -24,7 +24,7 @@ _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From c0dfe4cf6f01ae83662fdc2e80c4832c41e13c49 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 3 Mar 2020 08:00:18 -0800 Subject: [PATCH 187/242] Backport PR #12167: Set .py tempfile suffix when using vi/emacs editor shortcuts. --- IPython/terminal/interactiveshell.py | 1 + IPython/terminal/shortcuts.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 9f7d335ead8..7f18ac2d6bd 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -322,6 +322,7 @@ def prompt(): mouse_support=self.mouse_support, enable_open_in_editor=self.extra_open_editor_shortcuts, color_depth=self.color_depth, + tempfile_suffix=".py", **self._extra_prompt_options()) def _make_style_from_name_or_cls(self, name_or_cls): diff --git a/IPython/terminal/shortcuts.py b/IPython/terminal/shortcuts.py index e44e34277e2..24224d4f804 100644 --- a/IPython/terminal/shortcuts.py +++ b/IPython/terminal/shortcuts.py @@ -250,7 +250,6 @@ def newline_autoindent(event): def open_input_in_editor(event): - event.app.current_buffer.tempfile_suffix = ".py" event.app.current_buffer.open_in_editor() From 3ed12106b61e8d4c894434e4f460c8750e7379b7 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 21 Mar 2020 10:21:28 -0700 Subject: [PATCH 188/242] Backport PR #12198: Fix typos in 7.x series release notes --- docs/source/whatsnew/version7.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index f54f0e5ef0c..62990dfc1e7 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -7,9 +7,9 @@ IPython 7.13 ============ -IPython 7.13 is the first release of the 7.x branch since master is diverging +IPython 7.13 is the final release of the 7.x branch since master is diverging toward an 8.0. Exiting new features have already been merged in 8.0 and will -not be available on the 7.x branch. All the changes bellow have been backported +not be available on the 7.x branch. All the changes below have been backported from the master branch. From ae6e88f9deefeda6671c8ad80503828a252f04f9 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 28 Mar 2020 11:11:01 -0700 Subject: [PATCH 189/242] Backport PR #12208: support for unicode identifiers --- IPython/core/completerlib.py | 2 +- IPython/core/inputtransformer.py | 4 ++-- IPython/core/inputtransformer2.py | 4 ++-- IPython/core/magics/namespace.py | 6 ------ IPython/core/prefilter.py | 2 +- IPython/core/tests/test_inputtransformer.py | 1 + IPython/core/tests/test_inputtransformer2.py | 8 ++++++++ IPython/core/tests/test_magic.py | 2 ++ IPython/core/tests/test_prefilter.py | 10 ++++++++++ 9 files changed, 27 insertions(+), 12 deletions(-) diff --git a/IPython/core/completerlib.py b/IPython/core/completerlib.py index 9e592b0817e..7860cb67dcb 100644 --- a/IPython/core/completerlib.py +++ b/IPython/core/completerlib.py @@ -52,7 +52,7 @@ TIMEOUT_GIVEUP = 20 # Regular expression for the python import statement -import_re = re.compile(r'(?P[a-zA-Z_][a-zA-Z0-9_]*?)' +import_re = re.compile(r'(?P[^\W\d]\w*?)' r'(?P[/\\]__init__)?' r'(?P%s)$' % r'|'.join(re.escape(s) for s in _suffixes)) diff --git a/IPython/core/inputtransformer.py b/IPython/core/inputtransformer.py index 1c35eb64f32..afeca93cc0e 100644 --- a/IPython/core/inputtransformer.py +++ b/IPython/core/inputtransformer.py @@ -278,8 +278,8 @@ def escaped_commands(line): _initial_space_re = re.compile(r'\s*') _help_end_re = re.compile(r"""(%{0,2} - [a-zA-Z_*][\w*]* # Variable name - (\.[a-zA-Z_*][\w*]*)* # .etc.etc + (?!\d)[\w*]+ # Variable name + (\.(?!\d)[\w*]+)* # .etc.etc ) (\?\??)$ # ? or ?? """, diff --git a/IPython/core/inputtransformer2.py b/IPython/core/inputtransformer2.py index 4562fe01d2c..0443e6829b4 100644 --- a/IPython/core/inputtransformer2.py +++ b/IPython/core/inputtransformer2.py @@ -405,8 +405,8 @@ def transform(self, lines): return lines_before + [new_line] + lines_after _help_end_re = re.compile(r"""(%{0,2} - [a-zA-Z_*][\w*]* # Variable name - (\.[a-zA-Z_*][\w*]*)* # .etc.etc + (?!\d)[\w*]+ # Variable name + (\.(?!\d)[\w*]+)* # .etc.etc ) (\?\??)$ # ? or ?? """, diff --git a/IPython/core/magics/namespace.py b/IPython/core/magics/namespace.py index cef6ddba8d7..acc4620549b 100644 --- a/IPython/core/magics/namespace.py +++ b/IPython/core/magics/namespace.py @@ -208,12 +208,6 @@ def psearch(self, parameter_s=''): %psearch -l list all available object types """ - try: - parameter_s.encode('ascii') - except UnicodeEncodeError: - print('Python identifiers can only contain ascii characters.') - return - # default namespaces to be searched def_search = ['user_local', 'user_global', 'builtin'] diff --git a/IPython/core/prefilter.py b/IPython/core/prefilter.py index dbf185e6a42..bf801f999c4 100644 --- a/IPython/core/prefilter.py +++ b/IPython/core/prefilter.py @@ -37,7 +37,7 @@ class PrefilterError(Exception): # RegExp to identify potential function names -re_fun_name = re.compile(r'[a-zA-Z_]([a-zA-Z0-9_.]*) *$') +re_fun_name = re.compile(r'[^\W\d]([\w.]*) *$') # RegExp to exclude strings with this start from autocalling. In # particular, all binary operators should be excluded, so that if foo is diff --git a/IPython/core/tests/test_inputtransformer.py b/IPython/core/tests/test_inputtransformer.py index 90a1d5afd1e..0d97fd4d6b1 100644 --- a/IPython/core/tests/test_inputtransformer.py +++ b/IPython/core/tests/test_inputtransformer.py @@ -113,6 +113,7 @@ def transform_checker(tests, transformer, **kwargs): (u'%hist2??', "get_ipython().run_line_magic('pinfo2', '%hist2')"), (u'%%hist3?', "get_ipython().run_line_magic('pinfo', '%%hist3')"), (u'%%hist4??', "get_ipython().run_line_magic('pinfo2', '%%hist4')"), + (u'π.foo?', "get_ipython().run_line_magic('pinfo', 'π.foo')"), (u'f*?', "get_ipython().run_line_magic('psearch', 'f*')"), (u'ax.*aspe*?', "get_ipython().run_line_magic('psearch', 'ax.*aspe*')"), (u'a = abc?', "get_ipython().set_next_input('a = abc');" diff --git a/IPython/core/tests/test_inputtransformer2.py b/IPython/core/tests/test_inputtransformer2.py index cde9eca0d06..b29a0196d3f 100644 --- a/IPython/core/tests/test_inputtransformer2.py +++ b/IPython/core/tests/test_inputtransformer2.py @@ -119,6 +119,11 @@ def test(): [r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"] ) +HELP_UNICODE = ( + ["π.foo?\n"], (1, 0), + ["get_ipython().run_line_magic('pinfo', 'π.foo')\n"] +) + def null_cleanup_transformer(lines): """ @@ -223,6 +228,9 @@ def test_transform_help(): tf = ipt2.HelpEnd((1, 0), (2, 8)) nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2]) + tf = ipt2.HelpEnd((1, 0), (1, 0)) + nt.assert_equal(tf.transform(HELP_UNICODE[0]), HELP_UNICODE[2]) + def test_find_assign_op_dedent(): """ be careful that empty token like dedent are not counted as parens diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index 9882d052db1..877326ccfc3 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -625,6 +625,8 @@ def doctest_precision(): def test_psearch(): with tt.AssertPrints("dict.fromkeys"): _ip.run_cell("dict.fr*?") + with tt.AssertPrints("π.is_integer"): + _ip.run_cell("π = 3.14;\nπ.is_integ*?") def test_timeit_shlex(): """test shlex issues with timeit (#1109)""" diff --git a/IPython/core/tests/test_prefilter.py b/IPython/core/tests/test_prefilter.py index 0e61b4693f7..ca447b3d0b7 100644 --- a/IPython/core/tests/test_prefilter.py +++ b/IPython/core/tests/test_prefilter.py @@ -115,3 +115,13 @@ def __call__(self, x): finally: del ip.user_ns['x'] ip.magic('autocall 0') + + +def test_autocall_should_support_unicode(): + ip.magic('autocall 2') + ip.user_ns['π'] = lambda x: x + try: + nt.assert_equal(ip.prefilter('π 3'),'π(3)') + finally: + ip.magic('autocall 0') + del ip.user_ns['π'] From cfc7ad90136045f885e397bd97a41f37533b644a Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 28 Mar 2020 12:07:49 -0700 Subject: [PATCH 190/242] Backport PR #12210: update link to rpy2 object inventory --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index afdff72b5c4..3dde0f3ecfb 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -223,7 +223,7 @@ def is_stable(extra): htmlhelp_basename = 'ipythondoc' intersphinx_mapping = {'python': ('https://docs.python.org/3/', None), - 'rpy2': ('https://rpy2.readthedocs.io/en/latest/', None), + 'rpy2': ('https://rpy2.github.io/doc/latest/html/', None), 'jupyterclient': ('https://jupyter-client.readthedocs.io/en/latest/', None), 'jupyter': ('https://jupyter.readthedocs.io/en/latest/', None), 'jedi': ('https://jedi.readthedocs.io/en/latest/', None), From 036da90aa4bcaf583985300aebb4b40f94665487 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 17 Apr 2020 15:40:56 -0700 Subject: [PATCH 191/242] Backport PR #12235: Fix compatibility with the latest release of Sphinx --- .travis.yml | 5 ----- docs/source/conf.py | 23 ++++++++++++++++++++++ docs/source/development/wrapperkernels.rst | 2 +- docs/sphinxext/configtraits.py | 3 +-- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9bb3a9bb0a1..0a748ebdcde 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,11 +69,6 @@ matrix: python: "3.7" dist: xenial sudo: true - - arch: arm64 - python: "3.7" - dist: xenial - env: ARM64=True IPYTHON_TESTING_TIMEOUT_SCALE=2 - sudo: true - arch: amd64 python: "3.8-dev" dist: xenial diff --git a/docs/source/conf.py b/docs/source/conf.py index 3dde0f3ecfb..5012da2694e 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -121,6 +121,29 @@ def is_stable(extra): numpydoc_class_members_toctree = False warning_is_error = True +import logging + +class ConfigtraitFilter(logging.Filter): + """ + This is a filter to remove in sphinx 3+ the error about config traits being duplicated. + + As we autogenerate configuration traits from, subclasses have lots of + duplication and we want to silence them. Indeed we build on travis with + warnings-as-error set to True, so those duplicate items make the build fail. + """ + + def filter(self, record): + if record.args and record.args[0] == 'configtrait' and 'duplicate' in record.msg: + return False + return True + +ct_filter = ConfigtraitFilter() + +import sphinx.util +logger = sphinx.util.logging.getLogger('sphinx.domains.std').logger + +logger.addFilter(ct_filter) + # The default replacements for |version| and |release|, also used in various # other places throughout the built documents. # diff --git a/docs/source/development/wrapperkernels.rst b/docs/source/development/wrapperkernels.rst index eb0a0488807..d734c30ee99 100644 --- a/docs/source/development/wrapperkernels.rst +++ b/docs/source/development/wrapperkernels.rst @@ -116,7 +116,7 @@ You can override a number of other methods to improve the functionality of your kernel. All of these methods should return a dictionary as described in the relevant section of the :doc:`messaging spec `. -.. class:: MyKernel +.. class:: MyBetterKernel .. method:: do_complete(code, cusor_pos) diff --git a/docs/sphinxext/configtraits.py b/docs/sphinxext/configtraits.py index 4e767694097..2b05d2bf6c8 100644 --- a/docs/sphinxext/configtraits.py +++ b/docs/sphinxext/configtraits.py @@ -8,8 +8,7 @@ Cross reference like this: :configtrait:`Application.log_datefmt`. """ -from sphinx.locale import l_ -from sphinx.util.docfields import Field + def setup(app): app.add_object_type('configtrait', 'configtrait', objname='Config option') From ce107da41dbddbb2ff2a17eb2261855b993f902f Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 10 Mar 2020 12:58:00 -0700 Subject: [PATCH 192/242] Backport PR #12184: exclude broken matplotlib in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9bb3a9bb0a1..ed4fdad6141 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ install: - pip install setuptools --upgrade - pip install -e file://$PWD#egg=ipython[test] --upgrade - pip install trio curio --upgrade --upgrade-strategy eager - - pip install pytest matplotlib + - pip install pytest 'matplotlib !=3.2.0' - pip install codecov check-manifest --upgrade script: From 2f2473bd178a6f5cca6bfca8b9d24022a51cf1a0 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 18 Apr 2020 14:39:52 -0700 Subject: [PATCH 193/242] Backport PR #12232: ensure SVG(url=...) is set correctly --- IPython/core/display.py | 49 +++++++++++++++++++++--------- IPython/core/tests/test_display.py | 39 ++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 14 deletions(-) diff --git a/IPython/core/display.py b/IPython/core/display.py index 465c000c55a..d8bdbfa3eeb 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -615,9 +615,12 @@ def __init__(self, data=None, url=None, filename=None, metadata=None): filename = data data = None - self.data = data self.url = url self.filename = filename + # because of @data.setter methods in + # subclasses ensure url and filename are set + # before assigning to self.data + self.data = data if metadata is not None: self.metadata = metadata @@ -652,23 +655,36 @@ def reload(self): with open(self.filename, self._read_flags) as f: self.data = f.read() elif self.url is not None: - try: - # Deferred import - from urllib.request import urlopen - response = urlopen(self.url) - self.data = response.read() - # extract encoding from header, if there is one: - encoding = None + # Deferred import + from urllib.request import urlopen + response = urlopen(self.url) + data = response.read() + # extract encoding from header, if there is one: + encoding = None + if 'content-type' in response.headers: for sub in response.headers['content-type'].split(';'): sub = sub.strip() if sub.startswith('charset'): encoding = sub.split('=')[-1].strip() break - # decode data, if an encoding was specified - if encoding: - self.data = self.data.decode(encoding, 'replace') - except: - self.data = None + if 'content-encoding' in response.headers: + # TODO: do deflate? + if 'gzip' in response.headers['content-encoding']: + import gzip + from io import BytesIO + with gzip.open(BytesIO(data), 'rt', encoding=encoding) as fp: + encoding = None + data = fp.read() + + # decode data, if an encoding was specified + # We only touch self.data once since + # subclasses such as SVG have @data.setter methods + # that transform self.data into ... well svg. + if encoding: + self.data = data.decode(encoding, 'replace') + else: + self.data = data + class TextDisplayObject(DisplayObject): """Validate that display data is text""" @@ -736,6 +752,11 @@ def _repr_latex_(self): class SVG(DisplayObject): + """Embed an SVG into the display. + + Note if you just want to view a svg image via a URL use `:class:Image` with + a url=URL keyword argument. + """ _read_flags = 'rb' # wrap data in a property, which extracts the tag, discarding @@ -879,7 +900,7 @@ def data(self, data): data = str(data) if isinstance(data, str): - if getattr(self, 'filename', None) is None: + if self.filename is None and self.url is None: warnings.warn("JSON expects JSONable dict or list, not JSON strings") data = json.loads(data) self._data = data diff --git a/IPython/core/tests/test_display.py b/IPython/core/tests/test_display.py index 0ce4ad2b429..04c93258448 100644 --- a/IPython/core/tests/test_display.py +++ b/IPython/core/tests/test_display.py @@ -72,6 +72,45 @@ def test_retina_png(): nt.assert_equal(md['width'], 1) nt.assert_equal(md['height'], 1) +def test_embed_svg_url(): + import gzip + from io import BytesIO + svg_data = b'' + url = 'http://test.com/circle.svg' + + gzip_svg = BytesIO() + with gzip.open(gzip_svg, 'wb') as fp: + fp.write(svg_data) + gzip_svg = gzip_svg.getvalue() + + def mocked_urlopen(*args, **kwargs): + class MockResponse: + def __init__(self, svg): + self._svg_data = svg + self.headers = {'content-type': 'image/svg+xml'} + + def read(self): + return self._svg_data + + if args[0] == url: + return MockResponse(svg_data) + elif args[0] == url + 'z': + ret= MockResponse(gzip_svg) + ret.headers['content-encoding']= 'gzip' + return ret + return MockResponse(None) + + with mock.patch('urllib.request.urlopen', side_effect=mocked_urlopen): + svg = display.SVG(url=url) + nt.assert_true(svg._repr_svg_().startswith(' Date: Sat, 18 Apr 2020 14:40:51 -0700 Subject: [PATCH 194/242] Backport PR #12219: Catch ValueError --- IPython/core/debugger.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py index ebb8dcac0d8..5c2887ff7cf 100644 --- a/IPython/core/debugger.py +++ b/IPython/core/debugger.py @@ -622,7 +622,11 @@ def do_where(self, arg): Take a number as argument as an (optional) number of context line to print""" if arg: - context = int(arg) + try: + context = int(arg) + except ValueError as err: + self.error(err) + return self.print_stack_trace(context) else: self.print_stack_trace() From bef36f795cc639891ae2f78cddf3ea1bbbb6ba7f Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 18 Apr 2020 14:43:41 -0700 Subject: [PATCH 195/242] Backport PR #12230: Add pretty-printing for types.SimpleNamespace --- IPython/lib/pretty.py | 17 +++++++++++++++++ IPython/lib/tests/test_pretty.py | 20 ++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/IPython/lib/pretty.py b/IPython/lib/pretty.py index 3115d3f6dbf..1cb46b1413d 100644 --- a/IPython/lib/pretty.py +++ b/IPython/lib/pretty.py @@ -648,6 +648,22 @@ def _re_pattern_pprint(obj, p, cycle): p.text(')') +def _types_simplenamespace_pprint(obj, p, cycle): + """The pprint function for types.SimpleNamespace.""" + name = 'namespace' + with p.group(len(name) + 1, name + '(', ')'): + if cycle: + p.text('...') + else: + for idx, (attr, value) in enumerate(obj.__dict__.items()): + if idx: + p.text(',') + p.breakable() + attr_kwarg = '{}='.format(attr) + with p.group(len(attr_kwarg), attr_kwarg): + p.pretty(value) + + def _type_pprint(obj, p, cycle): """The pprint for classes and types.""" # Heap allocated types might not have the module attribute, @@ -741,6 +757,7 @@ def _exception_pprint(obj, p, cycle): types.FunctionType: _function_pprint, types.BuiltinFunctionType: _function_pprint, types.MethodType: _repr_pprint, + types.SimpleNamespace: _types_simplenamespace_pprint, datetime.datetime: _repr_pprint, datetime.timedelta: _repr_pprint, _exception_base: _exception_pprint diff --git a/IPython/lib/tests/test_pretty.py b/IPython/lib/tests/test_pretty.py index 695012d6fc9..ba4c3296694 100644 --- a/IPython/lib/tests/test_pretty.py +++ b/IPython/lib/tests/test_pretty.py @@ -407,6 +407,26 @@ def test_mappingproxy(): nt.assert_equal(pretty.pretty(obj), expected) +def test_simplenamespace(): + SN = types.SimpleNamespace + + sn_recursive = SN() + sn_recursive.first = sn_recursive + sn_recursive.second = sn_recursive + cases = [ + (SN(), "namespace()"), + (SN(x=SN()), "namespace(x=namespace())"), + (SN(a_long_name=[SN(s=string.ascii_lowercase)]*3, a_short_name=None), + "namespace(a_long_name=[namespace(s='abcdefghijklmnopqrstuvwxyz'),\n" + " namespace(s='abcdefghijklmnopqrstuvwxyz'),\n" + " namespace(s='abcdefghijklmnopqrstuvwxyz')],\n" + " a_short_name=None)"), + (sn_recursive, "namespace(first=namespace(...), second=namespace(...))"), + ] + for obj, expected in cases: + nt.assert_equal(pretty.pretty(obj), expected) + + def test_pretty_environ(): dict_repr = pretty.pretty(dict(os.environ)) # reindent to align with 'environ' prefix From 4f4b8aacf8b577b27a5f3a8a29e8ac573b76c416 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 18 Apr 2020 14:45:18 -0700 Subject: [PATCH 196/242] Backport PR #12212: Add option to change HTML attributes in display.Video --- IPython/core/display.py | 29 ++++++++++++------- .../whatsnew/pr/video-display-attributes.rst | 4 +++ 2 files changed, 23 insertions(+), 10 deletions(-) create mode 100644 docs/source/whatsnew/pr/video-display-attributes.rst diff --git a/IPython/core/display.py b/IPython/core/display.py index 465c000c55a..14a3b648d09 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -1308,7 +1308,7 @@ def _find_ext(self, s): class Video(DisplayObject): def __init__(self, data=None, url=None, filename=None, embed=False, - mimetype=None, width=None, height=None): + mimetype=None, width=None, height=None, html_attributes="controls"): """Create a video object given raw data or an URL. When this object is returned by an input cell or passed to the @@ -1346,14 +1346,22 @@ def __init__(self, data=None, url=None, filename=None, embed=False, height : int Height in pixels to which to constrain the video in html. If not supplied, defaults to the height of the video. + html_attributes : str + Attributes for the HTML `video element. - """.format(url, width, height) + """.format(url, self.html_attributes, width, height) return output # Embedded videos are base64-encoded. @@ -1411,10 +1420,10 @@ def _repr_html_(self): else: b64_video = b2a_base64(video).decode('ascii').rstrip() - output = """""".format(self.html_attributes, width, height, mimetype, b64_video) return output def reload(self): diff --git a/docs/source/whatsnew/pr/video-display-attributes.rst b/docs/source/whatsnew/pr/video-display-attributes.rst new file mode 100644 index 00000000000..1e7434e895d --- /dev/null +++ b/docs/source/whatsnew/pr/video-display-attributes.rst @@ -0,0 +1,4 @@ +Video HTML attributes +===================== + +Add an option to `IPython.display.Video` to change the attributes of the HTML display of the video. From 51cc220c314cbb79f97152d0ee0706c6861df398 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 18 Apr 2020 20:57:34 -0700 Subject: [PATCH 197/242] Backport PR #12250: remove deprecated usage of matplotlib quality parameter --- IPython/core/tests/test_pylabtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/tests/test_pylabtools.py b/IPython/core/tests/test_pylabtools.py index 7b724000f58..7b64aab111a 100644 --- a/IPython/core/tests/test_pylabtools.py +++ b/IPython/core/tests/test_pylabtools.py @@ -61,7 +61,7 @@ def test_figure_to_jpeg(): ax = fig.add_subplot(1,1,1) ax.plot([1,2,3]) plt.draw() - jpeg = pt.print_figure(fig, 'jpeg', quality=50)[:100].lower() + jpeg = pt.print_figure(fig, 'jpeg', pil_kwargs={'optimize': 50})[:100].lower() assert jpeg.startswith(_JPEG) def test_retina_figure(): From 937ada355f2d2123179a207e2f71fec0ca63be12 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 20 Apr 2020 13:04:54 -0700 Subject: [PATCH 198/242] Backport PR #12253: prepare release notes for 7.14 --- docs/source/whatsnew/development.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/source/whatsnew/development.rst b/docs/source/whatsnew/development.rst index 9d4493a3a53..9fab2b878ec 100644 --- a/docs/source/whatsnew/development.rst +++ b/docs/source/whatsnew/development.rst @@ -24,6 +24,28 @@ Need to be updated: .. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT. +As a reminder, IPython master has diverged from the 7.x branch, thus master may +have more feature and API changes. + +Important changes: + + - Fix compatibility with Sphinx 3+ :ghpull:`12235` + - Remove deprecated matplotlib parameter usage, compatibility with matplotlib + 3.3+ :`122250` + +Misc Changes + + - set ``.py`` extension when editing current buffer in vi/emacs. :ghpull:`12167` + - support for unicode identifiers in ``?``/``??`` :ghpull:`12208` + - add extra options to the ``Video`` Rich objects :ghpull:`12212` + - add pretty-printing to SimpleNamespace :ghpull:`12230` + +New Deprecation Warnings. + + - Many object present in ``IPython.core.display`` should be imported from + ``IPython.display`` by user and external libraries. Trying to import those + from ``IPython.core.display`` will trigger a deprecation warning in IPython + 7.14+ and _may_ raise an error in IPython 8+ Backwards incompatible changes ------------------------------ From e2f24a7a02ca3e374a31e43f8ed6751704315996 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 22 Apr 2020 18:38:57 -0700 Subject: [PATCH 199/242] Backport PR #12168: Make IPython.core.debugger interruptible by KeyboardInterrupt --- IPython/core/debugger.py | 28 +++++++++++++++++ IPython/core/tests/test_debugger.py | 31 +++++++++++++++++++ .../whatsnew/pr/interruptible-debugger.rst | 4 +++ 3 files changed, 63 insertions(+) create mode 100644 docs/source/whatsnew/pr/interruptible-debugger.rst diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py index 5c2887ff7cf..89eae383dd2 100644 --- a/IPython/core/debugger.py +++ b/IPython/core/debugger.py @@ -634,6 +634,34 @@ def do_where(self, arg): do_w = do_where +class InterruptiblePdb(Pdb): + """Version of debugger where KeyboardInterrupt exits the debugger altogether.""" + + def cmdloop(self): + """Wrap cmdloop() such that KeyboardInterrupt stops the debugger.""" + try: + return OldPdb.cmdloop(self) + except KeyboardInterrupt: + self.stop_here = lambda frame: False + self.do_quit("") + sys.settrace(None) + self.quitting = False + raise + + def _cmdloop(self): + while True: + try: + # keyboard interrupts allow for an easy way to cancel + # the current command, so allow them during interactive input + self.allow_kbdint = True + self.cmdloop() + self.allow_kbdint = False + break + except KeyboardInterrupt: + self.message('--KeyboardInterrupt--') + raise + + def set_trace(frame=None): """ Start debugging from `frame`. diff --git a/IPython/core/tests/test_debugger.py b/IPython/core/tests/test_debugger.py index dcfd9a42438..7f3720a5ab9 100644 --- a/IPython/core/tests/test_debugger.py +++ b/IPython/core/tests/test_debugger.py @@ -4,8 +4,16 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. +import signal import sys +import time import warnings +from tempfile import NamedTemporaryFile +from subprocess import check_output, CalledProcessError, PIPE +import subprocess +from unittest.mock import patch +import builtins +import bdb import nose.tools as nt @@ -223,3 +231,26 @@ def can_exit(): >>> sys.settrace(old_trace) ''' + + +def test_interruptible_core_debugger(): + """The debugger can be interrupted. + + The presumption is there is some mechanism that causes a KeyboardInterrupt + (this is implemented in ipykernel). We want to ensure the + KeyboardInterrupt cause debugging to cease. + """ + def raising_input(msg="", called=[0]): + called[0] += 1 + if called[0] == 1: + raise KeyboardInterrupt() + else: + raise AssertionError("input() should only be called once!") + + with patch.object(builtins, "input", raising_input): + debugger.InterruptiblePdb().set_trace() + # The way this test will fail is by set_trace() never exiting, + # resulting in a timeout by the test runner. The alternative + # implementation would involve a subprocess, but that adds issues with + # interrupting subprocesses that are rather complex, so it's simpler + # just to do it this way. diff --git a/docs/source/whatsnew/pr/interruptible-debugger.rst b/docs/source/whatsnew/pr/interruptible-debugger.rst new file mode 100644 index 00000000000..f501b3476b8 --- /dev/null +++ b/docs/source/whatsnew/pr/interruptible-debugger.rst @@ -0,0 +1,4 @@ +IPython.core.debugger.Pdb is now interruptible +============================================== + +A ``KeyboardInterrupt`` will now interrupt IPython's extended debugger, in order to make Jupyter able to interrupt it. From 68a63a052ca74e0817496b67d34d92b32527351a Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Thu, 23 Apr 2020 09:09:31 -0700 Subject: [PATCH 200/242] Backport PR #12261: Do not exit if self.interact is True --- IPython/core/shellapp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/IPython/core/shellapp.py b/IPython/core/shellapp.py index 33052c44d15..9e8bfbfbb81 100644 --- a/IPython/core/shellapp.py +++ b/IPython/core/shellapp.py @@ -418,7 +418,8 @@ def _run_cmd_line_code(self): fname = os.path.join(fname, "__main__.py") if not os.path.exists(fname): self.log.warning("File '%s' doesn't exist", fname) - self.exit(2) + if not self.interact: + self.exit(2) try: self._exec_file(fname, shell_futures=True) except: From 01aae92a85270fa59218e1e57eb94a112b87713e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 1 May 2020 11:52:47 -0700 Subject: [PATCH 201/242] Backport PR #12272: Update what's new for 7.14 release --- docs/source/whatsnew/development.rst | 20 -------------- docs/source/whatsnew/version7.rst | 39 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/docs/source/whatsnew/development.rst b/docs/source/whatsnew/development.rst index 9fab2b878ec..99821d1105c 100644 --- a/docs/source/whatsnew/development.rst +++ b/docs/source/whatsnew/development.rst @@ -27,26 +27,6 @@ Need to be updated: As a reminder, IPython master has diverged from the 7.x branch, thus master may have more feature and API changes. -Important changes: - - - Fix compatibility with Sphinx 3+ :ghpull:`12235` - - Remove deprecated matplotlib parameter usage, compatibility with matplotlib - 3.3+ :`122250` - -Misc Changes - - - set ``.py`` extension when editing current buffer in vi/emacs. :ghpull:`12167` - - support for unicode identifiers in ``?``/``??`` :ghpull:`12208` - - add extra options to the ``Video`` Rich objects :ghpull:`12212` - - add pretty-printing to SimpleNamespace :ghpull:`12230` - -New Deprecation Warnings. - - - Many object present in ``IPython.core.display`` should be imported from - ``IPython.display`` by user and external libraries. Trying to import those - from ``IPython.core.display`` will trigger a deprecation warning in IPython - 7.14+ and _may_ raise an error in IPython 8+ - Backwards incompatible changes ------------------------------ diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 62990dfc1e7..f68c8101404 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,45 @@ 7.x Series ============ +.. _version 714: + +IPython 7.14 +============ + +IPython 7.14 is a minor release that fix a couple of bugs and prepare +compatibility with new or future versions of some libraries. + +Important changes: +------------------ + + - Fix compatibility with Sphinx 3+ :ghpull:`12235` + - Remove deprecated matplotlib parameter usage, compatibility with matplotlib + 3.3+ :`122250` + +Misc Changes +------------ + + - set ``.py`` extension when editing current buffer in vi/emacs. :ghpull:`12167` + - support for unicode identifiers in ``?``/``??`` :ghpull:`12208` + - add extra options to the ``Video`` Rich objects :ghpull:`12212` + - add pretty-printing to ``SimpleNamespace`` :ghpull:`12230` + +Pending deprecated imports +-------------------------- + +Many object present in ``IPython.core.display`` are there for internal use only, +and should already been imported from ``IPython.display`` by users and external +libraries. Trying to import those from ``IPython.core.display`` is still possible +but will trigger a +deprecation warning in later versions of IPython and will become errors in the +future. + +This will simplify compatibility with other Python kernels (like Xeus-Python), +and simplify code base. + + + + .. _version 713: IPython 7.13 From e78905a3cb7222a00ea17a4b16a2cbc048e93e4f Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 1 May 2020 12:05:31 -0700 Subject: [PATCH 202/242] release 7.14.0 --- IPython/core/release.py | 4 ++-- docs/source/whatsnew/github-stats-7.rst | 25 ++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index feb09c0e335..c01d886b892 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 13 +_version_minor = 14 _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] diff --git a/docs/source/whatsnew/github-stats-7.rst b/docs/source/whatsnew/github-stats-7.rst index 91955bb2afa..2c0c5d81cf3 100644 --- a/docs/source/whatsnew/github-stats-7.rst +++ b/docs/source/whatsnew/github-stats-7.rst @@ -1,9 +1,31 @@ Issues closed in the 7.x development cycle ========================================== -Issues closed in 8.12 +Issues closed in 7.14 --------------------- +GitHub stats for 2020/02/29 - 2020/05/01 (tag: 7.13.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 0 issues and merged 30 pull requests. +The full list can be seen `on GitHub `__ + +The following 10 authors contributed 47 commits. + +* Eric Wieser +* foobarbyte +* Ian Castleden +* Itamar Turner-Trauring +* Lumir Balhar +* Markus Wageringel +* Matthias Bussonnier +* Matthieu Ancellin +* Quentin Peter +* Theo Ouzhinski + +Issues closed in 7.13 +--------------------- GitHub stats for 2020/02/01 - 2020/02/28 (tag: 7.12.0) @@ -27,6 +49,7 @@ The following 12 authors contributed 108 commits. * Nathan Goldbaum * Terry Davis + Issues closed in 7.12 --------------------- From 56db1609ad109d1d9dfd617dbaa8f3416a453676 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 1 May 2020 12:06:39 -0700 Subject: [PATCH 203/242] back to dev --- IPython/core/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index c01d886b892..1bc5595cac9 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 14 +_version_minor = 15 _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From b8beeb3a94f0cfb113132763912b3d5cb306eaa1 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 4 May 2020 15:22:45 -0700 Subject: [PATCH 204/242] Backport PR #12280: Unify API between completers. --- IPython/core/completer.py | 5 ++--- IPython/core/tests/test_completer.py | 15 +++++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 985ac5ab9eb..32144e1e9b6 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -1699,8 +1699,6 @@ def latex_matches(self, text): u"""Match Latex syntax for unicode characters. This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α`` - - Used on Python 3 only. """ slashpos = text.rfind('\\') if slashpos > -1: @@ -1713,7 +1711,8 @@ def latex_matches(self, text): # If a user has partially typed a latex symbol, give them # a full list of options \al -> [\aleph, \alpha] matches = [k for k in latex_symbols if k.startswith(s)] - return s, matches + if matches: + return s, matches return u'', [] def dispatch_custom_completer(self, text): diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index 2920d453936..2c19e2e0187 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -224,20 +224,27 @@ def test_latex_completions(self): nt.assert_in("\\alpha", matches) nt.assert_in("\\aleph", matches) + def test_latex_no_results(self): + """ + forward latex should really return nothing in either field if nothing is found. + """ + ip = get_ipython() + text, matches = ip.Completer.latex_matches("\\really_i_should_match_nothing") + nt.assert_equal(text, "") + nt.assert_equal(matches, []) + def test_back_latex_completion(self): ip = get_ipython() # do not return more than 1 matches fro \beta, only the latex one. name, matches = ip.complete("\\β") - nt.assert_equal(len(matches), 1) - nt.assert_equal(matches[0], "\\beta") + nt.assert_equal(matches, ['\\beta']) def test_back_unicode_completion(self): ip = get_ipython() name, matches = ip.complete("\\Ⅴ") - nt.assert_equal(len(matches), 1) - nt.assert_equal(matches[0], "\\ROMAN NUMERAL FIVE") + nt.assert_equal(matches, ["\\ROMAN NUMERAL FIVE"]) def test_forward_unicode_completion(self): ip = get_ipython() From db177d900add8dbdea1e7973f4300c19fa12b851 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 8 May 2020 09:49:50 -0700 Subject: [PATCH 205/242] Backport PR #12296: Delete 109-112 for removing external test --- IPython/core/tests/test_display.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/IPython/core/tests/test_display.py b/IPython/core/tests/test_display.py index 04c93258448..95f1eb622e4 100644 --- a/IPython/core/tests/test_display.py +++ b/IPython/core/tests/test_display.py @@ -105,11 +105,6 @@ def read(self): nt.assert_true(svg._repr_svg_().startswith(' Date: Sat, 9 May 2020 09:58:12 -0700 Subject: [PATCH 206/242] Backport PR #12284: Try to elide long completion based on user input. --- .travis.yml | 3 +- IPython/core/interactiveshell.py | 2 +- IPython/terminal/ptutils.py | 32 ++++++++++++++++--- .../terminal/tests/test_interactivshell.py | 30 +++++++++++++---- 4 files changed, 55 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index ad1aab7f253..c4a309e1ef0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ install: - pip install setuptools --upgrade - pip install -e file://$PWD#egg=ipython[test] --upgrade - pip install trio curio --upgrade --upgrade-strategy eager - - pip install pytest 'matplotlib !=3.2.0' + - pip install pytest 'matplotlib !=3.2.0' mypy - pip install codecov check-manifest --upgrade script: @@ -50,6 +50,7 @@ script: fi - cd /tmp && iptest --coverage xml && cd - - pytest IPython + - mypy --ignore-missing-imports -m IPython.terminal.ptutils # On the latest Python (on Linux) only, make sure that the docs build. - | if [[ "$TRAVIS_PYTHON_VERSION" == "3.7" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 848a14a481c..f5fdba6692b 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2217,7 +2217,7 @@ def set_custom_completer(self, completer, pos=0): The position argument (defaults to 0) is the index in the completers list where you want the completer to be inserted.""" - newcomp = types.MethodType(completer,self.Completer) + newcomp = types.MethodType(completer, self.Completer) self.Completer.custom_matchers.insert(pos,newcomp) def set_completer_frame(self, frame=None): diff --git a/IPython/terminal/ptutils.py b/IPython/terminal/ptutils.py index 47fd6f40612..ed7ad45eb45 100644 --- a/IPython/terminal/ptutils.py +++ b/IPython/terminal/ptutils.py @@ -23,7 +23,7 @@ _completion_sentinel = object() -def _elide(string, *, min_elide=30): +def _elide_point(string:str, *, min_elide=30)->str: """ If a string is long enough, and has at least 3 dots, replace the middle part with ellipses. @@ -53,6 +53,26 @@ def _elide(string, *, min_elide=30): return string +def _elide_typed(string:str, typed:str, *, min_elide:int=30)->str: + """ + Elide the middle of a long string if the beginning has already been typed. + """ + + if len(string) < min_elide: + return string + cut_how_much = len(typed)-3 + if cut_how_much < 7: + return string + if string.startswith(typed) and len(string)> len(typed): + return f"{string[:3]}\N{HORIZONTAL ELLIPSIS}{string[cut_how_much:]}" + return string + +def _elide(string:str, typed:str, min_elide=30)->str: + return _elide_typed( + _elide_point(string, min_elide=min_elide), + typed, min_elide=min_elide) + + def _adjust_completion_text_based_on_context(text, body, offset): if text.endswith('=') and len(body) > offset and body[offset] == '=': @@ -89,7 +109,11 @@ def get_completions(self, document, complete_event): cursor_col = document.cursor_position_col cursor_position = document.cursor_position offset = cursor_to_position(body, cursor_row, cursor_col) - yield from self._get_completions(body, offset, cursor_position, self.ipy_completer) + try: + yield from self._get_completions(body, offset, cursor_position, self.ipy_completer) + except Exception as e: + from traceback import print_tb + print_tb(e) @staticmethod def _get_completions(body, offset, cursor_position, ipyc): @@ -128,9 +152,9 @@ def _get_completions(body, offset, cursor_position, ipyc): adjusted_text = _adjust_completion_text_based_on_context(c.text, body, offset) if c.type == 'function': - yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text+'()'), display_meta=c.type+c.signature) + yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text+'()', body[c.start:c.end]), display_meta=c.type+c.signature) else: - yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text), display_meta=c.type) + yield Completion(adjusted_text, start_position=c.start - offset, display=_elide(display_text, body[c.start:c.end]), display_meta=c.type) class IPythonPTLexer(Lexer): """ diff --git a/IPython/terminal/tests/test_interactivshell.py b/IPython/terminal/tests/test_interactivshell.py index 6bacc8ea48a..640e5d482ae 100644 --- a/IPython/terminal/tests/test_interactivshell.py +++ b/IPython/terminal/tests/test_interactivshell.py @@ -17,14 +17,32 @@ class TestElide(unittest.TestCase): def test_elide(self): - _elide('concatenate((a1, a2, ...), axis') # do not raise - _elide('concatenate((a1, a2, ..), . axis') # do not raise - nt.assert_equal(_elide('aaaa.bbbb.ccccc.dddddd.eeeee.fffff.gggggg.hhhhhh'), 'aaaa.b…g.hhhhhh') - + _elide('concatenate((a1, a2, ...), axis', '') # do not raise + _elide('concatenate((a1, a2, ..), . axis', '') # do not raise + nt.assert_equal(_elide('aaaa.bbbb.ccccc.dddddd.eeeee.fffff.gggggg.hhhhhh',''), 'aaaa.b…g.hhhhhh') + test_string = os.sep.join(['', 10*'a', 10*'b', 10*'c', '']) expect_stirng = os.sep + 'a' + '\N{HORIZONTAL ELLIPSIS}' + 'b' + os.sep + 10*'c' - nt.assert_equal(_elide(test_string), expect_stirng) - + nt.assert_equal(_elide(test_string, ''), expect_stirng) + + def test_elide_typed_normal(self): + nt.assert_equal(_elide('the quick brown fox jumped over the lazy dog', 'the quick brown fox', min_elide=10), 'the…fox jumped over the lazy dog') + + + def test_elide_typed_short_match(self): + """ + if the match is too short we don't elide. + avoid the "the...the" + """ + nt.assert_equal(_elide('the quick brown fox jumped over the lazy dog', 'the', min_elide=10), 'the quick brown fox jumped over the lazy dog') + + def test_elide_typed_no_match(self): + """ + if the match is too short we don't elide. + avoid the "the...the" + """ + # here we typed red instead of brown + nt.assert_equal(_elide('the quick brown fox jumped over the lazy dog', 'the quick red fox', min_elide=10), 'the quick brown fox jumped over the lazy dog') class TestContextAwareCompletion(unittest.TestCase): From 4d9c6ba67ff2d9b1161f9bad97868de964f14bc4 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 10 May 2020 16:23:46 -0700 Subject: [PATCH 207/242] Backport PR #12303: Fix nightly parso error --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c4a309e1ef0..00c5e3f6bbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,7 @@ script: - | if [[ "$TRAVIS_PYTHON_VERSION" == "nightly" ]]; then # on nightly fake parso known the grammar - cp /home/travis/virtualenv/python3.8-dev/lib/python3.8/site-packages/parso/python/grammar37.txt /home/travis/virtualenv/python3.8-dev/lib/python3.8/site-packages/parso/python/grammar38.txt + cp /home/travis/virtualenv/python3.9-dev/lib/python3.9/site-packages/parso/python/grammar38.txt /home/travis/virtualenv/python3.9-dev/lib/python3.9/site-packages/parso/python/grammar39.txt fi - cd /tmp && iptest --coverage xml && cd - - pytest IPython From bd722769ea84ca8b1798cda7d3cc7c0862764d30 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 11 May 2020 17:04:40 -0700 Subject: [PATCH 208/242] Backport PR #12307: Fix some test on python 3.9 (nightly). --- IPython/core/tests/test_async_helpers.py | 13 ++++++++----- IPython/core/tests/test_oinspect.py | 8 ++++++++ IPython/core/tests/test_ultratb.py | 16 ++++++++++------ 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/IPython/core/tests/test_async_helpers.py b/IPython/core/tests/test_async_helpers.py index 86a516ccc1f..11c475874d7 100644 --- a/IPython/core/tests/test_async_helpers.py +++ b/IPython/core/tests/test_async_helpers.py @@ -8,7 +8,7 @@ from textwrap import dedent, indent from unittest import TestCase from IPython.testing.decorators import skip_without - +import sys iprc = lambda x: ip.run_cell(dedent(x)).raise_error() iprc_nr = lambda x: ip.run_cell(dedent(x)) @@ -275,10 +275,13 @@ def test_autoawait(self): await sleep(0.1) """ ) - - def test_memory_error(self): - with self.assertRaises(MemoryError): - iprc("(" * 200 + ")" * 200) + + if sys.version_info < (3,9): + # new pgen parser in 3.9 does not raise MemoryError on too many nested + # parens anymore + def test_memory_error(self): + with self.assertRaises(MemoryError): + iprc("(" * 200 + ")" * 200) @skip_without('curio') def test_autoawait_curio(self): diff --git a/IPython/core/tests/test_oinspect.py b/IPython/core/tests/test_oinspect.py index 8bdcfad98ce..19c6db7c4f8 100644 --- a/IPython/core/tests/test_oinspect.py +++ b/IPython/core/tests/test_oinspect.py @@ -421,6 +421,14 @@ def long_function( long_function.__name__, ) nt.assert_in(sig, [ + # Python >=3.9 + '''\ +long_function( + a_really_long_parameter: int, + and_another_long_one: bool = False, + let_us_make_sure_this_is_looong: Optional[str] = None, +) -> bool\ +''', # Python >=3.7 '''\ long_function( diff --git a/IPython/core/tests/test_ultratb.py b/IPython/core/tests/test_ultratb.py index 3ab0ce3cf00..3751117b692 100644 --- a/IPython/core/tests/test_ultratb.py +++ b/IPython/core/tests/test_ultratb.py @@ -252,12 +252,16 @@ def test_non_syntaxerror(self): with tt.AssertPrints('QWERTY'): ip.showsyntaxerror() - -class MemoryErrorTest(unittest.TestCase): - def test_memoryerror(self): - memoryerror_code = "(" * 200 + ")" * 200 - with tt.AssertPrints("MemoryError"): - ip.run_cell(memoryerror_code) +import sys +if sys.version_info < (3,9): + """ + New 3.9 Pgen Parser does not raise Memory error, except on failed malloc. + """ + class MemoryErrorTest(unittest.TestCase): + def test_memoryerror(self): + memoryerror_code = "(" * 200 + ")" * 200 + with tt.AssertPrints("MemoryError"): + ip.run_cell(memoryerror_code) class Python3ChainedExceptionsTest(unittest.TestCase): From e3cac25bd6134de06149c3c4446ba8ed25af6e43 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 18 May 2020 16:11:23 -0700 Subject: [PATCH 209/242] Backport PR #12314: Enable high DPI scaling in qt.py --- IPython/terminal/pt_inputhooks/qt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/IPython/terminal/pt_inputhooks/qt.py b/IPython/terminal/pt_inputhooks/qt.py index ce0a9da86bd..1504c3bde72 100644 --- a/IPython/terminal/pt_inputhooks/qt.py +++ b/IPython/terminal/pt_inputhooks/qt.py @@ -25,6 +25,7 @@ def inputhook(context): 'variable. Deactivate Qt5 code.' ) return + QtCore.QCoreApplication.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling) _appref = app = QtGui.QApplication([" "]) event_loop = QtCore.QEventLoop(app) From ddc7e997d11f9f5fad220ecb5888f6c23e8b430f Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 19 May 2020 08:25:52 -0700 Subject: [PATCH 210/242] Backport PR #12321: Fix broken link in docs --- docs/source/config/details.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/config/details.rst b/docs/source/config/details.rst index 6685b14d8f6..9e63232d81d 100644 --- a/docs/source/config/details.rst +++ b/docs/source/config/details.rst @@ -247,7 +247,7 @@ VI input mode to ``Normal`` when in insert mode:: For more information on filters and what you can do with the ``event`` object, `see the prompt_toolkit docs -`__. +`__. Enter to execute From 33036bd61631c789b2a397f2b401d8759323475a Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 23 May 2020 09:43:28 -0700 Subject: [PATCH 211/242] Backport PR #12328: document ipython and ipykernel global configuration --- docs/source/config/intro.rst | 98 ++++++++++++++++++++++++++++++------ 1 file changed, 83 insertions(+), 15 deletions(-) diff --git a/docs/source/config/intro.rst b/docs/source/config/intro.rst index 115315c9ddb..97dc954881b 100644 --- a/docs/source/config/intro.rst +++ b/docs/source/config/intro.rst @@ -19,27 +19,29 @@ To create the blank config files, run:: ipython profile create [profilename] If you leave out the profile name, the files will be created for the -``default`` profile (see :ref:`profiles`). These will typically be -located in :file:`~/.ipython/profile_default/`, and will be named -:file:`ipython_config.py`, :file:`ipython_notebook_config.py`, etc. -The settings in :file:`ipython_config.py` apply to all IPython commands. +``default`` profile (see :ref:`profiles`). These will typically be located in +:file:`~/.ipython/profile_default/`, and will be named +:file:`ipython_config.py`, for hitorical reasons you may also find fiels +named with IPytho nprefix instead of Jupyter: +:file:`ipython_notebook_config.py`, etc. The settings in +:file:`ipython_config.py` apply to all IPython commands. -The files typically start by getting the root config object:: - - c = get_config() +by default, configuration files are fully featured Python scripts that can +execute arbitrary code, the main usage is to set value on the config object +``c`` which exist in your configuration file. You can then configure class attributes like this:: c.InteractiveShell.automagic = False Be careful with spelling--incorrect names will simply be ignored, with -no error. +no error. -To add to a collection which may have already been defined elsewhere, -you can use methods like those found on lists, dicts and sets: append, -extend, :meth:`~traitlets.config.LazyConfigValue.prepend` (like -extend, but at the front), add and update (which works both for dicts -and sets):: +To add to a collection which may have already been defined elsewhere or have +default values, you can use methods like those found on lists, dicts and +sets: append, extend, :meth:`~traitlets.config.LazyConfigValue.prepend` (like +extend, but at the front), add and update (which works both for dicts and +sets):: c.InteractiveShellApp.extensions.append('Cython') @@ -52,7 +54,6 @@ Example config file :: # sample ipython_config.py - c = get_config() c.TerminalIPythonApp.display_banner = True c.InteractiveShellApp.log_level = 20 @@ -78,6 +79,38 @@ Example config file ('la', 'ls -al') ] +Json Configuration files +------------------------ + +in case where executability of configuration can be problematic, or +configurations need to be modified programatically, IPython also support a +limited set of functionalities via ``.json`` config files. + +You can defined most of the configuration options via a json object which +hierarchy represent the value you woudl normally set on the ``c`` object of +``.py`` configuration files. The following ``ipython_config.json`` file:: + + { + "InteractiveShell": { + "colors": "LightBG", + "editor": "nano" + }, + "InteractiveShellApp": { + "extensions": [ + "myextension" + ] + } + } + +Is equivalent to the following ``ipython_config.py``:: + + c.InteractiveShellApp.extensions = [ + 'myextension' + ] + + c.InteractiveShell.colors = 'LightBG' + c.InteractiveShell.editor = 'nano' + Command line arguments ---------------------- @@ -94,7 +127,7 @@ Many frequently used options have short aliases and flags, such as To see all of these abbreviated options, run:: ipython --help - ipython notebook --help + jupyter notebook --help # etc. Options specified at the command line, in either format, override @@ -163,3 +196,38 @@ the directory :file:`~/.ipython/` by default. To see where IPython is looking for the IPython directory, use the command ``ipython locate``, or the Python function :func:`IPython.paths.get_ipython_dir`. + + +Systemwide configuration +======================== + +It can be usefull to deploy systemwide ipython or ipykernel configuration +when managing environment for many users. At startup time IPython and +IPykernel will search for configuration file in multiple systemwide +locations, mainly: + + - ``/etc/ipython/`` + - ``/usr/local/etc/ipython/`` + +When the global install is a standalone python distribution it may also +search in distribution specific location, for example: + + - ``$ANACONDA_LOCATION/etc/ipython/`` + +In those locations, Terminal IPython will look for a file called +``ipython_config.py`` and ``ipython_config.json``, ipykernel will look for +``ipython_kernel_config.py`` and ``ipython_kernel.json``. + +Configuration files are loaded in order and merged with configuration on +later locatoin taking precedence on earlier locations (that is to say a user +can overwrite a systemwide configuration option). + +You can see all locations in which IPython is looking for configuration files +by starting ipython in debug mode:: + + $ ipython --debug -c 'exit()' + +Identically with ipykernel though the command is currently blocking until +this process is killed with ``Ctrl-\``:: + + $ python -m ipykernel --debug \ No newline at end of file From a816d9513623f6096b738ee11c460c9e6398e851 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 25 May 2020 19:54:04 -0700 Subject: [PATCH 212/242] Backport PR #12335: Better documentation for `space_for_menu`. --- IPython/terminal/interactiveshell.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 7f18ac2d6bd..b3bb5777450 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -102,7 +102,11 @@ class TerminalInteractiveShell(InteractiveShell): mime_renderers = Dict().tag(config=True) space_for_menu = Integer(6, help='Number of line at the bottom of the screen ' - 'to reserve for the completion menu' + 'to reserve for the tab completion menu, ' + 'search history, ...etc, the height of ' + 'these menus will at most this value. ' + 'Increase it is you prefer long and skinny ' + 'menus, decrease for short and wide.' ).tag(config=True) pt_app = None From 7f7f6b8a6bc1dba26b30faab35c8cf8053aebb51 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 25 May 2020 19:54:17 -0700 Subject: [PATCH 213/242] Backport PR #12336: Fix auto formatting to be called only once. --- IPython/terminal/shortcuts.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/IPython/terminal/shortcuts.py b/IPython/terminal/shortcuts.py index 24224d4f804..a23fa091a0e 100644 --- a/IPython/terminal/shortcuts.py +++ b/IPython/terminal/shortcuts.py @@ -130,8 +130,10 @@ def newline_or_execute(event): # if all we have after the cursor is whitespace: reformat current text # before cursor after_cursor = d.text[d.cursor_position:] + reformatted = False if not after_cursor.strip(): reformat_text_before_cursor(b, d, shell) + reformatted = True if not (d.on_last_line or d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end() ): @@ -142,7 +144,8 @@ def newline_or_execute(event): return if (status != 'incomplete') and b.accept_handler: - reformat_text_before_cursor(b, d, shell) + if not reformatted: + reformat_text_before_cursor(b, d, shell) b.validate_and_handle() else: if shell.autoindent: From c89c9b18168b7babbac66bf6d6c2ba08731c92ee Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 26 May 2020 09:13:32 -0700 Subject: [PATCH 214/242] Backport PR #12340: docs typoes --- docs/source/config/intro.rst | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/source/config/intro.rst b/docs/source/config/intro.rst index 97dc954881b..6dccde1348c 100644 --- a/docs/source/config/intro.rst +++ b/docs/source/config/intro.rst @@ -11,24 +11,24 @@ Many of IPython's classes have configurable attributes (see :doc:`options/index` for the list). These can be configured in several ways. -Python config files -------------------- +Python configuration files +-------------------------- -To create the blank config files, run:: +To create the blank configuration files, run:: ipython profile create [profilename] If you leave out the profile name, the files will be created for the ``default`` profile (see :ref:`profiles`). These will typically be located in :file:`~/.ipython/profile_default/`, and will be named -:file:`ipython_config.py`, for hitorical reasons you may also find fiels -named with IPytho nprefix instead of Jupyter: +:file:`ipython_config.py`, for historical reasons you may also find files +named with IPython prefix instead of Jupyter: :file:`ipython_notebook_config.py`, etc. The settings in :file:`ipython_config.py` apply to all IPython commands. -by default, configuration files are fully featured Python scripts that can -execute arbitrary code, the main usage is to set value on the config object -``c`` which exist in your configuration file. +By default, configuration files are fully featured Python scripts that can +execute arbitrary code, the main usage is to set value on the configuration +object ``c`` which exist in your configuration file. You can then configure class attributes like this:: @@ -48,8 +48,8 @@ sets):: .. versionadded:: 2.0 list, dict and set methods for config values -Example config file -``````````````````` +Example configuration file +`````````````````````````` :: @@ -79,15 +79,15 @@ Example config file ('la', 'ls -al') ] -Json Configuration files +JSON Configuration files ------------------------ -in case where executability of configuration can be problematic, or -configurations need to be modified programatically, IPython also support a -limited set of functionalities via ``.json`` config files. +In case where executability of configuration can be problematic, or +configurations need to be modified programmatically, IPython also support a +limited set of functionalities via ``.json`` configuration files. You can defined most of the configuration options via a json object which -hierarchy represent the value you woudl normally set on the ``c`` object of +hierarchy represent the value you would normally set on the ``c`` object of ``.py`` configuration files. The following ``ipython_config.json`` file:: { @@ -201,7 +201,7 @@ To see where IPython is looking for the IPython directory, use the command Systemwide configuration ======================== -It can be usefull to deploy systemwide ipython or ipykernel configuration +It can be useful to deploy systemwide ipython or ipykernel configuration when managing environment for many users. At startup time IPython and IPykernel will search for configuration file in multiple systemwide locations, mainly: @@ -219,7 +219,7 @@ In those locations, Terminal IPython will look for a file called ``ipython_kernel_config.py`` and ``ipython_kernel.json``. Configuration files are loaded in order and merged with configuration on -later locatoin taking precedence on earlier locations (that is to say a user +later location taking precedence on earlier locations (that is to say a user can overwrite a systemwide configuration option). You can see all locations in which IPython is looking for configuration files @@ -230,4 +230,4 @@ by starting ipython in debug mode:: Identically with ipykernel though the command is currently blocking until this process is killed with ``Ctrl-\``:: - $ python -m ipykernel --debug \ No newline at end of file + $ python -m ipykernel --debug From cd5bdd4968bf8fe0236aad1750d852cf41882c85 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 27 May 2020 16:29:29 -0700 Subject: [PATCH 215/242] Backport PR #12305: Fix ipython#11508: check if line_buffer is None --- IPython/core/completer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 32144e1e9b6..bc114f0f66b 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -1982,8 +1982,8 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, # if text is either None or an empty string, rely on the line buffer if (not line_buffer) and full_text: line_buffer = full_text.split('\n')[cursor_line] - if not text: - text = self.splitter.split_line(line_buffer, cursor_pos) + if not text: # issue #11508: check line_buffer before calling split_line + text = self.splitter.split_line(line_buffer, cursor_pos) if line_buffer else '' if self.backslash_combining_completions: # allow deactivation of these on windows. From d7d455601d9579e0d7addd655e6c5382fc4e5ec2 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 29 May 2020 08:00:40 -0700 Subject: [PATCH 216/242] Backport PR #12342: Fixed bug on windows where relative image path has wrong slashes --- IPython/sphinxext/ipython_directive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/sphinxext/ipython_directive.py b/IPython/sphinxext/ipython_directive.py index 0481624c42c..3d5b3768230 100644 --- a/IPython/sphinxext/ipython_directive.py +++ b/IPython/sphinxext/ipython_directive.py @@ -419,8 +419,8 @@ def process_image(self, decorator): # insert relative path to image file in source # as absolute path for Sphinx # sphinx expects a posix path, even on Windows - posix_path = pathlib.Path(savefig_dir,filename).as_posix() - outfile = '/' + os.path.relpath(posix_path, source_dir) + path = pathlib.Path(savefig_dir, filename) + outfile = '/' + path.relative_to(source_dir).as_posix() imagerows = ['.. image:: %s' % outfile] From 6cf72b2a0397204b9b93d24de1724e32c3ccbafe Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 29 May 2020 12:51:25 -0700 Subject: [PATCH 217/242] Backport PR #12347: What's new in 7.15+ Reproducible build working --- docs/source/coredev/index.rst | 17 +++++++ docs/source/whatsnew/version7.rst | 84 +++++++++++++++++++++++++++++++ setup.py | 2 +- tools/build_release | 8 +-- tools/make_tarball.py | 1 - tools/release | 5 +- tools/release_helper.sh | 37 +++++++++++--- tools/retar.py | 60 ++++++++++++++++++++++ 8 files changed, 194 insertions(+), 20 deletions(-) create mode 100644 tools/retar.py diff --git a/docs/source/coredev/index.rst b/docs/source/coredev/index.rst index 60f1cb0fdfd..ee1eadb9b1e 100644 --- a/docs/source/coredev/index.rst +++ b/docs/source/coredev/index.rst @@ -80,6 +80,13 @@ for the release you are actually making:: VERSION=5.0.0 BRANCH=master +For `reproducibility of builds `_, +we recommend setting ``SOURCE_DATE_EPOCH`` prior to running the build; record the used value +of ``SOURCE_DATE_EPOCH`` as it may not be available from build artifact. You +should be able to use ``date +%s`` to get a formatted timestamp:: + + SOURCE_DATE_EPOCH=$(date +%s) + 2. Create GitHub stats and finish release note ---------------------------------------------- @@ -229,6 +236,16 @@ uploading them to PyPI. We do not use an universal wheel as each wheel installs an ``ipython2`` or ``ipython3`` script, depending on the version of Python it is built for. Using an universal wheel would prevent this. +Check the shasum of files with:: + + shasum -a 256 dist/* + +and takes notes of them you might need them to update the conda-forge recipes. +Rerun the command and check the hash have not changed:: + + ./tools/release + shasum -a 256 dist/* + Use the following to actually upload the result of the build:: ./tools/release upload diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index f68c8101404..c35a4651904 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,90 @@ 7.x Series ============ +.. _version 715: + +IPython 7.15 +============ + +IPython 7.15 brings a number of bug fixes and user facing improvements. + +Misc Noticeable changes: +------------------------ + + - Long completion name have better elision in terminal :ghpull:`12284` + - I've started to test on Python 3.9 :ghpull:`12307` and fix some errors. + - Hi DPI scaling of figures when using qt eventloop :ghpull:`12314` + - Document the ability to have systemwide configuration for IPython. + :ghpull:`12328` + - Fix issues with input autoformatting :ghpull:`12336` + +Reproducible Build +------------------ + +Starting with IPython 7.15, I am attempting to provide reproducible builds, +that is to say you should be able from the source tree to generate an sdist +and wheel that are identical byte for byte with the publish version on PyPI. + +I've only tested on a couple of machines so far and the process is relatively +straightforward, so this mean that IPython not only have a deterministic build +process, but also I have either removed, or put under control all effects of +the build environments on the final artifact. I encourage you to attempt the +build process on your machine as documented in :ref:`core_developer_guide` +and let me know if you do not obtain an identical artifact. + +While reproducible builds is critical to check that the supply chain of (open +source) software has not been compromised, it can also help to speedup many +of the build processes in large environment (conda, apt...) by allowing +better caching of intermediate build steps. + +Learn more on ``_. `Reflections on trusting +trust `_ is also one of the +cornerstone and recommended reads on this subject. + +.. note:: + + The build commit from which the sdist is generated is also `signed + `_, so you should be able to + check it has not been compromised, and the git repository is a `merkle-tree + `_, you can check the consistency + with `git-fsck `_ which you likely `want + to enable by default + `_. + +NEP29: Last version to support Python 3.6 +----------------------------------------- + +IPython 7.15 will be the Last IPython version to officially support Python +3.6, as stated by `NumPy Enhancement Proposal 29 +`_. Starting with +next minor version of IPython I may stop testing on Python 3.6 and may stop +publishing release artifacts that install on Python 3.6 + +Highlighted features +-------------------- + +Highlighted features are not new, but seem to not be widely known, this +section will help you discover in more narrative form what you can do with +IPython. + +Increase Tab Completion Menu Height +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In terminal IPython it is possible to increase the hight of the tab-completion +menu. To do so set the value of +:configtrait:`TerminalInteractiveShell.space_for_menu`, this will reserve more +space at the bottom of the screen for various kind of menus in IPython including +tab completion and searching in history. + +Autoformat Code in the terminal +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you have a preferred code formatter, you can configure IPython to +reformat your code. Set the value of +:configtrait:`TerminalInteractiveShell.autoformatter` to for example ``'black'`` +and IPython will auto format your code when possible. + + .. _version 714: IPython 7.14 diff --git a/setup.py b/setup.py index 593e3a6a0af..daf5f55f000 100755 --- a/setup.py +++ b/setup.py @@ -225,7 +225,7 @@ for key, deps in extras_require.items(): if ':' not in key: everything.update(deps) -extras_require['all'] = everything +extras_require['all'] = list(sorted(everything)) if 'setuptools' in sys.modules: setuptools_extra_args['python_requires'] = '>=3.6' diff --git a/tools/build_release b/tools/build_release index 26dc9ec874a..51fd87d54d9 100755 --- a/tools/build_release +++ b/tools/build_release @@ -2,6 +2,7 @@ """IPython release build script. """ import os +import sys from shutil import rmtree from toollib import sh, pjoin, get_ipdir, cd, sdists, buildwheels @@ -12,15 +13,10 @@ def build_release(): ipdir = get_ipdir() cd(ipdir) - # Cleanup - for d in ['build', 'dist', pjoin('docs', 'build'), pjoin('docs', 'dist'), - pjoin('docs', 'source', 'api', 'generated')]: - if os.path.isdir(d): - rmtree(d) - # Build source and binary distros sh(sdists) buildwheels() + sh(' '.join([sys.executable, 'tools/retar.py', 'dist/*.gz'])) if __name__ == '__main__': build_release() diff --git a/tools/make_tarball.py b/tools/make_tarball.py index bdce25ba804..fb639f61f61 100755 --- a/tools/make_tarball.py +++ b/tools/make_tarball.py @@ -3,7 +3,6 @@ """ import subprocess -import os from toollib import cd, sh diff --git a/tools/release b/tools/release index 5c8686b3145..2de8e120070 100755 --- a/tools/release +++ b/tools/release @@ -81,13 +81,10 @@ else: sh('mv ipython-*.tgz %s' % ipbackupdir) # Build release files - sh('./build_release %s' % ipdir) + sh('./build_release') cd(ipdir) - # Upload all files - sh(sdists) - buildwheels() print("`./release upload` to upload source distribution on PyPI and ipython archive") sys.exit(0) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index 7489c16d756..6e2e8bf201a 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -98,12 +98,6 @@ then fi -echo -echo $BLUE"Attempting to build package..."$NOR - -tools/build_release -rm dist/* - if ask_section "Should we commit, tag, push... etc ? " then echo @@ -160,14 +154,41 @@ fi if ask_section "Should we build and release ?" then + + echo $BLUE"going to set SOURCE_DATE_EPOCH"$NOR + echo $BLUE'export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)'$NOR + echo $GREEN"Press enter to continue"$NOR + read + + export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD) + + echo $BLUE"SOURCE_DATE_EPOCH set to $SOURCE_DATE_EPOCH"$NOR + echo $GREEN"Press enter to continue"$NOR + read + + + + echo + echo $BLUE"Attempting to build package..."$NOR + + tools/release + + + echo $RED'$ shasum -a 256 dist/*' + shasum -a 256 dist/* + echo $NOR + + echo $BLUE"We are going to rebuild, node the hash above, and compare them to the rebuild"$NOR + echo $GREEN"Press enter to continue"$NOR + read echo echo $BLUE"Attempting to build package..."$NOR tools/release - echo $RED - echo '$ shasum -a 256 dist/*' + echo $RED"Check the shasum for SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" + echo $RED'$ shasum -a 256 dist/*' shasum -a 256 dist/* echo $NOR diff --git a/tools/retar.py b/tools/retar.py new file mode 100644 index 00000000000..dbdca0ed7f5 --- /dev/null +++ b/tools/retar.py @@ -0,0 +1,60 @@ +""" +Un-targz and retargz a targz file to ensure reproducible build. + +usage: + + $ export SOURCE_DATE_EPOCH=$(date +%s) + ... + $ python retar.py + +The process of creating an sdist can be non-reproducible: + - directory created during the process get a mtime of the creation date; + - gziping files embed the timestamp of fo zip creation. + +This will untar-retar; ensuring that all mtime > SOURCE_DATE_EPOCH will be set +equal to SOURCE_DATE_EPOCH. + +""" + +import tarfile +import sys +import os +import gzip +import io + +if len(sys.argv) > 2: + raise ValueError("Too many arguments") + + +timestamp = int(os.environ["SOURCE_DATE_EPOCH"]) + +old_buf = io.BytesIO() +with open(sys.argv[1], "rb") as f: + old_buf.write(f.read()) +old_buf.seek(0) +old = tarfile.open(fileobj=old_buf, mode="r:gz") + +buf = io.BytesIO() +new = tarfile.open(fileobj=buf, mode="w", format=tarfile.GNU_FORMAT) +for i, m in enumerate(old): + data = None + # mutation does not work, copy + if m.name.endswith('.DS_Store'): + continue + m2 = tarfile.TarInfo(m.name) + m2.mtime = min(timestamp, m.mtime) + m2.size = m.size + m2.type = m.type + m2.linkname = m.linkname + if m.isdir(): + data = old.extractfile(m) + new.addfile(m2, data) + else: + new.addfile(m2) +new.close() +old.close() + +buf.seek(0) +with open(sys.argv[1], "wb") as f: + with gzip.GzipFile('', "wb", fileobj=f, mtime=timestamp) as gzf: + gzf.write(buf.read()) From c19480ca30e1360d09572c1b595f66c5a13ce7de Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 29 May 2020 15:26:05 -0700 Subject: [PATCH 218/242] Backport PR #12352: Update with forgotten 7.14 release notes. --- docs/source/whatsnew/development.rst | 1 + .../source/whatsnew/pr/interruptible-debugger.rst | 4 ---- .../whatsnew/pr/video-display-attributes.rst | 4 ---- docs/source/whatsnew/version7.rst | 15 +++++++++++++++ 4 files changed, 16 insertions(+), 8 deletions(-) delete mode 100644 docs/source/whatsnew/pr/interruptible-debugger.rst delete mode 100644 docs/source/whatsnew/pr/video-display-attributes.rst diff --git a/docs/source/whatsnew/development.rst b/docs/source/whatsnew/development.rst index 99821d1105c..f2bde112f66 100644 --- a/docs/source/whatsnew/development.rst +++ b/docs/source/whatsnew/development.rst @@ -22,6 +22,7 @@ Need to be updated: pr/* + .. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT. As a reminder, IPython master has diverged from the 7.x branch, thus master may diff --git a/docs/source/whatsnew/pr/interruptible-debugger.rst b/docs/source/whatsnew/pr/interruptible-debugger.rst deleted file mode 100644 index f501b3476b8..00000000000 --- a/docs/source/whatsnew/pr/interruptible-debugger.rst +++ /dev/null @@ -1,4 +0,0 @@ -IPython.core.debugger.Pdb is now interruptible -============================================== - -A ``KeyboardInterrupt`` will now interrupt IPython's extended debugger, in order to make Jupyter able to interrupt it. diff --git a/docs/source/whatsnew/pr/video-display-attributes.rst b/docs/source/whatsnew/pr/video-display-attributes.rst deleted file mode 100644 index 1e7434e895d..00000000000 --- a/docs/source/whatsnew/pr/video-display-attributes.rst +++ /dev/null @@ -1,4 +0,0 @@ -Video HTML attributes -===================== - -Add an option to `IPython.display.Video` to change the attributes of the HTML display of the video. diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index c35a4651904..3d9abc96f6f 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -18,6 +18,10 @@ Misc Noticeable changes: - Document the ability to have systemwide configuration for IPython. :ghpull:`12328` - Fix issues with input autoformatting :ghpull:`12336` + - ``IPython.core.debugger.Pdb`` is now interruptible (:ghpull:`12168`, in 7.14 + but forgotten in release notes) + - Video HTML attributes (:ghpull:`12212`, in 7.14 but forgotten in release + notes) Reproducible Build ------------------ @@ -109,6 +113,17 @@ Misc Changes - add extra options to the ``Video`` Rich objects :ghpull:`12212` - add pretty-printing to ``SimpleNamespace`` :ghpull:`12230` +IPython.core.debugger.Pdb is now interruptible +---------------------------------------------- + +A ``KeyboardInterrupt`` will now interrupt IPython's extended debugger, in order to make Jupyter able to interrupt it. (:ghpull:`12168`) + +Video HTML attributes +--------------------- + +Add an option to `IPython.display.Video` to change the attributes of the HTML display of the video (:ghpull:`12212`) + + Pending deprecated imports -------------------------- From 6d868182949af86df9c042a995ae31b2cc368ef4 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 29 May 2020 15:39:32 -0700 Subject: [PATCH 219/242] Merge pull request #12354 from Carreau/docstats --- docs/source/whatsnew/github-stats-7.rst | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/source/whatsnew/github-stats-7.rst b/docs/source/whatsnew/github-stats-7.rst index 2c0c5d81cf3..d326771e8de 100644 --- a/docs/source/whatsnew/github-stats-7.rst +++ b/docs/source/whatsnew/github-stats-7.rst @@ -1,6 +1,25 @@ Issues closed in the 7.x development cycle ========================================== +Issues closed in 7.15 +--------------------- + +GitHub stats for 2020/05/01 - 2020/05/29 (tag: 7.14.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 1 issues and merged 29 pull requests. +The full list can be seen `on GitHub `__ + +The following 6 authors contributed 31 commits. + +* Blake Griffin +* Inception95 +* Marcio Mazza +* Matthias Bussonnier +* Talley Lambert +* Thomas + Issues closed in 7.14 --------------------- @@ -27,6 +46,29 @@ The following 10 authors contributed 47 commits. Issues closed in 7.13 --------------------- +GitHub stats for 2020/02/29 - 2020/05/01 (tag: 7.13.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 0 issues and merged 30 pull requests. +The full list can be seen `on GitHub `__ + +The following 10 authors contributed 47 commits. + +* Eric Wieser +* foobarbyte +* Ian Castleden +* Itamar Turner-Trauring +* Lumir Balhar +* Markus Wageringel +* Matthias Bussonnier +* Matthieu Ancellin +* Quentin Peter +* Theo Ouzhinski + +Issues closed in 7.13 +--------------------- + GitHub stats for 2020/02/01 - 2020/02/28 (tag: 7.12.0) These lists are automatically generated, and may be incomplete or contain duplicates. From 2922831acf6d7c185fb863b4436ffd7cfa665675 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 29 May 2020 15:46:12 -0700 Subject: [PATCH 220/242] release 7.15.0 --- IPython/core/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 1bc5595cac9..aae948bbb1b 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -24,7 +24,7 @@ _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 368973ccabcaf67824ea557e7428a871a5c62a5b Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 29 May 2020 15:47:19 -0700 Subject: [PATCH 221/242] back to dev --- IPython/core/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index aae948bbb1b..76bed55d664 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 15 +_version_minor = 16 _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 2874a21a1d2efca947ca1721cb39e5adcc256b09 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sun, 31 May 2020 15:57:22 -0700 Subject: [PATCH 222/242] Backport PR #12301: Exceptions raised when executing notebooks via the %run magic command --- IPython/core/magics/execution.py | 19 +++++++++++-------- IPython/core/tests/test_run.py | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/IPython/core/magics/execution.py b/IPython/core/magics/execution.py index 438d0b5f0ad..dc6cdf00e29 100644 --- a/IPython/core/magics/execution.py +++ b/IPython/core/magics/execution.py @@ -688,17 +688,16 @@ def run(self, parameter_s='', runner=None, modulename = opts["m"][0] modpath = find_mod(modulename) if modpath is None: - warn('%r is not a valid modulename on sys.path'%modulename) - return + msg = '%r is not a valid modulename on sys.path'%modulename + raise Exception(msg) arg_lst = [modpath] + arg_lst try: fpath = None # initialize to make sure fpath is in scope later fpath = arg_lst[0] filename = file_finder(fpath) except IndexError: - warn('you must provide at least a filename.') - print('\n%run:\n', oinspect.getdoc(self.run)) - return + msg = 'you must provide at least a filename.' + raise Exception(msg) except IOError as e: try: msg = str(e) @@ -706,13 +705,17 @@ def run(self, parameter_s='', runner=None, msg = e.message if os.name == 'nt' and re.match(r"^'.*'$",fpath): warn('For Windows, use double quotes to wrap a filename: %run "mypath\\myfile.py"') - error(msg) - return + raise Exception(msg) + except TypeError: + if fpath in sys.meta_path: + filename = "" + else: + raise if filename.lower().endswith(('.ipy', '.ipynb')): with preserve_keys(self.shell.user_ns, '__file__'): self.shell.user_ns['__file__'] = filename - self.shell.safe_execfile_ipy(filename) + self.shell.safe_execfile_ipy(filename, raise_exceptions=True) return # Control the response to exit() calls made by the script being run diff --git a/IPython/core/tests/test_run.py b/IPython/core/tests/test_run.py index 38d71b31740..eff832b3fc0 100644 --- a/IPython/core/tests/test_run.py +++ b/IPython/core/tests/test_run.py @@ -402,6 +402,25 @@ def test_run_nb(self): nt.assert_equal(_ip.user_ns['answer'], 42) + def test_run_nb_error(self): + """Test %run notebook.ipynb error""" + from nbformat import v4, writes + # %run when a file name isn't provided + nt.assert_raises(Exception, _ip.magic, "run") + + # %run when a file doesn't exist + nt.assert_raises(Exception, _ip.magic, "run foobar.ipynb") + + # %run on a notebook with an error + nb = v4.new_notebook( + cells=[ + v4.new_code_cell("0/0") + ] + ) + src = writes(nb, version=4) + self.mktmp(src, ext='.ipynb') + nt.assert_raises(Exception, _ip.magic, "run %s" % self.fname) + def test_file_options(self): src = ('import sys\n' 'a = " ".join(sys.argv[1:])\n') From 31133c3a5766f23a8b1c067319dc4d5c0c182355 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 2 Jun 2020 13:08:08 -0700 Subject: [PATCH 223/242] Backport PR #12360: use $SHELL in system_piped --- IPython/utils/_process_posix.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/IPython/utils/_process_posix.py b/IPython/utils/_process_posix.py index f3f93f774d6..a11cad7697c 100644 --- a/IPython/utils/_process_posix.py +++ b/IPython/utils/_process_posix.py @@ -59,11 +59,12 @@ class ProcessHandler(object): @property def sh(self): - if self._sh is None: - self._sh = pexpect.which('sh') + if self._sh is None: + shell_name = os.environ.get("SHELL", "sh") + self._sh = pexpect.which(shell_name) if self._sh is None: - raise OSError('"sh" shell not found') - + raise OSError('"{}" shell not found'.format(shell_name)) + return self._sh def __init__(self, logfile=None, read_timeout=None, terminate_timeout=None): From a7ca2e7881463080512976750d4c7f514745447d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 2 Jun 2020 16:38:55 -0700 Subject: [PATCH 224/242] Backport PR #12358: Fix retaring process for twine. --- tools/retar.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/retar.py b/tools/retar.py index dbdca0ed7f5..9b5e73c6baa 100644 --- a/tools/retar.py +++ b/tools/retar.py @@ -47,10 +47,10 @@ m2.type = m.type m2.linkname = m.linkname if m.isdir(): + new.addfile(m2) + else: data = old.extractfile(m) new.addfile(m2, data) - else: - new.addfile(m2) new.close() old.close() @@ -58,3 +58,7 @@ with open(sys.argv[1], "wb") as f: with gzip.GzipFile('', "wb", fileobj=f, mtime=timestamp) as gzf: gzf.write(buf.read()) + +# checks the archive is valid. +archive = tarfile.open(sys.argv[1]) +names = archive.getnames() From ee4be875ffae4f1b0c71182eafa89976d0127b5e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 3 Jun 2020 16:20:50 -0700 Subject: [PATCH 225/242] Backport PR #12339: Add instruction to list the API changes. --- tools/release_helper.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index 6e2e8bf201a..dd700b56329 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -81,6 +81,21 @@ then fi +if ask_section "Generate API difference (using frapuccino)" +then + echo $BLUE"Checking out $PREV_RELEASE"$NOR + git checkout $PREV_RELEASE + echo $BLUE"Saving API to file $PREV_RELEASE"$NOR + frappuccino IPython --save IPython-$PREV_RELEASE.json + echo $BLUE"comming back to $BRANCH"$NOR + git checkout $BRANCH + echo $BLUE"comparing ..."$NOR + frappuccino IPython --compare IPython-$PREV_RELEASE.json + echo $GREEN"Use the above guideline to write an API changelog ..."$NOR + echo $GREEN"Press any keys to continue"$NOR + read +fi + echo "Cleaning repository" git clean -xfdi From e769fcbd50ef648251128985232d4b04a86e839b Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 5 Jun 2020 10:31:16 -0700 Subject: [PATCH 226/242] Backport PR #12367: PR: Add handling for malformed pathext env var (Windows) --- IPython/core/magics/osm.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/IPython/core/magics/osm.py b/IPython/core/magics/osm.py index ddc9b6f5874..90da7e22803 100644 --- a/IPython/core/magics/osm.py +++ b/IPython/core/magics/osm.py @@ -25,6 +25,7 @@ from IPython.utils.process import abbrev_cwd from IPython.utils.terminal import set_term_title from traitlets import Bool +from warnings import warn @magics_class @@ -48,8 +49,15 @@ def __init__(self, shell=None, **kwargs): winext = os.environ['pathext'].replace(';','|').replace('.','') except KeyError: winext = 'exe|com|bat|py' - - self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE) + try: + self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE) + except re.error: + warn("Seems like your pathext environmental " + "variable is malformed. Please check it to " + "enable a proper handle of file extensions " + "managed for your system") + winext = 'exe|com|bat|py' + self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE) # call up the chain super().__init__(shell=shell, **kwargs) From 1d622dd149295be58bb9c8098cde8a8e9943b5e4 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 5 Jun 2020 11:16:27 -0700 Subject: [PATCH 227/242] Backport PR #12355: Fix GUI inputhook for qt 5.15.0 --- IPython/terminal/pt_inputhooks/qt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/terminal/pt_inputhooks/qt.py b/IPython/terminal/pt_inputhooks/qt.py index 1504c3bde72..9f5e1b41dcd 100644 --- a/IPython/terminal/pt_inputhooks/qt.py +++ b/IPython/terminal/pt_inputhooks/qt.py @@ -45,7 +45,7 @@ def inputhook(context): QtCore.QSocketNotifier.Read) try: # connect the callback we care about before we turn it on - notifier.activated.connect(event_loop.exit) + notifier.activated.connect(lambda: event_loop.exit()) notifier.setEnabled(True) # only start the event loop we are not already flipped if not context.input_is_ready(): From 4a742600e67590652dcf630425159f7fd8bd0182 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 8 Jun 2020 11:04:19 -0700 Subject: [PATCH 228/242] Backport PR #12375: BF: wx inputhook --- IPython/terminal/pt_inputhooks/wx.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/IPython/terminal/pt_inputhooks/wx.py b/IPython/terminal/pt_inputhooks/wx.py index 618f092b4f9..a0f4442c771 100644 --- a/IPython/terminal/pt_inputhooks/wx.py +++ b/IPython/terminal/pt_inputhooks/wx.py @@ -177,11 +177,13 @@ def inputhook_wxphoenix(context): # Use a wx.Timer to periodically check whether input is ready - as soon as # it is, we exit the main loop + timer = wx.Timer() + def poll(ev): if context.input_is_ready(): + timer.Stop() app.ExitMainLoop() - timer = wx.Timer() timer.Start(poll_interval) timer.Bind(wx.EVT_TIMER, poll) From 7238a928f1adc516b769028bd267fe00ef71a1d7 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 15 Jun 2020 15:18:00 -0700 Subject: [PATCH 229/242] Backport PR #12389: Minor: typos, grammar --- docs/source/interactive/autoawait.rst | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/source/interactive/autoawait.rst b/docs/source/interactive/autoawait.rst index f87379d2ca0..e4ed965a13e 100644 --- a/docs/source/interactive/autoawait.rst +++ b/docs/source/interactive/autoawait.rst @@ -8,7 +8,7 @@ Asynchronous in REPL: Autoawait This feature is experimental and behavior can change between python and IPython version without prior deprecation. -Starting with IPython 7.0, and when user Python 3.6 and above, IPython offer the +Starting with IPython 7.0, and when using Python 3.6 and above, IPython offer the ability to run asynchronous code from the REPL. Constructs which are :exc:`SyntaxError` s in the Python REPL can be used seamlessly in IPython. @@ -19,7 +19,7 @@ will differ between IPython, IPykernel and their versions. When a supported library is used, IPython will automatically allow Futures and Coroutines in the REPL to be ``await`` ed. This will happen if an :ref:`await -` (or any other async constructs like async-with, async-for) is use at +` (or any other async constructs like async-with, async-for) is used at top level scope, or if any structure valid only in `async def `_ function context are present. For example, the following being a syntax error in the @@ -73,7 +73,7 @@ By default IPython will assume integration with Python's provided :mod:`asyncio`, but integration with other libraries is provided. In particular we provide experimental integration with the ``curio`` and ``trio`` library. -You can switch current integration by using the +You can switch the current integration by using the ``c.InteractiveShell.loop_runner`` option or the ``autoawait `` magic. @@ -118,7 +118,7 @@ to your code. When using command line IPython, the default loop (or runner) does not process in the background, so top level asynchronous code must finish for the REPL to -allow you to enter more code. As with usual Python semantic, the awaitables are +allow you to enter more code. As with usual Python semantics, the awaitables are started only when awaited for the first time. That is to say, in first example, no network request is done between ``In[1]`` and ``In[2]``. @@ -131,8 +131,8 @@ a loop to run. By default IPython will use a fake coroutine runner which should allow ``IPython.embed()`` to be nested. Though this will prevent usage of the :magic:`%autoawait` feature when using IPython embed. -You can set explicitly a coroutine runner for ``embed()`` if you desire to run -asynchronous code, the exact behavior is though undefined. +You can set a coroutine runner explicitly for ``embed()`` if you want to run +asynchronous code, though the exact behavior is undefined. Effects on Magics ----------------- @@ -140,14 +140,14 @@ Effects on Magics A couple of magics (``%%timeit``, ``%timeit``, ``%%time``, ``%%prun``) have not yet been updated to work with asynchronous code and will raise syntax errors when trying to use top-level ``await``. We welcome any contribution to help fix -those, and extra cases we haven't caught yet. We hope for better support in Cor +those, and extra cases we haven't caught yet. We hope for better support in Core Python for top-level Async code. Internals --------- As running asynchronous code is not supported in interactive REPL (as of Python -3.7) we have to rely to a number of complex workaround and heuristic to allow +3.7) we have to rely to a number of complex workarounds and heuristics to allow this to happen. It is interesting to understand how this works in order to comprehend potential bugs, or provide a custom runner. @@ -179,16 +179,16 @@ significant overhead to this kind of code. By default the generated coroutine function will be consumed by Asyncio's ``loop_runner = asyncio.get_evenloop().run_until_complete()`` method if ``async`` mode is deemed necessary, otherwise the coroutine will just be -exhausted in a simple runner. It is though possible to change the default +exhausted in a simple runner. It is possible, though, to change the default runner. A loop runner is a *synchronous* function responsible from running a coroutine object. -The runner is responsible from ensuring that ``coroutine`` run to completion, -and should return the result of executing the coroutine. Let's write a +The runner is responsible for ensuring that ``coroutine`` runs to completion, +and it should return the result of executing the coroutine. Let's write a runner for ``trio`` that print a message when used as an exercise, ``trio`` is -special as it usually prefer to run a function object and make a coroutine by +special as it usually prefers to run a function object and make a coroutine by itself, we can get around this limitation by wrapping it in an async-def without parameters and passing this value to ``trio``:: @@ -246,8 +246,8 @@ Difference between terminal IPython and IPykernel The exact asynchronous code running behavior varies between Terminal IPython and IPykernel. The root cause of this behavior is due to IPykernel having a *persistent* `asyncio` loop running, while Terminal IPython starts and stops a -loop for each code block. This can lead to surprising behavior in some case if -you are used to manipulate asyncio loop yourself, see for example +loop for each code block. This can lead to surprising behavior in some cases if +you are used to manipulating asyncio loop yourself, see for example :ghissue:`11303` for a longer discussion but here are some of the astonishing cases. From ad4453ed63be4914c1abbd2e90db3a5d68f765f4 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Tue, 16 Jun 2020 14:48:50 -0700 Subject: [PATCH 230/242] Backport PR #12359: Implement understanding on __tracebackhide__ --- IPython/core/debugger.py | 155 ++++++++++++++++-- IPython/core/interactiveshell.py | 16 +- IPython/core/magics/basic.py | 14 +- IPython/core/tests/test_debugger.py | 78 ++++++++- IPython/core/ultratb.py | 27 ++- IPython/terminal/debugger.py | 10 ++ appveyor.yml | 7 - .../source/whatsnew/pr/skip_tracebackhide.rst | 11 ++ 8 files changed, 290 insertions(+), 28 deletions(-) create mode 100644 docs/source/whatsnew/pr/skip_tracebackhide.rst diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py index 89eae383dd2..a330baa450e 100644 --- a/IPython/core/debugger.py +++ b/IPython/core/debugger.py @@ -280,26 +280,31 @@ def __init__(self, color_scheme=None, completekey=None, # Set the prompt - the default prompt is '(Pdb)' self.prompt = prompt + self.skip_hidden = True def set_colors(self, scheme): """Shorthand access to the color table scheme selector method.""" self.color_scheme_table.set_active_scheme(scheme) self.parser.style = scheme + + def hidden_frames(self, stack): + """ + Given an index in the stack return wether it should be skipped. + + This is used in up/down and where to skip frames. + """ + ip_hide = [s[0].f_locals.get("__tracebackhide__", False) for s in stack] + ip_start = [i for i, s in enumerate(ip_hide) if s == "__ipython_bottom__"] + if ip_start: + ip_hide = [h if i > ip_start[0] else True for (i, h) in enumerate(ip_hide)] + return ip_hide + def interaction(self, frame, traceback): try: OldPdb.interaction(self, frame, traceback) except KeyboardInterrupt: - self.stdout.write('\n' + self.shell.get_exception_only()) - - def new_do_up(self, arg): - OldPdb.do_up(self, arg) - do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up) - - def new_do_down(self, arg): - OldPdb.do_down(self, arg) - - do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down) + self.stdout.write("\n" + self.shell.get_exception_only()) def new_do_frame(self, arg): OldPdb.do_frame(self, arg) @@ -320,6 +325,8 @@ def new_do_restart(self, arg): return self.do_quit(arg) def print_stack_trace(self, context=None): + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal if context is None: context = self.context try: @@ -329,8 +336,21 @@ def print_stack_trace(self, context=None): except (TypeError, ValueError): raise ValueError("Context must be a positive integer") try: - for frame_lineno in self.stack: + skipped = 0 + for hidden, frame_lineno in zip(self.hidden_frames(self.stack), self.stack): + if hidden and self.skip_hidden: + skipped += 1 + continue + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + skipped = 0 self.print_stack_entry(frame_lineno, context=context) + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) except KeyboardInterrupt: pass @@ -487,6 +507,16 @@ def print_list_lines(self, filename, first, last): except KeyboardInterrupt: pass + def do_skip_hidden(self, arg): + """ + Change whether or not we should skip frames with the + __tracebackhide__ attribute. + """ + if arg.strip().lower() in ("true", "yes"): + self.skip_hidden = True + elif arg.strip().lower() in ("false", "no"): + self.skip_hidden = False + def do_list(self, arg): """Print lines of code from the current stack frame """ @@ -633,6 +663,109 @@ def do_where(self, arg): do_w = do_where + def stop_here(self, frame): + hidden = False + if self.skip_hidden: + hidden = frame.f_locals.get("__tracebackhide__", False) + if hidden: + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + print(f"{Colors.excName} [... skipped 1 hidden frame]{ColorsNormal}\n") + + return super().stop_here(frame) + + def do_up(self, arg): + """u(p) [count] + Move the current frame count (default one) levels up in the + stack trace (to an older frame). + + Will skip hidden frames. + """ + ## modified version of upstream that skips + # frames with __tracebackide__ + if self.curindex == 0: + self.error("Oldest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + skipped = 0 + if count < 0: + _newframe = 0 + else: + _newindex = self.curindex + counter = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex - 1, -1, -1): + frame = self.stack[i][0] + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + # if no break occured. + self.error("all frames above hidden") + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + _newframe = i + self._select_frame(_newframe) + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + + def do_down(self, arg): + """d(own) [count] + Move the current frame count (default one) levels down in the + stack trace (to a newer frame). + + Will skip hidden frames. + """ + if self.curindex + 1 == len(self.stack): + self.error("Newest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + if count < 0: + _newframe = len(self.stack) - 1 + else: + _newindex = self.curindex + counter = 0 + skipped = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex + 1, len(self.stack)): + frame = self.stack[i][0] + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + self.error("all frames bellow hidden") + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + _newframe = i + + self._select_frame(_newframe) + + do_d = do_down + do_u = do_up class InterruptiblePdb(Pdb): """Version of debugger where KeyboardInterrupt exits the debugger altogether.""" diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index f5fdba6692b..ddb1b64ea78 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2211,11 +2211,20 @@ def complete(self, text, line=None, cursor_pos=None): with self.builtin_trap: return self.Completer.complete(text, line, cursor_pos) - def set_custom_completer(self, completer, pos=0): + def set_custom_completer(self, completer, pos=0) -> None: """Adds a new custom completer function. The position argument (defaults to 0) is the index in the completers - list where you want the completer to be inserted.""" + list where you want the completer to be inserted. + + `completer` should have the following signature:: + + def completion(self: Completer, text: string) -> List[str]: + raise NotImplementedError + + It will be bound to the current Completer instance and pass some text + and return a list with current completions to suggest to the user. + """ newcomp = types.MethodType(completer, self.Completer) self.Completer.custom_matchers.insert(pos,newcomp) @@ -3310,6 +3319,9 @@ async def run_code(self, code_obj, result=None, *, async_=False): False : successful execution. True : an error occurred. """ + # special value to say that anything above is IPython and should be + # hidden. + __tracebackhide__ = "__ipython_bottom__" # Set our own excepthook in case the user code tries to call it # directly, so that the IPython crash handler doesn't get triggered old_excepthook, sys.excepthook = sys.excepthook, self.excepthook diff --git a/IPython/core/magics/basic.py b/IPython/core/magics/basic.py index 5c8649b5408..a8feb755386 100644 --- a/IPython/core/magics/basic.py +++ b/IPython/core/magics/basic.py @@ -364,13 +364,25 @@ def xmode(self, parameter_s=''): Valid modes: Plain, Context, Verbose, and Minimal. - If called without arguments, acts as a toggle.""" + If called without arguments, acts as a toggle. + + When in verbose mode the value --show (and --hide) + will respectively show (or hide) frames with ``__tracebackhide__ = + True`` value set. + """ def xmode_switch_err(name): warn('Error changing %s exception modes.\n%s' % (name,sys.exc_info()[1])) shell = self.shell + if parameter_s.strip() == "--show": + shell.InteractiveTB.skip_hidden = False + return + if parameter_s.strip() == "--hide": + shell.InteractiveTB.skip_hidden = True + return + new_mode = parameter_s.strip().capitalize() try: shell.InteractiveTB.set_mode(mode=new_mode) diff --git a/IPython/core/tests/test_debugger.py b/IPython/core/tests/test_debugger.py index 7f3720a5ab9..9fdc944e4d0 100644 --- a/IPython/core/tests/test_debugger.py +++ b/IPython/core/tests/test_debugger.py @@ -4,20 +4,24 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. +import bdb +import builtins +import os import signal +import subprocess import sys import time import warnings +from subprocess import PIPE, CalledProcessError, check_output from tempfile import NamedTemporaryFile -from subprocess import check_output, CalledProcessError, PIPE -import subprocess +from textwrap import dedent from unittest.mock import patch -import builtins -import bdb import nose.tools as nt from IPython.core import debugger +from IPython.testing import IPYTHON_TESTING_TIMEOUT_SCALE +from IPython.testing.decorators import skip_win32 #----------------------------------------------------------------------------- # Helper classes, from CPython's Pdb test suite @@ -254,3 +258,69 @@ def raising_input(msg="", called=[0]): # implementation would involve a subprocess, but that adds issues with # interrupting subprocesses that are rather complex, so it's simpler # just to do it this way. + +@skip_win32 +def test_xmode_skip(): + """that xmode skip frames + + Not as a doctest as pytest does not run doctests. + """ + import pexpect + env = os.environ.copy() + env["IPY_TEST_SIMPLE_PROMPT"] = "1" + + child = pexpect.spawn( + sys.executable, ["-m", "IPython", "--colors=nocolor"], env=env + ) + child.timeout = 15 * IPYTHON_TESTING_TIMEOUT_SCALE + + child.expect("IPython") + child.expect("\n") + child.expect_exact("In [1]") + + block = dedent( + """ +def f(): + __tracebackhide__ = True + g() + +def g(): + raise ValueError + +f() + """ + ) + + for line in block.splitlines(): + child.sendline(line) + child.expect_exact(line) + child.expect_exact("skipping") + + block = dedent( + """ +def f(): + __tracebackhide__ = True + g() + +def g(): + from IPython.core.debugger import set_trace + set_trace() + +f() + """ + ) + + for line in block.splitlines(): + child.sendline(line) + child.expect_exact(line) + + child.expect("ipdb>") + child.sendline("w") + child.expect("hidden") + child.expect("ipdb>") + child.sendline("skip_hidden false") + child.sendline("w") + child.expect("__traceba") + child.expect("ipdb>") + + child.close() diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index 9e7e8fb390d..45e22bd7b94 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -879,13 +879,36 @@ def __init__(self, color_scheme='Linux', call_pdb=False, ostream=None, self.check_cache = check_cache self.debugger_cls = debugger_cls or debugger.Pdb + self.skip_hidden = True def format_records(self, records, last_unique, recursion_repeat): """Format the stack frames of the traceback""" frames = [] + + skipped = 0 for r in records[:last_unique+recursion_repeat+1]: - #print '*** record:',file,lnum,func,lines,index # dbg + if self.skip_hidden: + if r[0].f_locals.get("__tracebackhide__", 0): + skipped += 1 + continue + if skipped: + Colors = self.Colors # just a shorthand + quicker name lookup + ColorsNormal = Colors.Normal # used a lot + frames.append( + " %s[... skipping hidden %s frame]%s\n" + % (Colors.excName, skipped, ColorsNormal) + ) + skipped = 0 + frames.append(self.format_record(*r)) + + if skipped: + Colors = self.Colors # just a shorthand + quicker name lookup + ColorsNormal = Colors.Normal # used a lot + frames.append( + " %s[... skipping hidden %s frame]%s\n" + % (Colors.excName, skipped, ColorsNormal) + ) if recursion_repeat: frames.append('... last %d frames repeated, from the frame below ...\n' % recursion_repeat) @@ -1123,8 +1146,6 @@ def format_exception_as_a_whole(self, etype, evalue, etb, number_of_lines_of_con head = self.prepare_header(etype, self.long_header) records = self.get_records(etb, number_of_lines_of_context, tb_offset) - if records is None: - return "" last_unique, recursion_repeat = find_recursion(orig_etype, evalue, records) diff --git a/IPython/terminal/debugger.py b/IPython/terminal/debugger.py index f3268e70bd6..ffd53c65e78 100644 --- a/IPython/terminal/debugger.py +++ b/IPython/terminal/debugger.py @@ -41,6 +41,16 @@ def get_prompt_tokens(): global_namespace={}, parent=self.shell, ) + # add a completer for all the do_ methods + methods_names = [m[3:] for m in dir(self) if m.startswith("do_")] + + def gen_comp(self, text): + return [m for m in methods_names if m.startswith(text)] + import types + newcomp = types.MethodType(gen_comp, compl) + compl.custom_matchers.insert(0, newcomp) + # end add completer. + self._ptcomp = IPythonPTCompleter(compl) options = dict( diff --git a/appveyor.yml b/appveyor.yml index b79c7f26ff4..d20effdf815 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,13 +4,6 @@ matrix: environment: matrix: - - PYTHON: "C:\\Python36" - PYTHON_VERSION: "3.6.x" - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python36-x64" - PYTHON_VERSION: "3.6.x" - PYTHON_ARCH: "64" - PYTHON: "C:\\Python37-x64" PYTHON_VERSION: "3.7.x" diff --git a/docs/source/whatsnew/pr/skip_tracebackhide.rst b/docs/source/whatsnew/pr/skip_tracebackhide.rst new file mode 100644 index 00000000000..0acfecf127d --- /dev/null +++ b/docs/source/whatsnew/pr/skip_tracebackhide.rst @@ -0,0 +1,11 @@ +The default tracebackmode will now skip frames that are marked with +``__tracebackhide__ = True`` and show how many traceback frames have been +skipped. This can be toggled by using :magic:`xmode` with the ``--show`` or +``--hide`` attribute. It will have no effect on non verbose traceback modes. + +The ipython debugger also now understand ``__tracebackhide__`` as well and will +skip hidden frames when displaying. Movement up and down the stack will skip the +hidden frames and will show how many frames were hidden. Internal IPython frames +are also now hidden by default. The behavior can be changed with the +``skip_hidden`` command and accepts "yes", "no", "true" and "false" case +insensitive parameters. From 737e62eb8f90c0412feaf53a5195c362d1ace462 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Jun 2020 14:58:42 -0700 Subject: [PATCH 231/242] Backport PR #12404: Update what's new for 7.16 --- docs/source/whatsnew/development.rst | 1 + docs/source/whatsnew/github-stats-7.rst | 22 +++++++ .../source/whatsnew/pr/skip_tracebackhide.rst | 11 ---- docs/source/whatsnew/version7.rst | 61 +++++++++++++++++++ 4 files changed, 84 insertions(+), 11 deletions(-) delete mode 100644 docs/source/whatsnew/pr/skip_tracebackhide.rst diff --git a/docs/source/whatsnew/development.rst b/docs/source/whatsnew/development.rst index f2bde112f66..9e9b2984c6c 100644 --- a/docs/source/whatsnew/development.rst +++ b/docs/source/whatsnew/development.rst @@ -23,6 +23,7 @@ Need to be updated: pr/* + .. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT. As a reminder, IPython master has diverged from the 7.x branch, thus master may diff --git a/docs/source/whatsnew/github-stats-7.rst b/docs/source/whatsnew/github-stats-7.rst index d326771e8de..6bc36c0789c 100644 --- a/docs/source/whatsnew/github-stats-7.rst +++ b/docs/source/whatsnew/github-stats-7.rst @@ -1,6 +1,28 @@ Issues closed in the 7.x development cycle ========================================== + +Issues closed in 7.16 +--------------------- + +GitHub stats for 2020/05/29 - 2020/06/26 (tag: 7.15.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +We closed 0 issues and merged 18 pull requests. +The full list can be seen `on GitHub `__ + +The following 7 authors contributed 22 commits. + +* Benjamin Ragan-Kelley +* dalthviz +* Frank Tobia +* Matthias Bussonnier +* palewire +* Paul McCarthy +* Talley Lambert + + Issues closed in 7.15 --------------------- diff --git a/docs/source/whatsnew/pr/skip_tracebackhide.rst b/docs/source/whatsnew/pr/skip_tracebackhide.rst deleted file mode 100644 index 0acfecf127d..00000000000 --- a/docs/source/whatsnew/pr/skip_tracebackhide.rst +++ /dev/null @@ -1,11 +0,0 @@ -The default tracebackmode will now skip frames that are marked with -``__tracebackhide__ = True`` and show how many traceback frames have been -skipped. This can be toggled by using :magic:`xmode` with the ``--show`` or -``--hide`` attribute. It will have no effect on non verbose traceback modes. - -The ipython debugger also now understand ``__tracebackhide__`` as well and will -skip hidden frames when displaying. Movement up and down the stack will skip the -hidden frames and will show how many frames were hidden. Internal IPython frames -are also now hidden by default. The behavior can be changed with the -``skip_hidden`` command and accepts "yes", "no", "true" and "false" case -insensitive parameters. diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 3d9abc96f6f..286f16fb9f4 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,67 @@ 7.x Series ============ +.. _version 716: + +IPython 7.16 +============ + + +The default traceback mode will now skip frames that are marked with +``__tracebackhide__ = True`` and show how many traceback frames have been +skipped. This can be toggled by using :magic:`xmode` with the ``--show`` or +``--hide`` attribute. It will have no effect on non verbose traceback modes. + +The ipython debugger also now understands ``__tracebackhide__`` as well and will +skip hidden frames when displaying. Movement up and down the stack will skip the +hidden frames and will show how many frames were hidden. Internal IPython frames +are also now hidden by default. The behavior can be changed with the +``skip_hidden`` while in the debugger, command and accepts "yes", "no", "true" +and "false" case insensitive parameters. + + +Misc Noticeable changes: +------------------------ + +- Exceptions are now (re)raised when running notebooks via the :magic:`%run`, helping to catch issues in workflows and + pipelines. :ghpull:`12301` +- Fix inputhook for qt 5.15.0 :ghpull:`12355` +- Fix wx inputhook :ghpull:`12375` +- Add handling for malformed pathext env var (Windows) :ghpull:`12367` +- use $SHELL in system_piped :ghpull:`12360` for uniform behavior with + ipykernel. + +Reproducible Build +------------------ + +IPython 7.15 reproducible build did not work, so we try again this month +:ghpull:`12358`. + + +API Changes +----------- + +Change of API and exposed objects automatically detected using `frappuccino +`_ (still in beta): + + +The following items are new and mostly related to understanding ``__tracebackbhide__``:: + + + IPython.core.debugger.Pdb.do_down(self, arg) + + IPython.core.debugger.Pdb.do_skip_hidden(self, arg) + + IPython.core.debugger.Pdb.do_up(self, arg) + + IPython.core.debugger.Pdb.hidden_frames(self, stack) + + IPython.core.debugger.Pdb.stop_here(self, frame) + + +The following items have been removed:: + + - IPython.core.debugger.Pdb.new_do_down + - IPython.core.debugger.Pdb.new_do_up + +Those were implementation details. + + .. _version 715: IPython 7.15 From 329684047cfc82be54374e88f6f6785ac99e0bc2 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Jun 2020 15:03:46 -0700 Subject: [PATCH 232/242] release 7.16.0 --- IPython/core/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 76bed55d664..4e50e18b838 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -24,7 +24,7 @@ _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 5ecc01a7d617abca756be8c2bacd0a315ffcb59a Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Jun 2020 15:04:29 -0700 Subject: [PATCH 233/242] back to dev --- IPython/core/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 4e50e18b838..51597dbf83b 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 16 +_version_minor = 17 _version_patch = 0 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 20bdc6fc0db7f3cf1f0df4bad9e2f922ac5adc1e Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Jun 2020 15:31:39 -0700 Subject: [PATCH 234/242] fix conda build --- tools/retar.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/retar.py b/tools/retar.py index 9b5e73c6baa..efd8c0c6bbd 100644 --- a/tools/retar.py +++ b/tools/retar.py @@ -46,6 +46,7 @@ m2.size = m.size m2.type = m.type m2.linkname = m.linkname + m2.mode = m.mode if m.isdir(): new.addfile(m2) else: From 2486838d9eaa73413505ec7e72b01d6e262f30f9 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Jun 2020 15:42:20 -0700 Subject: [PATCH 235/242] release 7.16.1 --- IPython/core/release.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 51597dbf83b..bba11bbaa50 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 17 -_version_patch = 0 +_version_minor = 16 +_version_patch = 1 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 8fcdcd384b8b5e17f9b95a10e65a52f3ca78d63f Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Nov 2021 11:19:25 -0800 Subject: [PATCH 236/242] Pin Jedi to <0.17.2. This may not completely fix the issue, in particular pip may decide to downgrade IPython, but maybe this will lead to less errors. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index daf5f55f000..3e734bafe14 100755 --- a/setup.py +++ b/setup.py @@ -186,7 +186,7 @@ install_requires = [ 'setuptools>=18.5', - 'jedi>=0.10', + 'jedi>=0.10,<=0.17.2', 'decorator', 'pickleshare', 'traitlets>=4.2', From bcae8e07c7eeef79d185522411d8b4cb014ee3be Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Nov 2021 11:55:54 -0800 Subject: [PATCH 237/242] Backport PR #13335: What's new 7.16.2 --- docs/source/whatsnew/version7.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 286f16fb9f4..6211783c099 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -4,6 +4,19 @@ .. _version 716: +IPython 7.16.1, 7.16.2 +====================== + +IPython 7.16.1 was release immediately after 7.16.0 to fix a conda packaging issue. +The source is identical to 7.16.0 but the file permissions in the tar are different. + +IPython 7.16.2 pins jedi dependency to "<=0.17.2" which should prevent some +issues for users still on python 3.6. This may not be sufficient as pip may +still allow to downgrade IPython. + +Compatibility with Jedi > 0.17.2 was not added as this would have meant bumping +the minimal version to >0.16. + IPython 7.16 ============ From 138f2664f1fbd3d04c24bf098732efd8c6d1f2c6 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Nov 2021 14:23:39 -0800 Subject: [PATCH 238/242] bring back release helper from master branch --- tools/release_helper.sh | 45 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index dd700b56329..de88d0f0a69 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -7,7 +7,7 @@ python -c 'import keyring' python -c 'import twine' python -c 'import sphinx' python -c 'import sphinx_rtd_theme' -python -c 'import nose' +python -c 'import pytest' BLACK=$(tput setaf 1) @@ -21,6 +21,7 @@ WHITE=$(tput setaf 7) NOR=$(tput sgr0) +echo "Will use $EDITOR to edit files when necessary" echo -n "PREV_RELEASE (X.y.z) [$PREV_RELEASE]: " read input PREV_RELEASE=${input:-$PREV_RELEASE} @@ -38,18 +39,40 @@ ask_section(){ echo echo $BLUE"$1"$NOR echo -n $GREEN"Press Enter to continue, S to skip: "$NOR - read -n1 value - echo - if [ -z $value ] || [ $value = 'y' ] ; then + if [ "$ZSH_NAME" = "zsh" ] ; then + read -k1 value + value=${value%$'\n'} + else + read -n1 value + fi + if [ -z "$value" ] || [ $value = 'y' ]; then return 0 fi return 1 } +maybe_edit(){ + echo + echo $BLUE"$1"$NOR + echo -n $GREEN"Press e to Edit $1, any other keys to skip: "$NOR + if [ "$ZSH_NAME" = "zsh" ] ; then + read -k1 value + value=${value%$'\n'} + else + read -n1 value + fi + + echo + if [ $value = 'e' ] ; then + $EDITOR $1 + fi +} + + echo -if ask_section "Updating what's new with informations from docs/source/whatsnew/pr" +if ask_section "Updating what's new with information from docs/source/whatsnew/pr" then python tools/update_whatsnew.py @@ -100,6 +123,11 @@ echo "Cleaning repository" git clean -xfdi echo $GREEN"please update version number in ${RED}IPython/core/release.py${NOR} , Do not commit yet – we'll do it later."$NOR +echo $GREEN"I tried ${RED}sed -i bkp -e '/Uncomment/s/^# //g' IPython/core/release.py${NOR}" +sed -i bkp -e '/Uncomment/s/^# //g' IPython/core/release.py +rm IPython/core/release.pybkp +git diff | cat +maybe_edit IPython/core/release.py echo $GREEN"Press enter to continue"$NOR read @@ -142,7 +170,14 @@ then echo $GREEN"please update version number and back to .dev in ${RED}IPython/core/release.py" + echo $GREEN"I tried ${RED}sed -i bkp -e '/Uncomment/s/^/# /g' IPython/core/release.py${NOR}" + sed -i bkp -e '/Uncomment/s/^/# /g' IPython/core/release.py + rm IPython/core/release.pybkp + git diff | cat + echo $GREEN"Please bump ${RED}the minor version number${NOR}" + maybe_edit IPython/core/release.py echo ${BLUE}"Do not commit yet – we'll do it later."$NOR + echo $GREEN"Press enter to continue"$NOR read From 9f477b701e3b7ba0680e24c0f94cd8bf75bca90d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Nov 2021 14:39:42 -0800 Subject: [PATCH 239/242] release 7.16.2 --- IPython/core/release.py | 2 +- tools/release_helper.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index bba11bbaa50..c7326fe1b5f 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -21,7 +21,7 @@ # version _version_major = 7 _version_minor = 16 -_version_patch = 1 +_version_patch = 2 _version_extra = '.dev' # _version_extra = 'b1' _version_extra = '' # Uncomment this for full releases diff --git a/tools/release_helper.sh b/tools/release_helper.sh index de88d0f0a69..cf053d3459a 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -21,7 +21,7 @@ WHITE=$(tput setaf 7) NOR=$(tput sgr0) -echo "Will use $EDITOR to edit files when necessary" +echo "Will use '$EDITOR' to edit files when necessary" echo -n "PREV_RELEASE (X.y.z) [$PREV_RELEASE]: " read input PREV_RELEASE=${input:-$PREV_RELEASE} @@ -65,7 +65,7 @@ maybe_edit(){ echo if [ $value = 'e' ] ; then - $EDITOR $1 + $=EDITOR $1 fi } From 8df8971999886a3804a2b9231a47f6d3764b9997 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 26 Nov 2021 14:44:01 -0800 Subject: [PATCH 240/242] back to dev --- IPython/core/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index c7326fe1b5f..431086851b7 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -21,10 +21,10 @@ # version _version_major = 7 _version_minor = 16 -_version_patch = 2 +_version_patch = 3 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +# _version_extra = '' # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] From 5fa1e409d2dc126c456510c16ece18e08b524e5b Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 19 Jan 2022 14:21:24 +0100 Subject: [PATCH 241/242] Merge pull request from GHSA-pq7m-3gw7-gq5x FIX CVE-2022-21699 --- IPython/__init__.py | 4 +++ IPython/core/application.py | 2 +- IPython/core/profileapp.py | 7 ++-- IPython/core/profiledir.py | 4 +-- IPython/tests/cve.py | 56 +++++++++++++++++++++++++++++++ docs/source/whatsnew/version7.rst | 8 +++++ 6 files changed, 75 insertions(+), 6 deletions(-) create mode 100644 IPython/tests/cve.py diff --git a/IPython/__init__.py b/IPython/__init__.py index 4fb77107680..c17ec76a602 100644 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -65,6 +65,10 @@ __license__ = release.license __version__ = release.version version_info = release.version_info +# list of CVEs that should have been patched in this release. +# this is informational and should not be relied upon. +__patched_cves__ = {"CVE-2022-21699"} + def embed_kernel(module=None, local_ns=None, **kwargs): """Embed and start an IPython kernel in a given scope. diff --git a/IPython/core/application.py b/IPython/core/application.py index 93639d88e2c..4f679df18e3 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -133,7 +133,7 @@ def _config_file_name_changed(self, change): config_file_paths = List(Unicode()) @default('config_file_paths') def _config_file_paths_default(self): - return [os.getcwd()] + return [] extra_config_file = Unicode( help="""Path to an extra config file to load. diff --git a/IPython/core/profileapp.py b/IPython/core/profileapp.py index 97434e3d0b5..9a1bae55ac5 100644 --- a/IPython/core/profileapp.py +++ b/IPython/core/profileapp.py @@ -181,9 +181,10 @@ def list_profile_dirs(self): profiles = list_profiles_in(os.getcwd()) if profiles: print() - print("Available profiles in current directory (%s):" % os.getcwd()) - self._print_profiles(profiles) - + print( + "Profiles from CWD have been removed for security reason, see CVE-2022-21699:" + ) + print() print("To use any of the above profiles, start IPython with:") print(" ipython --profile=") diff --git a/IPython/core/profiledir.py b/IPython/core/profiledir.py index 3199dfd5d64..2c48e4c2f1c 100644 --- a/IPython/core/profiledir.py +++ b/IPython/core/profiledir.py @@ -186,7 +186,7 @@ def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None): is not found, a :class:`ProfileDirError` exception will be raised. The search path algorithm is: - 1. ``os.getcwd()`` + 1. ``os.getcwd()`` # removed for security reason. 2. ``ipython_dir`` Parameters @@ -198,7 +198,7 @@ def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None): will be "profile_". """ dirname = u'profile_' + name - paths = [os.getcwd(), ipython_dir] + paths = [ipython_dir] for p in paths: profile_dir = os.path.join(p, dirname) if os.path.isdir(profile_dir): diff --git a/IPython/tests/cve.py b/IPython/tests/cve.py new file mode 100644 index 00000000000..026415a57a4 --- /dev/null +++ b/IPython/tests/cve.py @@ -0,0 +1,56 @@ +""" +Test that CVEs stay fixed. +""" + +from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory +from pathlib import Path +import random +import sys +import os +import string +import subprocess +import time + +def test_cve_2022_21699(): + """ + Here we test CVE-2022-21699. + + We create a temporary directory, cd into it. + Make a profile file that should not be executed and start IPython in a subprocess, + checking for the value. + + + + """ + + dangerous_profile_dir = Path('profile_default') + + dangerous_startup_dir = dangerous_profile_dir / 'startup' + dangerous_expected = 'CVE-2022-21699-'+''.join([random.choice(string.ascii_letters) for i in range(10)]) + + with TemporaryWorkingDirectory() as t: + dangerous_startup_dir.mkdir(parents=True) + (dangerous_startup_dir/ 'foo.py').write_text(f'print("{dangerous_expected}")') + # 1 sec to make sure FS is flushed. + #time.sleep(1) + cmd = [sys.executable,'-m', 'IPython'] + env = os.environ.copy() + env['IPY_TEST_SIMPLE_PROMPT'] = '1' + + + # First we fake old behavior, making sure the profile is/was actually dangerous + p_dangerous = subprocess.Popen(cmd + [f'--profile-dir={dangerous_profile_dir}'], env=env, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out_dangerous, err_dangerouns = p_dangerous.communicate(b"exit\r") + assert dangerous_expected in out_dangerous.decode() + + # Now that we know it _would_ have been dangerous, we test it's not loaded + p = subprocess.Popen(cmd, env=env, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate(b"exit\r") + assert b'IPython' in out + assert dangerous_expected not in out.decode() + assert err == b'' + + + diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index 6211783c099..f8c88ab844b 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,14 @@ 7.x Series ============ +======= +.. _version 7.16.3: + +IPython 7.16.3 (CVE-2022-21699) +=============================== + +Fixed CVE-2022-21699, see IPython 8.0.1 release notes for informations. + .. _version 716: IPython 7.16.1, 7.16.2 From d43c7c79c1c8518f1917c3cd69e5384738ef0b37 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 19 Jan 2022 14:37:30 +0100 Subject: [PATCH 242/242] release 7.16.3 --- IPython/core/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/core/release.py b/IPython/core/release.py index 431086851b7..a6f3cf81f7d 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -24,7 +24,7 @@ _version_patch = 3 _version_extra = '.dev' # _version_extra = 'b1' -# _version_extra = '' # Uncomment this for full releases +_version_extra = "" # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch]