diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000000..70971c53b5a --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,8 @@ +# When making commits that are strictly formatting/style changes, add the +# commit hash here, so git blame can ignore the change. +# See docs for more details: +# https://git-scm.com/docs/git-config#Documentation/git-config.txt-blameignoreRevsFile + +# Example entries: +# # initial black-format +# # rename something internal diff --git a/.gitignore b/.gitignore index 4471ae7e307..1fc0e22a320 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ docs/source/api/generated docs/source/config/options docs/source/config/shortcuts/*.csv docs/source/interactive/magics-generated.txt -docs/source/config/shortcuts/*.csv docs/gh-pages jupyter_notebook/notebook/static/mathjax jupyter_notebook/static/style/*.map diff --git a/.meeseeksdev.yml b/.meeseeksdev.yml index 924dc8eeaeb..b52022dde07 100644 --- a/.meeseeksdev.yml +++ b/.meeseeksdev.yml @@ -12,6 +12,7 @@ special: config: tag: only: + - good first issue - async/await - backported - help wanted diff --git a/.travis.yml b/.travis.yml index 3369a9695f1..00c5e3f6bbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,6 @@ addons: python: - 3.6 - - 3.5 sudo: false @@ -38,8 +37,8 @@ install: - pip install pip --upgrade - pip install setuptools --upgrade - pip install -e file://$PWD#egg=ipython[test] --upgrade - - pip install trio curio - - pip install 'pytest<4' matplotlib + - pip install trio curio --upgrade --upgrade-strategy eager + - pip install pytest 'matplotlib !=3.2.0' mypy - pip install codecov check-manifest --upgrade script: @@ -47,10 +46,11 @@ script: - | if [[ "$TRAVIS_PYTHON_VERSION" == "nightly" ]]; then # on nightly fake parso known the grammar - cp /home/travis/virtualenv/python3.8-dev/lib/python3.8/site-packages/parso/python/grammar37.txt /home/travis/virtualenv/python3.8-dev/lib/python3.8/site-packages/parso/python/grammar38.txt + cp /home/travis/virtualenv/python3.9-dev/lib/python3.9/site-packages/parso/python/grammar38.txt /home/travis/virtualenv/python3.9-dev/lib/python3.9/site-packages/parso/python/grammar39.txt fi - cd /tmp && iptest --coverage xml && cd - - pytest IPython + - mypy --ignore-missing-imports -m IPython.terminal.ptutils # On the latest Python (on Linux) only, make sure that the docs build. - | if [[ "$TRAVIS_PYTHON_VERSION" == "3.7" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then @@ -66,18 +66,27 @@ after_success: matrix: include: - - python: "3.7" + - arch: amd64 + python: "3.7" dist: xenial sudo: true - - python: "3.8-dev" + - arch: amd64 + python: "3.8-dev" dist: xenial sudo: true - - python: "3.7-dev" + - arch: amd64 + python: "3.7-dev" dist: xenial sudo: true - - python: "nightly" + - arch: amd64 + python: "nightly" dist: xenial sudo: true + - arch: arm64 + python: "nightly" + dist: bionic + env: ARM64=True + sudo: true - os: osx language: generic python: 3.6 diff --git a/IPython/__init__.py b/IPython/__init__.py index 043a946ab26..c17ec76a602 100644 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -27,12 +27,13 @@ #----------------------------------------------------------------------------- # Don't forget to also update setup.py when this changes! -if sys.version_info < (3, 5): +if sys.version_info < (3, 6): raise ImportError( """ -IPython 7.0+ supports Python 3.5 and above. +IPython 7.10+ supports Python 3.6 and above. When using Python 2.7, please install IPython 5.x LTS Long Term Support version. Python 3.3 and 3.4 were supported up to IPython 6.x. +Python 3.5 was supported with IPython 7.0 to 7.9. See IPython `README.rst` file for more information: @@ -64,6 +65,10 @@ __license__ = release.license __version__ = release.version version_info = release.version_info +# list of CVEs that should have been patched in this release. +# this is informational and should not be relied upon. +__patched_cves__ = {"CVE-2022-21699"} + def embed_kernel(module=None, local_ns=None, **kwargs): """Embed and start an IPython kernel in a given scope. diff --git a/IPython/config.py b/IPython/config.py index cf2bacafad1..964f46f10ac 100644 --- a/IPython/config.py +++ b/IPython/config.py @@ -7,7 +7,7 @@ import sys from warnings import warn -from IPython.utils.shimmodule import ShimModule, ShimWarning +from .utils.shimmodule import ShimModule, ShimWarning warn("The `IPython.config` package has been deprecated since IPython 4.0. " "You should import from traitlets.config instead.", ShimWarning) diff --git a/IPython/conftest.py b/IPython/conftest.py index b9d1f065353..8b2af8c020a 100644 --- a/IPython/conftest.py +++ b/IPython/conftest.py @@ -6,11 +6,11 @@ import pathlib import shutil -from IPython.testing import tools +from .testing import tools def get_ipython(): - from IPython.terminal.interactiveshell import TerminalInteractiveShell + from .terminal.interactiveshell import TerminalInteractiveShell if TerminalInteractiveShell._instance: return TerminalInteractiveShell.instance() @@ -60,7 +60,7 @@ def inject(): builtins.ip = get_ipython() builtins.ip.system = types.MethodType(xsys, ip) builtins.ip.builtin_trap.activate() - from IPython.core import page + from .core import page page.pager_page = nopage # yield diff --git a/IPython/core/alias.py b/IPython/core/alias.py index 4577becf7f6..2ad990231a0 100644 --- a/IPython/core/alias.py +++ b/IPython/core/alias.py @@ -25,7 +25,7 @@ import sys from traitlets.config.configurable import Configurable -from IPython.core.error import UsageError +from .error import UsageError from traitlets import List, Instance from logging import error diff --git a/IPython/core/application.py b/IPython/core/application.py index 93639d88e2c..4f679df18e3 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -133,7 +133,7 @@ def _config_file_name_changed(self, change): config_file_paths = List(Unicode()) @default('config_file_paths') def _config_file_paths_default(self): - return [os.getcwd()] + return [] extra_config_file = Unicode( help="""Path to an extra config file to load. diff --git a/IPython/core/async_helpers.py b/IPython/core/async_helpers.py index 1a7d88991db..fb4cc193250 100644 --- a/IPython/core/async_helpers.py +++ b/IPython/core/async_helpers.py @@ -146,20 +146,20 @@ def _should_be_async(cell: str) -> bool: If it works, assume it should be async. Otherwise Return False. - Not handled yet: If the block of code has a return statement as the top + Not handled yet: If the block of code has a return statement as the top level, it will be seen as async. This is a know limitation. """ if sys.version_info > (3, 8): try: code = compile(cell, "<>", "exec", flags=getattr(ast,'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0)) return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE - except SyntaxError: + except (SyntaxError, MemoryError): return False try: # we can't limit ourself to ast.parse, as it __accepts__ to parse on # 3.7+, but just does not _compile_ code = compile(cell, "<>", "exec") - except SyntaxError: + except (SyntaxError, MemoryError): try: parse_tree = _async_parse_cell(cell) @@ -167,7 +167,7 @@ def _should_be_async(cell: str) -> bool: v = _AsyncSyntaxErrorVisitor() v.visit(parse_tree) - except SyntaxError: + except (SyntaxError, MemoryError): return False return True return False diff --git a/IPython/core/completer.py b/IPython/core/completer.py index e7bb0f8bb97..bc114f0f66b 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -126,7 +126,7 @@ from contextlib import contextmanager from importlib import import_module -from typing import Iterator, List, Tuple, Iterable, Union +from typing import Iterator, List, Tuple, Iterable from types import SimpleNamespace from traitlets.config.configurable import Configurable @@ -626,6 +626,8 @@ def __init__(self, namespace=None, global_namespace=None, **kwargs): else: self.global_namespace = global_namespace + self.custom_matchers = [] + super(Completer, self).__init__(**kwargs) def complete(self, text, state): @@ -1122,12 +1124,14 @@ def matchers(self): if self.use_jedi: return [ + *self.custom_matchers, self.file_matches, self.magic_matches, self.dict_key_matches, ] else: return [ + *self.custom_matchers, self.python_matches, self.file_matches, self.magic_matches, @@ -1371,18 +1375,18 @@ def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str): try_jedi = True try: - # should we check the type of the node is Error ? + # find the first token in the current tree -- if it is a ' or " then we are in a string + completing_string = False try: - # jedi < 0.11 - from jedi.parser.tree import ErrorLeaf - except ImportError: - # jedi >= 0.11 - from parso.tree import ErrorLeaf + first_child = next(c for c in interpreter._get_module().tree_node.children if hasattr(c, 'value')) + except StopIteration: + pass + else: + # note the value may be ', ", or it may also be ''' or """, or + # in some cases, """what/you/typed..., but all of these are + # strings. + completing_string = len(first_child.value) > 0 and first_child.value[0] in {"'", '"'} - next_to_last_tree = interpreter._get_module().tree_node.children[-2] - completing_string = False - if isinstance(next_to_last_tree, ErrorLeaf): - completing_string = next_to_last_tree.value.lstrip()[0] in {'"', "'"} # if we are in a string jedi is likely not the right candidate for # now. Skip it. try_jedi = not completing_string @@ -1695,8 +1699,6 @@ def latex_matches(self, text): u"""Match Latex syntax for unicode characters. This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α`` - - Used on Python 3 only. """ slashpos = text.rfind('\\') if slashpos > -1: @@ -1709,7 +1711,8 @@ def latex_matches(self, text): # If a user has partially typed a latex symbol, give them # a full list of options \al -> [\aleph, \alpha] matches = [k for k in latex_symbols if k.startswith(s)] - return s, matches + if matches: + return s, matches return u'', [] def dispatch_custom_completer(self, text): @@ -1979,8 +1982,8 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, # if text is either None or an empty string, rely on the line buffer if (not line_buffer) and full_text: line_buffer = full_text.split('\n')[cursor_line] - if not text: - text = self.splitter.split_line(line_buffer, cursor_pos) + if not text: # issue #11508: check line_buffer before calling split_line + text = self.splitter.split_line(line_buffer, cursor_pos) if line_buffer else '' if self.backslash_combining_completions: # allow deactivation of these on windows. @@ -2014,7 +2017,7 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, # Start with a clean slate of completions matches = [] - custom_res = self.dispatch_custom_completer(text) + # FIXME: we should extend our api to return a dict with completions for # different types of objects. The rlcomplete() method could then # simply collapse the dict into a list for readline, but we'd have @@ -2025,29 +2028,24 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, full_text = line_buffer completions = self._jedi_matches( cursor_pos, cursor_line, full_text) - if custom_res is not None: - # did custom completers produce something? - matches = [(m, 'custom') for m in custom_res] + + if self.merge_completions: + matches = [] + for matcher in self.matchers: + try: + matches.extend([(m, matcher.__qualname__) + for m in matcher(text)]) + except: + # Show the ugly traceback if the matcher causes an + # exception, but do NOT crash the kernel! + sys.excepthook(*sys.exc_info()) else: - # Extend the list of completions with the results of each - # matcher, so we return results to the user from all - # namespaces. - if self.merge_completions: - matches = [] - for matcher in self.matchers: - try: - matches.extend([(m, matcher.__qualname__) - for m in matcher(text)]) - except: - # Show the ugly traceback if the matcher causes an - # exception, but do NOT crash the kernel! - sys.excepthook(*sys.exc_info()) - else: - for matcher in self.matchers: - matches = [(m, matcher.__qualname__) - for m in matcher(text)] - if matches: - break + for matcher in self.matchers: + matches = [(m, matcher.__qualname__) + for m in matcher(text)] + if matches: + break + seen = set() filtered_matches = set() for m in matches: @@ -2056,17 +2054,20 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, filtered_matches.add(m) seen.add(t) - _filtered_matches = sorted( - set(filtered_matches), key=lambda x: completions_sorting_key(x[0]))\ - [:MATCHES_LIMIT] + _filtered_matches = sorted(filtered_matches, key=lambda x: completions_sorting_key(x[0])) + custom_res = [(m, 'custom') for m in self.dispatch_custom_completer(text) or []] + + _filtered_matches = custom_res or _filtered_matches + + _filtered_matches = _filtered_matches[:MATCHES_LIMIT] _matches = [m[0] for m in _filtered_matches] origins = [m[1] for m in _filtered_matches] self.matches = _matches return text, _matches, origins, completions - + def fwd_unicode_match(self, text:str) -> Tuple[str, list]: if self._names is None: self._names = [] diff --git a/IPython/core/completerlib.py b/IPython/core/completerlib.py index 9b14bf7c715..7860cb67dcb 100644 --- a/IPython/core/completerlib.py +++ b/IPython/core/completerlib.py @@ -30,9 +30,9 @@ from zipimport import zipimporter # Our own imports -from IPython.core.completer import expand_user, compress_user -from IPython.core.error import TryNext -from IPython.utils._process_common import arg_split +from .completer import expand_user, compress_user +from .error import TryNext +from ..utils._process_common import arg_split # FIXME: this should be pulled in with the right call via the component system from IPython import get_ipython @@ -52,7 +52,7 @@ TIMEOUT_GIVEUP = 20 # Regular expression for the python import statement -import_re = re.compile(r'(?P[a-zA-Z_][a-zA-Z0-9_]*?)' +import_re = re.compile(r'(?P[^\W\d]\w*?)' r'(?P[/\\]__init__)?' r'(?P%s)$' % r'|'.join(re.escape(s) for s in _suffixes)) diff --git a/IPython/core/crashhandler.py b/IPython/core/crashhandler.py index 2117edb5c0b..1e0b429d09a 100644 --- a/IPython/core/crashhandler.py +++ b/IPython/core/crashhandler.py @@ -29,6 +29,8 @@ from IPython.utils.sysinfo import sys_info from IPython.utils.py3compat import input +from IPython.core.release import __version__ as version + #----------------------------------------------------------------------------- # Code #----------------------------------------------------------------------------- @@ -68,7 +70,7 @@ """ _lite_message_template = """ -If you suspect this is an IPython bug, please report it at: +If you suspect this is an IPython {version} bug, please report it at: https://github.com/ipython/ipython/issues or send an email to the mailing list at {email} @@ -222,5 +224,5 @@ def crash_handler_lite(etype, evalue, tb): else: # we are not in a shell, show generic config config = "c." - print(_lite_message_template.format(email=author_email, config=config), file=sys.stderr) + print(_lite_message_template.format(email=author_email, config=config, version=version), file=sys.stderr) diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py index 9ce896ec634..a330baa450e 100644 --- a/IPython/core/debugger.py +++ b/IPython/core/debugger.py @@ -153,10 +153,7 @@ def __init__(self, colors=None): # at least raise that limit to 80 chars, which should be enough for # most interactive uses. try: - try: - from reprlib import aRepr # Py 3 - except ImportError: - from repr import aRepr # Py 2 + from reprlib import aRepr aRepr.maxstring = 80 except: # This is only a user-facing convenience, so any error we encounter @@ -283,26 +280,31 @@ def __init__(self, color_scheme=None, completekey=None, # Set the prompt - the default prompt is '(Pdb)' self.prompt = prompt + self.skip_hidden = True def set_colors(self, scheme): """Shorthand access to the color table scheme selector method.""" self.color_scheme_table.set_active_scheme(scheme) self.parser.style = scheme + + def hidden_frames(self, stack): + """ + Given an index in the stack return wether it should be skipped. + + This is used in up/down and where to skip frames. + """ + ip_hide = [s[0].f_locals.get("__tracebackhide__", False) for s in stack] + ip_start = [i for i, s in enumerate(ip_hide) if s == "__ipython_bottom__"] + if ip_start: + ip_hide = [h if i > ip_start[0] else True for (i, h) in enumerate(ip_hide)] + return ip_hide + def interaction(self, frame, traceback): try: OldPdb.interaction(self, frame, traceback) except KeyboardInterrupt: - self.stdout.write('\n' + self.shell.get_exception_only()) - - def new_do_up(self, arg): - OldPdb.do_up(self, arg) - do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up) - - def new_do_down(self, arg): - OldPdb.do_down(self, arg) - - do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down) + self.stdout.write("\n" + self.shell.get_exception_only()) def new_do_frame(self, arg): OldPdb.do_frame(self, arg) @@ -323,6 +325,8 @@ def new_do_restart(self, arg): return self.do_quit(arg) def print_stack_trace(self, context=None): + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal if context is None: context = self.context try: @@ -332,8 +336,21 @@ def print_stack_trace(self, context=None): except (TypeError, ValueError): raise ValueError("Context must be a positive integer") try: - for frame_lineno in self.stack: + skipped = 0 + for hidden, frame_lineno in zip(self.hidden_frames(self.stack), self.stack): + if hidden and self.skip_hidden: + skipped += 1 + continue + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + skipped = 0 self.print_stack_entry(frame_lineno, context=context) + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) except KeyboardInterrupt: pass @@ -490,6 +507,16 @@ def print_list_lines(self, filename, first, last): except KeyboardInterrupt: pass + def do_skip_hidden(self, arg): + """ + Change whether or not we should skip frames with the + __tracebackhide__ attribute. + """ + if arg.strip().lower() in ("true", "yes"): + self.skip_hidden = True + elif arg.strip().lower() in ("false", "no"): + self.skip_hidden = False + def do_list(self, arg): """Print lines of code from the current stack frame """ @@ -625,13 +652,148 @@ def do_where(self, arg): Take a number as argument as an (optional) number of context line to print""" if arg: - context = int(arg) + try: + context = int(arg) + except ValueError as err: + self.error(err) + return self.print_stack_trace(context) else: self.print_stack_trace() do_w = do_where + def stop_here(self, frame): + hidden = False + if self.skip_hidden: + hidden = frame.f_locals.get("__tracebackhide__", False) + if hidden: + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + print(f"{Colors.excName} [... skipped 1 hidden frame]{ColorsNormal}\n") + + return super().stop_here(frame) + + def do_up(self, arg): + """u(p) [count] + Move the current frame count (default one) levels up in the + stack trace (to an older frame). + + Will skip hidden frames. + """ + ## modified version of upstream that skips + # frames with __tracebackide__ + if self.curindex == 0: + self.error("Oldest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + skipped = 0 + if count < 0: + _newframe = 0 + else: + _newindex = self.curindex + counter = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex - 1, -1, -1): + frame = self.stack[i][0] + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + # if no break occured. + self.error("all frames above hidden") + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + _newframe = i + self._select_frame(_newframe) + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + + def do_down(self, arg): + """d(own) [count] + Move the current frame count (default one) levels down in the + stack trace (to a newer frame). + + Will skip hidden frames. + """ + if self.curindex + 1 == len(self.stack): + self.error("Newest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + if count < 0: + _newframe = len(self.stack) - 1 + else: + _newindex = self.curindex + counter = 0 + skipped = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex + 1, len(self.stack)): + frame = self.stack[i][0] + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + self.error("all frames bellow hidden") + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + _newframe = i + + self._select_frame(_newframe) + + do_d = do_down + do_u = do_up + +class InterruptiblePdb(Pdb): + """Version of debugger where KeyboardInterrupt exits the debugger altogether.""" + + def cmdloop(self): + """Wrap cmdloop() such that KeyboardInterrupt stops the debugger.""" + try: + return OldPdb.cmdloop(self) + except KeyboardInterrupt: + self.stop_here = lambda frame: False + self.do_quit("") + sys.settrace(None) + self.quitting = False + raise + + def _cmdloop(self): + while True: + try: + # keyboard interrupts allow for an easy way to cancel + # the current command, so allow them during interactive input + self.allow_kbdint = True + self.cmdloop() + self.allow_kbdint = False + break + except KeyboardInterrupt: + self.message('--KeyboardInterrupt--') + raise + def set_trace(frame=None): """ diff --git a/IPython/core/display.py b/IPython/core/display.py index d31350cb895..424414a662f 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -296,6 +296,13 @@ def display(*objs, include=None, exclude=None, metadata=None, transient=None, di if transient: kwargs['transient'] = transient + if not objs and display_id: + # if given no objects, but still a request for a display_id, + # we assume the user wants to insert an empty output that + # can be updated later + objs = [{}] + raw = True + if not raw: format = InteractiveShell.instance().display_formatter.format @@ -608,9 +615,12 @@ def __init__(self, data=None, url=None, filename=None, metadata=None): filename = data data = None - self.data = data self.url = url self.filename = filename + # because of @data.setter methods in + # subclasses ensure url and filename are set + # before assigning to self.data + self.data = data if metadata is not None: self.metadata = metadata @@ -645,23 +655,36 @@ def reload(self): with open(self.filename, self._read_flags) as f: self.data = f.read() elif self.url is not None: - try: - # Deferred import - from urllib.request import urlopen - response = urlopen(self.url) - self.data = response.read() - # extract encoding from header, if there is one: - encoding = None + # Deferred import + from urllib.request import urlopen + response = urlopen(self.url) + data = response.read() + # extract encoding from header, if there is one: + encoding = None + if 'content-type' in response.headers: for sub in response.headers['content-type'].split(';'): sub = sub.strip() if sub.startswith('charset'): encoding = sub.split('=')[-1].strip() break - # decode data, if an encoding was specified - if encoding: - self.data = self.data.decode(encoding, 'replace') - except: - self.data = None + if 'content-encoding' in response.headers: + # TODO: do deflate? + if 'gzip' in response.headers['content-encoding']: + import gzip + from io import BytesIO + with gzip.open(BytesIO(data), 'rt', encoding=encoding) as fp: + encoding = None + data = fp.read() + + # decode data, if an encoding was specified + # We only touch self.data once since + # subclasses such as SVG have @data.setter methods + # that transform self.data into ... well svg. + if encoding: + self.data = data.decode(encoding, 'replace') + else: + self.data = data + class TextDisplayObject(DisplayObject): """Validate that display data is text""" @@ -729,6 +752,11 @@ def _repr_latex_(self): class SVG(DisplayObject): + """Embed an SVG into the display. + + Note if you just want to view a svg image via a URL use `:class:Image` with + a url=URL keyword argument. + """ _read_flags = 'rb' # wrap data in a property, which extracts the tag, discarding @@ -872,7 +900,7 @@ def data(self, data): data = str(data) if isinstance(data, str): - if getattr(self, 'filename', None) is None: + if self.filename is None and self.url is None: warnings.warn("JSON expects JSONable dict or list, not JSON strings") data = json.loads(data) self._data = data @@ -1301,7 +1329,7 @@ def _find_ext(self, s): class Video(DisplayObject): def __init__(self, data=None, url=None, filename=None, embed=False, - mimetype=None, width=None, height=None): + mimetype=None, width=None, height=None, html_attributes="controls"): """Create a video object given raw data or an URL. When this object is returned by an input cell or passed to the @@ -1339,14 +1367,22 @@ def __init__(self, data=None, url=None, filename=None, embed=False, height : int Height in pixels to which to constrain the video in html. If not supplied, defaults to the height of the video. + html_attributes : str + Attributes for the HTML `video element. - """.format(url, width, height) + """.format(url, self.html_attributes, width, height) return output # Embedded videos are base64-encoded. @@ -1404,10 +1441,10 @@ def _repr_html_(self): else: b64_video = b2a_base64(video).decode('ascii').rstrip() - output = """""".format(self.html_attributes, width, height, mimetype, b64_video) return output def reload(self): diff --git a/IPython/core/displayhook.py b/IPython/core/displayhook.py index 2128c82d12f..3c06675e86e 100644 --- a/IPython/core/displayhook.py +++ b/IPython/core/displayhook.py @@ -153,7 +153,7 @@ def compute_format_data(self, result): # This can be set to True by the write_output_prompt method in a subclass prompt_end_newline = False - def write_format_data(self, format_dict, md_dict=None): + def write_format_data(self, format_dict, md_dict=None) -> None: """Write the format data dict to the frontend. This default version of this method simply writes the plain text @@ -198,7 +198,7 @@ def update_user_ns(self, result): """Update user_ns with various things like _, __, _1, etc.""" # Avoid recursive reference when displaying _oh/Out - if result is not self.shell.user_ns['_oh']: + if self.cache_size and result is not self.shell.user_ns['_oh']: if len(self.shell.user_ns['_oh']) >= self.cache_size and self.do_full_cache: self.cull_cache() diff --git a/IPython/core/displaypub.py b/IPython/core/displaypub.py index 9625da2a843..1da0458cf08 100644 --- a/IPython/core/displaypub.py +++ b/IPython/core/displaypub.py @@ -28,6 +28,7 @@ # Main payload class #----------------------------------------------------------------------------- + class DisplayPublisher(Configurable): """A traited class that publishes display data to frontends. @@ -35,6 +36,10 @@ class DisplayPublisher(Configurable): be accessed there. """ + def __init__(self, shell=None, *args, **kwargs): + self.shell = shell + super().__init__(*args, **kwargs) + def _validate_data(self, data, metadata=None): """Validate the display data. @@ -53,7 +58,7 @@ def _validate_data(self, data, metadata=None): raise TypeError('metadata must be a dict, got: %r' % data) # use * to indicate transient, update are keyword-only - def publish(self, data, metadata=None, source=None, *, transient=None, update=False, **kwargs): + def publish(self, data, metadata=None, source=None, *, transient=None, update=False, **kwargs) -> None: """Publish data and metadata to all frontends. See the ``display_data`` message in the messaging documentation for @@ -98,7 +103,15 @@ def publish(self, data, metadata=None, source=None, *, transient=None, update=Fa rather than creating a new output. """ - # The default is to simply write the plain text data using sys.stdout. + handlers = {} + if self.shell is not None: + handlers = getattr(self.shell, 'mime_renderers', {}) + + for mime, handler in handlers.items(): + if mime in data: + handler(data[mime], metadata.get(mime, None)) + return + if 'text/plain' in data: print(data['text/plain']) diff --git a/IPython/core/formatters.py b/IPython/core/formatters.py index fd93d3747b5..237b959b9a6 100644 --- a/IPython/core/formatters.py +++ b/IPython/core/formatters.py @@ -20,10 +20,10 @@ from decorator import decorator from traitlets.config.configurable import Configurable -from IPython.core.getipython import get_ipython -from IPython.utils.sentinel import Sentinel -from IPython.utils.dir2 import get_real_method -from IPython.lib import pretty +from .getipython import get_ipython +from ..utils.sentinel import Sentinel +from ..utils.dir2 import get_real_method +from ..lib import pretty from traitlets import ( Bool, Dict, Integer, Unicode, CUnicode, ObjectName, List, ForwardDeclaredInstance, @@ -1015,7 +1015,7 @@ def format_display_data(obj, include=None, exclude=None): data dict. If this is set all format types will be computed, except for those included in this argument. """ - from IPython.core.interactiveshell import InteractiveShell + from .interactiveshell import InteractiveShell return InteractiveShell.instance().display_formatter.format( obj, diff --git a/IPython/core/historyapp.py b/IPython/core/historyapp.py index 3bcc697a20c..a6437eff26e 100644 --- a/IPython/core/historyapp.py +++ b/IPython/core/historyapp.py @@ -9,9 +9,9 @@ import sqlite3 from traitlets.config.application import Application -from IPython.core.application import BaseIPythonApplication +from .application import BaseIPythonApplication from traitlets import Bool, Int, Dict -from IPython.utils.io import ask_yes_no +from ..utils.io import ask_yes_no trim_hist_help = """Trim the IPython history database to the last 1000 entries. diff --git a/IPython/core/hooks.py b/IPython/core/hooks.py index 66a544d7d8c..fa732f7ba82 100644 --- a/IPython/core/hooks.py +++ b/IPython/core/hooks.py @@ -37,10 +37,9 @@ def load_ipython_extension(ip): import os import subprocess -import warnings import sys -from IPython.core.error import TryNext +from .error import TryNext # List here all the default hooks. For now it's just the editor functions # but over time we'll move here all the public API for user-accessible things. @@ -82,44 +81,6 @@ def editor(self, filename, linenum=None, wait=True): if wait and proc.wait() != 0: raise TryNext() -import tempfile -from IPython.utils.decorators import undoc - -@undoc -def fix_error_editor(self,filename,linenum,column,msg): - """DEPRECATED - - Open the editor at the given filename, linenumber, column and - show an error message. This is used for correcting syntax errors. - The current implementation only has special support for the VIM editor, - and falls back on the 'editor' hook if VIM is not used. - - Call ip.set_hook('fix_error_editor',yourfunc) to use your own function, - """ - - warnings.warn(""" -`fix_error_editor` is deprecated as of IPython 6.0 and will be removed -in future versions. It appears to be used only for automatically fixing syntax -error that has been broken for a few years and has thus been removed. If you -happened to use this function and still need it please make your voice heard on -the mailing list ipython-dev@python.org , or on the GitHub Issue tracker: -https://github.com/ipython/ipython/issues/9649 """, UserWarning) - - def vim_quickfix_file(): - t = tempfile.NamedTemporaryFile() - t.write('%s:%d:%d:%s\n' % (filename,linenum,column,msg)) - t.flush() - return t - if os.path.basename(self.editor) != 'vim': - self.hooks.editor(filename,linenum) - return - t = vim_quickfix_file() - try: - if os.system('vim --cmd "set errorformat=%f:%l:%c:%m" -q ' + t.name): - raise TryNext() - finally: - t.close() - def synchronize_with_editor(self, filename, linenum, column): pass @@ -212,7 +173,7 @@ def pre_run_code_hook(self): def clipboard_get(self): """ Get text from the clipboard. """ - from IPython.lib.clipboard import ( + from ..lib.clipboard import ( osx_clipboard_get, tkinter_clipboard_get, win32_clipboard_get ) diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py index 84aa0a71f0f..e7bc6e7f5a3 100644 --- a/IPython/core/inputsplitter.py +++ b/IPython/core/inputsplitter.py @@ -31,7 +31,6 @@ import tokenize import warnings -from IPython.utils.py3compat import cast_unicode from IPython.core.inputtransformer import (leading_indent, classic_prompt, ipy_prompt, @@ -386,7 +385,7 @@ def check_complete(self, source): finally: self.reset() - def push(self, lines): + def push(self, lines:str) -> bool: """Push one or more lines of input. This stores the given lines and returns a status code indicating @@ -408,6 +407,7 @@ def push(self, lines): this value is also stored as a private attribute (``_is_complete``), so it can be queried at any time. """ + assert isinstance(lines, str) self._store(lines) source = self.source @@ -677,7 +677,7 @@ def transform_cell(self, cell): finally: self.reset() - def push(self, lines): + def push(self, lines:str) -> bool: """Push one or more lines of IPython input. This stores the given lines and returns a status code indicating @@ -700,9 +700,8 @@ def push(self, lines): this value is also stored as a private attribute (_is_complete), so it can be queried at any time. """ - + assert isinstance(lines, str) # We must ensure all input is pure unicode - lines = cast_unicode(lines, self.encoding) # ''.splitlines() --> [], but we need to push the empty line to transformers lines_list = lines.splitlines() if not lines_list: diff --git a/IPython/core/inputtransformer.py b/IPython/core/inputtransformer.py index 1c35eb64f32..afeca93cc0e 100644 --- a/IPython/core/inputtransformer.py +++ b/IPython/core/inputtransformer.py @@ -278,8 +278,8 @@ def escaped_commands(line): _initial_space_re = re.compile(r'\s*') _help_end_re = re.compile(r"""(%{0,2} - [a-zA-Z_*][\w*]* # Variable name - (\.[a-zA-Z_*][\w*]*)* # .etc.etc + (?!\d)[\w*]+ # Variable name + (\.(?!\d)[\w*]+)* # .etc.etc ) (\?\??)$ # ? or ?? """, diff --git a/IPython/core/inputtransformer2.py b/IPython/core/inputtransformer2.py index 04f423274da..0443e6829b4 100644 --- a/IPython/core/inputtransformer2.py +++ b/IPython/core/inputtransformer2.py @@ -405,8 +405,8 @@ def transform(self, lines): return lines_before + [new_line] + lines_after _help_end_re = re.compile(r"""(%{0,2} - [a-zA-Z_*][\w*]* # Variable name - (\.[a-zA-Z_*][\w*]*)* # .etc.etc + (?!\d)[\w*]+ # Variable name + (\.(?!\d)[\w*]+)* # .etc.etc ) (\?\??)$ # ? or ?? """, @@ -674,8 +674,8 @@ def check_complete(self, cell: str): while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types: tokens_by_line[-1].pop() - if len(tokens_by_line) == 1 and not tokens_by_line[-1]: - return 'incomplete', 0 + if not tokens_by_line[-1]: + return 'incomplete', find_last_indent(lines) if tokens_by_line[-1][-1].string == ':': # The last line starts a block (e.g. 'if foo:') diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index d1b59d76c0e..ddb1b64ea78 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -13,7 +13,6 @@ import abc import ast -import asyncio import atexit import builtins as builtin_mod import functools @@ -87,6 +86,7 @@ # NoOpContext is deprecated, but ipykernel imports it from here. # See https://github.com/ipython/ipykernel/issues/157 +# (2016, let's try to remove than in IPython 8.0) from IPython.utils.contexts import NoOpContext try: @@ -166,13 +166,7 @@ def removed_co_newlocals(function:types.FunctionType) -> types.FunctionType: # we still need to run things using the asyncio eventloop, but there is no # async integration from .async_helpers import (_asyncio_runner, _asyncify, _pseudo_sync_runner) -if sys.version_info > (3, 5): - from .async_helpers import _curio_runner, _trio_runner, _should_be_async -else : - _curio_runner = _trio_runner = None - - def _should_be_async(cell:str)->bool: - return False +from .async_helpers import _curio_runner, _trio_runner, _should_be_async def _ast_asyncify(cell:str, wrapper_name:str) -> ast.Module: @@ -701,6 +695,13 @@ def __init__(self, ipython_dir=None, profile_dir=None, self.events.trigger('shell_initialized', self) atexit.register(self.atexit_operations) + # The trio runner is used for running Trio in the foreground thread. It + # is different from `_trio_runner(async_fn)` in `async_helpers.py` + # which calls `trio.run()` for every cell. This runner runs all cells + # inside a single Trio event loop. If used, it is set from + # `ipykernel.kernelapp`. + self.trio_runner = None + def get_ipython(self): """Return the currently running IPython instance.""" return self @@ -721,6 +722,9 @@ def set_autoindent(self,value=None): else: self.autoindent = value + def set_trio_runner(self, tr): + self.trio_runner = tr + #------------------------------------------------------------------------- # init_* methods called by __init__ #------------------------------------------------------------------------- @@ -863,7 +867,7 @@ def init_display_formatter(self): self.configurables.append(self.display_formatter) def init_display_pub(self): - self.display_pub = self.display_pub_class(parent=self) + self.display_pub = self.display_pub_class(parent=self, shell=self) self.configurables.append(self.display_pub) def init_data_pub(self): @@ -2207,14 +2211,23 @@ def complete(self, text, line=None, cursor_pos=None): with self.builtin_trap: return self.Completer.complete(text, line, cursor_pos) - def set_custom_completer(self, completer, pos=0): + def set_custom_completer(self, completer, pos=0) -> None: """Adds a new custom completer function. The position argument (defaults to 0) is the index in the completers - list where you want the completer to be inserted.""" + list where you want the completer to be inserted. + + `completer` should have the following signature:: + + def completion(self: Completer, text: string) -> List[str]: + raise NotImplementedError + + It will be bound to the current Completer instance and pass some text + and return a list with current completions to suggest to the user. + """ - newcomp = types.MethodType(completer,self.Completer) - self.Completer.matchers.insert(pos,newcomp) + newcomp = types.MethodType(completer, self.Completer) + self.Completer.custom_matchers.insert(pos,newcomp) def set_completer_frame(self, frame=None): """Set the frame of the completer.""" @@ -2245,8 +2258,7 @@ def init_magics(self): m.NamespaceMagics, m.OSMagics, m.PackagingMagics, m.PylabMagics, m.ScriptMagics, ) - if sys.version_info >(3,5): - self.register_magics(m.AsyncMagics) + self.register_magics(m.AsyncMagics) # Register Magic Aliases mman = self.magics_manager @@ -2872,7 +2884,9 @@ def _run_cell(self, raw_cell:str, store_history:bool, silent:bool, shell_futures # when this is the case, we want to run it using the pseudo_sync_runner # so that code can invoke eventloops (for example via the %run , and # `%paste` magic. - if self.should_run_async(raw_cell): + if self.trio_runner: + runner = self.trio_runner + elif self.should_run_async(raw_cell): runner = self.loop_runner else: runner = _pseudo_sync_runner @@ -3305,6 +3319,9 @@ async def run_code(self, code_obj, result=None, *, async_=False): False : successful execution. True : an error occurred. """ + # special value to say that anything above is IPython and should be + # hidden. + __tracebackhide__ = "__ipython_bottom__" # Set our own excepthook in case the user code tries to call it # directly, so that the IPython crash handler doesn't get triggered old_excepthook, sys.excepthook = sys.excepthook, self.excepthook diff --git a/IPython/core/magic.py b/IPython/core/magic.py index 2b41617bed7..bc51677f083 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -17,13 +17,13 @@ from getopt import getopt, GetoptError from traitlets.config.configurable import Configurable -from IPython.core import oinspect -from IPython.core.error import UsageError -from IPython.core.inputtransformer2 import ESC_MAGIC, ESC_MAGIC2 +from . import oinspect +from .error import UsageError +from .inputtransformer2 import ESC_MAGIC, ESC_MAGIC2 from decorator import decorator -from IPython.utils.ipstruct import Struct -from IPython.utils.process import arg_split -from IPython.utils.text import dedent +from ..utils.ipstruct import Struct +from ..utils.process import arg_split +from ..utils.text import dedent from traitlets import Bool, Dict, Instance, observe from logging import error diff --git a/IPython/core/magics/basic.py b/IPython/core/magics/basic.py index 3721cbc4eb0..a8feb755386 100644 --- a/IPython/core/magics/basic.py +++ b/IPython/core/magics/basic.py @@ -5,7 +5,6 @@ from logging import error import io from pprint import pformat -import textwrap import sys from warnings import warn @@ -124,7 +123,7 @@ def alias_magic(self, line=''): In [6]: %whereami Out[6]: u'/home/testuser' - In [7]: %alias_magic h history -p "-l 30" --line + In [7]: %alias_magic h history "-p -l 30" --line Created `%h` as an alias for `%history -l 30`. """ @@ -365,13 +364,25 @@ def xmode(self, parameter_s=''): Valid modes: Plain, Context, Verbose, and Minimal. - If called without arguments, acts as a toggle.""" + If called without arguments, acts as a toggle. + + When in verbose mode the value --show (and --hide) + will respectively show (or hide) frames with ``__tracebackhide__ = + True`` value set. + """ def xmode_switch_err(name): warn('Error changing %s exception modes.\n%s' % (name,sys.exc_info()[1])) shell = self.shell + if parameter_s.strip() == "--show": + shell.InteractiveTB.skip_hidden = False + return + if parameter_s.strip() == "--hide": + shell.InteractiveTB.skip_hidden = True + return + new_mode = parameter_s.strip().capitalize() try: shell.InteractiveTB.set_mode(mode=new_mode) diff --git a/IPython/core/magics/code.py b/IPython/core/magics/code.py index 41aa37ca7cf..a1841384651 100644 --- a/IPython/core/magics/code.py +++ b/IPython/core/magics/code.py @@ -29,7 +29,6 @@ from IPython.core.magic import Magics, magics_class, line_magic from IPython.core.oinspect import find_file, find_source_lines from IPython.testing.skipdoctest import skip_doctest -from IPython.utils import py3compat from IPython.utils.contexts import preserve_keys from IPython.utils.path import get_py_filename from warnings import warn @@ -214,9 +213,9 @@ def save(self, parameter_s=''): force = 'f' in opts append = 'a' in opts mode = 'a' if append else 'w' - ext = u'.ipy' if raw else u'.py' + ext = '.ipy' if raw else '.py' fname, codefrom = args[0], " ".join(args[1:]) - if not fname.endswith((u'.py',u'.ipy')): + if not fname.endswith(('.py','.ipy')): fname += ext file_exists = os.path.isfile(fname) if file_exists and not force and not append: @@ -233,14 +232,13 @@ def save(self, parameter_s=''): except (TypeError, ValueError) as e: print(e.args[0]) return - out = py3compat.cast_unicode(cmds) with io.open(fname, mode, encoding="utf-8") as f: if not file_exists or not append: - f.write(u"# coding: utf-8\n") - f.write(out) + f.write("# coding: utf-8\n") + f.write(cmds) # make sure we end on a newline - if not out.endswith(u'\n'): - f.write(u'\n') + if not cmds.endswith('\n'): + f.write('\n') print('The following commands were written to file `%s`:' % fname) print(cmds) diff --git a/IPython/core/magics/execution.py b/IPython/core/magics/execution.py index d9e66b545a7..dc6cdf00e29 100644 --- a/IPython/core/magics/execution.py +++ b/IPython/core/magics/execution.py @@ -688,17 +688,16 @@ def run(self, parameter_s='', runner=None, modulename = opts["m"][0] modpath = find_mod(modulename) if modpath is None: - warn('%r is not a valid modulename on sys.path'%modulename) - return + msg = '%r is not a valid modulename on sys.path'%modulename + raise Exception(msg) arg_lst = [modpath] + arg_lst try: fpath = None # initialize to make sure fpath is in scope later fpath = arg_lst[0] filename = file_finder(fpath) except IndexError: - warn('you must provide at least a filename.') - print('\n%run:\n', oinspect.getdoc(self.run)) - return + msg = 'you must provide at least a filename.' + raise Exception(msg) except IOError as e: try: msg = str(e) @@ -706,13 +705,17 @@ def run(self, parameter_s='', runner=None, msg = e.message if os.name == 'nt' and re.match(r"^'.*'$",fpath): warn('For Windows, use double quotes to wrap a filename: %run "mypath\\myfile.py"') - error(msg) - return + raise Exception(msg) + except TypeError: + if fpath in sys.meta_path: + filename = "" + else: + raise if filename.lower().endswith(('.ipy', '.ipynb')): with preserve_keys(self.shell.user_ns, '__file__'): self.shell.user_ns['__file__'] = filename - self.shell.safe_execfile_ipy(filename) + self.shell.safe_execfile_ipy(filename, raise_exceptions=True) return # Control the response to exit() calls made by the script being run @@ -853,6 +856,8 @@ def run(): sys.argv = save_argv if restore_main: sys.modules['__main__'] = restore_main + if '__mp_main__' in sys.modules: + sys.modules['__mp_main__'] = restore_main else: # Remove from sys.modules the reference to main_mod we'd # added. Otherwise it will trap references to objects diff --git a/IPython/core/magics/namespace.py b/IPython/core/magics/namespace.py index cef6ddba8d7..acc4620549b 100644 --- a/IPython/core/magics/namespace.py +++ b/IPython/core/magics/namespace.py @@ -208,12 +208,6 @@ def psearch(self, parameter_s=''): %psearch -l list all available object types """ - try: - parameter_s.encode('ascii') - except UnicodeEncodeError: - print('Python identifiers can only contain ascii characters.') - return - # default namespaces to be searched def_search = ['user_local', 'user_global', 'builtin'] diff --git a/IPython/core/magics/osm.py b/IPython/core/magics/osm.py index c6d1539bd6d..90da7e22803 100644 --- a/IPython/core/magics/osm.py +++ b/IPython/core/magics/osm.py @@ -25,6 +25,7 @@ from IPython.utils.process import abbrev_cwd from IPython.utils.terminal import set_term_title from traitlets import Bool +from warnings import warn @magics_class @@ -48,8 +49,15 @@ def __init__(self, shell=None, **kwargs): winext = os.environ['pathext'].replace(';','|').replace('.','') except KeyError: winext = 'exe|com|bat|py' - - self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE) + try: + self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE) + except re.error: + warn("Seems like your pathext environmental " + "variable is malformed. Please check it to " + "enable a proper handle of file extensions " + "managed for your system") + winext = 'exe|com|bat|py' + self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE) # call up the chain super().__init__(shell=shell, **kwargs) @@ -446,7 +454,13 @@ def env(self, parameter_s=''): raise UsageError(err) if len(bits) > 1: return self.set_env(parameter_s) - return dict(os.environ) + env = dict(os.environ) + # hide likely secrets when printing the whole environment + for key in list(env): + if any(s in key.lower() for s in ('key', 'token', 'secret')): + env[key] = '' + + return env @line_magic def set_env(self, parameter_s): diff --git a/IPython/core/magics/packaging.py b/IPython/core/magics/packaging.py index 6477c7defc7..cfee7865f5d 100644 --- a/IPython/core/magics/packaging.py +++ b/IPython/core/magics/packaging.py @@ -12,7 +12,6 @@ import re import shlex import sys -from subprocess import Popen, PIPE from IPython.core.magic import Magics, magics_class, line_magic @@ -101,4 +100,4 @@ def conda(self, line): extra_args.extend(["--prefix", sys.prefix]) self.shell.system(' '.join([conda, command] + extra_args + args)) - print("\nNote: you may need to restart the kernel to use updated packages.") \ No newline at end of file + print("\nNote: you may need to restart the kernel to use updated packages.") diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index 749f97959e6..ab25eeeffca 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -22,7 +22,8 @@ from textwrap import dedent import types import io as stdlib_io -from itertools import zip_longest + +from typing import Union # IPython's own from IPython.core import page @@ -76,13 +77,13 @@ def pylight(code): 'call_def', 'call_docstring', # These won't be printed but will be used to determine how to # format the object - 'ismagic', 'isalias', 'isclass', 'argspec', 'found', 'name' + 'ismagic', 'isalias', 'isclass', 'found', 'name' ] def object_info(**kw): """Make an object info dict with all fields present.""" - infodict = dict(zip_longest(info_fields, [None])) + infodict = {k:None for k in info_fields} infodict.update(kw) return infodict @@ -110,7 +111,7 @@ def get_encoding(obj): encoding, lines = openpy.detect_encoding(buffer.readline) return encoding -def getdoc(obj): +def getdoc(obj) -> Union[str,None]: """Stable wrapper around inspect.getdoc. This can't crash because of attribute problems. @@ -128,11 +129,10 @@ def getdoc(obj): if isinstance(ds, str): return inspect.cleandoc(ds) docstr = inspect.getdoc(obj) - encoding = get_encoding(obj) - return py3compat.cast_unicode(docstr, encoding=encoding) + return docstr -def getsource(obj, oname=''): +def getsource(obj, oname='') -> Union[str,None]: """Wrapper around inspect.getsource. This can be modified by other projects to provide customized source @@ -158,18 +158,15 @@ def getsource(obj, oname=''): if fn is not None: encoding = get_encoding(fn) oname_prefix = ('%s.' % oname) if oname else '' - sources.append(cast_unicode( - ''.join(('# ', oname_prefix, attrname)), - encoding=encoding)) + sources.append(''.join(('# ', oname_prefix, attrname))) if inspect.isfunction(fn): sources.append(dedent(getsource(fn))) else: # Default str/repr only prints function name, # pretty.pretty prints module name too. - sources.append(cast_unicode( - '%s%s = %s\n' % ( - oname_prefix, attrname, pretty(fn)), - encoding=encoding)) + sources.append( + '%s%s = %s\n' % (oname_prefix, attrname, pretty(fn)) + ) if sources: return '\n'.join(sources) else: @@ -191,8 +188,7 @@ def getsource(obj, oname=''): except TypeError: return None - encoding = get_encoding(obj) - return cast_unicode(src, encoding=encoding) + return src def is_simple_callable(obj): @@ -200,26 +196,38 @@ def is_simple_callable(obj): return (inspect.isfunction(obj) or inspect.ismethod(obj) or \ isinstance(obj, _builtin_func_type) or isinstance(obj, _builtin_meth_type)) - +@undoc def getargspec(obj): - """Wrapper around :func:`inspect.getfullargspec` on Python 3, and - :func:inspect.getargspec` on Python 2. + """Wrapper around :func:`inspect.getfullargspec` In addition to functions and methods, this can also handle objects with a ``__call__`` attribute. + + DEPRECATED: Deprecated since 7.10. Do not use, will be removed. """ + + warnings.warn('`getargspec` function is deprecated as of IPython 7.10' + 'and will be removed in future versions.', DeprecationWarning, stacklevel=2) + if safe_hasattr(obj, '__call__') and not is_simple_callable(obj): obj = obj.__call__ return inspect.getfullargspec(obj) - +@undoc def format_argspec(argspec): """Format argspect, convenience wrapper around inspect's. This takes a dict instead of ordered arguments and calls inspect.format_argspec with the arguments in the necessary order. + + DEPRECATED: Do not use; will be removed in future versions. """ + + warnings.warn('`format_argspec` function is deprecated as of IPython 7.10' + 'and will be removed in future versions.', DeprecationWarning, stacklevel=2) + + return inspect.formatargspec(argspec['args'], argspec['varargs'], argspec['varkw'], argspec['defaults']) @@ -276,7 +284,7 @@ def _get_wrapped(obj): return orig_obj return obj -def find_file(obj): +def find_file(obj) -> str: """Find the absolute path to the file where an object was defined. This is essentially a robust wrapper around `inspect.getabsfile`. @@ -357,18 +365,17 @@ def __init__(self, color_table=InspectColors, self.str_detail_level = str_detail_level self.set_active_scheme(scheme) - def _getdef(self,obj,oname=''): + def _getdef(self,obj,oname='') -> Union[str,None]: """Return the call signature for any callable object. If any exception is generated, None is returned instead and the exception is suppressed.""" try: - hdef = _render_signature(signature(obj), oname) - return cast_unicode(hdef) + return _render_signature(signature(obj), oname) except: return None - def __head(self,h): + def __head(self,h) -> str: """Return a header string with proper colors.""" return '%s%s%s' % (self.color_table.active_colors.header,h, self.color_table.active_colors.normal) @@ -504,29 +511,8 @@ def pfile(self, obj, oname=''): # 0-offset, so we must adjust. page.page(self.format(openpy.read_py_file(ofile, skip_encoding_cookie=False)), lineno - 1) - def _format_fields(self, fields, title_width=0): - """Formats a list of fields for display. - Parameters - ---------- - fields : list - A list of 2-tuples: (field_title, field_content) - title_width : int - How many characters to pad titles to. Default to longest title. - """ - out = [] - header = self.__head - if title_width == 0: - title_width = max(len(title) + 2 for title, _ in fields) - for title, content in fields: - if len(content.splitlines()) > 1: - title = header(title + ':') + '\n' - else: - title = header((title + ':').ljust(title_width)) - out.append(cast_unicode(title) + cast_unicode(content)) - return "\n".join(out) - - def _mime_format(self, text, formatter=None): + def _mime_format(self, text:str, formatter=None) -> dict: """Return a mime bundle representation of the input text. - if `formatter` is None, the returned mime bundle has @@ -542,7 +528,6 @@ def _mime_format(self, text, formatter=None): Formatters returning strings are supported but this behavior is deprecated. """ - text = cast_unicode(text) defaults = { 'text/plain': text, 'text/html': '
' + text + '
' @@ -605,7 +590,7 @@ def _get_info(self, obj, oname='', formatter=None, info=None, detail_level=0): 'text/html': '', } - def append_field(bundle, title, key, formatter=None): + def append_field(bundle, title:str, key:str, formatter=None): field = info[key] if field is not None: formatted_field = self._mime_format(field, formatter) @@ -725,7 +710,8 @@ def _info(self, obj, oname='', info=None, detail_level=0) -> dict: Returns ======= - An object info dict with known fields from `info_fields`. + An object info dict with known fields from `info_fields`. Keys are + strings, values are string or None. """ if info is None: @@ -916,33 +902,6 @@ def _info(self, obj, oname='', info=None, detail_level=0) -> dict: if call_ds: out['call_docstring'] = call_ds - # Compute the object's argspec as a callable. The key is to decide - # whether to pull it from the object itself, from its __init__ or - # from its __call__ method. - - if inspect.isclass(obj): - # Old-style classes need not have an __init__ - callable_obj = getattr(obj, "__init__", None) - elif callable(obj): - callable_obj = obj - else: - callable_obj = None - - if callable_obj is not None: - try: - argspec = getargspec(callable_obj) - except Exception: - # For extensions/builtins we can't retrieve the argspec - pass - else: - # named tuples' _asdict() method returns an OrderedDict, but we - # we want a normal - out['argspec'] = argspec_dict = dict(argspec._asdict()) - # We called this varkw before argspec became a named tuple. - # With getfullargspec it's also called varkw. - if 'varkw' not in argspec_dict: - argspec_dict['varkw'] = argspec_dict.pop('keywords') - return object_info(**out) @staticmethod @@ -1029,7 +988,7 @@ def psearch(self,pattern,ns_table,ns_search=[], page.page('\n'.join(sorted(search_result))) -def _render_signature(obj_signature, obj_name): +def _render_signature(obj_signature, obj_name) -> str: """ This was mostly taken from inspect.Signature.__str__. Look there for the comments. diff --git a/IPython/core/page.py b/IPython/core/page.py index b5d5b1f56d1..ed16b617812 100644 --- a/IPython/core/page.py +++ b/IPython/core/page.py @@ -15,9 +15,11 @@ import os +import io import re import sys import tempfile +import subprocess from io import UnsupportedOperation @@ -208,9 +210,13 @@ def pager_page(strng, start=0, screen_lines=0, pager_cmd=None): else: try: retval = None - # if I use popen4, things hang. No idea why. - #pager,shell_out = os.popen4(pager_cmd) - pager = os.popen(pager_cmd, 'w') + # Emulate os.popen, but redirect stderr + proc = subprocess.Popen(pager_cmd, + shell=True, + stdin=subprocess.PIPE, + stderr=subprocess.DEVNULL + ) + pager = os._wrap_close(io.TextIOWrapper(proc.stdin), proc) try: pager_encoding = pager.encoding or sys.stdout.encoding pager.write(strng) @@ -335,32 +341,3 @@ def page_more(): return False else: return True - - -def snip_print(str,width = 75,print_full = 0,header = ''): - """Print a string snipping the midsection to fit in width. - - print_full: mode control: - - - 0: only snip long strings - - 1: send to page() directly. - - 2: snip long strings and ask for full length viewing with page() - - Return 1 if snipping was necessary, 0 otherwise.""" - - if print_full == 1: - page(header+str) - return 0 - - print(header, end=' ') - if len(str) < width: - print(str) - snip = 0 - else: - whalf = int((width -5)/2) - print(str[:whalf] + ' <...> ' + str[-whalf:]) - snip = 1 - if snip and print_full == 2: - if py3compat.input(header+' Snipped. View (y/n)? [N]').lower() == 'y': - page(str) - return snip diff --git a/IPython/core/prefilter.py b/IPython/core/prefilter.py index 0262a29bf3b..bf801f999c4 100644 --- a/IPython/core/prefilter.py +++ b/IPython/core/prefilter.py @@ -12,16 +12,16 @@ from keyword import iskeyword import re -from IPython.core.autocall import IPyAutocall +from .autocall import IPyAutocall from traitlets.config.configurable import Configurable -from IPython.core.inputtransformer2 import ( +from .inputtransformer2 import ( ESC_MAGIC, ESC_QUOTE, ESC_QUOTE2, ESC_PAREN, ) -from IPython.core.macro import Macro -from IPython.core.splitinput import LineInfo +from .macro import Macro +from .splitinput import LineInfo from traitlets import ( List, Integer, Unicode, Bool, Instance, CRegExp @@ -37,7 +37,7 @@ class PrefilterError(Exception): # RegExp to identify potential function names -re_fun_name = re.compile(r'[a-zA-Z_]([a-zA-Z0-9_.]*) *$') +re_fun_name = re.compile(r'[^\W\d]([\w.]*) *$') # RegExp to exclude strings with this start from autocalling. In # particular, all binary operators should be excluded, so that if foo is diff --git a/IPython/core/profileapp.py b/IPython/core/profileapp.py index 97434e3d0b5..9a1bae55ac5 100644 --- a/IPython/core/profileapp.py +++ b/IPython/core/profileapp.py @@ -181,9 +181,10 @@ def list_profile_dirs(self): profiles = list_profiles_in(os.getcwd()) if profiles: print() - print("Available profiles in current directory (%s):" % os.getcwd()) - self._print_profiles(profiles) - + print( + "Profiles from CWD have been removed for security reason, see CVE-2022-21699:" + ) + print() print("To use any of the above profiles, start IPython with:") print(" ipython --profile=") diff --git a/IPython/core/profiledir.py b/IPython/core/profiledir.py index 6ab600f3004..2c48e4c2f1c 100644 --- a/IPython/core/profiledir.py +++ b/IPython/core/profiledir.py @@ -9,8 +9,8 @@ import errno from traitlets.config.configurable import LoggingConfigurable -from IPython.paths import get_ipython_package_dir -from IPython.utils.path import expand_path, ensure_dir_exists +from ..paths import get_ipython_package_dir +from ..utils.path import expand_path, ensure_dir_exists from traitlets import Unicode, Bool, observe #----------------------------------------------------------------------------- @@ -186,7 +186,7 @@ def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None): is not found, a :class:`ProfileDirError` exception will be raised. The search path algorithm is: - 1. ``os.getcwd()`` + 1. ``os.getcwd()`` # removed for security reason. 2. ``ipython_dir`` Parameters @@ -198,7 +198,7 @@ def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None): will be "profile_". """ dirname = u'profile_' + name - paths = [os.getcwd(), ipython_dir] + paths = [ipython_dir] for p in paths: profile_dir = os.path.join(p, dirname) if os.path.isdir(profile_dir): diff --git a/IPython/core/pylabtools.py b/IPython/core/pylabtools.py index 4423ed5d408..cb1ce811984 100644 --- a/IPython/core/pylabtools.py +++ b/IPython/core/pylabtools.py @@ -123,14 +123,18 @@ def print_figure(fig, fmt='png', bbox_inches='tight', **kwargs): } # **kwargs get higher priority kw.update(kwargs) - + bytes_io = BytesIO() + if fig.canvas is None: + from matplotlib.backend_bases import FigureCanvasBase + FigureCanvasBase(fig) + fig.canvas.print_figure(bytes_io, **kw) data = bytes_io.getvalue() if fmt == 'svg': data = data.decode('utf-8') return data - + def retina_figure(fig, **kwargs): """format a figure as a pixel-doubled (retina) PNG""" pngdata = print_figure(fig, fmt='retina', **kwargs) @@ -310,12 +314,12 @@ def activate_matplotlib(backend): # magic of switch_backend(). matplotlib.rcParams['backend'] = backend - import matplotlib.pyplot - matplotlib.pyplot.switch_backend(backend) + # Due to circular imports, pyplot may be only partially initialised + # when this function runs. + # So avoid needing matplotlib attribute-lookup to access pyplot. + from matplotlib import pyplot as plt - # This must be imported last in the matplotlib series, after - # backend/interactivity choices have been made - import matplotlib.pyplot as plt + plt.switch_backend(backend) plt.show._needmain = False # We need to detect at runtime whether show() is called by the user. diff --git a/IPython/core/release.py b/IPython/core/release.py index 0ad67d00f56..a6f3cf81f7d 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -20,11 +20,11 @@ # release. 'dev' as a _version_extra string means this is a development # version _version_major = 7 -_version_minor = 8 -_version_patch = 0 +_version_minor = 16 +_version_patch = 3 _version_extra = '.dev' # _version_extra = 'b1' -_version_extra = '' # Uncomment this for full releases +_version_extra = "" # Uncomment this for full releases # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] diff --git a/IPython/core/shellapp.py b/IPython/core/shellapp.py index a09192f255c..9e8bfbfbb81 100644 --- a/IPython/core/shellapp.py +++ b/IPython/core/shellapp.py @@ -60,6 +60,10 @@ colours.""", "Disable using colors for info related things." ) +addflag('ignore-cwd', 'InteractiveShellApp.ignore_cwd', + "Exclude the current working directory from sys.path", + "Include the current working directory in sys.path", +) nosep_config = Config() nosep_config.InteractiveShell.separate_in = '' nosep_config.InteractiveShell.separate_out = '' @@ -168,6 +172,12 @@ class InteractiveShellApp(Configurable): When False, pylab mode should not import any names into the user namespace. """ ).tag(config=True) + ignore_cwd = Bool( + False, + help="""If True, IPython will not add the current working directory to sys.path. + When False, the current working directory is added to sys.path, allowing imports + of modules defined in the current directory.""" + ).tag(config=True) shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True) # whether interact-loop should start @@ -189,8 +199,10 @@ def init_path(self): .. versionchanged:: 7.2 Try to insert after the standard library, instead of first. + .. versionchanged:: 8.0 + Allow optionally not including the current directory in sys.path """ - if '' in sys.path: + if '' in sys.path or self.ignore_cwd: return for idx, path in enumerate(sys.path): parent, last_part = os.path.split(path) @@ -404,6 +416,10 @@ def _run_cmd_line_code(self): fname = self.file_to_run if os.path.isdir(fname): fname = os.path.join(fname, "__main__.py") + if not os.path.exists(fname): + self.log.warning("File '%s' doesn't exist", fname) + if not self.interact: + self.exit(2) try: self._exec_file(fname, shell_futures=True) except: diff --git a/IPython/core/tests/refbug.py b/IPython/core/tests/refbug.py index b8de4c81078..92e2eead347 100644 --- a/IPython/core/tests/refbug.py +++ b/IPython/core/tests/refbug.py @@ -16,7 +16,6 @@ #----------------------------------------------------------------------------- # Module imports #----------------------------------------------------------------------------- -import sys from IPython import get_ipython diff --git a/IPython/core/tests/test_async_helpers.py b/IPython/core/tests/test_async_helpers.py index f23eae06acc..11c475874d7 100644 --- a/IPython/core/tests/test_async_helpers.py +++ b/IPython/core/tests/test_async_helpers.py @@ -3,309 +3,314 @@ Should only trigger on python 3.5+ or will have syntax errors. """ -import sys from itertools import chain, repeat import nose.tools as nt from textwrap import dedent, indent from unittest import TestCase from IPython.testing.decorators import skip_without - +import sys iprc = lambda x: ip.run_cell(dedent(x)).raise_error() iprc_nr = lambda x: ip.run_cell(dedent(x)) -if sys.version_info > (3, 5): - from IPython.core.async_helpers import _should_be_async - - class AsyncTest(TestCase): - def test_should_be_async(self): - nt.assert_false(_should_be_async("False")) - nt.assert_true(_should_be_async("await bar()")) - nt.assert_true(_should_be_async("x = await bar()")) - nt.assert_false( - _should_be_async( - dedent( - """ - async def awaitable(): - pass - """ - ) +from IPython.core.async_helpers import _should_be_async + +class AsyncTest(TestCase): + def test_should_be_async(self): + nt.assert_false(_should_be_async("False")) + nt.assert_true(_should_be_async("await bar()")) + nt.assert_true(_should_be_async("x = await bar()")) + nt.assert_false( + _should_be_async( + dedent( + """ + async def awaitable(): + pass + """ ) ) - - def _get_top_level_cases(self): - # These are test cases that should be valid in a function - # but invalid outside of a function. - test_cases = [] - test_cases.append(('basic', "{val}")) - - # Note, in all conditional cases, I use True instead of - # False so that the peephole optimizer won't optimize away - # the return, so CPython will see this as a syntax error: - # - # while True: - # break - # return - # - # But not this: - # - # while False: - # return - # - # See https://bugs.python.org/issue1875 - - test_cases.append(('if', dedent(""" - if True: + ) + + def _get_top_level_cases(self): + # These are test cases that should be valid in a function + # but invalid outside of a function. + test_cases = [] + test_cases.append(('basic', "{val}")) + + # Note, in all conditional cases, I use True instead of + # False so that the peephole optimizer won't optimize away + # the return, so CPython will see this as a syntax error: + # + # while True: + # break + # return + # + # But not this: + # + # while False: + # return + # + # See https://bugs.python.org/issue1875 + + test_cases.append(('if', dedent(""" + if True: + {val} + """))) + + test_cases.append(('while', dedent(""" + while True: + {val} + break + """))) + + test_cases.append(('try', dedent(""" + try: + {val} + except: + pass + """))) + + test_cases.append(('except', dedent(""" + try: + pass + except: + {val} + """))) + + test_cases.append(('finally', dedent(""" + try: + pass + except: + pass + finally: + {val} + """))) + + test_cases.append(('for', dedent(""" + for _ in range(4): + {val} + """))) + + + test_cases.append(('nested', dedent(""" + if True: + while True: {val} - """))) + break + """))) - test_cases.append(('while', dedent(""" + test_cases.append(('deep-nested', dedent(""" + if True: while True: - {val} break - """))) + for x in range(3): + if True: + while True: + for x in range(3): + {val} + """))) - test_cases.append(('try', dedent(""" - try: - {val} - except: - pass - """))) + return test_cases - test_cases.append(('except', dedent(""" - try: - pass - except: - {val} - """))) + def _get_ry_syntax_errors(self): + # This is a mix of tests that should be a syntax error if + # return or yield whether or not they are in a function - test_cases.append(('finally', dedent(""" - try: - pass - except: - pass - finally: - {val} - """))) + test_cases = [] - test_cases.append(('for', dedent(""" - for _ in range(4): + test_cases.append(('class', dedent(""" + class V: + {val} + """))) + + test_cases.append(('nested-class', dedent(""" + class V: + class C: {val} - """))) + """))) + return test_cases - test_cases.append(('nested', dedent(""" - if True: - while True: - {val} - break - """))) - test_cases.append(('deep-nested', dedent(""" - if True: - while True: + def test_top_level_return_error(self): + tl_err_test_cases = self._get_top_level_cases() + tl_err_test_cases.extend(self._get_ry_syntax_errors()) + + vals = ('return', 'yield', 'yield from (_ for _ in range(3))', + dedent(''' + def f(): + pass + return + '''), + ) + + for test_name, test_case in tl_err_test_cases: + # This example should work if 'pass' is used as the value + with self.subTest((test_name, 'pass')): + iprc(test_case.format(val='pass')) + + # It should fail with all the values + for val in vals: + with self.subTest((test_name, val)): + msg = "Syntax error not raised for %s, %s" % (test_name, val) + with self.assertRaises(SyntaxError, msg=msg): + iprc(test_case.format(val=val)) + + def test_in_func_no_error(self): + # Test that the implementation of top-level return/yield + # detection isn't *too* aggressive, and works inside a function + func_contexts = [] + + func_contexts.append(('func', False, dedent(""" + def f():"""))) + + func_contexts.append(('method', False, dedent(""" + class MyClass: + def __init__(self): + """))) + + func_contexts.append(('async-func', True, dedent(""" + async def f():"""))) + + func_contexts.append(('async-method', True, dedent(""" + class MyClass: + async def f(self):"""))) + + func_contexts.append(('closure', False, dedent(""" + def f(): + def g(): + """))) + + def nest_case(context, case): + # Detect indentation + lines = context.strip().splitlines() + prefix_len = 0 + for c in lines[-1]: + if c != ' ': break - for x in range(3): - if True: - while True: - for x in range(3): - {val} - """))) + prefix_len += 1 - return test_cases + indented_case = indent(case, ' ' * (prefix_len + 4)) + return context + '\n' + indented_case - def _get_ry_syntax_errors(self): - # This is a mix of tests that should be a syntax error if - # return or yield whether or not they are in a function + # Gather and run the tests - test_cases = [] + # yield is allowed in async functions, starting in Python 3.6, + # and yield from is not allowed in any version + vals = ('return', 'yield', 'yield from (_ for _ in range(3))') + async_safe = (True, + True, + False) + vals = tuple(zip(vals, async_safe)) - test_cases.append(('class', dedent(""" - class V: - {val} - """))) - - test_cases.append(('nested-class', dedent(""" - class V: - class C: - {val} - """))) - - return test_cases - - - def test_top_level_return_error(self): - tl_err_test_cases = self._get_top_level_cases() - tl_err_test_cases.extend(self._get_ry_syntax_errors()) - - vals = ('return', 'yield', 'yield from (_ for _ in range(3))', - dedent(''' - def f(): - pass - return - '''), - ) - - for test_name, test_case in tl_err_test_cases: - # This example should work if 'pass' is used as the value - with self.subTest((test_name, 'pass')): - iprc(test_case.format(val='pass')) - - # It should fail with all the values - for val in vals: - with self.subTest((test_name, val)): - msg = "Syntax error not raised for %s, %s" % (test_name, val) - with self.assertRaises(SyntaxError, msg=msg): - iprc(test_case.format(val=val)) - - def test_in_func_no_error(self): - # Test that the implementation of top-level return/yield - # detection isn't *too* aggressive, and works inside a function - func_contexts = [] - - func_contexts.append(('func', False, dedent(""" - def f():"""))) - - func_contexts.append(('method', False, dedent(""" - class MyClass: - def __init__(self): - """))) - - func_contexts.append(('async-func', True, dedent(""" - async def f():"""))) - - func_contexts.append(('async-method', True, dedent(""" - class MyClass: - async def f(self):"""))) - - func_contexts.append(('closure', False, dedent(""" - def f(): - def g(): - """))) - - def nest_case(context, case): - # Detect indentation - lines = context.strip().splitlines() - prefix_len = 0 - for c in lines[-1]: - if c != ' ': - break - prefix_len += 1 - - indented_case = indent(case, ' ' * (prefix_len + 4)) - return context + '\n' + indented_case - - # Gather and run the tests - - # yield is allowed in async functions, starting in Python 3.6, - # and yield from is not allowed in any version - vals = ('return', 'yield', 'yield from (_ for _ in range(3))') - async_safe = (True, - sys.version_info >= (3, 6), - False) - vals = tuple(zip(vals, async_safe)) - - success_tests = zip(self._get_top_level_cases(), repeat(False)) - failure_tests = zip(self._get_ry_syntax_errors(), repeat(True)) - - tests = chain(success_tests, failure_tests) - - for context_name, async_func, context in func_contexts: - for (test_name, test_case), should_fail in tests: - nested_case = nest_case(context, test_case) - - for val, async_safe in vals: - val_should_fail = (should_fail or - (async_func and not async_safe)) - - test_id = (context_name, test_name, val) - cell = nested_case.format(val=val) - - with self.subTest(test_id): - if val_should_fail: - msg = ("SyntaxError not raised for %s" % - str(test_id)) - with self.assertRaises(SyntaxError, msg=msg): - iprc(cell) - - print(cell) - else: - iprc(cell) + success_tests = zip(self._get_top_level_cases(), repeat(False)) + failure_tests = zip(self._get_ry_syntax_errors(), repeat(True)) - def test_nonlocal(self): - # fails if outer scope is not a function scope or if var not defined - with self.assertRaises(SyntaxError): - iprc("nonlocal x") - iprc(""" - x = 1 - def f(): - nonlocal x - x = 10000 - yield x - """) - iprc(""" - def f(): - def g(): - nonlocal x - x = 10000 - yield x - """) - - # works if outer scope is a function scope and var exists - iprc(""" - def f(): - x = 20 - def g(): - nonlocal x - x = 10000 - yield x - """) + tests = chain(success_tests, failure_tests) + for context_name, async_func, context in func_contexts: + for (test_name, test_case), should_fail in tests: + nested_case = nest_case(context, test_case) - def test_execute(self): - iprc(""" - import asyncio - await asyncio.sleep(0.001) - """ - ) + for val, async_safe in vals: + val_should_fail = (should_fail or + (async_func and not async_safe)) - def test_autoawait(self): - iprc("%autoawait False") - iprc("%autoawait True") - iprc(""" - from asyncio import sleep - await sleep(0.1) - """ - ) + test_id = (context_name, test_name, val) + cell = nested_case.format(val=val) - @skip_without('curio') - def test_autoawait_curio(self): - iprc("%autoawait curio") + with self.subTest(test_id): + if val_should_fail: + msg = ("SyntaxError not raised for %s" % + str(test_id)) + with self.assertRaises(SyntaxError, msg=msg): + iprc(cell) - @skip_without('trio') - def test_autoawait_trio(self): - iprc("%autoawait trio") + print(cell) + else: + iprc(cell) - @skip_without('trio') - def test_autoawait_trio_wrong_sleep(self): - iprc("%autoawait trio") - res = iprc_nr(""" - import asyncio - await asyncio.sleep(0) + def test_nonlocal(self): + # fails if outer scope is not a function scope or if var not defined + with self.assertRaises(SyntaxError): + iprc("nonlocal x") + iprc(""" + x = 1 + def f(): + nonlocal x + x = 10000 + yield x """) - with nt.assert_raises(TypeError): - res.raise_error() - - @skip_without('trio') - def test_autoawait_asyncio_wrong_sleep(self): - iprc("%autoawait asyncio") - res = iprc_nr(""" - import trio - await trio.sleep(0) + iprc(""" + def f(): + def g(): + nonlocal x + x = 10000 + yield x """) - with nt.assert_raises(RuntimeError): - res.raise_error() - - def tearDown(self): - ip.loop_runner = "asyncio" + # works if outer scope is a function scope and var exists + iprc(""" + def f(): + x = 20 + def g(): + nonlocal x + x = 10000 + yield x + """) + + + def test_execute(self): + iprc(""" + import asyncio + await asyncio.sleep(0.001) + """ + ) + + def test_autoawait(self): + iprc("%autoawait False") + iprc("%autoawait True") + iprc(""" + from asyncio import sleep + await sleep(0.1) + """ + ) + + if sys.version_info < (3,9): + # new pgen parser in 3.9 does not raise MemoryError on too many nested + # parens anymore + def test_memory_error(self): + with self.assertRaises(MemoryError): + iprc("(" * 200 + ")" * 200) + + @skip_without('curio') + def test_autoawait_curio(self): + iprc("%autoawait curio") + + @skip_without('trio') + def test_autoawait_trio(self): + iprc("%autoawait trio") + + @skip_without('trio') + def test_autoawait_trio_wrong_sleep(self): + iprc("%autoawait trio") + res = iprc_nr(""" + import asyncio + await asyncio.sleep(0) + """) + with nt.assert_raises(TypeError): + res.raise_error() + + @skip_without('trio') + def test_autoawait_asyncio_wrong_sleep(self): + iprc("%autoawait asyncio") + res = iprc_nr(""" + import trio + await trio.sleep(0) + """) + with nt.assert_raises(RuntimeError): + res.raise_error() + + + def tearDown(self): + ip.loop_runner = "asyncio" diff --git a/IPython/core/tests/test_autocall.py b/IPython/core/tests/test_autocall.py index 10a4e0d477d..ded9f78858a 100644 --- a/IPython/core/tests/test_autocall.py +++ b/IPython/core/tests/test_autocall.py @@ -7,7 +7,6 @@ """ from IPython.core.splitinput import LineInfo from IPython.core.prefilter import AutocallChecker -from IPython.utils import py3compat def doctest_autocall(): """ diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py index d364d1c1a10..2c19e2e0187 100644 --- a/IPython/core/tests/test_completer.py +++ b/IPython/core/tests/test_completer.py @@ -175,6 +175,20 @@ def complete_A(a, existing_completions): ip.complete("x.") + def test_custom_completion_ordering(self): + """Test that errors from custom attribute completers are silenced.""" + ip = get_ipython() + + _, matches = ip.complete('in') + assert matches.index('input') < matches.index('int') + + def complete_example(a): + return ['example2', 'example1'] + + ip.Completer.custom_completers.add_re('ex*', complete_example) + _, matches = ip.complete('ex') + assert matches.index('example2') < matches.index('example1') + def test_unicode_completions(self): ip = get_ipython() # Some strings that trigger different types of completion. Check them both @@ -210,20 +224,27 @@ def test_latex_completions(self): nt.assert_in("\\alpha", matches) nt.assert_in("\\aleph", matches) + def test_latex_no_results(self): + """ + forward latex should really return nothing in either field if nothing is found. + """ + ip = get_ipython() + text, matches = ip.Completer.latex_matches("\\really_i_should_match_nothing") + nt.assert_equal(text, "") + nt.assert_equal(matches, []) + def test_back_latex_completion(self): ip = get_ipython() # do not return more than 1 matches fro \beta, only the latex one. name, matches = ip.complete("\\β") - nt.assert_equal(len(matches), 1) - nt.assert_equal(matches[0], "\\beta") + nt.assert_equal(matches, ['\\beta']) def test_back_unicode_completion(self): ip = get_ipython() name, matches = ip.complete("\\Ⅴ") - nt.assert_equal(len(matches), 1) - nt.assert_equal(matches[0], "\\ROMAN NUMERAL FIVE") + nt.assert_equal(matches, ["\\ROMAN NUMERAL FIVE"]) def test_forward_unicode_completion(self): ip = get_ipython() @@ -471,10 +492,9 @@ def _(line, cursor_pos, expect, message, completion): 5, 6, "real" ) - if sys.version_info > (3, 4): - yield _, "a[0].from_", 10, "a[0].from_bytes", "Should have completed on a[0].from_: %s", Completion( - 5, 10, "from_bytes" - ) + yield _, "a[0].from_", 10, "a[0].from_bytes", "Should have completed on a[0].from_: %s", Completion( + 5, 10, "from_bytes" + ) def test_omit__names(self): # also happens to test IPCompleter as a configurable diff --git a/IPython/core/tests/test_debugger.py b/IPython/core/tests/test_debugger.py index dcfd9a42438..9fdc944e4d0 100644 --- a/IPython/core/tests/test_debugger.py +++ b/IPython/core/tests/test_debugger.py @@ -4,12 +4,24 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. +import bdb +import builtins +import os +import signal +import subprocess import sys +import time import warnings +from subprocess import PIPE, CalledProcessError, check_output +from tempfile import NamedTemporaryFile +from textwrap import dedent +from unittest.mock import patch import nose.tools as nt from IPython.core import debugger +from IPython.testing import IPYTHON_TESTING_TIMEOUT_SCALE +from IPython.testing.decorators import skip_win32 #----------------------------------------------------------------------------- # Helper classes, from CPython's Pdb test suite @@ -223,3 +235,92 @@ def can_exit(): >>> sys.settrace(old_trace) ''' + + +def test_interruptible_core_debugger(): + """The debugger can be interrupted. + + The presumption is there is some mechanism that causes a KeyboardInterrupt + (this is implemented in ipykernel). We want to ensure the + KeyboardInterrupt cause debugging to cease. + """ + def raising_input(msg="", called=[0]): + called[0] += 1 + if called[0] == 1: + raise KeyboardInterrupt() + else: + raise AssertionError("input() should only be called once!") + + with patch.object(builtins, "input", raising_input): + debugger.InterruptiblePdb().set_trace() + # The way this test will fail is by set_trace() never exiting, + # resulting in a timeout by the test runner. The alternative + # implementation would involve a subprocess, but that adds issues with + # interrupting subprocesses that are rather complex, so it's simpler + # just to do it this way. + +@skip_win32 +def test_xmode_skip(): + """that xmode skip frames + + Not as a doctest as pytest does not run doctests. + """ + import pexpect + env = os.environ.copy() + env["IPY_TEST_SIMPLE_PROMPT"] = "1" + + child = pexpect.spawn( + sys.executable, ["-m", "IPython", "--colors=nocolor"], env=env + ) + child.timeout = 15 * IPYTHON_TESTING_TIMEOUT_SCALE + + child.expect("IPython") + child.expect("\n") + child.expect_exact("In [1]") + + block = dedent( + """ +def f(): + __tracebackhide__ = True + g() + +def g(): + raise ValueError + +f() + """ + ) + + for line in block.splitlines(): + child.sendline(line) + child.expect_exact(line) + child.expect_exact("skipping") + + block = dedent( + """ +def f(): + __tracebackhide__ = True + g() + +def g(): + from IPython.core.debugger import set_trace + set_trace() + +f() + """ + ) + + for line in block.splitlines(): + child.sendline(line) + child.expect_exact(line) + + child.expect("ipdb>") + child.sendline("w") + child.expect("hidden") + child.expect("ipdb>") + child.sendline("skip_hidden false") + child.sendline("w") + child.expect("__traceba") + child.expect("ipdb>") + + child.close() diff --git a/IPython/core/tests/test_display.py b/IPython/core/tests/test_display.py index 1fed51127a1..95f1eb622e4 100644 --- a/IPython/core/tests/test_display.py +++ b/IPython/core/tests/test_display.py @@ -14,7 +14,7 @@ from IPython.utils.io import capture_output from IPython.utils.tempdir import NamedFileInTemporaryDirectory from IPython import paths as ipath -from IPython.testing.tools import AssertPrints, AssertNotPrints +from IPython.testing.tools import AssertNotPrints import IPython.testing.decorators as dec @@ -72,6 +72,40 @@ def test_retina_png(): nt.assert_equal(md['width'], 1) nt.assert_equal(md['height'], 1) +def test_embed_svg_url(): + import gzip + from io import BytesIO + svg_data = b'' + url = 'http://test.com/circle.svg' + + gzip_svg = BytesIO() + with gzip.open(gzip_svg, 'wb') as fp: + fp.write(svg_data) + gzip_svg = gzip_svg.getvalue() + + def mocked_urlopen(*args, **kwargs): + class MockResponse: + def __init__(self, svg): + self._svg_data = svg + self.headers = {'content-type': 'image/svg+xml'} + + def read(self): + return self._svg_data + + if args[0] == url: + return MockResponse(svg_data) + elif args[0] == url + 'z': + ret= MockResponse(gzip_svg) + ret.headers['content-encoding']= 'gzip' + return ret + return MockResponse(None) + + with mock.patch('urllib.request.urlopen', side_effect=mocked_urlopen): + svg = display.SVG(url=url) + nt.assert_true(svg._repr_svg_().startswith('