diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000000..b26d4442ea1 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,19 @@ +# Top-most EditorConfig file +root = true + +[*] +# Unix-style newlines with a newline ending every file +end_of_line = lf +insert_final_newline = true +charset = utf-8 + +# Four-space indentation +indent_size = 4 +indent_style = space + +trim_trailing_whitespace = false + +[*.yml] +# Two-space indentation +indent_size = 2 +indent_style = space diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000000..70971c53b5a --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,8 @@ +# When making commits that are strictly formatting/style changes, add the +# commit hash here, so git blame can ignore the change. +# See docs for more details: +# https://git-scm.com/docs/git-config#Documentation/git-config.txt-blameignoreRevsFile + +# Example entries: +# # initial black-format +# # rename something internal diff --git a/.gitignore b/.gitignore index c10cded6d18..1fc0e22a320 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ _build docs/man/*.gz docs/source/api/generated docs/source/config/options +docs/source/config/shortcuts/*.csv docs/source/interactive/magics-generated.txt docs/gh-pages jupyter_notebook/notebook/static/mathjax @@ -19,5 +20,11 @@ __pycache__ .DS_Store \#*# .#* +.cache .coverage *.swp +.vscode +.pytest_cache +.python-version +venv*/ +.idea/ diff --git a/.mailmap b/.mailmap index 5e092be6876..8d4757e6865 100644 --- a/.mailmap +++ b/.mailmap @@ -1,4 +1,5 @@ A. J. Holyoake ajholyoake +Alok Singh Alok Singh <8325708+alok@users.noreply.github.com> Aaron Culich Aaron Culich Aron Ahmadia ahmadia Benjamin Ragan-Kelley @@ -68,6 +69,7 @@ Jonathan Frederic Jonathan Frederic jon Jonathan Frederic U-Jon-PC\Jon Jonathan March Jonathan March +Jean Cruypenynck Jean Cruypenynck Jonathan March jdmarch Jörgen Stenarson Jörgen Stenarson Jörgen Stenarson Jorgen Stenarson @@ -81,6 +83,10 @@ Julia Evans Julia Evans Kester Tong KesterTong Kyle Kelley Kyle Kelley Kyle Kelley rgbkrk +kd2718 +Kory Donati kory donati +Kory Donati Kory Donati +Kory Donati koryd Laurent Dufréchou Laurent Dufréchou Laurent Dufréchou laurent dufrechou <> @@ -88,6 +94,7 @@ Laurent Dufréchou laurent.dufrechou <> Laurent Dufréchou Laurent Dufrechou <> Laurent Dufréchou laurent.dufrechou@gmail.com <> Laurent Dufréchou ldufrechou +Luciana da Costa Marques luciana Lorena Pantano Lorena Luis Pedro Coelho Luis Pedro Coelho Marc Molla marcmolla @@ -96,6 +103,7 @@ Matthias Bussonnier Matthias BUSSONNIER Bussonnier Matthias Matthias Bussonnier Matthias BUSSONNIER Matthias Bussonnier Matthias Bussonnier +Matthias Bussonnier Matthias Bussonnier Michael Droettboom Michael Droettboom Nicholas Bollweg Nicholas Bollweg (Nick) Nicolas Rougier @@ -105,6 +113,7 @@ Omar Andrés Zapata Mesa Omar Andres Zapata Mesa Pankaj Pandey Pascal Schetelat pascal-schetelat Paul Ivanov Paul Ivanov +Paul Ivanov Paul Ivanov Pauli Virtanen Pauli Virtanen <> Pauli Virtanen Pauli Virtanen Pierre Gerold Pierre Gerold @@ -122,14 +131,17 @@ Satrajit Ghosh Satrajit Ghosh Scott Sanderson Scott Sanderson smithj1 smithj1 smithj1 smithj1 +Sang Min Park Sang Min Park Steven Johnson stevenJohnson Steven Silvester blink1073 S. Weber s8weber Stefan van der Walt Stefan van der Walt Silvia Vinyes Silvia Silvia Vinyes silviav12 +Srinivas Reddy Thatiparthy Srinivas Reddy Thatiparthy Sylvain Corlay Sylvain Corlay sylvain.corlay +Tamir Bahar Tamir Bahar Ted Drain TD22057 Théophile Studer Théophile Studer Thomas A Caswell Thomas A Caswell diff --git a/.meeseeksdev.yml b/.meeseeksdev.yml new file mode 100644 index 00000000000..b52022dde07 --- /dev/null +++ b/.meeseeksdev.yml @@ -0,0 +1,22 @@ +users: + LucianaMarques: + can: + - tag +special: + everyone: + can: + - say + - tag + - untag + - close + config: + tag: + only: + - good first issue + - async/await + - backported + - help wanted + - documentation + - notebook + - tab-completion + - windows diff --git a/.travis.yml b/.travis.yml index 47d42d6fd75..00c5e3f6bbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,44 +1,118 @@ # http://travis-ci.org/#!/ipython/ipython language: python +os: linux + +addons: + apt: + packages: + - graphviz + python: - - "nightly" - - 3.5 - - 3.4 - - 3.3 - - 2.7 - - pypy + - 3.6 + sudo: false + +env: + global: + - PATH=$TRAVIS_BUILD_DIR/pandoc:$PATH + +group: edge + before_install: - - git clone --quiet --depth 1 https://github.com/minrk/travis-wheels travis-wheels - - 'if [[ $GROUP != js* ]]; then COVERAGE=""; fi' + - | + # install Python on macOS + if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + env | sort + if ! which python$TRAVIS_PYTHON_VERSION; then + HOMEBREW_NO_AUTO_UPDATE=1 brew tap minrk/homebrew-python-frameworks + HOMEBREW_NO_AUTO_UPDATE=1 brew cask install python-framework-${TRAVIS_PYTHON_VERSION/./} + fi + python3 -m pip install virtualenv + python3 -m virtualenv -p $(which python$TRAVIS_PYTHON_VERSION) ~/travis-env + source ~/travis-env/bin/activate + fi + - python --version + install: - - pip install "setuptools>=18.5" - # Installs PyPy (+ its Numpy). Based on @frol comment at: - # https://github.com/travis-ci/travis-ci/issues/5027 - - | - if [ "$TRAVIS_PYTHON_VERSION" = "pypy" ]; then - export PYENV_ROOT="$HOME/.pyenv" - if [ -f "$PYENV_ROOT/bin/pyenv" ]; then - cd "$PYENV_ROOT" && git pull - else - rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT" - fi - export PYPY_VERSION="5.3.1" - "$PYENV_ROOT/bin/pyenv" install "pypy-$PYPY_VERSION" - virtualenv --python="$PYENV_ROOT/versions/pypy-$PYPY_VERSION/bin/python" "$HOME/virtualenvs/pypy-$PYPY_VERSION" - source "$HOME/virtualenvs/pypy-$PYPY_VERSION/bin/activate" - pip install https://bitbucket.org/pypy/numpy/get/master.zip - fi - - pip install -f travis-wheels/wheelhouse -e file://$PWD#egg=ipython[test] - - pip install codecov + - pip install pip --upgrade + - pip install setuptools --upgrade + - pip install -e file://$PWD#egg=ipython[test] --upgrade + - pip install trio curio --upgrade --upgrade-strategy eager + - pip install pytest 'matplotlib !=3.2.0' mypy + - pip install codecov check-manifest --upgrade + script: - - cd /tmp && iptest --coverage xml && cd - + - check-manifest + - | + if [[ "$TRAVIS_PYTHON_VERSION" == "nightly" ]]; then + # on nightly fake parso known the grammar + cp /home/travis/virtualenv/python3.9-dev/lib/python3.9/site-packages/parso/python/grammar38.txt /home/travis/virtualenv/python3.9-dev/lib/python3.9/site-packages/parso/python/grammar39.txt + fi + - cd /tmp && iptest --coverage xml && cd - + - pytest IPython + - mypy --ignore-missing-imports -m IPython.terminal.ptutils + # On the latest Python (on Linux) only, make sure that the docs build. + - | + if [[ "$TRAVIS_PYTHON_VERSION" == "3.7" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + pip install -r docs/requirements.txt + python tools/fixup_whats_new_pr.py + make -C docs/ html SPHINXOPTS="-W" + fi + after_success: - - cp /tmp/ipy_coverage.xml ./ - - cp /tmp/.coverage ./ - - codecov + - cp /tmp/ipy_coverage.xml ./ + - cp /tmp/.coverage ./ + - codecov matrix: - allow_failures: - - python: nightly - - python: pypy + include: + - arch: amd64 + python: "3.7" + dist: xenial + sudo: true + - arch: amd64 + python: "3.8-dev" + dist: xenial + sudo: true + - arch: amd64 + python: "3.7-dev" + dist: xenial + sudo: true + - arch: amd64 + python: "nightly" + dist: xenial + sudo: true + - arch: arm64 + python: "nightly" + dist: bionic + env: ARM64=True + sudo: true + - os: osx + language: generic + python: 3.6 + env: TRAVIS_PYTHON_VERSION=3.6 + - os: osx + language: generic + python: 3.7 + env: TRAVIS_PYTHON_VERSION=3.7 + allow_failures: + - python: nightly + +before_deploy: + - rm -rf dist/ + - python setup.py sdist + - python setup.py bdist_wheel + +deploy: + provider: releases + api_key: + secure: Y/Ae9tYs5aoBU8bDjN2YrwGG6tCbezj/h3Lcmtx8HQavSbBgXnhnZVRb2snOKD7auqnqjfT/7QMm4ZyKvaOEgyggGktKqEKYHC8KOZ7yp8I5/UMDtk6j9TnXpSqqBxPiud4MDV76SfRYEQiaDoG4tGGvSfPJ9KcNjKrNvSyyxns= + file: dist/* + file_glob: true + skip_cleanup: true + on: + repo: ipython/ipython + all_branches: true # Backports are released from e.g. 5.x branch + tags: true + python: 3.6 # Any version should work, but we only need one + condition: $TRAVIS_OS_NAME = "linux" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a6aab5bc20b..3aecb233319 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,32 @@ +## Triaging Issues + +On the IPython repository, we strive to trust users and give them responsibility. +By using one of our bots, any user can close issues or add/remove +labels by mentioning the bot and asking it to do things on your behalf. + +To close an issue (or PR), even if you did not create it, use the following: + +> @meeseeksdev close + +This command can be in the middle of another comment, but must start on its +own line. + +To add labels to an issue, ask the bot to `tag` with a comma-separated list of +tags to add: + +> @meeseeksdev tag windows, documentation + +Only already pre-created tags can be added. So far, the list is limited to: +`async/await`, `backported`, `help wanted`, `documentation`, `notebook`, +`tab-completion`, `windows` + +To remove a label, use the `untag` command: + +> @meeseeksdev untag windows, documentation + +We'll be adding additional capabilities for the bot and will share them here +when they are ready to be used. + ## Opening an Issue When opening a new Issue, please take the following steps: @@ -6,13 +35,13 @@ When opening a new Issue, please take the following steps: Keyword searches for your error messages are most helpful. 2. If possible, try updating to master and reproducing your issue, because we may have already fixed it. -3. Try to include a minimal reproducible test case +3. Try to include a minimal reproducible test case. 4. Include relevant system information. Start with the output of: python -c "import IPython; print(IPython.sys_info())" - And include any relevant package versions, depending on the issue, - such as matplotlib, numpy, Qt, Qt bindings (PyQt/PySide), tornado, web browser, etc. + And include any relevant package versions, depending on the issue, such as + matplotlib, numpy, Qt, Qt bindings (PyQt/PySide), tornado, web browser, etc. ## Pull Requests @@ -26,8 +55,8 @@ Some guidelines on contributing to IPython: The worst case is that the PR is closed. * Pull Requests should generally be made against master * Pull Requests should be tested, if feasible: - - bugfixes should include regression tests - - new behavior should at least get minimal exercise + - bugfixes should include regression tests. + - new behavior should at least get minimal exercise. * New features and backwards-incompatible changes should be documented by adding a new file to the [pr](docs/source/whatsnew/pr) directory, see [the README.md there](docs/source/whatsnew/pr/README.md) for details. @@ -37,9 +66,30 @@ Some guidelines on contributing to IPython: If you're making functional changes, you can clean up the specific pieces of code you're working on. -[Travis](http://travis-ci.org/#!/ipython/ipython) does a pretty good job testing IPython and Pull Requests, -but it may make sense to manually perform tests (possibly with our `test_pr` script), +[Travis](http://travis-ci.org/#!/ipython/ipython) does a pretty good job testing +IPython and Pull Requests, but it may make sense to manually perform tests, particularly for PRs that affect `IPython.parallel` or Windows. For more detailed information, see our [GitHub Workflow](https://github.com/ipython/ipython/wiki/Dev:-GitHub-workflow). +## Running Tests + +All the tests can by running +```shell +iptest +``` + +All the tests for a single module (for example **test_alias**) can be run by using the fully qualified path to the module. +```shell +iptest IPython.core.tests.test_alias +``` + +Only a single test (for example **test_alias_lifecycle**) within a single file can be run by adding the specific test after a `:` at the end: +```shell +iptest IPython.core.tests.test_alias:test_alias_lifecycle +``` + +For convenience, the full path to a file can often be used instead of the module path on unix systems. For example we can run all the tests by using +```shell +iptest IPython/core/tests/test_alias.py +``` diff --git a/COPYING.rst b/COPYING.rst index 59674acdc8d..e5c79ef38f0 100644 --- a/COPYING.rst +++ b/COPYING.rst @@ -3,39 +3,8 @@ ============================= IPython is licensed under the terms of the Modified BSD License (also known as -New or Revised or 3-Clause BSD), as follows: +New or Revised or 3-Clause BSD). See the LICENSE file. -- Copyright (c) 2008-2014, IPython Development Team -- Copyright (c) 2001-2007, Fernando Perez -- Copyright (c) 2001, Janko Hauser -- Copyright (c) 2001, Nathaniel Gray - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -Redistributions in binary form must reproduce the above copyright notice, this -list of conditions and the following disclaimer in the documentation and/or -other materials provided with the distribution. - -Neither the name of the IPython Development Team nor the names of its -contributors may be used to endorse or promote products derived from this -software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. About the IPython Development Team ---------------------------------- @@ -45,9 +14,7 @@ Fernando Perez began IPython in 2001 based on code from Janko Hauser the project lead. The IPython Development Team is the set of all contributors to the IPython -project. This includes all of the IPython subprojects. A full list with -details is kept in the documentation directory, in the file -``about/credits.txt``. +project. This includes all of the IPython subprojects. The core team that coordinates development on GitHub can be found here: https://github.com/ipython/. diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 994e9f01cdd..00000000000 --- a/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -# DEPRECATED: You probably want jupyter/notebook - -FROM jupyter/notebook - -MAINTAINER IPython Project - -ONBUILD RUN echo "ipython/ipython is deprecated, use jupyter/notebook" >&2 diff --git a/IPython/__init__.py b/IPython/__init__.py index 9b450da6a0c..c17ec76a602 100644 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -2,7 +2,7 @@ """ IPython: tools for interactive and parallel computing in Python. -http://ipython.org +https://ipython.org """ #----------------------------------------------------------------------------- # Copyright (c) 2008-2011, IPython Development Team. @@ -18,21 +18,28 @@ #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- -from __future__ import absolute_import import os import sys -import warnings #----------------------------------------------------------------------------- # Setup everything #----------------------------------------------------------------------------- # Don't forget to also update setup.py when this changes! -v = sys.version_info -if v[:2] < (2,7) or (v[0] >= 3 and v[:2] < (3,3)): - raise ImportError('IPython requires Python version 2.7 or 3.3 or above.') -del v +if sys.version_info < (3, 6): + raise ImportError( +""" +IPython 7.10+ supports Python 3.6 and above. +When using Python 2.7, please install IPython 5.x LTS Long Term Support version. +Python 3.3 and 3.4 were supported up to IPython 6.x. +Python 3.5 was supported with IPython 7.0 to 7.9. + +See IPython `README.rst` file for more information: + + https://github.com/ipython/ipython/blob/master/README.rst + +""") # Make it easy to import extensions - they are always directly on pythonpath. # Therefore, non-IPython modules can be added to extensions directory. @@ -58,6 +65,10 @@ __license__ = release.license __version__ = release.version version_info = release.version_info +# list of CVEs that should have been patched in this release. +# this is informational and should not be relied upon. +__patched_cves__ = {"CVE-2022-21699"} + def embed_kernel(module=None, local_ns=None, **kwargs): """Embed and start an IPython kernel in a given scope. @@ -69,7 +80,7 @@ def embed_kernel(module=None, local_ns=None, **kwargs): Parameters ---------- - module : ModuleType, optional + module : types.ModuleType, optional The module to load into IPython globals (default: caller) local_ns : dict, optional The namespace to load into IPython user namespace (default: caller) @@ -143,4 +154,3 @@ def start_kernel(argv=None, **kwargs): """ from IPython.kernel.zmq.kernelapp import launch_new_instance return launch_new_instance(argv=argv, **kwargs) - diff --git a/IPython/config.py b/IPython/config.py index c3a3e9190fc..964f46f10ac 100644 --- a/IPython/config.py +++ b/IPython/config.py @@ -7,9 +7,9 @@ import sys from warnings import warn -from IPython.utils.shimmodule import ShimModule, ShimWarning +from .utils.shimmodule import ShimModule, ShimWarning -warn("The `IPython.config` package has been deprecated. " +warn("The `IPython.config` package has been deprecated since IPython 4.0. " "You should import from traitlets.config instead.", ShimWarning) diff --git a/IPython/conftest.py b/IPython/conftest.py new file mode 100644 index 00000000000..8b2af8c020a --- /dev/null +++ b/IPython/conftest.py @@ -0,0 +1,69 @@ +import types +import sys +import builtins +import os +import pytest +import pathlib +import shutil + +from .testing import tools + + +def get_ipython(): + from .terminal.interactiveshell import TerminalInteractiveShell + if TerminalInteractiveShell._instance: + return TerminalInteractiveShell.instance() + + config = tools.default_config() + config.TerminalInteractiveShell.simple_prompt = True + + # Create and initialize our test-friendly IPython instance. + shell = TerminalInteractiveShell.instance(config=config) + return shell + + +@pytest.fixture(scope='session', autouse=True) +def work_path(): + path = pathlib.Path("./tmp-ipython-pytest-profiledir") + os.environ["IPYTHONDIR"] = str(path.absolute()) + if path.exists(): + raise ValueError('IPython dir temporary path already exists ! Did previous test run exit successfully ?') + path.mkdir() + yield + shutil.rmtree(str(path.resolve())) + + +def nopage(strng, start=0, screen_lines=0, pager_cmd=None): + if isinstance(strng, dict): + strng = strng.get("text/plain", "") + print(strng) + + +def xsys(self, cmd): + """Replace the default system call with a capturing one for doctest. + """ + # We use getoutput, but we need to strip it because pexpect captures + # the trailing newline differently from commands.getoutput + print(self.getoutput(cmd, split=False, depth=1).rstrip(), end="", file=sys.stdout) + sys.stdout.flush() + + +# for things to work correctly we would need this as a session fixture; +# unfortunately this will fail on some test that get executed as _collection_ +# time (before the fixture run), in particular parametrized test that contain +# yields. so for now execute at import time. +#@pytest.fixture(autouse=True, scope='session') +def inject(): + + builtins.get_ipython = get_ipython + builtins._ip = get_ipython() + builtins.ip = get_ipython() + builtins.ip.system = types.MethodType(xsys, ip) + builtins.ip.builtin_trap.activate() + from .core import page + + page.pager_page = nopage + # yield + + +inject() diff --git a/IPython/consoleapp.py b/IPython/consoleapp.py index 14903bdc74c..c2bbe1888f5 100644 --- a/IPython/consoleapp.py +++ b/IPython/consoleapp.py @@ -6,7 +6,7 @@ from warnings import warn -warn("The `IPython.consoleapp` package has been deprecated. " - "You should import from jupyter_client.consoleapp instead.") +warn("The `IPython.consoleapp` package has been deprecated since IPython 4.0." + "You should import from jupyter_client.consoleapp instead.", stacklevel=2) from jupyter_client.consoleapp import * diff --git a/IPython/core/alias.py b/IPython/core/alias.py index 28a9ccb00d6..2ad990231a0 100644 --- a/IPython/core/alias.py +++ b/IPython/core/alias.py @@ -25,9 +25,8 @@ import sys from traitlets.config.configurable import Configurable -from IPython.core.error import UsageError +from .error import UsageError -from IPython.utils.py3compat import string_types from traitlets import List, Instance from logging import error @@ -148,7 +147,7 @@ def validate(self): raise InvalidAliasError("The name %s can't be aliased " "because it is another magic command." % self.name) - if not (isinstance(self.cmd, string_types)): + if not (isinstance(self.cmd, str)): raise InvalidAliasError("An alias command must be a string, " "got: %r" % self.cmd) @@ -205,6 +204,8 @@ def __init__(self, shell=None, **kwargs): def init_aliases(self): # Load default & user aliases for name, cmd in self.default_aliases + self.user_aliases: + if cmd.startswith('ls ') and self.shell.colors == 'NoColor': + cmd = cmd.replace(' --color', '') self.soft_define_alias(name, cmd) @property diff --git a/IPython/core/application.py b/IPython/core/application.py index 18016279552..4f679df18e3 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -26,9 +26,8 @@ from IPython.core.profiledir import ProfileDir, ProfileDirError from IPython.paths import get_ipython_dir, get_ipython_package_dir from IPython.utils.path import ensure_dir_exists -from IPython.utils import py3compat from traitlets import ( - List, Unicode, Type, Bool, Dict, Set, Instance, Undefined, + List, Unicode, Type, Bool, Set, Instance, Undefined, default, observe, ) @@ -44,6 +43,14 @@ "/etc/ipython", ] + +ENV_CONFIG_DIRS = [] +_env_config_dir = os.path.join(sys.prefix, 'etc', 'ipython') +if _env_config_dir not in SYSTEM_CONFIG_DIRS: + # only add ENV_CONFIG if sys.prefix is not already included + ENV_CONFIG_DIRS.append(_env_config_dir) + + _envvar = os.environ.get('IPYTHON_SUPPRESS_CONFIG_ERRORS') if _envvar in {None, ''}: IPYTHON_SUPPRESS_CONFIG_ERRORS = None @@ -94,12 +101,12 @@ def load_subconfig(self, fname, path=None, profile=None): class BaseIPythonApplication(Application): - name = Unicode(u'ipython') + name = u'ipython' description = Unicode(u'IPython: an enhanced interactive Python shell.') version = Unicode(release.version) - aliases = Dict(base_aliases) - flags = Dict(base_flags) + aliases = base_aliases + flags = base_flags classes = List([ProfileDir]) # enable `load_subconfig('cfg.py', profile='name')` @@ -126,7 +133,7 @@ def _config_file_name_changed(self, change): config_file_paths = List(Unicode()) @default('config_file_paths') def _config_file_paths_default(self): - return [py3compat.getcwd()] + return [] extra_config_file = Unicode( help="""Path to an extra config file to load. @@ -215,7 +222,7 @@ def __init__(self, **kwargs): super(BaseIPythonApplication, self).__init__(**kwargs) # ensure current working directory exists try: - py3compat.getcwd() + os.getcwd() except: # exit if cwd doesn't exist self.log.error("Current working directory doesn't exist.") @@ -260,14 +267,10 @@ def _ipython_dir_changed(self, change): old = change['old'] new = change['new'] if old is not Undefined: - str_old = py3compat.cast_bytes_py2(os.path.abspath(old), - sys.getfilesystemencoding() - ) + str_old = os.path.abspath(old) if str_old in sys.path: sys.path.remove(str_old) - str_path = py3compat.cast_bytes_py2(os.path.abspath(new), - sys.getfilesystemencoding() - ) + str_path = os.path.abspath(new) sys.path.append(str_path) ensure_dir_exists(new) readme = os.path.join(new, 'README') @@ -290,7 +293,7 @@ def load_config_file(self, suppress_errors=IPYTHON_SUPPRESS_CONFIG_ERRORS): printed on screen. For testing, the suppress_errors option is set to False, so errors will make tests fail. - `supress_errors` default value is to be `None` in which case the + `suppress_errors` default value is to be `None` in which case the behavior default to the one of `traitlets.Application`. The default value can be set : @@ -403,6 +406,7 @@ def init_profile_dir(self): def init_config_files(self): """[optionally] copy default config files into profile dir.""" + self.config_file_paths.extend(ENV_CONFIG_DIRS) self.config_file_paths.extend(SYSTEM_CONFIG_DIRS) # copy config files path = self.builtin_profile_dir diff --git a/IPython/core/async_helpers.py b/IPython/core/async_helpers.py new file mode 100644 index 00000000000..fb4cc193250 --- /dev/null +++ b/IPython/core/async_helpers.py @@ -0,0 +1,173 @@ +""" +Async helper function that are invalid syntax on Python 3.5 and below. + +This code is best effort, and may have edge cases not behaving as expected. In +particular it contain a number of heuristics to detect whether code is +effectively async and need to run in an event loop or not. + +Some constructs (like top-level `return`, or `yield`) are taken care of +explicitly to actually raise a SyntaxError and stay as close as possible to +Python semantics. +""" + + +import ast +import sys +import inspect +from textwrap import dedent, indent + + +class _AsyncIORunner: + + def __call__(self, coro): + """ + Handler for asyncio autoawait + """ + import asyncio + + return asyncio.get_event_loop().run_until_complete(coro) + + def __str__(self): + return 'asyncio' + +_asyncio_runner = _AsyncIORunner() + + +def _curio_runner(coroutine): + """ + handler for curio autoawait + """ + import curio + + return curio.run(coroutine) + + +def _trio_runner(async_fn): + import trio + + async def loc(coro): + """ + We need the dummy no-op async def to protect from + trio's internal. See https://github.com/python-trio/trio/issues/89 + """ + return await coro + + return trio.run(loc, async_fn) + + +def _pseudo_sync_runner(coro): + """ + A runner that does not really allow async execution, and just advance the coroutine. + + See discussion in https://github.com/python-trio/trio/issues/608, + + Credit to Nathaniel Smith + + """ + try: + coro.send(None) + except StopIteration as exc: + return exc.value + else: + # TODO: do not raise but return an execution result with the right info. + raise RuntimeError( + "{coro_name!r} needs a real async loop".format(coro_name=coro.__name__) + ) + + +def _asyncify(code: str) -> str: + """wrap code in async def definition. + + And setup a bit of context to run it later. + """ + res = dedent( + """ + async def __wrapper__(): + try: + {usercode} + finally: + locals() + """ + ).format(usercode=indent(code, " " * 8)) + return res + + +class _AsyncSyntaxErrorVisitor(ast.NodeVisitor): + """ + Find syntax errors that would be an error in an async repl, but because + the implementation involves wrapping the repl in an async function, it + is erroneously allowed (e.g. yield or return at the top level) + """ + def __init__(self): + if sys.version_info >= (3,8): + raise ValueError('DEPRECATED in Python 3.8+') + self.depth = 0 + super().__init__() + + def generic_visit(self, node): + func_types = (ast.FunctionDef, ast.AsyncFunctionDef) + invalid_types_by_depth = { + 0: (ast.Return, ast.Yield, ast.YieldFrom), + 1: (ast.Nonlocal,) + } + + should_traverse = self.depth < max(invalid_types_by_depth.keys()) + if isinstance(node, func_types) and should_traverse: + self.depth += 1 + super().generic_visit(node) + self.depth -= 1 + elif isinstance(node, invalid_types_by_depth[self.depth]): + raise SyntaxError() + else: + super().generic_visit(node) + + +def _async_parse_cell(cell: str) -> ast.AST: + """ + This is a compatibility shim for pre-3.7 when async outside of a function + is a syntax error at the parse stage. + + It will return an abstract syntax tree parsed as if async and await outside + of a function were not a syntax error. + """ + if sys.version_info < (3, 7): + # Prior to 3.7 you need to asyncify before parse + wrapped_parse_tree = ast.parse(_asyncify(cell)) + return wrapped_parse_tree.body[0].body[0] + else: + return ast.parse(cell) + + +def _should_be_async(cell: str) -> bool: + """Detect if a block of code need to be wrapped in an `async def` + + Attempt to parse the block of code, it it compile we're fine. + Otherwise we wrap if and try to compile. + + If it works, assume it should be async. Otherwise Return False. + + Not handled yet: If the block of code has a return statement as the top + level, it will be seen as async. This is a know limitation. + """ + if sys.version_info > (3, 8): + try: + code = compile(cell, "<>", "exec", flags=getattr(ast,'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0)) + return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE + except (SyntaxError, MemoryError): + return False + try: + # we can't limit ourself to ast.parse, as it __accepts__ to parse on + # 3.7+, but just does not _compile_ + code = compile(cell, "<>", "exec") + except (SyntaxError, MemoryError): + try: + parse_tree = _async_parse_cell(cell) + + # Raise a SyntaxError if there are top-level return or yields + v = _AsyncSyntaxErrorVisitor() + v.visit(parse_tree) + + except (SyntaxError, MemoryError): + return False + return True + return False diff --git a/IPython/core/builtin_trap.py b/IPython/core/builtin_trap.py index 909a555c733..a8ea4abcd9d 100644 --- a/IPython/core/builtin_trap.py +++ b/IPython/core/builtin_trap.py @@ -1,31 +1,14 @@ """ -A context manager for managing things injected into :mod:`__builtin__`. - -Authors: - -* Brian Granger -* Fernando Perez +A context manager for managing things injected into :mod:`builtins`. """ -#----------------------------------------------------------------------------- -# Copyright (C) 2010-2011 The IPython Development Team. -# -# Distributed under the terms of the BSD License. -# -# Complete license in the file COPYING.txt, distributed with this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. +import builtins as builtin_mod from traitlets.config.configurable import Configurable -from IPython.utils.py3compat import builtin_mod, iteritems from traitlets import Instance -#----------------------------------------------------------------------------- -# Classes and functions -#----------------------------------------------------------------------------- class __BuiltinUndefined(object): pass BuiltinUndefined = __BuiltinUndefined() @@ -52,17 +35,6 @@ def __init__(self, shell=None): 'quit': HideBuiltin, 'get_ipython': self.shell.get_ipython, } - # Recursive reload function - try: - from IPython.lib import deepreload - if self.shell.deep_reload: - from warnings import warn - warn("Automatically replacing builtin `reload` by `deepreload.reload` is deprecated since IPython 4.0, please import `reload` explicitly from `IPython.lib.deepreload", DeprecationWarning) - self.auto_builtins['reload'] = deepreload._dreload - else: - self.auto_builtins['dreload']= deepreload._dreload - except ImportError: - pass def __enter__(self): if self._nested_level == 0: @@ -101,14 +73,14 @@ def activate(self): """Store ipython references in the __builtin__ namespace.""" add_builtin = self.add_builtin - for name, func in iteritems(self.auto_builtins): + for name, func in self.auto_builtins.items(): add_builtin(name, func) def deactivate(self): """Remove any builtins which might have been added by add_builtins, or restore overwritten ones to their previous values.""" remove_builtin = self.remove_builtin - for key, val in iteritems(self._orig_builtins): + for key, val in self._orig_builtins.items(): remove_builtin(key, val) self._orig_builtins.clear() self._builtins_added = False diff --git a/IPython/core/compilerop.py b/IPython/core/compilerop.py index e39ded68d79..c4771af7303 100644 --- a/IPython/core/compilerop.py +++ b/IPython/core/compilerop.py @@ -25,7 +25,6 @@ #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- -from __future__ import print_function # Stdlib imports import __future__ @@ -36,12 +35,13 @@ import linecache import operator import time +from contextlib import contextmanager #----------------------------------------------------------------------------- # Constants #----------------------------------------------------------------------------- -# Roughtly equal to PyCF_MASK | PyCF_MASK_OBSOLETE as defined in pythonrun.h, +# Roughly equal to PyCF_MASK | PyCF_MASK_OBSOLETE as defined in pythonrun.h, # this is used as a bitmask to extract future-related code flags. PyCF_MASK = functools.reduce(operator.or_, (getattr(__future__, fname).compiler_flag @@ -53,10 +53,10 @@ def code_name(code, number=0): """ Compute a (probably) unique name for code for caching. - + This now expects code to be unicode. """ - hash_digest = hashlib.md5(code.encode("utf-8")).hexdigest() + hash_digest = hashlib.sha1(code.encode("utf-8")).hexdigest() # Include the number and 12 characters of the hash in the name. It's # pretty much impossible that in a single session we'll have collisions # even with truncated hashes, and the full one makes tracebacks too long @@ -72,7 +72,7 @@ class CachingCompiler(codeop.Compile): def __init__(self): codeop.Compile.__init__(self) - + # This is ugly, but it must be done this way to allow multiple # simultaneous ipython instances to coexist. Since Python itself # directly accesses the data structures in the linecache module, and @@ -91,10 +91,11 @@ def __init__(self): # stdlib that call it outside our control go through our codepath # (otherwise we'd lose our tracebacks). linecache.checkcache = check_linecache_ipython - + + def ast_parse(self, source, filename='', symbol='exec'): """Parse code to an AST with the current compiler flags active. - + Arguments are exactly the same as ast.parse (in the standard library), and are passed to the built-in compile function.""" return compile(source, filename, symbol, self.flags | PyCF_ONLY_AST, 1) @@ -110,10 +111,10 @@ def compiler_flags(self): """Flags currently active in the compilation process. """ return self.flags - + def cache(self, code, number=0): """Make a name for a block of code, and cache the code. - + Parameters ---------- code : str @@ -121,7 +122,7 @@ def cache(self, code, number=0): number : int A number which forms part of the code's name. Used for the execution counter. - + Returns ------- The name of the cached code (as a string). Pass this as the filename @@ -134,10 +135,25 @@ def cache(self, code, number=0): linecache._ipython_cache[name] = entry return name + @contextmanager + def extra_flags(self, flags): + ## bits that we'll set to 1 + turn_on_bits = ~self.flags & flags + + + self.flags = self.flags | flags + try: + yield + finally: + # turn off only the bits we turned on so that something like + # __future__ that set flags stays. + self.flags &= ~turn_on_bits + + def check_linecache_ipython(*args): """Call linecache.checkcache() safely protecting our cached values. """ - # First call the orignal checkcache as intended + # First call the original checkcache as intended linecache._checkcache_ori(*args) # Then, update back the cache with our data, so that tracebacks related # to our compiled codes can be produced. diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 87b13b1c5d8..bc114f0f66b 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -1,22 +1,119 @@ -# encoding: utf-8 -"""Word completion for IPython. +"""Completion for IPython. This module started as fork of the rlcompleter module in the Python standard library. The original enhancements made to rlcompleter have been sent upstream and were accepted as of Python 2.3, +This module now support a wide variety of completion mechanism both available +for normal classic Python code, as well as completer for IPython specific +Syntax like magics. + +Latex and Unicode completion +============================ + +IPython and compatible frontends not only can complete your code, but can help +you to input a wide range of characters. In particular we allow you to insert +a unicode character using the tab completion mechanism. + +Forward latex/unicode completion +-------------------------------- + +Forward completion allows you to easily type a unicode character using its latex +name, or unicode long description. To do so type a backslash follow by the +relevant name and press tab: + + +Using latex completion: + +.. code:: + + \\alpha + α + +or using unicode completion: + + +.. code:: + + \\greek small letter alpha + α + + +Only valid Python identifiers will complete. Combining characters (like arrow or +dots) are also available, unlike latex they need to be put after the their +counterpart that is to say, `F\\\\vec` is correct, not `\\\\vecF`. + +Some browsers are known to display combining characters incorrectly. + +Backward latex completion +------------------------- + +It is sometime challenging to know how to type a character, if you are using +IPython, or any compatible frontend you can prepend backslash to the character +and press `` to expand it to its latex form. + +.. code:: + + \\α + \\alpha + + +Both forward and backward completions can be deactivated by setting the +``Completer.backslash_combining_completions`` option to ``False``. + + +Experimental +============ + +Starting with IPython 6.0, this module can make use of the Jedi library to +generate completions both using static analysis of the code, and dynamically +inspecting multiple namespaces. Jedi is an autocompletion and static analysis +for Python. The APIs attached to this new mechanism is unstable and will +raise unless use in an :any:`provisionalcompleter` context manager. + +You will find that the following are experimental: + + - :any:`provisionalcompleter` + - :any:`IPCompleter.completions` + - :any:`Completion` + - :any:`rectify_completions` + +.. note:: + + better name for :any:`rectify_completions` ? + +We welcome any feedback on these new API, and we also encourage you to try this +module in debug mode (start IPython with ``--Completer.debug=True``) in order +to have extra logging information if :any:`jedi` is crashing, or if current +IPython completer pending deprecations are returning results not yet handled +by :any:`jedi` + +Using Jedi for tab completion allow snippets like the following to work without +having to execute any code: + + >>> myvar = ['hello', 42] + ... myvar[1].bi + +Tab completion will be able to infer that ``myvar[1]`` is a real number without +executing any code unlike the previously available ``IPCompleter.greedy`` +option. + +Be sure to update :any:`jedi` to the latest stable version or to try the +current development version to get better completions. """ + # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. # # Some of this code originated from rlcompleter in the Python standard library # Copyright (C) 2001 Python Software Foundation, www.python.org -from __future__ import print_function import __main__ +import builtins as builtin_mod import glob +import time import inspect import itertools import keyword @@ -25,20 +122,34 @@ import sys import unicodedata import string +import warnings + +from contextlib import contextmanager +from importlib import import_module +from typing import Iterator, List, Tuple, Iterable +from types import SimpleNamespace from traitlets.config.configurable import Configurable from IPython.core.error import TryNext -from IPython.core.inputsplitter import ESC_MAGIC +from IPython.core.inputtransformer2 import ESC_MAGIC from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol +from IPython.core.oinspect import InspectColors from IPython.utils import generics -from IPython.utils.decorators import undoc from IPython.utils.dir2 import dir2, get_real_method from IPython.utils.process import arg_split -from IPython.utils.py3compat import builtin_mod, string_types, PY3, cast_unicode_py2 -from traitlets import Bool, Enum, observe - -from functools import wraps - +from traitlets import Bool, Enum, observe, Int + +# skip module docstests +skip_doctest = True + +try: + import jedi + jedi.settings.case_insensitive_completion = False + import jedi.api.helpers + import jedi.api.classes + JEDI_INSTALLED = True +except ImportError: + JEDI_INSTALLED = False #----------------------------------------------------------------------------- # Globals #----------------------------------------------------------------------------- @@ -51,48 +162,52 @@ else: PROTECTABLES = ' ()[]{}?=\\|;:\'#*"^&' +# Protect against returning an enormous number of completions which the frontend +# may have trouble processing. +MATCHES_LIMIT = 500 + +_deprecation_readline_sentinel = object() -#----------------------------------------------------------------------------- -# Work around BUG decorators. -#----------------------------------------------------------------------------- -def _strip_single_trailing_space(complete): +class ProvisionalCompleterWarning(FutureWarning): """ - This is a workaround for a weird IPython/Prompt_toolkit behavior, - that can be removed once we rely on a slightly more recent prompt_toolkit - version (likely > 1.0.3). So this can likely be removed in IPython 6.0 + Exception raise by an experimental feature in this module. - cf https://github.com/ipython/ipython/issues/9658 - and https://github.com/jonathanslenders/python-prompt-toolkit/pull/328 + Wrap code in :any:`provisionalcompleter` context manager if you + are certain you want to use an unstable feature. + """ + pass - The bug is due to the fact that in PTK the completer will reinvoke itself - after trying to completer to the longuest common prefix of all the - completions, unless only one completion is available. +warnings.filterwarnings('error', category=ProvisionalCompleterWarning) - This logic is faulty if the completion ends with space, which can happen in - case like:: +@contextmanager +def provisionalcompleter(action='ignore'): + """ - from foo import im - which only matching completion is `import `. Note the leading space at the - end. So leaving a space at the end is a reasonable request, but for now - we'll strip it. - """ + This context manager has to be used in any place where unstable completer + behavior and API may be called. - @wraps(complete) - def comp(*args, **kwargs): - text, matches = complete(*args, **kwargs) - if len(matches) == 1: - return text, [matches[0].rstrip()] - return text, matches + >>> with provisionalcompleter(): + ... completer.do_experimental_things() # works - return comp + >>> completer.do_experimental_things() # raises. + .. note:: Unstable + By using this context manager you agree that the API in use may change + without warning, and that you won't complain if they do so. + + You also understand that, if the API is not to your liking, you should report + a bug to explain your use case upstream. + + We'll be happy to get your feedback, feature requests, and improvements on + any of the unstable APIs! + """ + with warnings.catch_warnings(): + warnings.filterwarnings(action, category=ProvisionalCompleterWarning) + yield -#----------------------------------------------------------------------------- -# Main functions and classes -#----------------------------------------------------------------------------- def has_open_quotes(s): """Return whether a string has open quotes. @@ -115,19 +230,19 @@ def has_open_quotes(s): return False -def protect_filename(s): +def protect_filename(s, protectables=PROTECTABLES): """Escape a string to protect certain characters.""" - if set(s) & set(PROTECTABLES): + if set(s) & set(protectables): if sys.platform == "win32": return '"' + s + '"' else: - return "".join(("\\" + c if c in PROTECTABLES else c) for c in s) + return "".join(("\\" + c if c in protectables else c) for c in s) else: return s -def expand_user(path): - """Expand '~'-style usernames in strings. +def expand_user(path:str) -> Tuple[str, bool, str]: + """Expand ``~``-style usernames in strings. This is similar to :func:`os.path.expanduser`, but it computes and returns extra information that will be useful if the input was being used in @@ -166,7 +281,7 @@ def expand_user(path): return newpath, tilde_expand, tilde_val -def compress_user(path, tilde_expand, tilde_val): +def compress_user(path:str, tilde_expand:bool, tilde_val:str) -> str: """Does the opposite of expand_user, with its outputs. """ if tilde_expand: @@ -180,15 +295,10 @@ def completions_sorting_key(word): This does several things: - - Lowercase all completions, so they are sorted alphabetically with - upper and lower case words mingled - Demote any completions starting with underscores to the end - Insert any %magic and %%cellmagic completions in the alphabetical order by their name """ - # Case insensitive sort - word = word.lower() - prio1, prio2 = 0, 0 if word.startswith('__'): @@ -212,8 +322,186 @@ def completions_sorting_key(word): return prio1, word, prio2 -@undoc -class Bunch(object): pass +class _FakeJediCompletion: + """ + This is a workaround to communicate to the UI that Jedi has crashed and to + report a bug. Will be used only id :any:`IPCompleter.debug` is set to true. + + Added in IPython 6.0 so should likely be removed for 7.0 + + """ + + def __init__(self, name): + + self.name = name + self.complete = name + self.type = 'crashed' + self.name_with_symbols = name + self.signature = '' + self._origin = 'fake' + + def __repr__(self): + return '' + + +class Completion: + """ + Completion object used and return by IPython completers. + + .. warning:: Unstable + + This function is unstable, API may change without warning. + It will also raise unless use in proper context manager. + + This act as a middle ground :any:`Completion` object between the + :any:`jedi.api.classes.Completion` object and the Prompt Toolkit completion + object. While Jedi need a lot of information about evaluator and how the + code should be ran/inspected, PromptToolkit (and other frontend) mostly + need user facing information. + + - Which range should be replaced replaced by what. + - Some metadata (like completion type), or meta information to displayed to + the use user. + + For debugging purpose we can also store the origin of the completion (``jedi``, + ``IPython.python_matches``, ``IPython.magics_matches``...). + """ + + __slots__ = ['start', 'end', 'text', 'type', 'signature', '_origin'] + + def __init__(self, start: int, end: int, text: str, *, type: str=None, _origin='', signature='') -> None: + warnings.warn("``Completion`` is a provisional API (as of IPython 6.0). " + "It may change without warnings. " + "Use in corresponding context manager.", + category=ProvisionalCompleterWarning, stacklevel=2) + + self.start = start + self.end = end + self.text = text + self.type = type + self.signature = signature + self._origin = _origin + + def __repr__(self): + return '' % \ + (self.start, self.end, self.text, self.type or '?', self.signature or '?') + + def __eq__(self, other)->Bool: + """ + Equality and hash do not hash the type (as some completer may not be + able to infer the type), but are use to (partially) de-duplicate + completion. + + Completely de-duplicating completion is a bit tricker that just + comparing as it depends on surrounding text, which Completions are not + aware of. + """ + return self.start == other.start and \ + self.end == other.end and \ + self.text == other.text + + def __hash__(self): + return hash((self.start, self.end, self.text)) + + +_IC = Iterable[Completion] + + +def _deduplicate_completions(text: str, completions: _IC)-> _IC: + """ + Deduplicate a set of completions. + + .. warning:: Unstable + + This function is unstable, API may change without warning. + + Parameters + ---------- + text: str + text that should be completed. + completions: Iterator[Completion] + iterator over the completions to deduplicate + + Yields + ------ + `Completions` objects + + + Completions coming from multiple sources, may be different but end up having + the same effect when applied to ``text``. If this is the case, this will + consider completions as equal and only emit the first encountered. + + Not folded in `completions()` yet for debugging purpose, and to detect when + the IPython completer does return things that Jedi does not, but should be + at some point. + """ + completions = list(completions) + if not completions: + return + + new_start = min(c.start for c in completions) + new_end = max(c.end for c in completions) + + seen = set() + for c in completions: + new_text = text[new_start:c.start] + c.text + text[c.end:new_end] + if new_text not in seen: + yield c + seen.add(new_text) + + +def rectify_completions(text: str, completions: _IC, *, _debug=False)->_IC: + """ + Rectify a set of completions to all have the same ``start`` and ``end`` + + .. warning:: Unstable + + This function is unstable, API may change without warning. + It will also raise unless use in proper context manager. + + Parameters + ---------- + text: str + text that should be completed. + completions: Iterator[Completion] + iterator over the completions to rectify + + + :any:`jedi.api.classes.Completion` s returned by Jedi may not have the same start and end, though + the Jupyter Protocol requires them to behave like so. This will readjust + the completion to have the same ``start`` and ``end`` by padding both + extremities with surrounding text. + + During stabilisation should support a ``_debug`` option to log which + completion are return by the IPython completer and not found in Jedi in + order to make upstream bug report. + """ + warnings.warn("`rectify_completions` is a provisional API (as of IPython 6.0). " + "It may change without warnings. " + "Use in corresponding context manager.", + category=ProvisionalCompleterWarning, stacklevel=2) + + completions = list(completions) + if not completions: + return + starts = (c.start for c in completions) + ends = (c.end for c in completions) + + new_start = min(starts) + new_end = max(ends) + + seen_jedi = set() + seen_python_matches = set() + for c in completions: + new_text = text[new_start:c.start] + c.text + text[c.end:new_end] + if c._origin == 'jedi': + seen_jedi.add(new_text) + elif c._origin == 'IPCompleter.python_matches': + seen_python_matches.add(new_text) + yield Completion(new_start, new_end, new_text, type=c.type, _origin=c._origin, signature=c.signature) + diff = seen_python_matches.difference(seen_jedi) + if diff and _debug: + print('IPython.python matches have extras:', diff) if sys.platform == 'win32': @@ -234,7 +522,7 @@ class CompletionSplitter(object): entire line. What characters are used as splitting delimiters can be controlled by - setting the `delims` attribute (this is a property that internally + setting the ``delims`` attribute (this is a property that internally automatically builds the necessary regular expression)""" # Private interface @@ -275,6 +563,7 @@ def split_line(self, line, cursor_pos=None): return self._delim_re.split(l)[-1] + class Completer(Configurable): greedy = Bool(False, @@ -285,7 +574,28 @@ class Completer(Configurable): but can be unsafe because the code is actually evaluated on TAB. """ ).tag(config=True) - + + use_jedi = Bool(default_value=JEDI_INSTALLED, + help="Experimental: Use Jedi to generate autocompletions. " + "Default to True if jedi is installed.").tag(config=True) + + jedi_compute_type_timeout = Int(default_value=400, + help="""Experimental: restrict time (in milliseconds) during which Jedi can compute types. + Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt + performance by preventing jedi to build its cache. + """).tag(config=True) + + debug = Bool(default_value=False, + help='Enable debug for the Completer. Mostly print extra ' + 'information for experimental jedi integration.')\ + .tag(config=True) + + backslash_combining_completions = Bool(True, + help="Enable unicode completions, e.g. \\alpha . " + "Includes completion of latex commands, unicode names, and expanding " + "unicode characters back to latex commands.").tag(config=True) + + def __init__(self, namespace=None, global_namespace=None, **kwargs): """Create a new completer for the command line. @@ -299,20 +609,15 @@ def __init__(self, namespace=None, global_namespace=None, **kwargs): An optional second namespace can be given. This allows the completer to handle cases where both the local and global scopes need to be distinguished. - - Completer instances should be used as the completion mechanism of - readline via the set_completer() call: - - readline.set_completer(Completer(my_namespace).complete) """ # Don't bind to namespace quite yet, but flag whether the user wants a # specific namespace or to use __main__.__dict__. This will allow us # to bind to __main__.__dict__ at completion time, not now. if namespace is None: - self.use_main_ns = 1 + self.use_main_ns = True else: - self.use_main_ns = 0 + self.use_main_ns = False self.namespace = namespace # The global namespace, if given, can be bound directly @@ -321,6 +626,8 @@ def __init__(self, namespace=None, global_namespace=None, **kwargs): else: self.global_namespace = global_namespace + self.custom_matchers = [] + super(Completer, self).__init__(**kwargs) def complete(self, text, state): @@ -360,7 +667,16 @@ def global_matches(self, text): for word in lst: if word[:n] == text and word != "__builtins__": match_append(word) - return [cast_unicode_py2(m) for m in matches] + + snake_case_re = re.compile(r"[^_]+(_[^_]+)+?\Z") + for lst in [self.namespace.keys(), + self.global_namespace.keys()]: + shortened = {"_".join([sub[0] for sub in word.split('_')]) : word + for word in lst if snake_case_re.match(word)} + for word in shortened.keys(): + if word[:n] == text and word != "__builtins__": + match_append(shortened[word]) + return matches def attr_matches(self, text): """Compute matches when text contains a dot. @@ -368,7 +684,7 @@ def attr_matches(self, text): Assuming the text is of the form NAME.NAME....[NAME], and is evaluatable in self.namespace or self.global_namespace, it will be evaluated and its attributes (as revealed by dir()) are used as - possible completions. (For class instances, class members are are + possible completions. (For class instances, class members are also considered.) WARNING: this can still invoke arbitrary C code, if an object @@ -378,7 +694,7 @@ def attr_matches(self, text): # Another option, seems to work great. Catches things like ''. m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text) - + if m: expr, attr = m.group(1, 3) elif self.greedy: @@ -388,7 +704,7 @@ def attr_matches(self, text): expr, attr = m2.group(1,2) else: return [] - + try: obj = eval(expr, self.namespace) except: @@ -399,13 +715,15 @@ def attr_matches(self, text): if self.limit_to__all__ and hasattr(obj, '__all__'): words = get__all__entries(obj) - else: + else: words = dir2(obj) try: words = generics.complete_object(obj, words) except TryNext: pass + except AssertionError: + raise except Exception: # Silence errors from completion function #raise # dbg @@ -421,15 +739,34 @@ def get__all__entries(obj): words = getattr(obj, '__all__') except: return [] - - return [cast_unicode_py2(w) for w in words if isinstance(w, string_types)] + return [w for w in words if isinstance(w, str)] + + +def match_dict_keys(keys: List[str], prefix: str, delims: str): + """Used by dict_key_matches, matching the prefix to a list of keys + + Parameters + ========== + keys: + list of keys in dictionary currently being completed. + prefix: + Part of the text already typed by the user. e.g. `mydict[b'fo` + delims: + String of delimiters to consider when finding the current key. + + Returns + ======= -def match_dict_keys(keys, prefix, delims): - """Used by dict_key_matches, matching the prefix to a list of keys""" + A tuple of three elements: ``quote``, ``token_start``, ``matched``, with + ``quote`` being the quote that need to be used to close current string. + ``token_start`` the position where the replacement should start occurring, + ``matches`` a list of replacement/completion + + """ if not prefix: return None, 0, [repr(k) for k in keys - if isinstance(k, (string_types, bytes))] + if isinstance(k, (str, bytes))] quote_match = re.search('["\']', prefix) quote = quote_match.group() try: @@ -442,7 +779,6 @@ def match_dict_keys(keys, prefix, delims): token_start = token_match.start() token_prefix = token_match.group() - # TODO: support bytes in Py3k matched = [] for key in keys: try: @@ -455,7 +791,7 @@ def match_dict_keys(keys, prefix, delims): # reformat remainder of key to begin with prefix rem = key[len(prefix_str):] # force repr wrapped in ' - rem_repr = repr(rem + '"') + rem_repr = repr(rem + '"') if isinstance(rem, str) else repr(rem + b'"') if rem_repr.startswith('u') and prefix[0] not in 'uU': # Found key is unicode, but prefix is Py2 string. # Therefore attempt to interpret key as string. @@ -476,23 +812,90 @@ def match_dict_keys(keys, prefix, delims): return quote, token_start, matched +def cursor_to_position(text:str, line:int, column:int)->int: + """ + + Convert the (line,column) position of the cursor in text to an offset in a + string. + + Parameters + ---------- + + text : str + The text in which to calculate the cursor offset + line : int + Line of the cursor; 0-indexed + column : int + Column of the cursor 0-indexed + + Return + ------ + Position of the cursor in ``text``, 0-indexed. + + See Also + -------- + position_to_cursor: reciprocal of this function + + """ + lines = text.split('\n') + assert line <= len(lines), '{} <= {}'.format(str(line), str(len(lines))) + + return sum(len(l) + 1 for l in lines[:line]) + column + +def position_to_cursor(text:str, offset:int)->Tuple[int, int]: + """ + Convert the position of the cursor in text (0 indexed) to a line + number(0-indexed) and a column number (0-indexed) pair + + Position should be a valid position in ``text``. + + Parameters + ---------- + + text : str + The text in which to calculate the cursor offset + offset : int + Position of the cursor in ``text``, 0-indexed. + + Return + ------ + (line, column) : (int, int) + Line of the cursor; 0-indexed, column of the cursor 0-indexed + + + See Also + -------- + cursor_to_position : reciprocal of this function + + + """ + + assert 0 <= offset <= len(text) , "0 <= %s <= %s" % (offset , len(text)) + + before = text[:offset] + blines = before.split('\n') # ! splitnes trim trailing \n + line = before.count('\n') + col = len(blines[-1]) + return line, col + + def _safe_isinstance(obj, module, class_name): """Checks if obj is an instance of module.class_name if loaded """ return (module in sys.modules and - isinstance(obj, getattr(__import__(module), class_name))) + isinstance(obj, getattr(import_module(module), class_name))) def back_unicode_name_matches(text): u"""Match unicode characters back to unicode name - - This does ☃ -> \\snowman + + This does ``☃`` -> ``\\snowman`` Note that snowman is not a valid python3 combining character but will be expanded. Though it will not recombine back to the snowman character by the completion machinery. This will not either back-complete standard sequences like \\n, \\b ... - + Used on Python 3 only. """ if len(text)<2: @@ -513,10 +916,10 @@ def back_unicode_name_matches(text): pass return u'', () -def back_latex_name_matches(text): - u"""Match latex characters back to unicode name - - This does ->\\sqrt +def back_latex_name_matches(text:str): + """Match latex characters back to unicode name + + This does ``\\ℵ`` -> ``\\aleph`` Used on Python 3 only. """ @@ -541,9 +944,57 @@ def back_latex_name_matches(text): return u'', () +def _formatparamchildren(parameter) -> str: + """ + Get parameter name and value from Jedi Private API + + Jedi does not expose a simple way to get `param=value` from its API. + + Parameter + ========= + + parameter: + Jedi's function `Param` + + Returns + ======= + + A string like 'a', 'b=1', '*args', '**kwargs' + + + """ + description = parameter.description + if not description.startswith('param '): + raise ValueError('Jedi function parameter description have change format.' + 'Expected "param ...", found %r".' % description) + return description[6:] + +def _make_signature(completion)-> str: + """ + Make the signature from a jedi completion + + Parameter + ========= + + completion: jedi.Completion + object does not complete a function type + + Returns + ======= + + a string consisting of the function signature, with the parenthesis but + without the function name. example: + `(a, *args, b=1, **kwargs)` + + """ + + return '(%s)'% ', '.join([f for f in (_formatparamchildren(p) for p in completion.params) if f]) + class IPCompleter(Completer): """Extension of the completer class with IPython-specific features""" - + + _names = None + @observe('greedy') def _greedy_changed(self, change): """update the splitter and readline delims when greedy is changed""" @@ -552,76 +1003,81 @@ def _greedy_changed(self, change): else: self.splitter.delims = DELIMS - if self.readline: - self.readline.set_completer_delims(self.splitter.delims) - + dict_keys_only = Bool(False, + help="""Whether to show dict key matches only""") + merge_completions = Bool(True, help="""Whether to merge completion results into a single list - + If False, only the completion results from the first non-empty completer will be returned. """ ).tag(config=True) omit__names = Enum((0,1,2), default_value=2, help="""Instruct the completer to omit private method names - + Specifically, when completing on ``object.``. - + When 2 [default]: all names that start with '_' will be excluded. - + When 1: all 'magic' names (``__foo__``) will be excluded. - + When 0: nothing will be excluded. """ ).tag(config=True) limit_to__all__ = Bool(False, help=""" DEPRECATED as of version 5.0. - + Instruct the completer to use __all__ for the completion - + Specifically, when completing on ``object.``. - + When True: only those names in obj.__all__ will be included. - - When False [default]: the __all__ attribute is ignored + + When False [default]: the __all__ attribute is ignored """, ).tag(config=True) + @observe('limit_to__all__') + def _limit_to_all_changed(self, change): + warnings.warn('`IPython.core.IPCompleter.limit_to__all__` configuration ' + 'value has been deprecated since IPython 5.0, will be made to have ' + 'no effects and then removed in future version of IPython.', + UserWarning) + def __init__(self, shell=None, namespace=None, global_namespace=None, - use_readline=True, config=None, **kwargs): + use_readline=_deprecation_readline_sentinel, config=None, **kwargs): """IPCompleter() -> completer - Return a completer object suitable for use by the readline library - via readline.set_completer(). + Return a completer object. - Inputs: + Parameters + ---------- - - shell: a pointer to the ipython shell itself. This is needed - because this completer knows about magic functions, and those can - only be accessed via the ipython instance. + shell + a pointer to the ipython shell itself. This is needed + because this completer knows about magic functions, and those can + only be accessed via the ipython instance. - - namespace: an optional dict where completions are performed. + namespace : dict, optional + an optional dict where completions are performed. - - global_namespace: secondary optional dict for completions, to - handle cases (such as IPython embedded inside functions) where - both Python scopes are visible. + global_namespace : dict, optional + secondary optional dict for completions, to + handle cases (such as IPython embedded inside functions) where + both Python scopes are visible. use_readline : bool, optional - If true, use the readline library. This completer can still function - without readline, though in that case callers must provide some extra - information on each call about the current line.""" + DEPRECATED, ignored since IPython 6.0, will have no effects + """ self.magic_escape = ESC_MAGIC self.splitter = CompletionSplitter() - # Readline configuration, only used by the rlcompleter method. - if use_readline: - # We store the right version of readline so that later code - import IPython.utils.rlineimpl as readline - self.readline = readline - else: - self.readline = None + if use_readline is not _deprecation_readline_sentinel: + warnings.warn('The `use_readline` parameter is deprecated and ignored since IPython 6.0.', + DeprecationWarning, stacklevel=2) # _greedy_changed() depends on splitter and readline being defined: Completer.__init__(self, namespace=namespace, global_namespace=global_namespace, @@ -652,22 +1108,46 @@ def __init__(self, shell=None, namespace=None, global_namespace=None, #use this if positional argument name is also needed #= re.compile(r'[\s|\[]*(\w+)(?:\s*=?\s*.*)') - # All active matcher routines for completion - self.matchers = [ - self.python_matches, - self.file_matches, - self.magic_matches, - self.python_func_kw_matches, - self.dict_key_matches, - ] + self.magic_arg_matchers = [ + self.magic_config_matches, + self.magic_color_matches, + ] # This is set externally by InteractiveShell self.custom_completers = None - def all_completions(self, text): + @property + def matchers(self): + """All active matcher routines for completion""" + if self.dict_keys_only: + return [self.dict_key_matches] + + if self.use_jedi: + return [ + *self.custom_matchers, + self.file_matches, + self.magic_matches, + self.dict_key_matches, + ] + else: + return [ + *self.custom_matchers, + self.python_matches, + self.file_matches, + self.magic_matches, + self.python_func_kw_matches, + self.dict_key_matches, + ] + + def all_completions(self, text) -> List[str]: """ - Wrapper around the complete method for the benefit of emacs. + Wrapper around the completion methods for the benefit of emacs. """ + prefix = text.rpartition('.')[0] + with provisionalcompleter(): + return ['.'.join([prefix, c.text]) if prefix and self.use_jedi else c.text + for c in self.completions(text, len(text))] + return self.complete(text)[1] def _clean_glob(self, text): @@ -730,7 +1210,7 @@ def file_matches(self, text): text = os.path.expanduser(text) if text == "": - return [text_prefix + cast_unicode_py2(protect_filename(f)) for f in self.glob("*")] + return [text_prefix + protect_filename(f) for f in self.glob("*")] # Compute the matches from the filesystem if sys.platform == 'win32': @@ -748,15 +1228,16 @@ def file_matches(self, text): else: if open_quotes: # if we have a string with an open quote, we don't need to - # protect the names at all (and we _shouldn't_, as it - # would cause bugs when the filesystem call is made). - matches = m0 + # protect the names beyond the quote (and we _shouldn't_, as + # it would cause bugs when the filesystem call is made). + matches = m0 if sys.platform == "win32" else\ + [protect_filename(f, open_quotes) for f in m0] else: matches = [text_prefix + protect_filename(f) for f in m0] # Mark directories in input list by appending '/' to their names. - return [cast_unicode_py2(x+'/') if os.path.isdir(x) else x for x in matches] + return [x+'/' if os.path.isdir(x) else x for x in matches] def magic_matches(self, text): """Match magics""" @@ -767,18 +1248,162 @@ def magic_matches(self, text): cell_magics = lsm['cell'] pre = self.magic_escape pre2 = pre+pre - + + explicit_magic = text.startswith(pre) + # Completion logic: # - user gives %%: only do cell magics # - user gives %: do both line and cell magics # - no prefix: do both # In other words, line magics are skipped if the user gives %% explicitly + # + # We also exclude magics that match any currently visible names: + # https://github.com/ipython/ipython/issues/4877, unless the user has + # typed a %: + # https://github.com/ipython/ipython/issues/10754 bare_text = text.lstrip(pre) - comp = [ pre2+m for m in cell_magics if m.startswith(bare_text)] + global_matches = self.global_matches(bare_text) + if not explicit_magic: + def matches(magic): + """ + Filter magics, in particular remove magics that match + a name present in global namespace. + """ + return ( magic.startswith(bare_text) and + magic not in global_matches ) + else: + def matches(magic): + return magic.startswith(bare_text) + + comp = [ pre2+m for m in cell_magics if matches(m)] if not text.startswith(pre2): - comp += [ pre+m for m in line_magics if m.startswith(bare_text)] - return [cast_unicode_py2(c) for c in comp] + comp += [ pre+m for m in line_magics if matches(m)] + + return comp + + def magic_config_matches(self, text:str) -> List[str]: + """ Match class names and attributes for %config magic """ + texts = text.strip().split() + + if len(texts) > 0 and (texts[0] == 'config' or texts[0] == '%config'): + # get all configuration classes + classes = sorted(set([ c for c in self.shell.configurables + if c.__class__.class_traits(config=True) + ]), key=lambda x: x.__class__.__name__) + classnames = [ c.__class__.__name__ for c in classes ] + + # return all classnames if config or %config is given + if len(texts) == 1: + return classnames + + # match classname + classname_texts = texts[1].split('.') + classname = classname_texts[0] + classname_matches = [ c for c in classnames + if c.startswith(classname) ] + + # return matched classes or the matched class with attributes + if texts[1].find('.') < 0: + return classname_matches + elif len(classname_matches) == 1 and \ + classname_matches[0] == classname: + cls = classes[classnames.index(classname)].__class__ + help = cls.class_get_help() + # strip leading '--' from cl-args: + help = re.sub(re.compile(r'^--', re.MULTILINE), '', help) + return [ attr.split('=')[0] + for attr in help.strip().splitlines() + if attr.startswith(texts[1]) ] + return [] + def magic_color_matches(self, text:str) -> List[str] : + """ Match color schemes for %colors magic""" + texts = text.split() + if text.endswith(' '): + # .split() strips off the trailing whitespace. Add '' back + # so that: '%colors ' -> ['%colors', ''] + texts.append('') + + if len(texts) == 2 and (texts[0] == 'colors' or texts[0] == '%colors'): + prefix = texts[1] + return [ color for color in InspectColors.keys() + if color.startswith(prefix) ] + return [] + + def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str): + """ + + Return a list of :any:`jedi.api.Completions` object from a ``text`` and + cursor position. + + Parameters + ---------- + cursor_column : int + column position of the cursor in ``text``, 0-indexed. + cursor_line : int + line position of the cursor in ``text``, 0-indexed + text : str + text to complete + + Debugging + --------- + + If ``IPCompleter.debug`` is ``True`` may return a :any:`_FakeJediCompletion` + object containing a string with the Jedi debug information attached. + """ + namespaces = [self.namespace] + if self.global_namespace is not None: + namespaces.append(self.global_namespace) + + completion_filter = lambda x:x + offset = cursor_to_position(text, cursor_line, cursor_column) + # filter output if we are completing for object members + if offset: + pre = text[offset-1] + if pre == '.': + if self.omit__names == 2: + completion_filter = lambda c:not c.name.startswith('_') + elif self.omit__names == 1: + completion_filter = lambda c:not (c.name.startswith('__') and c.name.endswith('__')) + elif self.omit__names == 0: + completion_filter = lambda x:x + else: + raise ValueError("Don't understand self.omit__names == {}".format(self.omit__names)) + + interpreter = jedi.Interpreter( + text[:offset], namespaces, column=cursor_column, line=cursor_line + 1) + try_jedi = True + + try: + # find the first token in the current tree -- if it is a ' or " then we are in a string + completing_string = False + try: + first_child = next(c for c in interpreter._get_module().tree_node.children if hasattr(c, 'value')) + except StopIteration: + pass + else: + # note the value may be ', ", or it may also be ''' or """, or + # in some cases, """what/you/typed..., but all of these are + # strings. + completing_string = len(first_child.value) > 0 and first_child.value[0] in {"'", '"'} + + # if we are in a string jedi is likely not the right candidate for + # now. Skip it. + try_jedi = not completing_string + except Exception as e: + # many of things can go wrong, we are using private API just don't crash. + if self.debug: + print("Error detecting if completing a non-finished string :", e, '|') + + if not try_jedi: + return [] + try: + return filter(completion_filter, interpreter.completions()) + except Exception as e: + if self.debug: + return [_FakeJediCompletion('Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' % (e))] + else: + return [] def python_matches(self, text): """Match attributes or global python names""" @@ -837,7 +1462,7 @@ def _default_arguments(self, obj): pass elif not (inspect.isfunction(obj) or inspect.ismethod(obj)): if inspect.isclass(obj): - #for cython embededsignature=True the constructor docstring + #for cython embedsignature=True the constructor docstring #belongs to the object itself not __init__ ret += self._default_arguments_from_docstring( getattr(obj, '__doc__', '')) @@ -850,18 +1475,11 @@ def _default_arguments(self, obj): ret += self._default_arguments_from_docstring( getattr(call_obj, '__doc__', '')) - if PY3: - _keeps = (inspect.Parameter.KEYWORD_ONLY, - inspect.Parameter.POSITIONAL_OR_KEYWORD) - signature = inspect.signature - else: - import IPython.utils.signatures - _keeps = (IPython.utils.signatures.Parameter.KEYWORD_ONLY, - IPython.utils.signatures.Parameter.POSITIONAL_OR_KEYWORD) - signature = IPython.utils.signatures.signature + _keeps = (inspect.Parameter.KEYWORD_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD) try: - sig = signature(call_obj) + sig = inspect.signature(call_obj) ret.extend(k for k, v in sig.parameters.items() if v.kind in _keeps) except ValueError: @@ -886,8 +1504,7 @@ def python_func_kw_matches(self,text): # parenthesis before the cursor # e.g. for "foo (1+bar(x), pa,a=1)", the candidate is "foo" tokens = regexp.findall(self.text_until_cursor) - tokens.reverse() - iterTokens = iter(tokens); openPar = 0 + iterTokens = reversed(tokens); openPar = 0 for token in iterTokens: if token == ')': @@ -912,23 +1529,38 @@ def python_func_kw_matches(self,text): break except StopIteration: break - # lookup the candidate callable matches either using global_matches - # or attr_matches for dotted names - if len(ids) == 1: - callableMatches = self.global_matches(ids[0]) - else: - callableMatches = self.attr_matches('.'.join(ids[::-1])) - argMatches = [] - for callableMatch in callableMatches: - try: - namedArgs = self._default_arguments(eval(callableMatch, - self.namespace)) - except: + + # Find all named arguments already assigned to, as to avoid suggesting + # them again + usedNamedArgs = set() + par_level = -1 + for token, next_token in zip(tokens, tokens[1:]): + if token == '(': + par_level += 1 + elif token == ')': + par_level -= 1 + + if par_level != 0: + continue + + if next_token != '=': continue - for namedArg in namedArgs: + usedNamedArgs.add(token) + + argMatches = [] + try: + callableObj = '.'.join(ids[::-1]) + namedArgs = self._default_arguments(eval(callableObj, + self.namespace)) + + # Remove used named arguments from the list, no need to show twice + for namedArg in set(namedArgs) - usedNamedArgs: if namedArg.startswith(text): argMatches.append(u"%s=" %namedArg) + except: + pass + return argMatches def dict_key_matches(self, text): @@ -971,7 +1603,7 @@ def get_keys(obj): $ ''' regexps = self.__dict_key_regexps = { - False: re.compile(dict_key_re_fmt % ''' + False: re.compile(dict_key_re_fmt % r''' # identifiers separated by . (?!\d)\w+ (?:\.(?!\d)\w+)* @@ -1000,7 +1632,7 @@ def get_keys(obj): closing_quote, token_offset, matches = match_dict_keys(keys, prefix, self.splitter.delims) if not matches: return matches - + # get the cursor position of # - the text being completed # - the start of the key text @@ -1011,13 +1643,13 @@ def get_keys(obj): completion_start = key_start + token_offset else: key_start = completion_start = match.end() - + # grab the leading prefix, to make sure all completions start with `text` if text_start > key_start: leading = '' else: leading = text[text_start:completion_start] - + # the index of the `[` character bracket_idx = match.end(1) @@ -1036,18 +1668,18 @@ def get_keys(obj): # brackets were opened inside text, maybe close them if not continuation.startswith(']'): suf += ']' - + return [leading + k + suf for k in matches] def unicode_name_matches(self, text): - u"""Match Latex-like syntax for unicode characters base + u"""Match Latex-like syntax for unicode characters base on the name of the character. - - This does \\GREEK SMALL LETTER ETA -> η - Works only on valid python 3 identifier, or on combining characters that + This does ``\\GREEK SMALL LETTER ETA`` -> ``η`` + + Works only on valid python 3 identifier, or on combining characters that will combine to form a valid identifier. - + Used on Python 3 only. """ slashpos = text.rfind('\\') @@ -1063,14 +1695,10 @@ def unicode_name_matches(self, text): return u'', [] - - def latex_matches(self, text): u"""Match Latex syntax for unicode characters. - - This does both \\alp -> \\alpha and \\alpha -> α - - Used on Python 3 only. + + This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α`` """ slashpos = text.rfind('\\') if slashpos > -1: @@ -1083,7 +1711,8 @@ def latex_matches(self, text): # If a user has partially typed a latex symbol, give them # a full list of options \al -> [\aleph, \alpha] matches = [k for k in latex_symbols if k.startswith(s)] - return s, matches + if matches: + return s, matches return u'', [] def dispatch_custom_completer(self, text): @@ -1096,7 +1725,7 @@ def dispatch_custom_completer(self, text): # Create a little structure to pass all the relevant information about # the current completion to any custom completer. - event = Bunch() + event = SimpleNamespace() event.line = line event.symbol = text cmd = line.split(None,1)[0] @@ -1117,18 +1746,170 @@ def dispatch_custom_completer(self, text): res = c(event) if res: # first, try case sensitive match - withcase = [cast_unicode_py2(r) for r in res if r.startswith(text)] + withcase = [r for r in res if r.startswith(text)] if withcase: return withcase # if none, then case insensitive ones are ok too text_low = text.lower() - return [cast_unicode_py2(r) for r in res if r.lower().startswith(text_low)] + return [r for r in res if r.lower().startswith(text_low)] except TryNext: pass + except KeyboardInterrupt: + """ + If custom completer take too long, + let keyboard interrupt abort and return nothing. + """ + break return None - @_strip_single_trailing_space + def completions(self, text: str, offset: int)->Iterator[Completion]: + """ + Returns an iterator over the possible completions + + .. warning:: Unstable + + This function is unstable, API may change without warning. + It will also raise unless use in proper context manager. + + Parameters + ---------- + + text:str + Full text of the current input, multi line string. + offset:int + Integer representing the position of the cursor in ``text``. Offset + is 0-based indexed. + + Yields + ------ + :any:`Completion` object + + + The cursor on a text can either be seen as being "in between" + characters or "On" a character depending on the interface visible to + the user. For consistency the cursor being on "in between" characters X + and Y is equivalent to the cursor being "on" character Y, that is to say + the character the cursor is on is considered as being after the cursor. + + Combining characters may span more that one position in the + text. + + + .. note:: + + If ``IPCompleter.debug`` is :any:`True` will yield a ``--jedi/ipython--`` + fake Completion token to distinguish completion returned by Jedi + and usual IPython completion. + + .. note:: + + Completions are not completely deduplicated yet. If identical + completions are coming from different sources this function does not + ensure that each completion object will only be present once. + """ + warnings.warn("_complete is a provisional API (as of IPython 6.0). " + "It may change without warnings. " + "Use in corresponding context manager.", + category=ProvisionalCompleterWarning, stacklevel=2) + + seen = set() + try: + for c in self._completions(text, offset, _timeout=self.jedi_compute_type_timeout/1000): + if c and (c in seen): + continue + yield c + seen.add(c) + except KeyboardInterrupt: + """if completions take too long and users send keyboard interrupt, + do not crash and return ASAP. """ + pass + + def _completions(self, full_text: str, offset: int, *, _timeout)->Iterator[Completion]: + """ + Core completion module.Same signature as :any:`completions`, with the + extra `timeout` parameter (in seconds). + + + Computing jedi's completion ``.type`` can be quite expensive (it is a + lazy property) and can require some warm-up, more warm up than just + computing the ``name`` of a completion. The warm-up can be : + + - Long warm-up the first time a module is encountered after + install/update: actually build parse/inference tree. + + - first time the module is encountered in a session: load tree from + disk. + + We don't want to block completions for tens of seconds so we give the + completer a "budget" of ``_timeout`` seconds per invocation to compute + completions types, the completions that have not yet been computed will + be marked as "unknown" an will have a chance to be computed next round + are things get cached. + + Keep in mind that Jedi is not the only thing treating the completion so + keep the timeout short-ish as if we take more than 0.3 second we still + have lots of processing to do. + + """ + deadline = time.monotonic() + _timeout + + + before = full_text[:offset] + cursor_line, cursor_column = position_to_cursor(full_text, offset) + + matched_text, matches, matches_origin, jedi_matches = self._complete( + full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column) + + iter_jm = iter(jedi_matches) + if _timeout: + for jm in iter_jm: + try: + type_ = jm.type + except Exception: + if self.debug: + print("Error in Jedi getting type of ", jm) + type_ = None + delta = len(jm.name_with_symbols) - len(jm.complete) + if type_ == 'function': + signature = _make_signature(jm) + else: + signature = '' + yield Completion(start=offset - delta, + end=offset, + text=jm.name_with_symbols, + type=type_, + signature=signature, + _origin='jedi') + + if time.monotonic() > deadline: + break + + for jm in iter_jm: + delta = len(jm.name_with_symbols) - len(jm.complete) + yield Completion(start=offset - delta, + end=offset, + text=jm.name_with_symbols, + type='', # don't compute type for speed + _origin='jedi', + signature='') + + + start_offset = before.rfind(matched_text) + + # TODO: + # Suppress this, right now just for debug. + if jedi_matches and matches and self.debug: + yield Completion(start=start_offset, end=offset, text='--jedi/ipython--', + _origin='debug', type='none', signature='') + + # I'm unsure if this is always true, so let's assert and see if it + # crash + assert before.endswith(matched_text) + for m, t in zip(matches, matches_origin): + yield Completion(start=start_offset, end=offset, text=m, _origin=t, signature='', type='') + + def complete(self, text=None, line_buffer=None, cursor_pos=None): """Find completions for the given text and line context. @@ -1158,28 +1939,67 @@ def complete(self, text=None, line_buffer=None, cursor_pos=None): matches : list A list of completion matches. + + + .. note:: + + This API is likely to be deprecated and replaced by + :any:`IPCompleter.completions` in the future. + + """ + warnings.warn('`Completer.complete` is pending deprecation since ' + 'IPython 6.0 and will be replaced by `Completer.completions`.', + PendingDeprecationWarning) + # potential todo, FOLD the 3rd throw away argument of _complete + # into the first 2 one. + return self._complete(line_buffer=line_buffer, cursor_pos=cursor_pos, text=text, cursor_line=0)[:2] + + def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None, + full_text=None) -> Tuple[str, List[str], List[str], Iterable[_FakeJediCompletion]]: + """ + + Like complete but can also returns raw jedi completions as well as the + origin of the completion text. This could (and should) be made much + cleaner but that will be simpler once we drop the old (and stateful) + :any:`complete` API. + + + With current provisional API, cursor_pos act both (depending on the + caller) as the offset in the ``text`` or ``line_buffer``, or as the + ``column`` when passing multiline strings this could/should be renamed + but would add extra noise. + """ + # if the cursor position isn't given, the only sane assumption we can # make is that it's at the end of the line (the common case) if cursor_pos is None: cursor_pos = len(line_buffer) if text is None else len(text) - if PY3: + if self.use_main_ns: + self.namespace = __main__.__dict__ + # if text is either None or an empty string, rely on the line buffer + if (not line_buffer) and full_text: + line_buffer = full_text.split('\n')[cursor_line] + if not text: # issue #11508: check line_buffer before calling split_line + text = self.splitter.split_line(line_buffer, cursor_pos) if line_buffer else '' + + if self.backslash_combining_completions: + # allow deactivation of these on windows. base_text = text if not line_buffer else line_buffer[:cursor_pos] latex_text, latex_matches = self.latex_matches(base_text) if latex_matches: - return latex_text, latex_matches + return latex_text, latex_matches, ['latex_matches']*len(latex_matches), () name_text = '' name_matches = [] - for meth in (self.unicode_name_matches, back_latex_name_matches, back_unicode_name_matches): + # need to add self.fwd_unicode_match() function here when done + for meth in (self.unicode_name_matches, back_latex_name_matches, back_unicode_name_matches, self.fwd_unicode_match): name_text, name_matches = meth(base_text) if name_text: - return name_text, name_matches - - # if text is either None or an empty string, rely on the line buffer - if not text: - text = self.splitter.split_line(line_buffer, cursor_pos) + return name_text, name_matches[:MATCHES_LIMIT], \ + [meth.__qualname__]*min(len(name_matches), MATCHES_LIMIT), () + # If no line buffer is given, assume the input text is all there was if line_buffer is None: @@ -1188,34 +2008,85 @@ def complete(self, text=None, line_buffer=None, cursor_pos=None): self.line_buffer = line_buffer self.text_until_cursor = self.line_buffer[:cursor_pos] + # Do magic arg matches + for matcher in self.magic_arg_matchers: + matches = list(matcher(line_buffer))[:MATCHES_LIMIT] + if matches: + origins = [matcher.__qualname__] * len(matches) + return text, matches, origins, () + # Start with a clean slate of completions - self.matches[:] = [] - custom_res = self.dispatch_custom_completer(text) - if custom_res is not None: - # did custom completers produce something? - self.matches = custom_res - else: - # Extend the list of completions with the results of each - # matcher, so we return results to the user from all - # namespaces. - if self.merge_completions: - self.matches = [] - for matcher in self.matchers: - try: - self.matches.extend(matcher(text)) - except: - # Show the ugly traceback if the matcher causes an - # exception, but do NOT crash the kernel! - sys.excepthook(*sys.exc_info()) - else: - for matcher in self.matchers: - self.matches = matcher(text) - if self.matches: - break + matches = [] + # FIXME: we should extend our api to return a dict with completions for # different types of objects. The rlcomplete() method could then # simply collapse the dict into a list for readline, but we'd have - # richer completion semantics in other evironments. - self.matches = sorted(set(self.matches), key=completions_sorting_key) + # richer completion semantics in other environments. + completions = () + if self.use_jedi: + if not full_text: + full_text = line_buffer + completions = self._jedi_matches( + cursor_pos, cursor_line, full_text) + + if self.merge_completions: + matches = [] + for matcher in self.matchers: + try: + matches.extend([(m, matcher.__qualname__) + for m in matcher(text)]) + except: + # Show the ugly traceback if the matcher causes an + # exception, but do NOT crash the kernel! + sys.excepthook(*sys.exc_info()) + else: + for matcher in self.matchers: + matches = [(m, matcher.__qualname__) + for m in matcher(text)] + if matches: + break + + seen = set() + filtered_matches = set() + for m in matches: + t, c = m + if t not in seen: + filtered_matches.add(m) + seen.add(t) + + _filtered_matches = sorted(filtered_matches, key=lambda x: completions_sorting_key(x[0])) + + custom_res = [(m, 'custom') for m in self.dispatch_custom_completer(text) or []] + + _filtered_matches = custom_res or _filtered_matches + + _filtered_matches = _filtered_matches[:MATCHES_LIMIT] + _matches = [m[0] for m in _filtered_matches] + origins = [m[1] for m in _filtered_matches] + + self.matches = _matches + + return text, _matches, origins, completions + + def fwd_unicode_match(self, text:str) -> Tuple[str, list]: + if self._names is None: + self._names = [] + for c in range(0,0x10FFFF + 1): + try: + self._names.append(unicodedata.name(chr(c))) + except ValueError: + pass - return text, self.matches + slashpos = text.rfind('\\') + # if text starts with slash + if slashpos > -1: + s = text[slashpos+1:] + candidates = [x for x in self._names if x.startswith(s)] + if candidates: + return s, candidates + else: + return '', () + + # if text does not start with slash + else: + return u'', () diff --git a/IPython/core/completerlib.py b/IPython/core/completerlib.py index 3fbc7e6cc56..7860cb67dcb 100644 --- a/IPython/core/completerlib.py +++ b/IPython/core/completerlib.py @@ -14,7 +14,6 @@ #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- -from __future__ import print_function # Stdlib imports import glob @@ -22,31 +21,28 @@ import os import re import sys +from importlib import import_module +from importlib.machinery import all_suffixes -try: - # Python >= 3.3 - from importlib.machinery import all_suffixes - _suffixes = all_suffixes() -except ImportError: - from imp import get_suffixes - _suffixes = [ s[0] for s in get_suffixes() ] # Third-party imports from time import time from zipimport import zipimporter # Our own imports -from IPython.core.completer import expand_user, compress_user -from IPython.core.error import TryNext -from IPython.utils._process_common import arg_split -from IPython.utils.py3compat import string_types +from .completer import expand_user, compress_user +from .error import TryNext +from ..utils._process_common import arg_split # FIXME: this should be pulled in with the right call via the component system from IPython import get_ipython +from typing import List + #----------------------------------------------------------------------------- # Globals and constants #----------------------------------------------------------------------------- +_suffixes = all_suffixes() # Time in seconds after which the rootmodules will be stored permanently in the # ipython ip.db database (kept in the user's .ipython dir). @@ -56,7 +52,7 @@ TIMEOUT_GIVEUP = 20 # Regular expression for the python import statement -import_re = re.compile(r'(?P[a-zA-Z_][a-zA-Z0-9_]*?)' +import_re = re.compile(r'(?P[^\W\d]\w*?)' r'(?P[/\\]__init__)?' r'(?P%s)$' % r'|'.join(re.escape(s) for s in _suffixes)) @@ -116,6 +112,11 @@ def get_root_modules(): ip.db['rootmodules_cache'] maps sys.path entries to list of modules. """ ip = get_ipython() + if ip is None: + # No global shell instance to store cached list of modules. + # Don't try to scan for modules every time. + return list(sys.builtin_module_names) + rootmodules_cache = ip.db.get('rootmodules_cache', {}) rootmodules = list(sys.builtin_module_names) start_time = time() @@ -153,16 +154,18 @@ def is_importable(module, attr, only_modules): else: return not(attr[:2] == '__' and attr[-2:] == '__') -def try_import(mod, only_modules=False): + +def try_import(mod: str, only_modules=False) -> List[str]: + """ + Try to import given module and return list of potential completions. + """ + mod = mod.rstrip('.') try: - m = __import__(mod) + m = import_module(mod) except: return [] - mods = mod.split('.') - for module in mods[1:]: - m = getattr(m, module) - m_is_init = hasattr(m, '__file__') and '__init__' in m.__file__ + m_is_init = '__init__' in (getattr(m, '__file__', '') or '') completions = [] if (not hasattr(m, '__file__')) or (not only_modules) or m_is_init: @@ -172,9 +175,9 @@ def try_import(mod, only_modules=False): completions.extend(getattr(m, '__all__', [])) if m_is_init: completions.extend(module_list(os.path.dirname(m.__file__))) - completions = {c for c in completions if isinstance(c, string_types)} - completions.discard('__init__') - return list(completions) + completions_set = {c for c in completions if isinstance(c, str)} + completions_set.discard('__init__') + return list(completions_set) #----------------------------------------------------------------------------- @@ -182,7 +185,7 @@ def try_import(mod, only_modules=False): #----------------------------------------------------------------------------- def quick_completer(cmd, completions): - """ Easily create a trivial completer for a command. + r""" Easily create a trivial completer for a command. Takes either a list of completions, or all completions in string (that will be split on whitespace). @@ -196,7 +199,7 @@ def quick_completer(cmd, completions): [d:\ipython]|3> foo ba """ - if isinstance(completions, string_types): + if isinstance(completions, str): completions = completions.split() def do_complete(self, event): diff --git a/IPython/core/crashhandler.py b/IPython/core/crashhandler.py index 2cbe13311e0..1e0b429d09a 100644 --- a/IPython/core/crashhandler.py +++ b/IPython/core/crashhandler.py @@ -18,7 +18,6 @@ #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- -from __future__ import print_function import os import sys @@ -28,7 +27,9 @@ from IPython.core import ultratb from IPython.core.release import author_email from IPython.utils.sysinfo import sys_info -from IPython.utils.py3compat import input, getcwd +from IPython.utils.py3compat import input + +from IPython.core.release import __version__ as version #----------------------------------------------------------------------------- # Code @@ -54,12 +55,22 @@ If you want to do it now, the following command will work (under Unix): mail -s '{app_name} Crash Report' {contact_email} < {crash_report_fname} +In your email, please also include information about: +- The operating system under which the crash happened: Linux, macOS, Windows, + other, and which exact version (for example: Ubuntu 16.04.3, macOS 10.13.2, + Windows 10 Pro), and whether it is 32-bit or 64-bit; +- How {app_name} was installed: using pip or conda, from GitHub, as part of + a Docker container, or other, providing more detail if possible; +- How to reproduce the crash: what exact sequence of instructions can one + input to get the same crash? Ideally, find a minimal yet complete sequence + of instructions that yields the crash. + To ensure accurate tracking of this issue, please file a report about it at: {bug_tracker} """ _lite_message_template = """ -If you suspect this is an IPython bug, please report it at: +If you suspect this is an IPython {version} bug, please report it at: https://github.com/ipython/ipython/issues or send an email to the mailing list at {email} @@ -140,9 +151,9 @@ def __call__(self, etype, evalue, etb): try: rptdir = self.app.ipython_dir except: - rptdir = getcwd() + rptdir = os.getcwd() if rptdir is None or not os.path.isdir(rptdir): - rptdir = getcwd() + rptdir = os.getcwd() report_name = os.path.join(rptdir,self.crash_report_fname) # write the report filename into the instance dict so it can get # properly expanded out in the user message template @@ -170,13 +181,14 @@ def __call__(self, etype, evalue, etb): print('Could not create crash report on disk.', file=sys.stderr) return - # Inform user on stderr of what happened - print('\n'+'*'*70+'\n', file=sys.stderr) - print(self.message_template.format(**self.info), file=sys.stderr) + with report: + # Inform user on stderr of what happened + print('\n'+'*'*70+'\n', file=sys.stderr) + print(self.message_template.format(**self.info), file=sys.stderr) + + # Construct report on disk + report.write(self.make_report(traceback)) - # Construct report on disk - report.write(self.make_report(traceback)) - report.close() input("Hit to quit (your terminal may close):") def make_report(self,traceback): @@ -212,5 +224,5 @@ def crash_handler_lite(etype, evalue, tb): else: # we are not in a shell, show generic config config = "c." - print(_lite_message_template.format(email=author_email, config=config), file=sys.stderr) + print(_lite_message_template.format(email=author_email, config=config, version=version), file=sys.stderr) diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py index 591e4c3b8ca..a330baa450e 100644 --- a/IPython/core/debugger.py +++ b/IPython/core/debugger.py @@ -13,7 +13,8 @@ changes. Licensing should therefore be under the standard Python terms. For details on the PSF (Python Software Foundation) standard license, see: -http://www.python.org/2.2.3/license.html""" +https://docs.python.org/2/license.html +""" #***************************************************************************** # @@ -24,16 +25,17 @@ # # #***************************************************************************** -from __future__ import print_function import bdb import functools import inspect +import linecache import sys import warnings +import re from IPython import get_ipython -from IPython.utils import PyColorize, ulinecache +from IPython.utils import PyColorize from IPython.utils import coloransi, py3compat from IPython.core.excolors import exception_colors from IPython.testing.skipdoctest import skip_doctest @@ -64,7 +66,7 @@ def BdbQuit_excepthook(et, ev, tb, excepthook=None): parameter. """ warnings.warn("`BdbQuit_excepthook` is deprecated since version 5.1", - DeprecationWarning) + DeprecationWarning, stacklevel=2) if et==bdb.BdbQuit: print('Exiting Debugger.') elif excepthook is not None: @@ -77,7 +79,7 @@ def BdbQuit_excepthook(et, ev, tb, excepthook=None): def BdbQuit_IPython_excepthook(self,et,ev,tb,tb_offset=None): warnings.warn( "`BdbQuit_IPython_excepthook` is deprecated since version 5.1", - DeprecationWarning) + DeprecationWarning, stacklevel=2) print('Exiting Debugger.') @@ -129,7 +131,7 @@ def __init__(self, colors=None): """ warnings.warn("`Tracer` is deprecated since version 5.1, directly use " "`IPython.core.debugger.Pdb.set_trace()`", - DeprecationWarning) + DeprecationWarning, stacklevel=2) ip = get_ipython() if ip is None: @@ -151,10 +153,7 @@ def __init__(self, colors=None): # at least raise that limit to 80 chars, which should be enough for # most interactive uses. try: - try: - from reprlib import aRepr # Py 3 - except ImportError: - from repr import aRepr # Py 2 + from reprlib import aRepr aRepr.maxstring = 80 except: # This is only a user-facing convenience, so any error we encounter @@ -174,6 +173,13 @@ def __call__(self): self.debugger.set_trace(sys._getframe().f_back) +RGX_EXTRA_INDENT = re.compile(r'(?<=\n)\s+') + + +def strip_indentation(multiline_string): + return RGX_EXTRA_INDENT.sub('', multiline_string) + + def decorate_fn_with_doc(new_fn, old_fn, additional_text=""): """Make new_fn have old_fn's doc string. This is particularly useful for the ``do_...`` commands that hook into the help system. @@ -182,27 +188,11 @@ def decorate_fn_with_doc(new_fn, old_fn, additional_text=""): def wrapper(*args, **kw): return new_fn(*args, **kw) if old_fn.__doc__: - wrapper.__doc__ = old_fn.__doc__ + additional_text + wrapper.__doc__ = strip_indentation(old_fn.__doc__) + additional_text return wrapper -def _file_lines(fname): - """Return the contents of a named file as a list of lines. - - This function never raises an IOError exception: if the file can't be - read, it simply returns an empty list.""" - - try: - outfile = open(fname) - except IOError: - return [] - else: - out = outfile.readlines() - outfile.close() - return out - - -class Pdb(OldPdb, object): +class Pdb(OldPdb): """Modified Pdb class, does not load readline. for a standalone version that uses prompt_toolkit, see @@ -211,7 +201,19 @@ class Pdb(OldPdb, object): """ def __init__(self, color_scheme=None, completekey=None, - stdin=None, stdout=None, context=5): + stdin=None, stdout=None, context=5, **kwargs): + """Create a new IPython debugger. + + :param color_scheme: Deprecated, do not use. + :param completekey: Passed to pdb.Pdb. + :param stdin: Passed to pdb.Pdb. + :param stdout: Passed to pdb.Pdb. + :param context: Number of lines of source code context to show when + displaying stacktrace information. + :param kwargs: Passed to pdb.Pdb. + The possibilities are python version dependent, see the python + docs for more info. + """ # Parent constructor: try: @@ -221,21 +223,26 @@ def __init__(self, color_scheme=None, completekey=None, except (TypeError, ValueError): raise ValueError("Context must be a positive integer") - OldPdb.__init__(self, completekey, stdin, stdout) + # `kwargs` ensures full compatibility with stdlib's `pdb.Pdb`. + OldPdb.__init__(self, completekey, stdin, stdout, **kwargs) # IPython changes... self.shell = get_ipython() if self.shell is None: + save_main = sys.modules['__main__'] # No IPython instance running, we must create one from IPython.terminal.interactiveshell import \ TerminalInteractiveShell self.shell = TerminalInteractiveShell.instance() + # needed by any code which calls __import__("__main__") after + # the debugger was entered. See also #9941. + sys.modules['__main__'] = save_main if color_scheme is not None: warnings.warn( "The `color_scheme` argument is deprecated since version 5.1", - DeprecationWarning) + DeprecationWarning, stacklevel=2) else: color_scheme = self.shell.colors @@ -265,58 +272,39 @@ def __init__(self, color_scheme=None, completekey=None, cst['Neutral'].colors.breakpoint_enabled = C.LightRed cst['Neutral'].colors.breakpoint_disabled = C.Red - self.set_colors(color_scheme) # Add a python parser so we can syntax highlight source while # debugging. - self.parser = PyColorize.Parser() + self.parser = PyColorize.Parser(style=color_scheme) + self.set_colors(color_scheme) # Set the prompt - the default prompt is '(Pdb)' self.prompt = prompt + self.skip_hidden = True def set_colors(self, scheme): """Shorthand access to the color table scheme selector method.""" self.color_scheme_table.set_active_scheme(scheme) + self.parser.style = scheme - def trace_dispatch(self, frame, event, arg): - try: - return super(Pdb, self).trace_dispatch(frame, event, arg) - except bdb.BdbQuit: - pass + + def hidden_frames(self, stack): + """ + Given an index in the stack return wether it should be skipped. + + This is used in up/down and where to skip frames. + """ + ip_hide = [s[0].f_locals.get("__tracebackhide__", False) for s in stack] + ip_start = [i for i, s in enumerate(ip_hide) if s == "__ipython_bottom__"] + if ip_start: + ip_hide = [h if i > ip_start[0] else True for (i, h) in enumerate(ip_hide)] + return ip_hide def interaction(self, frame, traceback): try: OldPdb.interaction(self, frame, traceback) except KeyboardInterrupt: - sys.stdout.write('\n' + self.shell.get_exception_only()) - - def parseline(self, line): - if line.startswith("!!"): - # Force standard behavior. - return super(Pdb, self).parseline(line[2:]) - # "Smart command mode" from pdb++: don't execute commands if a variable - # with the same name exists. - cmd, arg, newline = super(Pdb, self).parseline(line) - # Fix for #9611: Do not trigger smart command if the command is `exit` - # or `quit` and it would resolve to their *global* value (the - # `ExitAutocall` object). Just checking that it is not present in the - # locals dict is not enough as locals and globals match at the - # toplevel. - if ((cmd in self.curframe.f_locals or cmd in self.curframe.f_globals) - and not (cmd in ["exit", "quit"] - and (self.curframe.f_locals is self.curframe.f_globals - or cmd not in self.curframe.f_locals))): - return super(Pdb, self).parseline("!" + line) - return super(Pdb, self).parseline(line) - - def new_do_up(self, arg): - OldPdb.do_up(self, arg) - do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up) - - def new_do_down(self, arg): - OldPdb.do_down(self, arg) - - do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down) + self.stdout.write("\n" + self.shell.get_exception_only()) def new_do_frame(self, arg): OldPdb.do_frame(self, arg) @@ -337,6 +325,8 @@ def new_do_restart(self, arg): return self.do_quit(arg) def print_stack_trace(self, context=None): + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal if context is None: context = self.context try: @@ -346,12 +336,25 @@ def print_stack_trace(self, context=None): except (TypeError, ValueError): raise ValueError("Context must be a positive integer") try: - for frame_lineno in self.stack: + skipped = 0 + for hidden, frame_lineno in zip(self.hidden_frames(self.stack), self.stack): + if hidden and self.skip_hidden: + skipped += 1 + continue + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + skipped = 0 self.print_stack_entry(frame_lineno, context=context) + if skipped: + print( + f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n" + ) except KeyboardInterrupt: pass - def print_stack_entry(self,frame_lineno, prompt_prefix='\n-> ', + def print_stack_entry(self, frame_lineno, prompt_prefix='\n-> ', context=None): if context is None: context = self.context @@ -361,7 +364,7 @@ def print_stack_entry(self,frame_lineno, prompt_prefix='\n-> ', raise ValueError("Context must be a positive integer") except (TypeError, ValueError): raise ValueError("Context must be a positive integer") - print(self.format_stack_entry(frame_lineno, '', context)) + print(self.format_stack_entry(frame_lineno, '', context), file=self.stdout) # vds: >> frame, lineno = frame_lineno @@ -375,9 +378,9 @@ def format_stack_entry(self, frame_lineno, lprefix=': ', context=None): try: context=int(context) if context <= 0: - print("Context must be a positive integer") + print("Context must be a positive integer", file=self.stdout) except (TypeError, ValueError): - print("Context must be a positive integer") + print("Context must be a positive integer", file=self.stdout) try: import reprlib # Py 3 except ImportError: @@ -428,7 +431,7 @@ def format_stack_entry(self, frame_lineno, lprefix=': ', context=None): ret.append(u'%s(%s)%s\n' % (link,lineno,call)) start = lineno - 1 - context//2 - lines = ulinecache.getlines(filename) + lines = linecache.getlines(filename) start = min(start, len(lines) - context) start = max(start, 0) lines = lines[start : start + context] @@ -447,9 +450,9 @@ def __format_line(self, tpl_line, filename, lineno, line, arrow = False): bp_mark = "" bp_mark_color = "" - scheme = self.color_scheme_table.active_scheme_name - new_line, err = self.parser.format2(line, 'str', scheme) - if not err: line = new_line + new_line, err = self.parser.format2(line, 'str') + if not err: + line = new_line bp = None if lineno in self.get_file_breaks(filename): @@ -487,7 +490,7 @@ def print_list_lines(self, filename, first, last): filename = self._exec_filename for lineno in range(first, last+1): - line = ulinecache.getline(filename, lineno) + line = linecache.getline(filename, lineno) if not line: break @@ -499,12 +502,24 @@ def print_list_lines(self, filename, first, last): src.append(line) self.lineno = lineno - print(''.join(src)) + print(''.join(src), file=self.stdout) except KeyboardInterrupt: pass + def do_skip_hidden(self, arg): + """ + Change whether or not we should skip frames with the + __tracebackhide__ attribute. + """ + if arg.strip().lower() in ("true", "yes"): + self.skip_hidden = True + elif arg.strip().lower() in ("false", "no"): + self.skip_hidden = False + def do_list(self, arg): + """Print lines of code from the current stack frame + """ self.lastcmd = 'list' last = None if arg: @@ -520,7 +535,7 @@ def do_list(self, arg): else: first = max(1, int(x) - 5) except: - print('*** Error in argument:', repr(arg)) + print('*** Error in argument:', repr(arg), file=self.stdout) return elif self.lineno is None: first = max(1, self.curframe.f_lineno - 5) @@ -548,6 +563,10 @@ def getsourcelines(self, obj): return inspect.getblock(lines[lineno:]), lineno+1 def do_longlist(self, arg): + """Print lines of code from the current stack frame. + + Shows more lines than 'list' does. + """ self.lastcmd = 'longlist' try: lines, lineno = self.getsourcelines(self.curframe) @@ -558,6 +577,25 @@ def do_longlist(self, arg): self.print_list_lines(self.curframe.f_code.co_filename, lineno, last) do_ll = do_longlist + def do_debug(self, arg): + """debug code + Enter a recursive debugger that steps through the code + argument (which is an arbitrary expression or statement to be + executed in the current environment). + """ + sys.settrace(None) + globals = self.curframe.f_globals + locals = self.curframe_locals + p = self.__class__(completekey=self.completekey, + stdin=self.stdin, stdout=self.stdout) + p.use_rawinput = self.use_rawinput + p.prompt = "(%s) " % self.prompt.strip() + self.message("ENTERING RECURSIVE DEBUGGER") + sys.call_tracing(p.run, (arg, globals, locals)) + self.message("LEAVING RECURSIVE DEBUGGER") + sys.settrace(self.trace_dispatch) + self.lastcmd = p.lastcmd + def do_pdef(self, arg): """Print the call signature for any callable object. @@ -605,19 +643,162 @@ def do_psource(self, arg): ('Globals', self.curframe.f_globals)] self.shell.find_line_magic('psource')(arg, namespaces=namespaces) - if sys.version_info > (3, ): - def do_where(self, arg): - """w(here) - Print a stack trace, with the most recent frame at the bottom. - An arrow indicates the "current frame", which determines the - context of most commands. 'bt' is an alias for this command. + def do_where(self, arg): + """w(here) + Print a stack trace, with the most recent frame at the bottom. + An arrow indicates the "current frame", which determines the + context of most commands. 'bt' is an alias for this command. - Take a number as argument as an (optional) number of context line to - print""" - if arg: + Take a number as argument as an (optional) number of context line to + print""" + if arg: + try: context = int(arg) - self.print_stack_trace(context) + except ValueError as err: + self.error(err) + return + self.print_stack_trace(context) + else: + self.print_stack_trace() + + do_w = do_where + + def stop_here(self, frame): + hidden = False + if self.skip_hidden: + hidden = frame.f_locals.get("__tracebackhide__", False) + if hidden: + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + print(f"{Colors.excName} [... skipped 1 hidden frame]{ColorsNormal}\n") + + return super().stop_here(frame) + + def do_up(self, arg): + """u(p) [count] + Move the current frame count (default one) levels up in the + stack trace (to an older frame). + + Will skip hidden frames. + """ + ## modified version of upstream that skips + # frames with __tracebackide__ + if self.curindex == 0: + self.error("Oldest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + skipped = 0 + if count < 0: + _newframe = 0 + else: + _newindex = self.curindex + counter = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex - 1, -1, -1): + frame = self.stack[i][0] + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break else: - self.print_stack_trace() + # if no break occured. + self.error("all frames above hidden") + return - do_w = do_where + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + _newframe = i + self._select_frame(_newframe) + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + + def do_down(self, arg): + """d(own) [count] + Move the current frame count (default one) levels down in the + stack trace (to a newer frame). + + Will skip hidden frames. + """ + if self.curindex + 1 == len(self.stack): + self.error("Newest frame") + return + try: + count = int(arg or 1) + except ValueError: + self.error("Invalid frame count (%s)" % arg) + return + if count < 0: + _newframe = len(self.stack) - 1 + else: + _newindex = self.curindex + counter = 0 + skipped = 0 + hidden_frames = self.hidden_frames(self.stack) + for i in range(self.curindex + 1, len(self.stack)): + frame = self.stack[i][0] + if hidden_frames[i] and self.skip_hidden: + skipped += 1 + continue + counter += 1 + if counter >= count: + break + else: + self.error("all frames bellow hidden") + return + + Colors = self.color_scheme_table.active_colors + ColorsNormal = Colors.Normal + if skipped: + print( + f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n" + ) + _newframe = i + + self._select_frame(_newframe) + + do_d = do_down + do_u = do_up + +class InterruptiblePdb(Pdb): + """Version of debugger where KeyboardInterrupt exits the debugger altogether.""" + + def cmdloop(self): + """Wrap cmdloop() such that KeyboardInterrupt stops the debugger.""" + try: + return OldPdb.cmdloop(self) + except KeyboardInterrupt: + self.stop_here = lambda frame: False + self.do_quit("") + sys.settrace(None) + self.quitting = False + raise + + def _cmdloop(self): + while True: + try: + # keyboard interrupts allow for an easy way to cancel + # the current command, so allow them during interactive input + self.allow_kbdint = True + self.cmdloop() + self.allow_kbdint = False + break + except KeyboardInterrupt: + self.message('--KeyboardInterrupt--') + raise + + +def set_trace(frame=None): + """ + Start debugging from `frame`. + + If frame is not specified, debugging starts from caller's frame. + """ + Pdb().set_trace(frame or sys._getframe().f_back) diff --git a/IPython/core/display.py b/IPython/core/display.py index 4a943e4dbb4..424414a662f 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -4,30 +4,28 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. -from __future__ import print_function - -try: - from base64 import encodebytes as base64_encode -except ImportError: - from base64 import encodestring as base64_encode +from binascii import b2a_hex, b2a_base64, hexlify import json import mimetypes import os import struct import sys import warnings +from copy import deepcopy +from os.path import splitext +from pathlib import Path, PurePath -from IPython.utils.py3compat import (string_types, cast_bytes_py2, cast_unicode, - unicode_type) +from IPython.utils.py3compat import cast_unicode from IPython.testing.skipdoctest import skip_doctest __all__ = ['display', 'display_pretty', 'display_html', 'display_markdown', 'display_svg', 'display_png', 'display_jpeg', 'display_latex', 'display_json', 'display_javascript', 'display_pdf', 'DisplayObject', 'TextDisplayObject', -'Pretty', 'HTML', 'Markdown', 'Math', 'Latex', 'SVG', 'JSON', 'Javascript', -'Image', 'clear_output', 'set_matplotlib_formats', 'set_matplotlib_close', -'publish_display_data'] +'Pretty', 'HTML', 'Markdown', 'Math', 'Latex', 'SVG', 'ProgressBar', 'JSON', +'GeoJSON', 'Javascript', 'Image', 'clear_output', 'set_matplotlib_formats', +'set_matplotlib_close', 'publish_display_data', 'update_display', 'DisplayHandle', +'Video'] #----------------------------------------------------------------------------- # utility functions @@ -79,23 +77,14 @@ def _display_mimetype(mimetype, objs, raw=False, metadata=None): # Main functions #----------------------------------------------------------------------------- -def publish_display_data(data, metadata=None, source=None): +# use * to indicate transient is keyword-only +def publish_display_data(data, metadata=None, source=None, *, transient=None, **kwargs): """Publish data and metadata to all frontends. See the ``display_data`` message in the messaging documentation for more details about this message type. - The following MIME types are currently implemented: - - * text/plain - * text/html - * text/markdown - * text/latex - * application/json - * application/javascript - * image/png - * image/jpeg - * image/svg+xml + Keys of data and metadata can be any mime-type. Parameters ---------- @@ -114,19 +103,40 @@ def publish_display_data(data, metadata=None, source=None): to specify metadata about particular representations. source : str, deprecated Unused. + transient : dict, keyword-only + A dictionary of transient data, such as display_id. """ from IPython.core.interactiveshell import InteractiveShell - InteractiveShell.instance().display_pub.publish( + + display_pub = InteractiveShell.instance().display_pub + + # only pass transient if supplied, + # to avoid errors with older ipykernel. + # TODO: We could check for ipykernel version and provide a detailed upgrade message. + if transient: + kwargs['transient'] = transient + + display_pub.publish( data=data, metadata=metadata, + **kwargs ) -def display(*objs, **kwargs): + +def _new_id(): + """Generate a new random text id with urandom""" + return b2a_hex(os.urandom(16)).decode('ascii') + + +def display(*objs, include=None, exclude=None, metadata=None, transient=None, display_id=None, **kwargs): """Display a Python object in all frontends. By default all representations will be computed and sent to the frontends. Frontends can decide which representation is used and how. + In terminal IPython this will be similar to using :func:`print`, for use in richer + frontends see Jupyter notebook examples with rich display logic. + Parameters ---------- objs : tuple of objects @@ -134,11 +144,11 @@ def display(*objs, **kwargs): raw : bool, optional Are the objects to be displayed already mimetype-keyed dicts of raw display data, or Python objects that need to be formatted before display? [default: False] - include : list or tuple, optional + include : list, tuple or set, optional A list of format type strings (MIME types) to include in the format data dict. If this is set *only* the format types included in this list will be computed. - exclude : list or tuple, optional + exclude : list, tuple or set, optional A list of format type strings (MIME types) to exclude in the format data dict. If this is set all format types will be computed, except for those included in this argument. @@ -146,20 +156,159 @@ def display(*objs, **kwargs): A dictionary of metadata to associate with the output. mime-type keys in this dictionary will be associated with the individual representation formats, if they exist. - """ - raw = kwargs.get('raw', False) - include = kwargs.get('include') - exclude = kwargs.get('exclude') - metadata = kwargs.get('metadata') + transient : dict, optional + A dictionary of transient data to associate with the output. + Data in this dict should not be persisted to files (e.g. notebooks). + display_id : str, bool optional + Set an id for the display. + This id can be used for updating this display area later via update_display. + If given as `True`, generate a new `display_id` + kwargs: additional keyword-args, optional + Additional keyword-arguments are passed through to the display publisher. + + Returns + ------- + + handle: DisplayHandle + Returns a handle on updatable displays for use with :func:`update_display`, + if `display_id` is given. Returns :any:`None` if no `display_id` is given + (default). + + Examples + -------- + + >>> class Json(object): + ... def __init__(self, json): + ... self.json = json + ... def _repr_pretty_(self, pp, cycle): + ... import json + ... pp.text(json.dumps(self.json, indent=2)) + ... def __repr__(self): + ... return str(self.json) + ... + + >>> d = Json({1:2, 3: {4:5}}) + + >>> print(d) + {1: 2, 3: {4: 5}} + + >>> display(d) + { + "1": 2, + "3": { + "4": 5 + } + } + + >>> def int_formatter(integer, pp, cycle): + ... pp.text('I'*integer) + + >>> plain = get_ipython().display_formatter.formatters['text/plain'] + >>> plain.for_type(int, int_formatter) + + >>> display(7-5) + II + + >>> del plain.type_printers[int] + >>> display(7-5) + 2 + + See Also + -------- + + :func:`update_display` + + Notes + ----- + + In Python, objects can declare their textual representation using the + `__repr__` method. IPython expands on this idea and allows objects to declare + other, rich representations including: + + - HTML + - JSON + - PNG + - JPEG + - SVG + - LaTeX + + A single object can declare some or all of these representations; all are + handled by IPython's display system. + + The main idea of the first approach is that you have to implement special + display methods when you define your class, one for each representation you + want to use. Here is a list of the names of the special methods and the + values they must return: + + - `_repr_html_`: return raw HTML as a string, or a tuple (see below). + - `_repr_json_`: return a JSONable dict, or a tuple (see below). + - `_repr_jpeg_`: return raw JPEG data, or a tuple (see below). + - `_repr_png_`: return raw PNG data, or a tuple (see below). + - `_repr_svg_`: return raw SVG data as a string, or a tuple (see below). + - `_repr_latex_`: return LaTeX commands in a string surrounded by "$", + or a tuple (see below). + - `_repr_mimebundle_`: return a full mimebundle containing the mapping + from all mimetypes to data. + Use this for any mime-type not listed above. + + The above functions may also return the object's metadata alonside the + data. If the metadata is available, the functions will return a tuple + containing the data and metadata, in that order. If there is no metadata + available, then the functions will return the data only. + + When you are directly writing your own classes, you can adapt them for + display in IPython by following the above approach. But in practice, you + often need to work with existing classes that you can't easily modify. + + You can refer to the documentation on integrating with the display system in + order to register custom formatters for already existing types + (:ref:`integrating_rich_display`). + + .. versionadded:: 5.4 display available without import + .. versionadded:: 6.1 display available without import + + Since IPython 5.4 and 6.1 :func:`display` is automatically made available to + the user without import. If you are using display in a document that might + be used in a pure python context or with older version of IPython, use the + following import at the top of your file:: + + from IPython.display import display + """ from IPython.core.interactiveshell import InteractiveShell + + if not InteractiveShell.initialized(): + # Directly print objects. + print(*objs) + return + + raw = kwargs.pop('raw', False) + if transient is None: + transient = {} + if metadata is None: + metadata={} + if display_id: + if display_id is True: + display_id = _new_id() + transient['display_id'] = display_id + if kwargs.get('update') and 'display_id' not in transient: + raise TypeError('display_id required for update_display') + if transient: + kwargs['transient'] = transient + + if not objs and display_id: + # if given no objects, but still a request for a display_id, + # we assume the user wants to insert an empty output that + # can be updated later + objs = [{}] + raw = True if not raw: format = InteractiveShell.instance().display_formatter.format for obj in objs: if raw: - publish_display_data(data=obj, metadata=metadata) + publish_display_data(data=obj, metadata=metadata, **kwargs) else: format_dict, md_dict = format(obj, include=include, exclude=exclude) if not format_dict: @@ -168,7 +317,80 @@ def display(*objs, **kwargs): if metadata: # kwarg-specified metadata gets precedence _merge(md_dict, metadata) - publish_display_data(data=format_dict, metadata=md_dict) + publish_display_data(data=format_dict, metadata=md_dict, **kwargs) + if display_id: + return DisplayHandle(display_id) + + +# use * for keyword-only display_id arg +def update_display(obj, *, display_id, **kwargs): + """Update an existing display by id + + Parameters + ---------- + + obj: + The object with which to update the display + display_id: keyword-only + The id of the display to update + + See Also + -------- + + :func:`display` + """ + kwargs['update'] = True + display(obj, display_id=display_id, **kwargs) + + +class DisplayHandle(object): + """A handle on an updatable display + + Call `.update(obj)` to display a new object. + + Call `.display(obj`) to add a new instance of this display, + and update existing instances. + + See Also + -------- + + :func:`display`, :func:`update_display` + + """ + + def __init__(self, display_id=None): + if display_id is None: + display_id = _new_id() + self.display_id = display_id + + def __repr__(self): + return "<%s display_id=%s>" % (self.__class__.__name__, self.display_id) + + def display(self, obj, **kwargs): + """Make a new display with my id, updating existing instances. + + Parameters + ---------- + + obj: + object to display + **kwargs: + additional keyword arguments passed to display + """ + display(obj, display_id=self.display_id, **kwargs) + + def update(self, obj, **kwargs): + """Update existing displays with my id + + Parameters + ---------- + + obj: + object to display + **kwargs: + additional keyword arguments passed to update_display + """ + update_display(obj, display_id=self.display_id, **kwargs) def display_pretty(*objs, **kwargs): @@ -190,7 +412,7 @@ def display_pretty(*objs, **kwargs): def display_html(*objs, **kwargs): """Display the HTML representation of an object. - + Note: If raw=False and the object does not have a HTML representation, no HTML will be shown. @@ -357,8 +579,9 @@ class DisplayObject(object): _read_flags = 'r' _show_mem_addr = False + metadata = None - def __init__(self, data=None, url=None, filename=None): + def __init__(self, data=None, url=None, filename=None, metadata=None): """Create a display object given raw data. When this object is returned by an expression or passed to the @@ -376,8 +599,13 @@ def __init__(self, data=None, url=None, filename=None): A URL to download the data from. filename : unicode Path to a local file to load the data from. + metadata : dict + Dict of metadata associated to be the object when displayed """ - if data is not None and isinstance(data, string_types): + if isinstance(data, (Path, PurePath)): + data = str(data) + + if data is not None and isinstance(data, str): if data.startswith('http') and url is None: url = data filename = None @@ -387,9 +615,17 @@ def __init__(self, data=None, url=None, filename=None): filename = data data = None - self.data = data self.url = url - self.filename = None if filename is None else unicode_type(filename) + self.filename = filename + # because of @data.setter methods in + # subclasses ensure url and filename are set + # before assigning to self.data + self.data = data + + if metadata is not None: + self.metadata = metadata + elif self.metadata is None: + self.metadata = {} self.reload() self._check_data() @@ -406,48 +642,83 @@ def _check_data(self): """Override in subclasses if there's something to check.""" pass + def _data_and_metadata(self): + """shortcut for returning metadata with shape information, if defined""" + if self.metadata: + return self.data, deepcopy(self.metadata) + else: + return self.data + def reload(self): """Reload the raw data from file or URL.""" if self.filename is not None: with open(self.filename, self._read_flags) as f: self.data = f.read() elif self.url is not None: - try: - try: - from urllib.request import urlopen # Py3 - except ImportError: - from urllib2 import urlopen - response = urlopen(self.url) - self.data = response.read() - # extract encoding from header, if there is one: - encoding = None + # Deferred import + from urllib.request import urlopen + response = urlopen(self.url) + data = response.read() + # extract encoding from header, if there is one: + encoding = None + if 'content-type' in response.headers: for sub in response.headers['content-type'].split(';'): sub = sub.strip() if sub.startswith('charset'): encoding = sub.split('=')[-1].strip() break - # decode data, if an encoding was specified - if encoding: - self.data = self.data.decode(encoding, 'replace') - except: - self.data = None + if 'content-encoding' in response.headers: + # TODO: do deflate? + if 'gzip' in response.headers['content-encoding']: + import gzip + from io import BytesIO + with gzip.open(BytesIO(data), 'rt', encoding=encoding) as fp: + encoding = None + data = fp.read() + + # decode data, if an encoding was specified + # We only touch self.data once since + # subclasses such as SVG have @data.setter methods + # that transform self.data into ... well svg. + if encoding: + self.data = data.decode(encoding, 'replace') + else: + self.data = data + class TextDisplayObject(DisplayObject): """Validate that display data is text""" def _check_data(self): - if self.data is not None and not isinstance(self.data, string_types): + if self.data is not None and not isinstance(self.data, str): raise TypeError("%s expects text, not %r" % (self.__class__.__name__, self.data)) class Pretty(TextDisplayObject): - def _repr_pretty_(self): - return self.data + def _repr_pretty_(self, pp, cycle): + return pp.text(self.data) class HTML(TextDisplayObject): + def __init__(self, data=None, url=None, filename=None, metadata=None): + def warn(): + if not data: + return False + + # + # Avoid calling lower() on the entire data, because it could be a + # long string and we're only interested in its beginning and end. + # + prefix = data[:10].lower() + suffix = data[-10:].lower() + return prefix.startswith("') + m_warn.assert_not_called() + + display.HTML('') + m_warn.assert_called_with('Consider using IPython.display.IFrame instead') + + m_warn.reset_mock() + display.HTML('') + m_warn.assert_called_with('Consider using IPython.display.IFrame instead') + +def test_progress(): + p = display.ProgressBar(10) + nt.assert_in('0/10',repr(p)) + p.html_width = '100%' + p.progress = 5 + nt.assert_equal(p._repr_html_(), "") + +def test_progress_iter(): + with capture_output(display=False) as captured: + for i in display.ProgressBar(5): + out = captured.stdout + nt.assert_in('{0}/5'.format(i), out) + out = captured.stdout + nt.assert_in('5/5', out) + def test_json(): d = {'a': 5} lis = [d] - j = display.JSON(d) - nt.assert_equal(j._repr_json_(), d) - + metadata = [ + {'expanded': False, 'root': 'root'}, + {'expanded': True, 'root': 'root'}, + {'expanded': False, 'root': 'custom'}, + {'expanded': True, 'root': 'custom'}, + ] + json_objs = [ + display.JSON(d), + display.JSON(d, expanded=True), + display.JSON(d, root='custom'), + display.JSON(d, expanded=True, root='custom'), + ] + for j, md in zip(json_objs, metadata): + nt.assert_equal(j._repr_json_(), (d, md)) + with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") j = display.JSON(json.dumps(d)) nt.assert_equal(len(w), 1) - nt.assert_equal(j._repr_json_(), d) - - j = display.JSON(lis) - nt.assert_equal(j._repr_json_(), lis) - + nt.assert_equal(j._repr_json_(), (d, metadata[0])) + + json_objs = [ + display.JSON(lis), + display.JSON(lis, expanded=True), + display.JSON(lis, root='custom'), + display.JSON(lis, expanded=True, root='custom'), + ] + for j, md in zip(json_objs, metadata): + nt.assert_equal(j._repr_json_(), (lis, md)) + with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") j = display.JSON(json.dumps(lis)) nt.assert_equal(len(w), 1) - nt.assert_equal(j._repr_json_(), lis) - + nt.assert_equal(j._repr_json_(), (lis, metadata[0])) + def test_video_embedding(): """use a tempfile, with dummy-data, to ensure that video embedding doesn't crash""" v = display.Video("http://ignored") @@ -172,15 +320,15 @@ def test_video_embedding(): assert not v.embed html = v._repr_html_() nt.assert_not_in('src="data:', html) - + v = display.Video(f.name, embed=True) html = v._repr_html_() nt.assert_in('src="data:video/mp4;base64,YWJj"',html) - + v = display.Video(f.name, embed=True, mimetype='video/other') html = v._repr_html_() nt.assert_in('src="data:video/other;base64,YWJj"',html) - + v = display.Video(b'abc', embed=True, mimetype='video/mp4') html = v._repr_html_() nt.assert_in('src="data:video/mp4;base64,YWJj"',html) @@ -189,3 +337,119 @@ def test_video_embedding(): html = v._repr_html_() nt.assert_in('src="data:video/xyz;base64,YWJj"',html) +def test_html_metadata(): + s = "

Test

" + h = display.HTML(s, metadata={"isolated": True}) + nt.assert_equal(h._repr_html_(), (s, {"isolated": True})) + +def test_display_id(): + ip = get_ipython() + with mock.patch.object(ip.display_pub, 'publish') as pub: + handle = display.display('x') + nt.assert_is(handle, None) + handle = display.display('y', display_id='secret') + nt.assert_is_instance(handle, display.DisplayHandle) + handle2 = display.display('z', display_id=True) + nt.assert_is_instance(handle2, display.DisplayHandle) + nt.assert_not_equal(handle.display_id, handle2.display_id) + + nt.assert_equal(pub.call_count, 3) + args, kwargs = pub.call_args_list[0] + nt.assert_equal(args, ()) + nt.assert_equal(kwargs, { + 'data': { + 'text/plain': repr('x') + }, + 'metadata': {}, + }) + args, kwargs = pub.call_args_list[1] + nt.assert_equal(args, ()) + nt.assert_equal(kwargs, { + 'data': { + 'text/plain': repr('y') + }, + 'metadata': {}, + 'transient': { + 'display_id': handle.display_id, + }, + }) + args, kwargs = pub.call_args_list[2] + nt.assert_equal(args, ()) + nt.assert_equal(kwargs, { + 'data': { + 'text/plain': repr('z') + }, + 'metadata': {}, + 'transient': { + 'display_id': handle2.display_id, + }, + }) + + +def test_update_display(): + ip = get_ipython() + with mock.patch.object(ip.display_pub, 'publish') as pub: + with nt.assert_raises(TypeError): + display.update_display('x') + display.update_display('x', display_id='1') + display.update_display('y', display_id='2') + args, kwargs = pub.call_args_list[0] + nt.assert_equal(args, ()) + nt.assert_equal(kwargs, { + 'data': { + 'text/plain': repr('x') + }, + 'metadata': {}, + 'transient': { + 'display_id': '1', + }, + 'update': True, + }) + args, kwargs = pub.call_args_list[1] + nt.assert_equal(args, ()) + nt.assert_equal(kwargs, { + 'data': { + 'text/plain': repr('y') + }, + 'metadata': {}, + 'transient': { + 'display_id': '2', + }, + 'update': True, + }) + + +def test_display_handle(): + ip = get_ipython() + handle = display.DisplayHandle() + nt.assert_is_instance(handle.display_id, str) + handle = display.DisplayHandle('my-id') + nt.assert_equal(handle.display_id, 'my-id') + with mock.patch.object(ip.display_pub, 'publish') as pub: + handle.display('x') + handle.update('y') + + args, kwargs = pub.call_args_list[0] + nt.assert_equal(args, ()) + nt.assert_equal(kwargs, { + 'data': { + 'text/plain': repr('x') + }, + 'metadata': {}, + 'transient': { + 'display_id': handle.display_id, + } + }) + args, kwargs = pub.call_args_list[1] + nt.assert_equal(args, ()) + nt.assert_equal(kwargs, { + 'data': { + 'text/plain': repr('y') + }, + 'metadata': {}, + 'transient': { + 'display_id': handle.display_id, + }, + 'update': True, + }) + diff --git a/IPython/core/tests/test_displayhook.py b/IPython/core/tests/test_displayhook.py index 3cca42a8089..6ad89793442 100644 --- a/IPython/core/tests/test_displayhook.py +++ b/IPython/core/tests/test_displayhook.py @@ -1,6 +1,7 @@ +import sys from IPython.testing.tools import AssertPrints, AssertNotPrints - -ip = get_ipython() +from IPython.core.displayhook import CapturingDisplayHook +from IPython.utils.capture import CapturedIO def test_output_displayed(): """Checking to make sure that output is displayed""" @@ -26,3 +27,86 @@ def test_output_quiet(): with AssertNotPrints('2'): ip.run_cell('1+1;\n#commented_out_function()', store_history=True) + +def test_underscore_no_overrite_user(): + ip.run_cell('_ = 42', store_history=True) + ip.run_cell('1+1', store_history=True) + + with AssertPrints('42'): + ip.run_cell('print(_)', store_history=True) + + ip.run_cell('del _', store_history=True) + ip.run_cell('6+6', store_history=True) + with AssertPrints('12'): + ip.run_cell('_', store_history=True) + + +def test_underscore_no_overrite_builtins(): + ip.run_cell("import gettext ; gettext.install('foo')", store_history=True) + ip.run_cell('3+3', store_history=True) + + with AssertPrints('gettext'): + ip.run_cell('print(_)', store_history=True) + + ip.run_cell('_ = "userset"', store_history=True) + + with AssertPrints('userset'): + ip.run_cell('print(_)', store_history=True) + ip.run_cell('import builtins; del builtins._') + + +def test_interactivehooks_ast_modes(): + """ + Test that ast nodes can be triggered with different modes + """ + saved_mode = ip.ast_node_interactivity + ip.ast_node_interactivity = 'last_expr_or_assign' + + try: + with AssertPrints('2'): + ip.run_cell('a = 1+1', store_history=True) + + with AssertPrints('9'): + ip.run_cell('b = 1+8 # comment with a semicolon;', store_history=False) + + with AssertPrints('7'): + ip.run_cell('c = 1+6\n#commented_out_function();', store_history=True) + + ip.run_cell('d = 11', store_history=True) + with AssertPrints('12'): + ip.run_cell('d += 1', store_history=True) + + with AssertNotPrints('42'): + ip.run_cell('(u,v) = (41+1, 43-1)') + + finally: + ip.ast_node_interactivity = saved_mode + +def test_interactivehooks_ast_modes_semi_suppress(): + """ + Test that ast nodes can be triggered with different modes and suppressed + by semicolon + """ + saved_mode = ip.ast_node_interactivity + ip.ast_node_interactivity = 'last_expr_or_assign' + + try: + with AssertNotPrints('2'): + ip.run_cell('x = 1+1;', store_history=True) + + with AssertNotPrints('7'): + ip.run_cell('y = 1+6; # comment with a semicolon', store_history=True) + + with AssertNotPrints('9'): + ip.run_cell('z = 1+8;\n#commented_out_function()', store_history=True) + + finally: + ip.ast_node_interactivity = saved_mode + +def test_capture_display_hook_format(): + """Tests that the capture display hook conforms to the CapturedIO output format""" + hook = CapturingDisplayHook(ip) + hook({"foo": "bar"}) + captured = CapturedIO(sys.stdout, sys.stderr, hook.outputs) + # Should not raise with RichOutput transformation error + captured.outputs diff --git a/IPython/core/tests/test_events.py b/IPython/core/tests/test_events.py index 3053a705a7f..a4211ecea4c 100644 --- a/IPython/core/tests/test_events.py +++ b/IPython/core/tests/test_events.py @@ -1,19 +1,27 @@ import unittest -try: # Python 3.3 + - from unittest.mock import Mock -except ImportError: - from mock import Mock +from unittest.mock import Mock +import nose.tools as nt from IPython.core import events import IPython.testing.tools as tt + +@events._define_event def ping_received(): pass + +@events._define_event +def event_with_argument(argument): + pass + + class CallbackTests(unittest.TestCase): def setUp(self): - self.em = events.EventManager(get_ipython(), {'ping_received': ping_received}) - + self.em = events.EventManager(get_ipython(), + {'ping_received': ping_received, + 'event_with_argument': event_with_argument}) + def test_register_unregister(self): cb = Mock() @@ -24,13 +32,30 @@ def test_register_unregister(self): self.em.unregister('ping_received', cb) self.em.trigger('ping_received') self.assertEqual(cb.call_count, 1) - + + def test_bare_function_missed_unregister(self): + def cb1(): + ... + + def cb2(): + ... + + self.em.register('ping_received', cb1) + nt.assert_raises(ValueError, self.em.unregister, 'ping_received', cb2) + self.em.unregister('ping_received', cb1) + def test_cb_error(self): cb = Mock(side_effect=ValueError) self.em.register('ping_received', cb) with tt.AssertPrints("Error in callback"): self.em.trigger('ping_received') + def test_cb_keyboard_interrupt(self): + cb = Mock(side_effect=KeyboardInterrupt) + self.em.register('ping_received', cb) + with tt.AssertPrints("Error in callback"): + self.em.trigger('ping_received') + def test_unregister_during_callback(self): invoked = [False] * 3 @@ -52,3 +77,16 @@ def func3(*_): self.em.trigger('ping_received') self.assertEqual([True, True, False], invoked) self.assertEqual([func3], self.em.callbacks['ping_received']) + + def test_ignore_event_arguments_if_no_argument_required(self): + call_count = [0] + def event_with_no_argument(): + call_count[0] += 1 + + self.em.register('event_with_argument', event_with_no_argument) + self.em.trigger('event_with_argument', 'the argument') + self.assertEqual(call_count[0], 1) + + self.em.unregister('event_with_argument', event_with_no_argument) + self.em.trigger('ping_received') + self.assertEqual(call_count[0], 1) diff --git a/IPython/core/tests/test_formatters.py b/IPython/core/tests/test_formatters.py index ab63ba43e15..cde43c94a92 100644 --- a/IPython/core/tests/test_formatters.py +++ b/IPython/core/tests/test_formatters.py @@ -49,7 +49,7 @@ def test_pretty(): f = PlainTextFormatter() f.for_type(A, foo_printer) nt.assert_equal(f(A()), 'foo') - nt.assert_equal(f(B()), 'foo') + nt.assert_equal(f(B()), 'B()') nt.assert_equal(f(GoodPretty()), 'foo') # Just don't raise an exception for the following: f(BadPretty()) @@ -116,8 +116,6 @@ def test_for_type(): def test_for_type_string(): f = PlainTextFormatter() - mod = C.__module__ - type_str = '%s.%s' % (C.__module__, 'C') # initial return, None @@ -166,7 +164,6 @@ def test_lookup_by_type(): f = PlainTextFormatter() f.for_type(C, foo_printer) nt.assert_is(f.lookup_by_type(C), foo_printer) - type_str = '%s.%s' % (C.__module__, 'C') with nt.assert_raises(KeyError): f.lookup_by_type(A) @@ -407,6 +404,9 @@ def __repr__(self): def _ipython_display_(self): raise NotImplementedError + save_enabled = f.ipython_display_formatter.enabled + f.ipython_display_formatter.enabled = True + yes = SelfDisplaying() no = NotSelfDisplaying() @@ -420,6 +420,9 @@ def _ipython_display_(self): nt.assert_equal(md, {}) nt.assert_equal(catcher, [yes]) + f.ipython_display_formatter.enabled = save_enabled + + def test_json_as_string_deprecated(): class JSONString(object): def _repr_json_(self): @@ -430,4 +433,101 @@ def _repr_json_(self): d = f(JSONString()) nt.assert_equal(d, {}) nt.assert_equal(len(w), 1) - \ No newline at end of file + + +def test_repr_mime(): + class HasReprMime(object): + def _repr_mimebundle_(self, include=None, exclude=None): + return { + 'application/json+test.v2': { + 'x': 'y' + }, + 'plain/text' : '', + 'image/png' : 'i-overwrite' + } + + def _repr_png_(self): + return 'should-be-overwritten' + def _repr_html_(self): + return 'hi!' + + f = get_ipython().display_formatter + html_f = f.formatters['text/html'] + save_enabled = html_f.enabled + html_f.enabled = True + obj = HasReprMime() + d, md = f.format(obj) + html_f.enabled = save_enabled + + nt.assert_equal(sorted(d), ['application/json+test.v2', + 'image/png', + 'plain/text', + 'text/html', + 'text/plain']) + nt.assert_equal(md, {}) + + d, md = f.format(obj, include={'image/png'}) + nt.assert_equal(list(d.keys()), ['image/png'], + 'Include should filter out even things from repr_mimebundle') + nt.assert_equal(d['image/png'], 'i-overwrite', '_repr_mimebundle_ take precedence') + + + +def test_pass_correct_include_exclude(): + class Tester(object): + + def __init__(self, include=None, exclude=None): + self.include = include + self.exclude = exclude + + def _repr_mimebundle_(self, include, exclude, **kwargs): + if include and (include != self.include): + raise ValueError('include got modified: display() may be broken.') + if exclude and (exclude != self.exclude): + raise ValueError('exclude got modified: display() may be broken.') + + return None + + include = {'a', 'b', 'c'} + exclude = {'c', 'e' , 'f'} + + f = get_ipython().display_formatter + f.format(Tester(include=include, exclude=exclude), include=include, exclude=exclude) + f.format(Tester(exclude=exclude), exclude=exclude) + f.format(Tester(include=include), include=include) + + +def test_repr_mime_meta(): + class HasReprMimeMeta(object): + def _repr_mimebundle_(self, include=None, exclude=None): + data = { + 'image/png': 'base64-image-data', + } + metadata = { + 'image/png': { + 'width': 5, + 'height': 10, + } + } + return (data, metadata) + + f = get_ipython().display_formatter + obj = HasReprMimeMeta() + d, md = f.format(obj) + nt.assert_equal(sorted(d), ['image/png', 'text/plain']) + nt.assert_equal(md, { + 'image/png': { + 'width': 5, + 'height': 10, + } + }) + +def test_repr_mime_failure(): + class BadReprMime(object): + def _repr_mimebundle_(self, include=None, exclude=None): + raise RuntimeError + + f = get_ipython().display_formatter + obj = BadReprMime() + d, md = f.format(obj) + nt.assert_in('text/plain', d) diff --git a/IPython/core/tests/test_handlers.py b/IPython/core/tests/test_handlers.py index 20f956a6997..19248177295 100644 --- a/IPython/core/tests/test_handlers.py +++ b/IPython/core/tests/test_handlers.py @@ -10,15 +10,12 @@ # our own packages from IPython.core import autocall from IPython.testing import tools as tt -from IPython.testing.globalipapp import get_ipython -from IPython.utils import py3compat #----------------------------------------------------------------------------- # Globals #----------------------------------------------------------------------------- # Get the public instance of IPython -ip = get_ipython() failures = [] num_tests = 0 @@ -51,11 +48,11 @@ def test_handlers(): # For many of the below, we're also checking that leading whitespace # turns off the esc char, which it should unless there is a continuation # line. - run([(i,py3compat.u_format(o)) for i,o in \ + run( [('"no change"', '"no change"'), # normal - (u"lsmagic", "get_ipython().magic({u}'lsmagic ')"), # magic + (u"lsmagic", "get_ipython().run_line_magic('lsmagic', '')"), # magic #("a = b # PYTHON-MODE", '_i'), # emacs -- avoids _in cache - ]]) + ]) # Objects which are instances of IPyAutocall are *always* autocalled autocallable = Autocallable() diff --git a/IPython/core/tests/test_history.py b/IPython/core/tests/test_history.py index e4496d27a0f..f4f080dc83f 100644 --- a/IPython/core/tests/test_history.py +++ b/IPython/core/tests/test_history.py @@ -11,6 +11,7 @@ import sys import tempfile from datetime import datetime +import sqlite3 # third party import nose.tools as nt @@ -19,11 +20,12 @@ from traitlets.config.loader import Config from IPython.utils.tempdir import TemporaryDirectory from IPython.core.history import HistoryManager, extract_hist_ranges -from IPython.utils import py3compat +from IPython.testing.decorators import skipif -def setUp(): - nt.assert_equal(sys.getdefaultencoding(), "utf-8" if py3compat.PY3 else "ascii") +def test_proper_default_encoding(): + nt.assert_equal(sys.getdefaultencoding(), "utf-8") +@skipif(sqlite3.sqlite_version_info > (3,24,0)) def test_history(): ip = get_ipython() with TemporaryDirectory() as tmpdir: @@ -41,7 +43,7 @@ def test_history(): ip.history_manager.store_output(3) nt.assert_equal(ip.history_manager.input_hist_raw, [''] + hist) - + # Detailed tests for _get_range_session grs = ip.history_manager._get_range_session nt.assert_equal(list(grs(start=2,stop=-1)), list(zip([0], [2], hist[1:-1]))) @@ -51,7 +53,7 @@ def test_history(): # Check whether specifying a range beyond the end of the current # session results in an error (gh-804) ip.magic('%hist 2-500') - + # Check that we can write non-ascii characters to a file ip.magic("%%hist -f %s" % os.path.join(tmpdir, "test1")) ip.magic("%%hist -pf %s" % os.path.join(tmpdir, "test2")) @@ -86,6 +88,7 @@ def test_history(): nt.assert_equal(list(gothist), expected) # Check get_hist_search + gothist = ip.history_manager.search("*test*") nt.assert_equal(list(gothist), [(1,2,hist[1])] ) diff --git a/IPython/core/tests/test_hooks.py b/IPython/core/tests/test_hooks.py index f44674cd9af..35d3f315b74 100644 --- a/IPython/core/tests/test_hooks.py +++ b/IPython/core/tests/test_hooks.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """Tests for CommandChainDispatcher.""" -from __future__ import absolute_import #----------------------------------------------------------------------------- # Imports diff --git a/IPython/core/tests/test_imports.py b/IPython/core/tests/test_imports.py index 88caef0ad36..7aa278fb63a 100644 --- a/IPython/core/tests/test_imports.py +++ b/IPython/core/tests/test_imports.py @@ -45,9 +45,6 @@ def test_import_prompts(): def test_import_release(): from IPython.core import release -def test_import_shadowns(): - from IPython.core import shadowns - def test_import_ultratb(): from IPython.core import ultratb diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py index 511003298a5..a39943aed80 100644 --- a/IPython/core/tests/test_inputsplitter.py +++ b/IPython/core/tests/test_inputsplitter.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """Tests for the inputsplitter module.""" -from __future__ import print_function # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. @@ -15,8 +14,6 @@ from IPython.core.inputtransformer import InputTransformer from IPython.core.tests.test_inputtransformer import syntax, syntax_ml from IPython.testing import tools as tt -from IPython.utils import py3compat -from IPython.utils.py3compat import string_types, input #----------------------------------------------------------------------------- # Semi-complete examples (also used as tests) @@ -38,7 +35,7 @@ def mini_interactive_loop(input_func): # input indefinitely, until some exit/quit command was issued. Here we # only illustrate the basic inner loop. while isp.push_accepts_more(): - indent = ' '*isp.indent_spaces + indent = ' '*isp.get_indent_spaces() prompt = '>>> ' + indent line = indent + input_func(prompt) isp.push(line) @@ -101,7 +98,7 @@ def test_remove_comments(): def test_get_input_encoding(): encoding = isp.get_input_encoding() - nt.assert_true(isinstance(encoding, string_types)) + nt.assert_true(isinstance(encoding, str)) # simple-minded check that at least encoding a simple string works with the # encoding we got. nt.assert_equal(u'test'.encode(encoding), b'test') @@ -133,7 +130,7 @@ def test_reset(self): isp.push('x=1') isp.reset() self.assertEqual(isp._buffer, []) - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) self.assertEqual(isp.source, '') self.assertEqual(isp.code, None) self.assertEqual(isp._is_complete, False) @@ -142,7 +139,7 @@ def test_source(self): self.isp._store('1') self.isp._store('2') self.assertEqual(self.isp.source, '1\n2\n') - self.assertTrue(len(self.isp._buffer)>0) + self.assertEqual(len(self.isp._buffer)>0, True) self.assertEqual(self.isp.source_reset(), '1\n2\n') self.assertEqual(self.isp._buffer, []) self.assertEqual(self.isp.source, '') @@ -150,21 +147,21 @@ def test_source(self): def test_indent(self): isp = self.isp # shorthand isp.push('x=1') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) isp.push('if 1:\n x=1') - self.assertEqual(isp.indent_spaces, 4) + self.assertEqual(isp.get_indent_spaces(), 4) isp.push('y=2\n') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) def test_indent2(self): isp = self.isp isp.push('if 1:') - self.assertEqual(isp.indent_spaces, 4) + self.assertEqual(isp.get_indent_spaces(), 4) isp.push(' x=1') - self.assertEqual(isp.indent_spaces, 4) + self.assertEqual(isp.get_indent_spaces(), 4) # Blank lines shouldn't change the indent level isp.push(' '*2) - self.assertEqual(isp.indent_spaces, 4) + self.assertEqual(isp.get_indent_spaces(), 4) def test_indent3(self): isp = self.isp @@ -172,111 +169,111 @@ def test_indent3(self): # shouldn't get confused. isp.push("if 1:") isp.push(" x = (1+\n 2)") - self.assertEqual(isp.indent_spaces, 4) + self.assertEqual(isp.get_indent_spaces(), 4) def test_indent4(self): isp = self.isp # whitespace after ':' should not screw up indent level isp.push('if 1: \n x=1') - self.assertEqual(isp.indent_spaces, 4) + self.assertEqual(isp.get_indent_spaces(), 4) isp.push('y=2\n') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) isp.push('if 1:\t\n x=1') - self.assertEqual(isp.indent_spaces, 4) + self.assertEqual(isp.get_indent_spaces(), 4) isp.push('y=2\n') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) def test_dedent_pass(self): isp = self.isp # shorthand # should NOT cause dedent isp.push('if 1:\n passes = 5') - self.assertEqual(isp.indent_spaces, 4) + self.assertEqual(isp.get_indent_spaces(), 4) isp.push('if 1:\n pass') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) isp.push('if 1:\n pass ') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) def test_dedent_break(self): isp = self.isp # shorthand # should NOT cause dedent isp.push('while 1:\n breaks = 5') - self.assertEqual(isp.indent_spaces, 4) + self.assertEqual(isp.get_indent_spaces(), 4) isp.push('while 1:\n break') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) isp.push('while 1:\n break ') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) def test_dedent_continue(self): isp = self.isp # shorthand # should NOT cause dedent isp.push('while 1:\n continues = 5') - self.assertEqual(isp.indent_spaces, 4) + self.assertEqual(isp.get_indent_spaces(), 4) isp.push('while 1:\n continue') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) isp.push('while 1:\n continue ') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) def test_dedent_raise(self): isp = self.isp # shorthand # should NOT cause dedent isp.push('if 1:\n raised = 4') - self.assertEqual(isp.indent_spaces, 4) + self.assertEqual(isp.get_indent_spaces(), 4) isp.push('if 1:\n raise TypeError()') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) isp.push('if 1:\n raise') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) isp.push('if 1:\n raise ') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) def test_dedent_return(self): isp = self.isp # shorthand # should NOT cause dedent isp.push('if 1:\n returning = 4') - self.assertEqual(isp.indent_spaces, 4) + self.assertEqual(isp.get_indent_spaces(), 4) isp.push('if 1:\n return 5 + 493') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) isp.push('if 1:\n return') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) isp.push('if 1:\n return ') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) isp.push('if 1:\n return(0)') - self.assertEqual(isp.indent_spaces, 0) + self.assertEqual(isp.get_indent_spaces(), 0) def test_push(self): isp = self.isp - self.assertTrue(isp.push('x=1')) + self.assertEqual(isp.push('x=1'), True) def test_push2(self): isp = self.isp - self.assertFalse(isp.push('if 1:')) + self.assertEqual(isp.push('if 1:'), False) for line in [' x=1', '# a comment', ' y=2']: print(line) - self.assertTrue(isp.push(line)) + self.assertEqual(isp.push(line), True) def test_push3(self): isp = self.isp isp.push('if True:') isp.push(' a = 1') - self.assertFalse(isp.push('b = [1,')) + self.assertEqual(isp.push('b = [1,'), False) def test_push_accepts_more(self): isp = self.isp isp.push('x=1') - self.assertFalse(isp.push_accepts_more()) + self.assertEqual(isp.push_accepts_more(), False) def test_push_accepts_more2(self): isp = self.isp isp.push('if 1:') - self.assertTrue(isp.push_accepts_more()) + self.assertEqual(isp.push_accepts_more(), True) isp.push(' x=1') - self.assertTrue(isp.push_accepts_more()) + self.assertEqual(isp.push_accepts_more(), True) isp.push('') - self.assertFalse(isp.push_accepts_more()) + self.assertEqual(isp.push_accepts_more(), False) def test_push_accepts_more3(self): isp = self.isp isp.push("x = (2+\n3)") - self.assertFalse(isp.push_accepts_more()) + self.assertEqual(isp.push_accepts_more(), False) def test_push_accepts_more4(self): isp = self.isp @@ -290,11 +287,11 @@ def test_push_accepts_more4(self): isp.push("if 1:") isp.push(" x = (2+") isp.push(" 3)") - self.assertTrue(isp.push_accepts_more()) + self.assertEqual(isp.push_accepts_more(), True) isp.push(" y = 3") - self.assertTrue(isp.push_accepts_more()) + self.assertEqual(isp.push_accepts_more(), True) isp.push('') - self.assertFalse(isp.push_accepts_more()) + self.assertEqual(isp.push_accepts_more(), False) def test_push_accepts_more5(self): isp = self.isp @@ -304,14 +301,14 @@ def test_push_accepts_more5(self): isp.push(' raise') # We want to be able to add an else: block at this point, so it should # wait for a blank line. - self.assertTrue(isp.push_accepts_more()) + self.assertEqual(isp.push_accepts_more(), True) def test_continuation(self): isp = self.isp isp.push("import os, \\") - self.assertTrue(isp.push_accepts_more()) + self.assertEqual(isp.push_accepts_more(), True) isp.push("sys") - self.assertFalse(isp.push_accepts_more()) + self.assertEqual(isp.push_accepts_more(), False) def test_syntax_error(self): isp = self.isp @@ -319,7 +316,7 @@ def test_syntax_error(self): # Python can be sent to the kernel for evaluation with possible ipython # special-syntax conversion. isp.push('run foo') - self.assertFalse(isp.push_accepts_more()) + self.assertEqual(isp.push_accepts_more(), False) def test_unicode(self): self.isp.push(u"Pérez") @@ -331,16 +328,16 @@ def test_line_continuation(self): isp = self.isp # A blank line after a line continuation should not accept more isp.push("1 \\\n\n") - self.assertFalse(isp.push_accepts_more()) + self.assertEqual(isp.push_accepts_more(), False) # Whitespace after a \ is a SyntaxError. The only way to test that # here is to test that push doesn't accept more (as with # test_syntax_error() above). isp.push(r"1 \ ") - self.assertFalse(isp.push_accepts_more()) + self.assertEqual(isp.push_accepts_more(), False) # Even if the line is continuable (c.f. the regular Python # interpreter) isp.push(r"(1 \ ") - self.assertFalse(isp.push_accepts_more()) + self.assertEqual(isp.push_accepts_more(), False) def test_check_complete(self): isp = self.isp @@ -480,11 +477,11 @@ def reset(self): for raw, expected in [ ("a=5", "a=5#"), - ("%ls foo", "get_ipython().magic(%r)" % u'ls foo#'), - ("!ls foo\n%ls bar", "get_ipython().system(%r)\nget_ipython().magic(%r)" % ( - u'ls foo#', u'ls bar#' + ("%ls foo", "get_ipython().run_line_magic(%r, %r)" % (u'ls', u'foo#')), + ("!ls foo\n%ls bar", "get_ipython().system(%r)\nget_ipython().run_line_magic(%r, %r)" % ( + u'ls foo#', u'ls', u'bar#' )), - ("1\n2\n3\n%ls foo\n4\n5", "1#\n2#\n3#\nget_ipython().magic(%r)\n4#\n5#" % u'ls foo#'), + ("1\n2\n3\n%ls foo\n4\n5", "1#\n2#\n3#\nget_ipython().run_line_magic(%r, %r)\n4#\n5#" % (u'ls', u'foo#')), ]: out = isp.transform_cell(raw) self.assertEqual(out.rstrip(), expected.rstrip()) @@ -509,7 +506,7 @@ def reset(self): while True: prompt = start_prompt while isp.push_accepts_more(): - indent = ' '*isp.indent_spaces + indent = ' '*isp.get_indent_spaces() if autoindent: line = indent + input(prompt+indent) else: @@ -569,8 +566,8 @@ class CellMagicsCommon(object): def test_whole_cell(self): src = "%%cellm line\nbody\n" out = self.sp.transform_cell(src) - ref = u"get_ipython().run_cell_magic({u}'cellm', {u}'line', {u}'body')\n" - nt.assert_equal(out, py3compat.u_format(ref)) + ref = "get_ipython().run_cell_magic('cellm', 'line', 'body')\n" + nt.assert_equal(out, ref) def test_cellmagic_help(self): self.sp.push('%%cellm?') @@ -613,3 +610,30 @@ def test_incremental(self): sp.push('\n') # In this case, a blank line should end the cell magic nt.assert_false(sp.push_accepts_more()) #2 + +indentation_samples = [ + ('a = 1', 0), + ('for a in b:', 4), + ('def f():', 4), + ('def f(): #comment', 4), + ('a = ":#not a comment"', 0), + ('def f():\n a = 1', 4), + ('def f():\n return 1', 0), + ('for a in b:\n' + ' if a < 0:' + ' continue', 3), + ('a = {', 4), + ('a = {\n' + ' 1,', 5), + ('b = """123', 0), + ('', 0), + ('def f():\n pass', 0), + ('class Bar:\n def f():\n pass', 4), + ('class Bar:\n def f():\n raise', 4), +] + +def test_find_next_indent(): + for code, exp in indentation_samples: + res = isp.find_next_indent(code) + msg = "{!r} != {!r} (expected)\n Code: {!r}".format(res, exp, code) + assert res == exp, msg diff --git a/IPython/core/tests/test_inputtransformer.py b/IPython/core/tests/test_inputtransformer.py index ebad8ab6d36..0d97fd4d6b1 100644 --- a/IPython/core/tests/test_inputtransformer.py +++ b/IPython/core/tests/test_inputtransformer.py @@ -39,16 +39,16 @@ def transform_checker(tests, transformer, **kwargs): syntax = \ dict(assign_system = [(i,py3compat.u_format(o)) for i,o in \ - [(u'a =! ls', "a = get_ipython().getoutput({u}'ls')"), - (u'b = !ls', "b = get_ipython().getoutput({u}'ls')"), - (u'c= !ls', "c = get_ipython().getoutput({u}'ls')"), + [(u'a =! ls', "a = get_ipython().getoutput('ls')"), + (u'b = !ls', "b = get_ipython().getoutput('ls')"), + (u'c= !ls', "c = get_ipython().getoutput('ls')"), (u'd == !ls', u'd == !ls'), # Invalid syntax, but we leave == alone. ('x=1', 'x=1'), # normal input is unmodified (' ',' '), # blank lines are kept intact # Tuple unpacking - (u"a, b = !echo 'a\\nb'", u"a, b = get_ipython().getoutput({u}\"echo 'a\\\\nb'\")"), - (u"a,= !echo 'a'", u"a, = get_ipython().getoutput({u}\"echo 'a'\")"), - (u"a, *bc = !echo 'a\\nb\\nc'", u"a, *bc = get_ipython().getoutput({u}\"echo 'a\\\\nb\\\\nc'\")"), + (u"a, b = !echo 'a\\nb'", u"a, b = get_ipython().getoutput(\"echo 'a\\\\nb'\")"), + (u"a,= !echo 'a'", u"a, = get_ipython().getoutput(\"echo 'a'\")"), + (u"a, *bc = !echo 'a\\nb\\nc'", u"a, *bc = get_ipython().getoutput(\"echo 'a\\\\nb\\\\nc'\")"), # Tuple unpacking with regular Python expressions, not our syntax. (u"a, b = range(2)", u"a, b = range(2)"), (u"a, = range(1)", u"a, = range(1)"), @@ -57,13 +57,13 @@ def transform_checker(tests, transformer, **kwargs): assign_magic = [(i,py3compat.u_format(o)) for i,o in \ - [(u'a =% who', "a = get_ipython().magic({u}'who')"), - (u'b = %who', "b = get_ipython().magic({u}'who')"), - (u'c= %ls', "c = get_ipython().magic({u}'ls')"), + [(u'a =% who', "a = get_ipython().run_line_magic('who', '')"), + (u'b = %who', "b = get_ipython().run_line_magic('who', '')"), + (u'c= %ls', "c = get_ipython().run_line_magic('ls', '')"), (u'd == %ls', u'd == %ls'), # Invalid syntax, but we leave == alone. ('x=1', 'x=1'), # normal input is unmodified (' ',' '), # blank lines are kept intact - (u"a, b = %foo", u"a, b = get_ipython().magic({u}'foo')"), + (u"a, b = %foo", u"a, b = get_ipython().run_line_magic('foo', '')"), ]], classic_prompt = @@ -78,13 +78,6 @@ def transform_checker(tests, transformer, **kwargs): (' ',' '), # blank lines are kept intact ], - strip_encoding_cookie = - [ - ('# -*- encoding: utf-8 -*-', ''), - ('# coding: latin-1', ''), - ], - - # Tests for the escape transformer to leave normal code alone escaped_noesc = [ (' ', ' '), @@ -94,53 +87,54 @@ def transform_checker(tests, transformer, **kwargs): # System calls escaped_shell = [(i,py3compat.u_format(o)) for i,o in \ - [ (u'!ls', "get_ipython().system({u}'ls')"), + [ (u'!ls', "get_ipython().system('ls')"), # Double-escape shell, this means to capture the output of the # subprocess and return it - (u'!!ls', "get_ipython().getoutput({u}'ls')"), + (u'!!ls', "get_ipython().getoutput('ls')"), ]], # Help/object info escaped_help = [(i,py3compat.u_format(o)) for i,o in \ [ (u'?', 'get_ipython().show_usage()'), - (u'?x1', "get_ipython().magic({u}'pinfo x1')"), - (u'??x2', "get_ipython().magic({u}'pinfo2 x2')"), - (u'?a.*s', "get_ipython().magic({u}'psearch a.*s')"), - (u'?%hist1', "get_ipython().magic({u}'pinfo %hist1')"), - (u'?%%hist2', "get_ipython().magic({u}'pinfo %%hist2')"), - (u'?abc = qwe', "get_ipython().magic({u}'pinfo abc')"), + (u'?x1', "get_ipython().run_line_magic('pinfo', 'x1')"), + (u'??x2', "get_ipython().run_line_magic('pinfo2', 'x2')"), + (u'?a.*s', "get_ipython().run_line_magic('psearch', 'a.*s')"), + (u'?%hist1', "get_ipython().run_line_magic('pinfo', '%hist1')"), + (u'?%%hist2', "get_ipython().run_line_magic('pinfo', '%%hist2')"), + (u'?abc = qwe', "get_ipython().run_line_magic('pinfo', 'abc')"), ]], end_help = [(i,py3compat.u_format(o)) for i,o in \ - [ (u'x3?', "get_ipython().magic({u}'pinfo x3')"), - (u'x4??', "get_ipython().magic({u}'pinfo2 x4')"), - (u'%hist1?', "get_ipython().magic({u}'pinfo %hist1')"), - (u'%hist2??', "get_ipython().magic({u}'pinfo2 %hist2')"), - (u'%%hist3?', "get_ipython().magic({u}'pinfo %%hist3')"), - (u'%%hist4??', "get_ipython().magic({u}'pinfo2 %%hist4')"), - (u'f*?', "get_ipython().magic({u}'psearch f*')"), - (u'ax.*aspe*?', "get_ipython().magic({u}'psearch ax.*aspe*')"), - (u'a = abc?', "get_ipython().set_next_input({u}'a = abc');" - "get_ipython().magic({u}'pinfo abc')"), - (u'a = abc.qe??', "get_ipython().set_next_input({u}'a = abc.qe');" - "get_ipython().magic({u}'pinfo2 abc.qe')"), - (u'a = *.items?', "get_ipython().set_next_input({u}'a = *.items');" - "get_ipython().magic({u}'psearch *.items')"), - (u'plot(a?', "get_ipython().set_next_input({u}'plot(a');" - "get_ipython().magic({u}'pinfo a')"), + [ (u'x3?', "get_ipython().run_line_magic('pinfo', 'x3')"), + (u'x4??', "get_ipython().run_line_magic('pinfo2', 'x4')"), + (u'%hist1?', "get_ipython().run_line_magic('pinfo', '%hist1')"), + (u'%hist2??', "get_ipython().run_line_magic('pinfo2', '%hist2')"), + (u'%%hist3?', "get_ipython().run_line_magic('pinfo', '%%hist3')"), + (u'%%hist4??', "get_ipython().run_line_magic('pinfo2', '%%hist4')"), + (u'π.foo?', "get_ipython().run_line_magic('pinfo', 'π.foo')"), + (u'f*?', "get_ipython().run_line_magic('psearch', 'f*')"), + (u'ax.*aspe*?', "get_ipython().run_line_magic('psearch', 'ax.*aspe*')"), + (u'a = abc?', "get_ipython().set_next_input('a = abc');" + "get_ipython().run_line_magic('pinfo', 'abc')"), + (u'a = abc.qe??', "get_ipython().set_next_input('a = abc.qe');" + "get_ipython().run_line_magic('pinfo2', 'abc.qe')"), + (u'a = *.items?', "get_ipython().set_next_input('a = *.items');" + "get_ipython().run_line_magic('psearch', '*.items')"), + (u'plot(a?', "get_ipython().set_next_input('plot(a');" + "get_ipython().run_line_magic('pinfo', 'a')"), (u'a*2 #comment?', 'a*2 #comment?'), ]], # Explicit magic calls escaped_magic = [(i,py3compat.u_format(o)) for i,o in \ - [ (u'%cd', "get_ipython().magic({u}'cd')"), - (u'%cd /home', "get_ipython().magic({u}'cd /home')"), + [ (u'%cd', "get_ipython().run_line_magic('cd', '')"), + (u'%cd /home', "get_ipython().run_line_magic('cd', '/home')"), # Backslashes need to be escaped. - (u'%cd C:\\User', "get_ipython().magic({u}'cd C:\\\\User')"), - (u' %magic', " get_ipython().magic({u}'magic')"), + (u'%cd C:\\User', "get_ipython().run_line_magic('cd', 'C:\\\\User')"), + (u' %magic', " get_ipython().run_line_magic('magic', '')"), ]], # Quoting with separate arguments @@ -170,11 +164,11 @@ def transform_checker(tests, transformer, **kwargs): # Check that we transform prompts before other transforms mixed = [(i,py3compat.u_format(o)) for i,o in \ - [ (u'In [1]: %lsmagic', "get_ipython().magic({u}'lsmagic')"), - (u'>>> %lsmagic', "get_ipython().magic({u}'lsmagic')"), - (u'In [2]: !ls', "get_ipython().system({u}'ls')"), - (u'In [3]: abs?', "get_ipython().magic({u}'pinfo abs')"), - (u'In [4]: b = %who', "b = get_ipython().magic({u}'who')"), + [ (u'In [1]: %lsmagic', "get_ipython().run_line_magic('lsmagic', '')"), + (u'>>> %lsmagic', "get_ipython().run_line_magic('lsmagic', '')"), + (u'In [2]: !ls', "get_ipython().system('ls')"), + (u'In [3]: abs?', "get_ipython().run_line_magic('pinfo', 'abs')"), + (u'In [4]: b = %who', "b = get_ipython().run_line_magic('who', '')"), ]], ) @@ -255,20 +249,6 @@ def transform_checker(tests, transformer, **kwargs): ], ], - strip_encoding_cookie = - [ - [ - ('# -*- coding: utf-8 -*-', ''), - ('foo', 'foo'), - ], - [ - ('#!/usr/bin/env python', '#!/usr/bin/env python'), - ('# -*- coding: latin-1 -*-', ''), - # only the first-two lines - ('# -*- coding: latin-1 -*-', '# -*- coding: latin-1 -*-'), - ], - ], - multiline_datastructure_prompt = [ [('>>> a = [1,','a = [1,'), ('... 2]','2]'), @@ -304,11 +284,11 @@ def transform_checker(tests, transformer, **kwargs): cellmagic = [ [(u'%%foo a', None), - (None, u_fmt("get_ipython().run_cell_magic({u}'foo', {u}'a', {u}'')")), + (None, u_fmt("get_ipython().run_cell_magic('foo', 'a', '')")), ], [(u'%%bar 123', None), (u'hello', None), - (None , u_fmt("get_ipython().run_cell_magic({u}'bar', {u}'123', {u}'hello')")), + (None , u_fmt("get_ipython().run_cell_magic('bar', '123', 'hello')")), ], [(u'a=5', 'a=5'), (u'%%cellmagic', '%%cellmagic'), @@ -317,31 +297,31 @@ def transform_checker(tests, transformer, **kwargs): escaped = [ [('%abc def \\', None), - ('ghi', u_fmt("get_ipython().magic({u}'abc def ghi')")), + ('ghi', u_fmt("get_ipython().run_line_magic('abc', 'def ghi')")), ], [('%abc def \\', None), ('ghi\\', None), - (None, u_fmt("get_ipython().magic({u}'abc def ghi')")), + (None, u_fmt("get_ipython().run_line_magic('abc', 'def ghi')")), ], ], assign_magic = [ [(u'a = %bc de \\', None), - (u'fg', u_fmt("a = get_ipython().magic({u}'bc de fg')")), + (u'fg', u_fmt("a = get_ipython().run_line_magic('bc', 'de fg')")), ], [(u'a = %bc de \\', None), (u'fg\\', None), - (None, u_fmt("a = get_ipython().magic({u}'bc de fg')")), + (None, u_fmt("a = get_ipython().run_line_magic('bc', 'de fg')")), ], ], assign_system = [ [(u'a = !bc de \\', None), - (u'fg', u_fmt("a = get_ipython().getoutput({u}'bc de fg')")), + (u'fg', u_fmt("a = get_ipython().getoutput('bc de fg')")), ], [(u'a = !bc de \\', None), (u'fg\\', None), - (None, u_fmt("a = get_ipython().getoutput({u}'bc de fg')")), + (None, u_fmt("a = get_ipython().getoutput('bc de fg')")), ], ], ) @@ -379,11 +359,6 @@ def test_ipy_prompt(): (u'In [1]: bar', 'In [1]: bar'), ], ipt.ipy_prompt) -def test_coding_cookie(): - tt.check_pairs(transform_and_reset(ipt.strip_encoding_cookie), syntax['strip_encoding_cookie']) - for example in syntax_ml['strip_encoding_cookie']: - transform_checker(example, ipt.strip_encoding_cookie) - def test_assemble_logical_lines(): tests = \ [ [(u"a = \\", None), @@ -415,6 +390,11 @@ def test_assemble_python_lines(): (u"2,", None), (None, u"a = [1,\n2,"), ], + [(u"a = '''", None), # Test line continuation within a multi-line string + (u"abc\\", None), + (u"def", None), + (u"'''", u"a = '''\nabc\\\ndef\n'''"), + ], ] + syntax_ml['multiline_datastructure'] for example in tests: transform_checker(example, ipt.assemble_python_lines) @@ -457,7 +437,7 @@ def test_cellmagic(): line_example = [(u'%%bar 123', None), (u'hello', None), - (u'' , u_fmt("get_ipython().run_cell_magic({u}'bar', {u}'123', {u}'hello')")), + (u'' , u_fmt("get_ipython().run_cell_magic('bar', '123', 'hello')")), ] transform_checker(line_example, ipt.cellmagic, end_on_blank_line=True) @@ -495,16 +475,16 @@ def decistmt(tokens): def test_token_input_transformer(): - tests = [(u'1.2', u_fmt(u"Decimal ({u}'1.2')")), + tests = [(u'1.2', u_fmt(u"Decimal ('1.2')")), (u'"1.2"', u'"1.2"'), ] tt.check_pairs(transform_and_reset(decistmt), tests) ml_tests = \ [ [(u"a = 1.2; b = '''x", None), - (u"y'''", u_fmt(u"a =Decimal ({u}'1.2');b ='''x\ny'''")), + (u"y'''", u_fmt(u"a =Decimal ('1.2');b ='''x\ny'''")), ], [(u"a = [1.2,", None), - (u"3]", u_fmt(u"a =[Decimal ({u}'1.2'),\n3 ]")), + (u"3]", u_fmt(u"a =[Decimal ('1.2'),\n3 ]")), ], [(u"a = '''foo", None), # Test resetting when within a multi-line string (u"bar", None), diff --git a/IPython/core/tests/test_inputtransformer2.py b/IPython/core/tests/test_inputtransformer2.py new file mode 100644 index 00000000000..b29a0196d3f --- /dev/null +++ b/IPython/core/tests/test_inputtransformer2.py @@ -0,0 +1,292 @@ +"""Tests for the token-based transformers in IPython.core.inputtransformer2 + +Line-based transformers are the simpler ones; token-based transformers are +more complex. See test_inputtransformer2_line for tests for line-based +transformations. +""" +import nose.tools as nt +import string + +from IPython.core import inputtransformer2 as ipt2 +from IPython.core.inputtransformer2 import make_tokens_by_line, _find_assign_op + +from textwrap import dedent + +MULTILINE_MAGIC = ("""\ +a = f() +%foo \\ +bar +g() +""".splitlines(keepends=True), (2, 0), """\ +a = f() +get_ipython().run_line_magic('foo', ' bar') +g() +""".splitlines(keepends=True)) + +INDENTED_MAGIC = ("""\ +for a in range(5): + %ls +""".splitlines(keepends=True), (2, 4), """\ +for a in range(5): + get_ipython().run_line_magic('ls', '') +""".splitlines(keepends=True)) + +MULTILINE_MAGIC_ASSIGN = ("""\ +a = f() +b = %foo \\ + bar +g() +""".splitlines(keepends=True), (2, 4), """\ +a = f() +b = get_ipython().run_line_magic('foo', ' bar') +g() +""".splitlines(keepends=True)) + +MULTILINE_SYSTEM_ASSIGN = ("""\ +a = f() +b = !foo \\ + bar +g() +""".splitlines(keepends=True), (2, 4), """\ +a = f() +b = get_ipython().getoutput('foo bar') +g() +""".splitlines(keepends=True)) + +##### + +MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT = ("""\ +def test(): + for i in range(1): + print(i) + res =! ls +""".splitlines(keepends=True), (4, 7), '''\ +def test(): + for i in range(1): + print(i) + res =get_ipython().getoutput(\' ls\') +'''.splitlines(keepends=True)) + +###### + +AUTOCALL_QUOTE = ( + [",f 1 2 3\n"], (1, 0), + ['f("1", "2", "3")\n'] +) + +AUTOCALL_QUOTE2 = ( + [";f 1 2 3\n"], (1, 0), + ['f("1 2 3")\n'] +) + +AUTOCALL_PAREN = ( + ["/f 1 2 3\n"], (1, 0), + ['f(1, 2, 3)\n'] +) + +SIMPLE_HELP = ( + ["foo?\n"], (1, 0), + ["get_ipython().run_line_magic('pinfo', 'foo')\n"] +) + +DETAILED_HELP = ( + ["foo??\n"], (1, 0), + ["get_ipython().run_line_magic('pinfo2', 'foo')\n"] +) + +MAGIC_HELP = ( + ["%foo?\n"], (1, 0), + ["get_ipython().run_line_magic('pinfo', '%foo')\n"] +) + +HELP_IN_EXPR = ( + ["a = b + c?\n"], (1, 0), + ["get_ipython().set_next_input('a = b + c');" + "get_ipython().run_line_magic('pinfo', 'c')\n"] +) + +HELP_CONTINUED_LINE = ("""\ +a = \\ +zip? +""".splitlines(keepends=True), (1, 0), +[r"get_ipython().set_next_input('a = \\\nzip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"] +) + +HELP_MULTILINE = ("""\ +(a, +b) = zip? +""".splitlines(keepends=True), (1, 0), +[r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"] +) + +HELP_UNICODE = ( + ["π.foo?\n"], (1, 0), + ["get_ipython().run_line_magic('pinfo', 'π.foo')\n"] +) + + +def null_cleanup_transformer(lines): + """ + A cleanup transform that returns an empty list. + """ + return [] + +def check_make_token_by_line_never_ends_empty(): + """ + Check that not sequence of single or double characters ends up leading to en empty list of tokens + """ + from string import printable + for c in printable: + nt.assert_not_equal(make_tokens_by_line(c)[-1], []) + for k in printable: + nt.assert_not_equal(make_tokens_by_line(c+k)[-1], []) + +def check_find(transformer, case, match=True): + sample, expected_start, _ = case + tbl = make_tokens_by_line(sample) + res = transformer.find(tbl) + if match: + # start_line is stored 0-indexed, expected values are 1-indexed + nt.assert_equal((res.start_line+1, res.start_col), expected_start) + return res + else: + nt.assert_is(res, None) + +def check_transform(transformer_cls, case): + lines, start, expected = case + transformer = transformer_cls(start) + nt.assert_equal(transformer.transform(lines), expected) + +def test_continued_line(): + lines = MULTILINE_MAGIC_ASSIGN[0] + nt.assert_equal(ipt2.find_end_of_continued_line(lines, 1), 2) + + nt.assert_equal(ipt2.assemble_continued_line(lines, (1, 5), 2), "foo bar") + +def test_find_assign_magic(): + check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN) + check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False) + check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT, match=False) + +def test_transform_assign_magic(): + check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN) + +def test_find_assign_system(): + check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN) + check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT) + check_find(ipt2.SystemAssign, (["a = !ls\n"], (1, 5), None)) + check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None)) + check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False) + +def test_transform_assign_system(): + check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN) + check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT) + +def test_find_magic_escape(): + check_find(ipt2.EscapedCommand, MULTILINE_MAGIC) + check_find(ipt2.EscapedCommand, INDENTED_MAGIC) + check_find(ipt2.EscapedCommand, MULTILINE_MAGIC_ASSIGN, match=False) + +def test_transform_magic_escape(): + check_transform(ipt2.EscapedCommand, MULTILINE_MAGIC) + check_transform(ipt2.EscapedCommand, INDENTED_MAGIC) + +def test_find_autocalls(): + for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]: + print("Testing %r" % case[0]) + check_find(ipt2.EscapedCommand, case) + +def test_transform_autocall(): + for case in [AUTOCALL_QUOTE, AUTOCALL_QUOTE2, AUTOCALL_PAREN]: + print("Testing %r" % case[0]) + check_transform(ipt2.EscapedCommand, case) + +def test_find_help(): + for case in [SIMPLE_HELP, DETAILED_HELP, MAGIC_HELP, HELP_IN_EXPR]: + check_find(ipt2.HelpEnd, case) + + tf = check_find(ipt2.HelpEnd, HELP_CONTINUED_LINE) + nt.assert_equal(tf.q_line, 1) + nt.assert_equal(tf.q_col, 3) + + tf = check_find(ipt2.HelpEnd, HELP_MULTILINE) + nt.assert_equal(tf.q_line, 1) + nt.assert_equal(tf.q_col, 8) + + # ? in a comment does not trigger help + check_find(ipt2.HelpEnd, (["foo # bar?\n"], None, None), match=False) + # Nor in a string + check_find(ipt2.HelpEnd, (["foo = '''bar?\n"], None, None), match=False) + +def test_transform_help(): + tf = ipt2.HelpEnd((1, 0), (1, 9)) + nt.assert_equal(tf.transform(HELP_IN_EXPR[0]), HELP_IN_EXPR[2]) + + tf = ipt2.HelpEnd((1, 0), (2, 3)) + nt.assert_equal(tf.transform(HELP_CONTINUED_LINE[0]), HELP_CONTINUED_LINE[2]) + + tf = ipt2.HelpEnd((1, 0), (2, 8)) + nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2]) + + tf = ipt2.HelpEnd((1, 0), (1, 0)) + nt.assert_equal(tf.transform(HELP_UNICODE[0]), HELP_UNICODE[2]) + +def test_find_assign_op_dedent(): + """ + be careful that empty token like dedent are not counted as parens + """ + class Tk: + def __init__(self, s): + self.string = s + + nt.assert_equal(_find_assign_op([Tk(s) for s in ('','a','=','b')]), 2) + nt.assert_equal(_find_assign_op([Tk(s) for s in ('','(', 'a','=','b', ')', '=' ,'5')]), 6) + +def test_check_complete(): + cc = ipt2.TransformerManager().check_complete + nt.assert_equal(cc("a = 1"), ('complete', None)) + nt.assert_equal(cc("for a in range(5):"), ('incomplete', 4)) + nt.assert_equal(cc("for a in range(5):\n if a > 0:"), ('incomplete', 8)) + nt.assert_equal(cc("raise = 2"), ('invalid', None)) + nt.assert_equal(cc("a = [1,\n2,"), ('incomplete', 0)) + nt.assert_equal(cc(")"), ('incomplete', 0)) + nt.assert_equal(cc("\\\r\n"), ('incomplete', 0)) + nt.assert_equal(cc("a = '''\n hi"), ('incomplete', 3)) + nt.assert_equal(cc("def a():\n x=1\n global x"), ('invalid', None)) + nt.assert_equal(cc("a \\ "), ('invalid', None)) # Nothing allowed after backslash + nt.assert_equal(cc("1\\\n+2"), ('complete', None)) + nt.assert_equal(cc("exit"), ('complete', None)) + + example = dedent(""" + if True: + a=1""" ) + + nt.assert_equal(cc(example), ('incomplete', 4)) + nt.assert_equal(cc(example+'\n'), ('complete', None)) + nt.assert_equal(cc(example+'\n '), ('complete', None)) + + # no need to loop on all the letters/numbers. + short = '12abAB'+string.printable[62:] + for c in short: + # test does not raise: + cc(c) + for k in short: + cc(c+k) + + nt.assert_equal(cc("def f():\n x=0\n \\\n "), ('incomplete', 2)) + +def test_check_complete_II(): + """ + Test that multiple line strings are properly handled. + + Separate test function for convenience + + """ + cc = ipt2.TransformerManager().check_complete + nt.assert_equal(cc('''def foo():\n """'''), ('incomplete', 4)) + + +def test_null_cleanup_transformer(): + manager = ipt2.TransformerManager() + manager.cleanup_transforms.insert(0, null_cleanup_transformer) + nt.assert_is(manager.transform_cell(""), "") diff --git a/IPython/core/tests/test_inputtransformer2_line.py b/IPython/core/tests/test_inputtransformer2_line.py new file mode 100644 index 00000000000..41b6ed2935c --- /dev/null +++ b/IPython/core/tests/test_inputtransformer2_line.py @@ -0,0 +1,116 @@ +"""Tests for the line-based transformers in IPython.core.inputtransformer2 + +Line-based transformers are the simpler ones; token-based transformers are +more complex. See test_inputtransformer2 for tests for token-based transformers. +""" +import nose.tools as nt + +from IPython.core import inputtransformer2 as ipt2 + +CELL_MAGIC = ("""\ +%%foo arg +body 1 +body 2 +""", """\ +get_ipython().run_cell_magic('foo', 'arg', 'body 1\\nbody 2\\n') +""") + +def test_cell_magic(): + for sample, expected in [CELL_MAGIC]: + nt.assert_equal(ipt2.cell_magic(sample.splitlines(keepends=True)), + expected.splitlines(keepends=True)) + +CLASSIC_PROMPT = ("""\ +>>> for a in range(5): +... print(a) +""", """\ +for a in range(5): + print(a) +""") + +CLASSIC_PROMPT_L2 = ("""\ +for a in range(5): +... print(a) +... print(a ** 2) +""", """\ +for a in range(5): + print(a) + print(a ** 2) +""") + +def test_classic_prompt(): + for sample, expected in [CLASSIC_PROMPT, CLASSIC_PROMPT_L2]: + nt.assert_equal(ipt2.classic_prompt(sample.splitlines(keepends=True)), + expected.splitlines(keepends=True)) + +IPYTHON_PROMPT = ("""\ +In [1]: for a in range(5): + ...: print(a) +""", """\ +for a in range(5): + print(a) +""") + +IPYTHON_PROMPT_L2 = ("""\ +for a in range(5): + ...: print(a) + ...: print(a ** 2) +""", """\ +for a in range(5): + print(a) + print(a ** 2) +""") + +def test_ipython_prompt(): + for sample, expected in [IPYTHON_PROMPT, IPYTHON_PROMPT_L2]: + nt.assert_equal(ipt2.ipython_prompt(sample.splitlines(keepends=True)), + expected.splitlines(keepends=True)) + +INDENT_SPACES = ("""\ + if True: + a = 3 +""", """\ +if True: + a = 3 +""") + +INDENT_TABS = ("""\ +\tif True: +\t\tb = 4 +""", """\ +if True: +\tb = 4 +""") + +def test_leading_indent(): + for sample, expected in [INDENT_SPACES, INDENT_TABS]: + nt.assert_equal(ipt2.leading_indent(sample.splitlines(keepends=True)), + expected.splitlines(keepends=True)) + +LEADING_EMPTY_LINES = ("""\ + \t + +if True: + a = 3 + +b = 4 +""", """\ +if True: + a = 3 + +b = 4 +""") + +ONLY_EMPTY_LINES = ("""\ + \t + +""", """\ + \t + +""") + +def test_leading_empty_lines(): + for sample, expected in [LEADING_EMPTY_LINES, ONLY_EMPTY_LINES]: + nt.assert_equal( + ipt2.leading_empty_lines(sample.splitlines(keepends=True)), + expected.splitlines(keepends=True)) diff --git a/IPython/core/tests/test_interactiveshell.py b/IPython/core/tests/test_interactiveshell.py index db22c158b66..496e3bd02bc 100644 --- a/IPython/core/tests/test_interactiveshell.py +++ b/IPython/core/tests/test_interactiveshell.py @@ -9,6 +9,7 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. +import asyncio import ast import os import signal @@ -16,34 +17,25 @@ import sys import tempfile import unittest -try: - from unittest import mock -except ImportError: - import mock +from unittest import mock + from os.path import join import nose.tools as nt from IPython.core.error import InputRejected from IPython.core.inputtransformer import InputTransformer +from IPython.core import interactiveshell from IPython.testing.decorators import ( skipif, skip_win32, onlyif_unicode_paths, onlyif_cmds_exist, ) from IPython.testing import tools as tt from IPython.utils.process import find_cmd -from IPython.utils import py3compat -from IPython.utils.py3compat import unicode_type, PY3 - -if PY3: - from io import StringIO -else: - from StringIO import StringIO #----------------------------------------------------------------------------- # Globals #----------------------------------------------------------------------------- # This is used by every single test, no point repeating it ad nauseam -ip = get_ipython() #----------------------------------------------------------------------------- # Tests @@ -134,8 +126,8 @@ def test_gh_597(self): """Pretty-printing lists of objects with non-ascii reprs may cause problems.""" class Spam(object): - def __repr__(self): - return "\xe9"*50 + def __repr__(self): + return "\xe9"*50 import IPython.core.formatters f = IPython.core.formatters.PlainTextFormatter() f([Spam(),Spam()]) @@ -143,26 +135,14 @@ def __repr__(self): def test_future_flags(self): """Check that future flags are used for parsing code (gh-777)""" - ip.run_cell('from __future__ import print_function') + ip.run_cell('from __future__ import barry_as_FLUFL') try: - ip.run_cell('prfunc_return_val = print(1,2, sep=" ")') + ip.run_cell('prfunc_return_val = 1 <> 2') assert 'prfunc_return_val' in ip.user_ns finally: # Reset compiler flags so we don't mess up other tests. ip.compile.reset_compiler_flags() - def test_future_unicode(self): - """Check that unicode_literals is imported from __future__ (gh #786)""" - try: - ip.run_cell(u'byte_str = "a"') - assert isinstance(ip.user_ns['byte_str'], str) # string literals are byte strings by default - ip.run_cell('from __future__ import unicode_literals') - ip.run_cell(u'unicode_str = "a"') - assert isinstance(ip.user_ns['unicode_str'], unicode_type) # strings literals are now unicode - finally: - # Reset compiler flags so we don't mess up other tests. - ip.compile.reset_compiler_flags() - def test_can_pickle(self): "Can we pickle objects defined interactively (GH-29)" ip = get_ipython() @@ -230,11 +210,13 @@ def test_var_expand(self): self.assertEqual(ip.var_expand(u'echo {f}'), u'echo Ca\xf1o') self.assertEqual(ip.var_expand(u'echo {f[:-1]}'), u'echo Ca\xf1') self.assertEqual(ip.var_expand(u'echo {1*2}'), u'echo 2') + + self.assertEqual(ip.var_expand(u"grep x | awk '{print $1}'"), u"grep x | awk '{print $1}'") ip.user_ns['f'] = b'Ca\xc3\xb1o' # This should not raise any exception: ip.var_expand(u'echo $f') - + def test_var_expand_local(self): """Test local variable expansion in !system and %magic calls""" # !system @@ -280,6 +262,7 @@ def test_silent_postexec(self): pre_always = mock.Mock() post_explicit = mock.Mock() post_always = mock.Mock() + all_mocks = [pre_explicit, pre_always, post_explicit, post_always] ip.events.register('pre_run_cell', pre_explicit) ip.events.register('pre_execute', pre_always) @@ -297,6 +280,19 @@ def test_silent_postexec(self): ip.run_cell("1") assert pre_explicit.called assert post_explicit.called + info, = pre_explicit.call_args[0] + result, = post_explicit.call_args[0] + self.assertEqual(info, result.info) + # check that post hooks are always called + [m.reset_mock() for m in all_mocks] + ip.run_cell("syntax error") + assert pre_always.called + assert pre_explicit.called + assert post_always.called + assert post_explicit.called + info, = pre_explicit.call_args[0] + result, = post_explicit.call_args[0] + self.assertEqual(info, result.info) finally: # remove post-exec ip.events.unregister('pre_run_cell', pre_explicit) @@ -338,19 +334,6 @@ def failing_hook(*args, **kwargs): finally: trap.hook = save_hook - @skipif(sys.version_info[0] >= 3, "softspace removed in py3") - def test_print_softspace(self): - """Verify that softspace is handled correctly when executing multiple - statements. - - In [1]: print 1; print 2 - 1 - 2 - - In [2]: print 1,; print 2 - 1 2 - """ - def test_ofind_line_magic(self): from IPython.core.magic import register_line_magic @@ -466,22 +449,6 @@ def my_handler(shell, etype, value, tb, tb_offset=None): # Reset the custom exception hook ip.set_custom_exc((), None) - @skipif(sys.version_info[0] >= 3, "no differences with __future__ in py3") - def test_future_environment(self): - "Can we run code with & without the shell's __future__ imports?" - ip.run_cell("from __future__ import division") - ip.run_cell("a = 1/2", shell_futures=True) - self.assertEqual(ip.user_ns['a'], 0.5) - ip.run_cell("b = 1/2", shell_futures=False) - self.assertEqual(ip.user_ns['b'], 0) - - ip.compile.reset_compiler_flags() - # This shouldn't leak to the shell's compiler - ip.run_cell("from __future__ import division \nc=1/2", shell_futures=False) - self.assertEqual(ip.user_ns['c'], 0.5) - ip.run_cell("d = 1/2", shell_futures=True) - self.assertEqual(ip.user_ns['d'], 0) - def test_mktempfile(self): filename = ip.mktempfile() # Check that we can open the file again on Windows @@ -509,15 +476,33 @@ def test_get_exception_only(self): raise DerivedInterrupt("foo") except KeyboardInterrupt: msg = ip.get_exception_only() - if sys.version_info[0] <= 2: - self.assertEqual(msg, 'DerivedInterrupt: foo\n') - else: - self.assertEqual(msg, 'IPython.core.tests.test_interactiveshell.DerivedInterrupt: foo\n') + self.assertEqual(msg, 'IPython.core.tests.test_interactiveshell.DerivedInterrupt: foo\n') def test_inspect_text(self): ip.run_cell('a = 5') text = ip.object_inspect_text('a') - self.assertIsInstance(text, unicode_type) + self.assertIsInstance(text, str) + + def test_last_execution_result(self): + """ Check that last execution result gets set correctly (GH-10702) """ + result = ip.run_cell('a = 5; a') + self.assertTrue(ip.last_execution_succeeded) + self.assertEqual(ip.last_execution_result.result, 5) + + result = ip.run_cell('a = x_invalid_id_x') + self.assertFalse(ip.last_execution_succeeded) + self.assertFalse(ip.last_execution_result.success) + self.assertIsInstance(ip.last_execution_result.error_in_exec, NameError) + + def test_reset_aliasing(self): + """ Check that standard posix aliases work after %reset. """ + if os.name != 'posix': + return + + ip.reset() + for cmd in ('clear', 'more', 'less', 'man'): + res = ip.run_cell('%' + cmd) + self.assertEqual(res.success, True) class TestSafeExecfileNonAsciiPath(unittest.TestCase): @@ -529,7 +514,7 @@ def setUp(self): os.mkdir(self.TESTDIR) with open(join(self.TESTDIR, u"åäötestscript.py"), "w") as sfile: sfile.write("pass\n") - self.oldpath = py3compat.getcwd() + self.oldpath = os.getcwd() os.chdir(self.TESTDIR) self.fname = u"åäötestscript.py" @@ -544,6 +529,10 @@ def test_1(self): ip.safe_execfile(self.fname, {}, raise_exceptions=True) class ExitCodeChecks(tt.TempFileMixin): + + def setUp(self): + self.system = ip.system_raw + def test_exit_code_ok(self): self.system('exit 0') self.assertEqual(ip.user_ns['_exit_code'], 0) @@ -572,8 +561,12 @@ def test_exit_code_signal_csh(self): else: del os.environ['SHELL'] -class TestSystemRaw(unittest.TestCase, ExitCodeChecks): - system = ip.system_raw + +class TestSystemRaw(ExitCodeChecks): + + def setUp(self): + super().setUp() + self.system = ip.system_raw @onlyif_unicode_paths def test_1(self): @@ -593,8 +586,11 @@ def test_control_c(self, *mocks): self.assertEqual(ip.user_ns['_exit_code'], -signal.SIGINT) # TODO: Exit codes are currently ignored on Windows. -class TestSystemPipedExitCode(unittest.TestCase, ExitCodeChecks): - system = ip.system_piped +class TestSystemPipedExitCode(ExitCodeChecks): + + def setUp(self): + super().setUp() + self.system = ip.system_piped @skip_win32 def test_exit_code_ok(self): @@ -608,7 +604,7 @@ def test_exit_code_error(self): def test_exit_code_signal(self): ExitCodeChecks.test_exit_code_signal(self) -class TestModules(unittest.TestCase, tt.TempFileMixin): +class TestModules(tt.TempFileMixin): def test_extraneous_loads(self): """Test we're not loading modules on startup that we shouldn't. """ @@ -622,10 +618,18 @@ def test_extraneous_loads(self): class Negator(ast.NodeTransformer): """Negates all number literals in an AST.""" + + # for python 3.7 and earlier def visit_Num(self, node): node.n = -node.n return node + # for python 3.8+ + def visit_Constant(self, node): + if isinstance(node.value, int): + return self.visit_Num(node) + return node + class TestAstTransform(unittest.TestCase): def setUp(self): self.negator = Negator() @@ -649,12 +653,12 @@ def f(x): called.add(x) ip.push({'f':f}) - with tt.AssertPrints("best of "): + with tt.AssertPrints("std. dev. of"): ip.run_line_magic("timeit", "-n1 f(1)") self.assertEqual(called, {-1}) called.clear() - - with tt.AssertPrints("best of "): + + with tt.AssertPrints("std. dev. of"): ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)") self.assertEqual(called, {-2, -3}) @@ -687,12 +691,23 @@ def test_macro(self): class IntegerWrapper(ast.NodeTransformer): """Wraps all integers in a call to Integer()""" + + # for Python 3.7 and earlier + + # for Python 3.7 and earlier def visit_Num(self, node): if isinstance(node.n, int): return ast.Call(func=ast.Name(id='Integer', ctx=ast.Load()), args=[node], keywords=[]) return node + # For Python 3.8+ + def visit_Constant(self, node): + if isinstance(node.value, int): + return self.visit_Num(node) + return node + + class TestAstTransform2(unittest.TestCase): def setUp(self): self.intwrapper = IntegerWrapper() @@ -721,27 +736,36 @@ def test_timeit(self): def f(x): called.add(x) ip.push({'f':f}) - - with tt.AssertPrints("best of "): + + with tt.AssertPrints("std. dev. of"): ip.run_line_magic("timeit", "-n1 f(1)") self.assertEqual(called, {(1,)}) called.clear() - - with tt.AssertPrints("best of "): + + with tt.AssertPrints("std. dev. of"): ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)") self.assertEqual(called, {(2,), (3,)}) class ErrorTransformer(ast.NodeTransformer): """Throws an error when it sees a number.""" + + # for Python 3.7 and earlier def visit_Num(self, node): raise ValueError("test") + # for Python 3.8+ + def visit_Constant(self, node): + if isinstance(node.value, int): + return self.visit_Num(node) + return node + + class TestAstTransformError(unittest.TestCase): def test_unregistering(self): err_transformer = ErrorTransformer() ip.ast_transformers.append(err_transformer) - with tt.AssertPrints("unregister", channel='stderr'): + with self.assertWarnsRegex(UserWarning, "It will be unregistered"): ip.run_cell("1 + 2") # This should have been removed. @@ -754,10 +778,17 @@ class StringRejector(ast.NodeTransformer): Used to verify that NodeTransformers can signal that a piece of code should not be executed by throwing an InputRejected. """ - + + #for python 3.7 and earlier def visit_Str(self, node): raise InputRejected("test") + # 3.8 only + def visit_Constant(self, node): + if isinstance(node.value, str): + raise InputRejected("test") + return node + class TestAstTransformInputRejection(unittest.TestCase): @@ -850,36 +881,27 @@ def test_user_expression(): # back to text only ip.display_formatter.active_types = ['text/plain'] - - - class TestSyntaxErrorTransformer(unittest.TestCase): """Check that SyntaxError raised by an input transformer is handled by run_cell()""" - class SyntaxErrorTransformer(InputTransformer): - - def push(self, line): + @staticmethod + def transformer(lines): + for line in lines: pos = line.find('syntaxerror') if pos >= 0: e = SyntaxError('input contains "syntaxerror"') e.text = line e.offset = pos + 1 raise e - return line - - def reset(self): - pass + return lines def setUp(self): - self.transformer = TestSyntaxErrorTransformer.SyntaxErrorTransformer() - ip.input_splitter.python_line_transforms.append(self.transformer) - ip.input_transformer_manager.python_line_transforms.append(self.transformer) + ip.input_transformers_post.append(self.transformer) def tearDown(self): - ip.input_splitter.python_line_transforms.remove(self.transformer) - ip.input_transformer_manager.python_line_transforms.remove(self.transformer) + ip.input_transformers_post.remove(self.transformer) def test_syntaxerror_input_transformer(self): with tt.AssertPrints('1234'): @@ -892,25 +914,25 @@ def test_syntaxerror_input_transformer(self): ip.run_cell('3456') - -def test_warning_suppression(): - ip.run_cell("import warnings") - try: - with tt.AssertPrints("UserWarning: asdf", channel="stderr"): - ip.run_cell("warnings.warn('asdf')") - # Here's the real test -- if we run that again, we should get the - # warning again. Traditionally, each warning was only issued once per - # IPython session (approximately), even if the user typed in new and - # different code that should have also triggered the warning, leading - # to much confusion. - with tt.AssertPrints("UserWarning: asdf", channel="stderr"): - ip.run_cell("warnings.warn('asdf')") - finally: - ip.run_cell("del warnings") +class TestWarningSuppression(unittest.TestCase): + def test_warning_suppression(self): + ip.run_cell("import warnings") + try: + with self.assertWarnsRegex(UserWarning, "asdf"): + ip.run_cell("warnings.warn('asdf')") + # Here's the real test -- if we run that again, we should get the + # warning again. Traditionally, each warning was only issued once per + # IPython session (approximately), even if the user typed in new and + # different code that should have also triggered the warning, leading + # to much confusion. + with self.assertWarnsRegex(UserWarning, "asdf"): + ip.run_cell("warnings.warn('asdf')") + finally: + ip.run_cell("del warnings") -def test_deprecation_warning(): - ip.run_cell(""" + def test_deprecation_warning(self): + ip.run_cell(""" import warnings def wrn(): warnings.warn( @@ -918,17 +940,17 @@ def wrn(): DeprecationWarning ) """) - try: - with tt.AssertPrints("I AM A WARNING", channel="stderr"): - ip.run_cell("wrn()") - finally: - ip.run_cell("del warnings") - ip.run_cell("del wrn") + try: + with self.assertWarnsRegex(DeprecationWarning, "I AM A WARNING"): + ip.run_cell("wrn()") + finally: + ip.run_cell("del warnings") + ip.run_cell("del wrn") class TestImportNoDeprecate(tt.TempFileMixin): - def setup(self): + def setUp(self): """Make a valid python temp file.""" self.mktmp(""" import warnings @@ -938,6 +960,7 @@ def wrn(): DeprecationWarning ) """) + super().setUp() def test_no_dep(self): """ @@ -948,3 +971,48 @@ def test_no_dep(self): with tt.AssertNotPrints("I AM A WARNING"): ip.run_cell("wrn()") ip.run_cell("del wrn") + + +def test_custom_exc_count(): + hook = mock.Mock(return_value=None) + ip.set_custom_exc((SyntaxError,), hook) + before = ip.execution_count + ip.run_cell("def foo()", store_history=True) + # restore default excepthook + ip.set_custom_exc((), None) + nt.assert_equal(hook.call_count, 1) + nt.assert_equal(ip.execution_count, before + 1) + + +def test_run_cell_async(): + loop = asyncio.get_event_loop() + ip.run_cell("import asyncio") + coro = ip.run_cell_async("await asyncio.sleep(0.01)\n5") + assert asyncio.iscoroutine(coro) + result = loop.run_until_complete(coro) + assert isinstance(result, interactiveshell.ExecutionResult) + assert result.result == 5 + + +def test_should_run_async(): + assert not ip.should_run_async("a = 5") + assert ip.should_run_async("await x") + assert ip.should_run_async("import asyncio; await asyncio.sleep(1)") + + +def test_set_custom_completer(): + num_completers = len(ip.Completer.matchers) + + def foo(*args, **kwargs): + return "I'm a completer!" + + ip.set_custom_completer(foo, 0) + + # check that we've really added a new completer + assert len(ip.Completer.matchers) == num_completers + 1 + + # check that the first completer is the function we defined + assert ip.Completer.matchers[0]() == "I'm a completer!" + + # clean up + ip.Completer.custom_matchers.pop() diff --git a/IPython/core/tests/test_iplib.py b/IPython/core/tests/test_iplib.py index f8101e6f466..adadae56ab9 100644 --- a/IPython/core/tests/test_iplib.py +++ b/IPython/core/tests/test_iplib.py @@ -8,14 +8,6 @@ import nose.tools as nt # our own packages -from IPython.testing.globalipapp import get_ipython - -#----------------------------------------------------------------------------- -# Globals -#----------------------------------------------------------------------------- - -# Get the public instance of IPython -ip = get_ipython() #----------------------------------------------------------------------------- # Test functions @@ -72,7 +64,7 @@ def doctest_tb_context(): --------------------------------------------------------------------------- ZeroDivisionError Traceback (most recent call last) -... in () +... in 30 mode = 'div' 31 ---> 32 bar(mode) @@ -104,7 +96,7 @@ def doctest_tb_verbose(): --------------------------------------------------------------------------- ZeroDivisionError Traceback (most recent call last) -... in () +... in 30 mode = 'div' 31 ---> 32 bar(mode) @@ -161,7 +153,7 @@ def doctest_tb_sysexit(): --------------------------------------------------------------------------- SystemExit Traceback (most recent call last) -...() +... 30 mode = 'div' 31 ---> 32 bar(mode) @@ -189,7 +181,7 @@ def doctest_tb_sysexit(): --------------------------------------------------------------------------- SystemExit Traceback (most recent call last) -... in () +... in 30 mode = 'div' 31 ---> 32 bar(mode) diff --git a/IPython/core/tests/test_logger.py b/IPython/core/tests/test_logger.py index 4d61ff2433d..ebebac16cfe 100644 --- a/IPython/core/tests/test_logger.py +++ b/IPython/core/tests/test_logger.py @@ -6,8 +6,6 @@ import nose.tools as nt from IPython.utils.tempdir import TemporaryDirectory -_ip = get_ipython() - def test_logstart_inaccessible_file(): try: _ip.logger.logstart(logfname="/") # Opening that filename will fail. diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index eb40331b71e..877326ccfc3 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -3,43 +3,36 @@ Needs to be run by nose (to make ipython session available). """ -from __future__ import absolute_import import io import os +import re import sys import warnings +from textwrap import dedent from unittest import TestCase - -try: - from importlib import invalidate_caches # Required from Python 3.3 -except ImportError: - def invalidate_caches(): - pass +from unittest import mock +from importlib import invalidate_caches +from io import StringIO import nose.tools as nt +import shlex + from IPython import get_ipython from IPython.core import magic from IPython.core.error import UsageError from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic, register_line_magic, register_cell_magic) -from IPython.core.magics import execution, script, code +from IPython.core.magics import execution, script, code, logging, osm from IPython.testing import decorators as dec from IPython.testing import tools as tt -from IPython.utils import py3compat from IPython.utils.io import capture_output -from IPython.utils.tempdir import TemporaryDirectory +from IPython.utils.tempdir import (TemporaryDirectory, + TemporaryWorkingDirectory) from IPython.utils.process import find_cmd -if py3compat.PY3: - from io import StringIO -else: - from StringIO import StringIO - - -_ip = get_ipython() @magic.magics_class class DummyMagics(magic.Magics): pass @@ -80,15 +73,65 @@ def test_extract_symbols_raises_exception_with_non_python_code(): with nt.assert_raises(SyntaxError): code.extract_symbols(source, "hello") + +def test_magic_not_found(): + # magic not found raises UsageError + with nt.assert_raises(UsageError): + _ip.magic('doesntexist') + + # ensure result isn't success when a magic isn't found + result = _ip.run_cell('%doesntexist') + assert isinstance(result.error_in_exec, UsageError) + + +def test_cell_magic_not_found(): + # magic not found raises UsageError + with nt.assert_raises(UsageError): + _ip.run_cell_magic('doesntexist', 'line', 'cell') + + # ensure result isn't success when a magic isn't found + result = _ip.run_cell('%%doesntexist') + assert isinstance(result.error_in_exec, UsageError) + + +def test_magic_error_status(): + def fail(shell): + 1/0 + _ip.register_magic_function(fail) + result = _ip.run_cell('%fail') + assert isinstance(result.error_in_exec, ZeroDivisionError) + + def test_config(): """ test that config magic does not raise can happen if Configurable init is moved too early into - Magics.__init__ as then a Config object will be registerd as a + Magics.__init__ as then a Config object will be registered as a magic. """ ## should not raise. _ip.magic('config') +def test_config_available_configs(): + """ test that config magic prints available configs in unique and + sorted order. """ + with capture_output() as captured: + _ip.magic('config') + + stdout = captured.stdout + config_classes = stdout.strip().split('\n')[1:] + nt.assert_list_equal(config_classes, sorted(set(config_classes))) + +def test_config_print_class(): + """ test that config with a classname prints the class's options. """ + with capture_output() as captured: + _ip.magic('config TerminalInteractiveShell') + + stdout = captured.stdout + if not re.match("TerminalInteractiveShell.* options", stdout.splitlines()[0]): + print(stdout) + raise AssertionError("1st line of stdout not like " + "'TerminalInteractiveShell.* options'") + def test_rehashx(): # clear up everything _ip.alias_manager.clear_aliases() @@ -105,6 +148,7 @@ def test_rehashx(): # rehashx must fill up syscmdlist scoms = _ip.db['syscmdlist'] nt.assert_true(len(scoms) > 10) + def test_magic_parse_options(): @@ -260,12 +304,10 @@ def test_macro_run(): """Test that we can run a multi-line macro successfully.""" ip = get_ipython() ip.history_manager.reset() - cmds = ["a=10", "a+=1", py3compat.doctest_refactor_print("print a"), - "%macro test 2-3"] + cmds = ["a=10", "a+=1", "print(a)", "%macro test 2-3"] for cmd in cmds: ip.run_cell(cmd, store_history=True) - nt.assert_equal(ip.user_ns["test"].value, - py3compat.doctest_refactor_print("a+=1\nprint a\n")) + nt.assert_equal(ip.user_ns["test"].value, "a+=1\nprint(a)\n") with tt.AssertPrints("12"): ip.run_cell("test") with tt.AssertPrints("13"): @@ -328,6 +370,31 @@ def test_reset_in_length(): _ip.run_cell("reset -f in") nt.assert_equal(len(_ip.user_ns['In']), _ip.displayhook.prompt_count+1) +class TestResetErrors(TestCase): + + def test_reset_redefine(self): + + @magics_class + class KernelMagics(Magics): + @line_magic + def less(self, shell): pass + + _ip.register_magics(KernelMagics) + + with self.assertLogs() as cm: + # hack, we want to just capture logs, but assertLogs fails if not + # logs get produce. + # so log one things we ignore. + import logging as log_mod + log = log_mod.getLogger() + log.info('Nothing') + # end hack. + _ip.run_cell("reset -f") + + assert len(cm.output) == 1 + for out in cm.output: + assert "Invalid alias" not in out + def test_tb_syntaxerror(): """test %tb after a SyntaxError""" ip = get_ipython() @@ -359,6 +426,15 @@ def test_time(): with tt.AssertPrints("hihi", suppress=False): ip.run_cell("f('hi')") +def test_time_last_not_expression(): + ip.run_cell("%%time\n" + "var_1 = 1\n" + "var_2 = 2\n") + assert ip.user_ns['var_1'] == 1 + del ip.user_ns['var_1'] + assert ip.user_ns['var_2'] == 2 + del ip.user_ns['var_2'] + @dec.skip_win32 def test_time2(): @@ -377,17 +453,28 @@ def test_time3(): "run = 0\n" "run += 1") -@dec.skipif(sys.version_info[0] >= 3, "no differences with __future__ in py3") -def test_time_futures(): - "Test %time with __future__ environments" +def test_multiline_time(): + """Make sure last statement from time return a value.""" ip = get_ipython() - ip.autocall = 0 - ip.run_cell("from __future__ import division") - with tt.AssertPrints('0.25'): - ip.run_line_magic('time', 'print(1/4)') - ip.compile.reset_compiler_flags() - with tt.AssertNotPrints('0.25'): - ip.run_line_magic('time', 'print(1/4)') + ip.user_ns.pop('run', None) + + ip.run_cell(dedent("""\ + %%time + a = "ho" + b = "hey" + a+b + """)) + nt.assert_equal(ip.user_ns_hidden['_'], 'hohey') + +def test_time_local_ns(): + """ + Test that local_ns is actually global_ns when running a cell magic + """ + ip = get_ipython() + ip.run_cell("%%time\n" + "myvar = 1") + nt.assert_equal(ip.user_ns['myvar'], 1) + del ip.user_ns['myvar'] def test_doctest_mode(): "Toggle doctest_mode twice, it should be a no-op and run without error" @@ -403,12 +490,12 @@ def test_parse_options(): nt.assert_equal(m.parse_options('foo', '')[1], 'foo') nt.assert_equal(m.parse_options(u'foo', '')[1], u'foo') - + def test_dirops(): """Test various directory handling operations.""" - # curpath = lambda :os.path.splitdrive(py3compat.getcwd())[1].replace('\\','/') - curpath = py3compat.getcwd - startdir = py3compat.getcwd() + # curpath = lambda :os.path.splitdrive(os.getcwd())[1].replace('\\','/') + curpath = os.getcwd + startdir = os.getcwd() ipdir = os.path.realpath(_ip.ipython_dir) try: _ip.magic('cd "%s"' % ipdir) @@ -423,10 +510,27 @@ def test_dirops(): os.chdir(startdir) +def test_cd_force_quiet(): + """Test OSMagics.cd_force_quiet option""" + _ip.config.OSMagics.cd_force_quiet = True + osmagics = osm.OSMagics(shell=_ip) + + startdir = os.getcwd() + ipdir = os.path.realpath(_ip.ipython_dir) + + try: + with tt.AssertNotPrints(ipdir): + osmagics.cd('"%s"' % ipdir) + with tt.AssertNotPrints(startdir): + osmagics.cd('-') + finally: + os.chdir(startdir) + + def test_xmode(): # Calling xmode three times should be a no-op xmode = _ip.InteractiveTB.mode - for i in range(3): + for i in range(4): _ip.magic("xmode") nt.assert_equal(_ip.InteractiveTB.mode, xmode) @@ -500,28 +604,29 @@ def __repr__(self): _ip.user_ns['a'] = A() _ip.magic("whos") -@py3compat.u_format def doctest_precision(): """doctest for %precision In [1]: f = get_ipython().display_formatter.formatters['text/plain'] In [2]: %precision 5 - Out[2]: {u}'%.5f' + Out[2]: '%.5f' In [3]: f.float_format - Out[3]: {u}'%.5f' + Out[3]: '%.5f' In [4]: %precision %e - Out[4]: {u}'%e' + Out[4]: '%e' In [5]: f(3.1415927) - Out[5]: {u}'3.141593e+00' + Out[5]: '3.141593e+00' """ def test_psearch(): with tt.AssertPrints("dict.fromkeys"): _ip.run_cell("dict.fr*?") + with tt.AssertPrints("π.is_integer"): + _ip.run_cell("π = 3.14;\nπ.is_integ*?") def test_timeit_shlex(): """test shlex issues with timeit (#1109)""" @@ -534,11 +639,6 @@ def test_timeit_shlex(): _ip.magic('timeit -r1 -n1 f("a " + "b ")') -def test_timeit_arguments(): - "Test valid timeit arguments, should not cause SyntaxError (GH #1269)" - _ip.magic("timeit ('#')") - - def test_timeit_special_syntax(): "Test %%timeit with IPython special syntax" @register_line_magic @@ -555,7 +655,7 @@ def lmagic(line): def test_timeit_return(): """ - test wether timeit -o return object + test whether timeit -o return object """ res = _ip.run_line_magic('timeit','-n10 -r10 -o 1') @@ -573,16 +673,9 @@ def test_timeit_return_quiet(): res = _ip.run_line_magic('timeit', '-n1 -r1 -q -o 1') assert (res is not None) -@dec.skipif(sys.version_info[0] >= 3, "no differences with __future__ in py3") -def test_timeit_futures(): - "Test %timeit with __future__ environments" - ip = get_ipython() - ip.run_cell("from __future__ import division") - with tt.AssertPrints('0.25'): - ip.run_line_magic('timeit', '-n1 -r1 print(1/4)') - ip.compile.reset_compiler_flags() - with tt.AssertNotPrints('0.25'): - ip.run_line_magic('timeit', '-n1 -r1 print(1/4)') +def test_timeit_invalid_return(): + with nt.assert_raises_regex(SyntaxError, "outside function"): + _ip.run_line_magic('timeit', 'return') @dec.skipif(execution.profile is None) def test_prun_special_syntax(): @@ -643,6 +736,24 @@ def test_env(self): env = _ip.magic("env") self.assertTrue(isinstance(env, dict)) + def test_env_secret(self): + env = _ip.magic("env") + hidden = "" + with mock.patch.dict( + os.environ, + { + "API_KEY": "abc123", + "SECRET_THING": "ssshhh", + "JUPYTER_TOKEN": "", + "VAR": "abc" + } + ): + env = _ip.magic("env") + assert env["API_KEY"] == hidden + assert env["SECRET_THING"] == hidden + assert env["JUPYTER_TOKEN"] == hidden + assert env["VAR"] == "abc" + def test_env_get_set_simple(self): env = _ip.magic("env var val1") self.assertEqual(env, None) @@ -675,8 +786,8 @@ def check_ident(self, magic): out = _ip.run_cell_magic(magic, 'a', 'b') nt.assert_equal(out, ('a','b')) # Via run_cell, it goes into the user's namespace via displayhook - _ip.run_cell('%%' + magic +' c\nd') - nt.assert_equal(_ip.user_ns['_'], ('c','d')) + _ip.run_cell('%%' + magic +' c\nd\n') + nt.assert_equal(_ip.user_ns['_'], ('c','d\n')) def test_cell_magic_func_deco(self): "Cell magic using simple decorator" @@ -722,11 +833,41 @@ def cellm33(self, line, cell): nt.assert_equal(c33, None) def test_file(): - """Basic %%file""" + """Basic %%writefile""" ip = get_ipython() with TemporaryDirectory() as td: fname = os.path.join(td, 'file1') - ip.run_cell_magic("file", fname, u'\n'.join([ + ip.run_cell_magic("writefile", fname, u'\n'.join([ + 'line1', + 'line2', + ])) + with open(fname) as f: + s = f.read() + nt.assert_in('line1\n', s) + nt.assert_in('line2', s) + +@dec.skip_win32 +def test_file_single_quote(): + """Basic %%writefile with embedded single quotes""" + ip = get_ipython() + with TemporaryDirectory() as td: + fname = os.path.join(td, '\'file1\'') + ip.run_cell_magic("writefile", fname, u'\n'.join([ + 'line1', + 'line2', + ])) + with open(fname) as f: + s = f.read() + nt.assert_in('line1\n', s) + nt.assert_in('line2', s) + +@dec.skip_win32 +def test_file_double_quote(): + """Basic %%writefile with embedded double quotes""" + ip = get_ipython() + with TemporaryDirectory() as td: + fname = os.path.join(td, '"file1"') + ip.run_cell_magic("writefile", fname, u'\n'.join([ 'line1', 'line2', ])) @@ -736,12 +877,12 @@ def test_file(): nt.assert_in('line2', s) def test_file_var_expand(): - """%%file $filename""" + """%%writefile $filename""" ip = get_ipython() with TemporaryDirectory() as td: fname = os.path.join(td, 'file1') ip.user_ns['filename'] = fname - ip.run_cell_magic("file", '$filename', u'\n'.join([ + ip.run_cell_magic("writefile", '$filename', u'\n'.join([ 'line1', 'line2', ])) @@ -751,11 +892,11 @@ def test_file_var_expand(): nt.assert_in('line2', s) def test_file_unicode(): - """%%file with unicode cell""" + """%%writefile with unicode cell""" ip = get_ipython() with TemporaryDirectory() as td: fname = os.path.join(td, 'file1') - ip.run_cell_magic("file", fname, u'\n'.join([ + ip.run_cell_magic("writefile", fname, u'\n'.join([ u'liné1', u'liné2', ])) @@ -765,15 +906,15 @@ def test_file_unicode(): nt.assert_in(u'liné2', s) def test_file_amend(): - """%%file -a amends files""" + """%%writefile -a amends files""" ip = get_ipython() with TemporaryDirectory() as td: fname = os.path.join(td, 'file2') - ip.run_cell_magic("file", fname, u'\n'.join([ + ip.run_cell_magic("writefile", fname, u'\n'.join([ 'line1', 'line2', ])) - ip.run_cell_magic("file", "-a %s" % fname, u'\n'.join([ + ip.run_cell_magic("writefile", "-a %s" % fname, u'\n'.join([ 'line3', 'line4', ])) @@ -781,7 +922,20 @@ def test_file_amend(): s = f.read() nt.assert_in('line1\n', s) nt.assert_in('line3\n', s) - + +def test_file_spaces(): + """%%file with spaces in filename""" + ip = get_ipython() + with TemporaryWorkingDirectory() as td: + fname = "file name" + ip.run_cell_magic("file", '"%s"'%fname, u'\n'.join([ + 'line1', + 'line2', + ])) + with open(fname) as f: + s = f.read() + nt.assert_in('line1\n', s) + nt.assert_in('line2', s) def test_script_config(): ip = get_ipython() @@ -812,13 +966,16 @@ def test_script_out_err(): def test_script_bg_out(): ip = get_ipython() ip.run_cell_magic("script", "--bg --out output sh", "echo 'hi'") + nt.assert_equal(ip.user_ns['output'].read(), b'hi\n') + ip.user_ns['output'].close() @dec.skip_win32 def test_script_bg_err(): ip = get_ipython() ip.run_cell_magic("script", "--bg --err error sh", "echo 'hello' >&2") nt.assert_equal(ip.user_ns['error'].read(), b'hello\n') + ip.user_ns['error'].close() @dec.skip_win32 def test_script_bg_out_err(): @@ -826,6 +983,8 @@ def test_script_bg_out_err(): ip.run_cell_magic("script", "--bg --out output --err error sh", "echo 'hi'\necho 'hello' >&2") nt.assert_equal(ip.user_ns['output'].read(), b'hi\n') nt.assert_equal(ip.user_ns['error'].read(), b'hello\n') + ip.user_ns['output'].close() + ip.user_ns['error'].close() def test_script_defaults(): ip = get_ipython() @@ -899,6 +1058,11 @@ def test_alias_magic(): nt.assert_equal(ip.run_line_magic('env', ''), ip.run_line_magic('env_alias', '')) + # Test that line alias with parameters passed in is created successfully. + ip.run_line_magic('alias_magic', '--line history_alias history --params ' + shlex.quote('3')) + nt.assert_in('history_alias', mm.magics['line']) + + def test_save(): """Test %save.""" ip = get_ipython() @@ -1009,3 +1173,53 @@ def sii(s): nt.assert_equal(sii(" a = 1\nb = 2"), "a = 1\nb = 2") nt.assert_equal(sii(" a\n b\nc"), "a\n b\nc") nt.assert_equal(sii("a\n b"), "a\n b") + +def test_logging_magic_quiet_from_arg(): + _ip.config.LoggingMagics.quiet = False + lm = logging.LoggingMagics(shell=_ip) + with TemporaryDirectory() as td: + try: + with tt.AssertNotPrints(re.compile("Activating.*")): + lm.logstart('-q {}'.format( + os.path.join(td, "quiet_from_arg.log"))) + finally: + _ip.logger.logstop() + +def test_logging_magic_quiet_from_config(): + _ip.config.LoggingMagics.quiet = True + lm = logging.LoggingMagics(shell=_ip) + with TemporaryDirectory() as td: + try: + with tt.AssertNotPrints(re.compile("Activating.*")): + lm.logstart(os.path.join(td, "quiet_from_config.log")) + finally: + _ip.logger.logstop() + + +def test_logging_magic_not_quiet(): + _ip.config.LoggingMagics.quiet = False + lm = logging.LoggingMagics(shell=_ip) + with TemporaryDirectory() as td: + try: + with tt.AssertPrints(re.compile("Activating.*")): + lm.logstart(os.path.join(td, "not_quiet.log")) + finally: + _ip.logger.logstop() + + +def test_time_no_var_expand(): + _ip.user_ns['a'] = 5 + _ip.user_ns['b'] = [] + _ip.magic('time b.append("{a}")') + assert _ip.user_ns['b'] == ['{a}'] + + +# this is slow, put at the end for local testing. +def test_timeit_arguments(): + "Test valid timeit arguments, should not cause SyntaxError (GH #1269)" + if sys.version_info < (3,7): + _ip.magic("timeit -n1 -r1 ('#')") + else: + # 3.7 optimize no-op statement like above out, and complain there is + # nothing in the for loop. + _ip.magic("timeit -n1 -r1 a=('#')") diff --git a/IPython/core/tests/test_magic_terminal.py b/IPython/core/tests/test_magic_terminal.py index 9643ecf217c..79e2d3ed4a5 100644 --- a/IPython/core/tests/test_magic_terminal.py +++ b/IPython/core/tests/test_magic_terminal.py @@ -2,29 +2,18 @@ Needs to be run by nose (to make ipython session available). """ -from __future__ import absolute_import #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- import sys +from io import StringIO from unittest import TestCase import nose.tools as nt from IPython.testing import tools as tt -from IPython.utils.py3compat import PY3 - -if PY3: - from io import StringIO -else: - from StringIO import StringIO - -#----------------------------------------------------------------------------- -# Globals -#----------------------------------------------------------------------------- -ip = get_ipython() #----------------------------------------------------------------------------- # Test functions begin @@ -57,8 +46,6 @@ def check_cpaste(code, should_fail=False): finally: sys.stdin = stdin_save -PY31 = sys.version_info[:2] == (3,1) - def test_cpaste(): """Test cpaste magic""" @@ -77,13 +64,8 @@ def runf(): ], 'fail': ["1 + runf()", + "++ runf()", ]} - - # I don't know why this is failing specifically on Python 3.1. I've - # checked it manually interactively, but we don't care enough about 3.1 - # to spend time fiddling with the tests, so we just skip it. - if not PY31: - tests['fail'].append("++ runf()") ip.user_ns['runf'] = runf @@ -183,7 +165,7 @@ def test_paste_echo(self): ip.write = writer nt.assert_equal(ip.user_ns['a'], 100) nt.assert_equal(ip.user_ns['b'], 200) - nt.assert_equal(out, code+"\n## -- End pasted text --\n") + assert out == code+"\n## -- End pasted text --\n" def test_paste_leading_commas(self): "Test multiline strings with leading commas" diff --git a/IPython/core/tests/test_oinspect.py b/IPython/core/tests/test_oinspect.py index eb753f1f758..19c6db7c4f8 100644 --- a/IPython/core/tests/test_oinspect.py +++ b/IPython/core/tests/test_oinspect.py @@ -4,33 +4,31 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. -from __future__ import print_function +from inspect import signature, Signature, Parameter import os import re -import sys import nose.tools as nt from .. import oinspect -from IPython.core.magic import (Magics, magics_class, line_magic, - cell_magic, line_cell_magic, - register_line_magic, register_cell_magic, - register_line_cell_magic) + from decorator import decorator -from IPython.testing.decorators import skipif -from IPython.testing.tools import AssertPrints + +from IPython.testing.tools import AssertPrints, AssertNotPrints from IPython.utils.path import compress_user -from IPython.utils import py3compat -from IPython.utils.signatures import Signature, Parameter #----------------------------------------------------------------------------- # Globals and constants #----------------------------------------------------------------------------- -inspector = oinspect.Inspector() -ip = get_ipython() +inspector = None + +def setup_module(): + global inspector + inspector = oinspect.Inspector() + #----------------------------------------------------------------------------- # Local utilities @@ -40,10 +38,15 @@ # defined, if any code is inserted above, the following line will need to be # updated. Do NOT insert any whitespace between the next line and the function # definition below. -THIS_LINE_NUMBER = 43 # Put here the actual number of this line -def test_find_source_lines(): - nt.assert_equal(oinspect.find_source_lines(test_find_source_lines), - THIS_LINE_NUMBER+1) +THIS_LINE_NUMBER = 41 # Put here the actual number of this line + +from unittest import TestCase + +class Test(TestCase): + + def test_find_source_lines(self): + self.assertEqual(oinspect.find_source_lines(Test.test_find_source_lines), + THIS_LINE_NUMBER+6) # A couple of utilities to ensure these tests work the same from a source or a @@ -64,7 +67,7 @@ def test_find_file_decorated1(): @decorator def noop1(f): - def wrapper(): + def wrapper(*a, **kw): return f(*a, **kw) return wrapper @@ -124,49 +127,6 @@ def method(self, x, z=2): """Some method's docstring""" -class OldStyle: - """An old-style class for testing.""" - pass - - -def f(x, y=2, *a, **kw): - """A simple function.""" - - -def g(y, z=3, *a, **kw): - pass # no docstring - - -@register_line_magic -def lmagic(line): - "A line magic" - - -@register_cell_magic -def cmagic(line, cell): - "A cell magic" - - -@register_line_cell_magic -def lcmagic(line, cell=None): - "A line/cell magic" - - -@magics_class -class SimpleMagics(Magics): - @line_magic - def Clmagic(self, cline): - "A class-based line magic" - - @cell_magic - def Ccmagic(self, cline, ccell): - "A class-based cell magic" - - @line_cell_magic - def Clcmagic(self, cline, ccell=None): - "A class-based line/cell magic" - - class Awkward(object): def __getattr__(self, name): raise Exception(name) @@ -201,77 +161,17 @@ def __init__(self, max_fibbing_twig, lies_told=0): def __getattr__(self, item): return SerialLiar(self.max_fibbing_twig, self.lies_told + 1) - -def check_calltip(obj, name, call, docstring): - """Generic check pattern all calltip tests will use""" - info = inspector.info(obj, name) - call_line, ds = oinspect.call_tip(info) - nt.assert_equal(call_line, call) - nt.assert_equal(ds, docstring) - #----------------------------------------------------------------------------- # Tests #----------------------------------------------------------------------------- -def test_calltip_class(): - check_calltip(Call, 'Call', 'Call(x, y=1)', Call.__init__.__doc__) - - -def test_calltip_instance(): - c = Call(1) - check_calltip(c, 'c', 'c(*a, **kw)', c.__call__.__doc__) - - -def test_calltip_method(): - c = Call(1) - check_calltip(c.method, 'c.method', 'c.method(x, z=2)', c.method.__doc__) - - -def test_calltip_function(): - check_calltip(f, 'f', 'f(x, y=2, *a, **kw)', f.__doc__) - - -def test_calltip_function2(): - check_calltip(g, 'g', 'g(y, z=3, *a, **kw)', '') - - -@skipif(sys.version_info >= (3, 5)) -def test_calltip_builtin(): - check_calltip(sum, 'sum', None, sum.__doc__) - - -def test_calltip_line_magic(): - check_calltip(lmagic, 'lmagic', 'lmagic(line)', "A line magic") - - -def test_calltip_cell_magic(): - check_calltip(cmagic, 'cmagic', 'cmagic(line, cell)', "A cell magic") - - -def test_calltip_line_cell_magic(): - check_calltip(lcmagic, 'lcmagic', 'lcmagic(line, cell=None)', - "A line/cell magic") - - -def test_class_magics(): - cm = SimpleMagics(ip) - ip.register_magics(cm) - check_calltip(cm.Clmagic, 'Clmagic', 'Clmagic(cline)', - "A class-based line magic") - check_calltip(cm.Ccmagic, 'Ccmagic', 'Ccmagic(cline, ccell)', - "A class-based cell magic") - check_calltip(cm.Clcmagic, 'Clcmagic', 'Clcmagic(cline, ccell=None)', - "A class-based line/cell magic") - - def test_info(): "Check that Inspector.info fills out various fields as expected." i = inspector.info(Call, oname='Call') nt.assert_equal(i['type_name'], 'type') expted_class = str(type(type)) # (Python 3) or nt.assert_equal(i['base_class'], expted_class) - if sys.version_info > (3,): - nt.assert_regex(i['string_form'], "") + nt.assert_regex(i['string_form'], "") fname = __file__ if fname.endswith(".pyc"): fname = fname[:-1] @@ -282,8 +182,7 @@ def test_info(): nt.assert_equal(i['docstring'], Call.__doc__) nt.assert_equal(i['source'], None) nt.assert_true(i['isclass']) - _self_py2 = '' if py3compat.PY3 else 'self, ' - nt.assert_equal(i['init_definition'], "Call(%sx, y=1)" % _self_py2) + nt.assert_equal(i['init_definition'], "Call(x, y=1)") nt.assert_equal(i['init_docstring'], Call.__init__.__doc__) i = inspector.info(Call, detail_level=1) @@ -299,15 +198,6 @@ def test_info(): nt.assert_equal(i['init_docstring'], Call.__init__.__doc__) nt.assert_equal(i['call_docstring'], Call.__call__.__doc__) - # Test old-style classes, which for example may not have an __init__ method. - if not py3compat.PY3: - i = inspector.info(OldStyle) - nt.assert_equal(i['type_name'], 'classobj') - - i = inspector.info(OldStyle()) - nt.assert_equal(i['type_name'], 'instance') - nt.assert_equal(i['docstring'], OldStyle.__doc__) - def test_class_signature(): info = inspector.info(HasSignature, 'HasSignature') nt.assert_equal(info['init_definition'], "HasSignature(test)") @@ -322,23 +212,24 @@ def test_bool_raise(): def test_info_serialliar(): fib_tracker = [0] - i = inspector.info(SerialLiar(fib_tracker)) + inspector.info(SerialLiar(fib_tracker)) # Nested attribute access should be cut off at 100 levels deep to avoid # infinite loops: https://github.com/ipython/ipython/issues/9122 nt.assert_less(fib_tracker[0], 9000) +def support_function_one(x, y=2, *a, **kw): + """A simple function.""" + def test_calldef_none(): # We should ignore __call__ for all of these. - for obj in [f, SimpleClass().method, any, str.upper]: - print(obj) + for obj in [support_function_one, SimpleClass().method, any, str.upper]: i = inspector.info(obj) nt.assert_is(i['call_def'], None) -if py3compat.PY3: - exec("def f_kwarg(pos, *, kwonly): pass") +def f_kwarg(pos, *, kwonly): + pass -@skipif(not py3compat.PY3) def test_definition_kwonlyargs(): i = inspector.info(f_kwarg, oname='f_kwarg') # analysis:ignore nt.assert_equal(i['definition'], "f_kwarg(pos, *, kwonly)") @@ -373,8 +264,13 @@ def test_empty_property_has_no_source(): def test_property_sources(): - import zlib - + import posixpath + # A simple adder whose source and signature stays + # the same across Python distributions + def simple_add(a, b): + "Adds two numbers" + return a + b + class A(object): @property def foo(self): @@ -382,18 +278,18 @@ def foo(self): foo = foo.setter(lambda self, v: setattr(self, 'bar', v)) - id = property(id) - compress = property(zlib.compress) + dname = property(posixpath.dirname) + adder = property(simple_add) i = inspector.info(A.foo, detail_level=1) nt.assert_in('def foo(self):', i['source']) nt.assert_in('lambda self, v:', i['source']) - i = inspector.info(A.id, detail_level=1) - nt.assert_in('fget = ', i['source']) - - i = inspector.info(A.compress, detail_level=1) - nt.assert_in('fget = ', i['source']) + i = inspector.info(A.dname, detail_level=1) + nt.assert_in('def dirname(p)', i['source']) + + i = inspector.info(A.adder, detail_level=1) + nt.assert_in('def simple_add(a, b)', i['source']) def test_property_docstring_is_in_info_for_detail_level_0(): @@ -404,12 +300,12 @@ def foobar(self): pass ip.user_ns['a_obj'] = A() - nt.assert_equals( + nt.assert_equal( 'This is `foobar` property.', ip.object_inspect('a_obj.foobar', detail_level=0)['docstring']) ip.user_ns['a_cls'] = A - nt.assert_equals( + nt.assert_equal( 'This is `foobar` property.', ip.object_inspect('a_cls.foobar', detail_level=0)['docstring']) @@ -426,6 +322,60 @@ def test_pinfo_nonascii(): ip.user_ns['nonascii2'] = nonascii2 ip._inspect('pinfo', 'nonascii2', detail_level=1) +def test_pinfo_type(): + """ + type can fail in various edge case, for example `type.__subclass__()` + """ + ip._inspect('pinfo', 'type') + + +def test_pinfo_docstring_no_source(): + """Docstring should be included with detail_level=1 if there is no source""" + with AssertPrints('Docstring:'): + ip._inspect('pinfo', 'str.format', detail_level=0) + with AssertPrints('Docstring:'): + ip._inspect('pinfo', 'str.format', detail_level=1) + + +def test_pinfo_no_docstring_if_source(): + """Docstring should not be included with detail_level=1 if source is found""" + def foo(): + """foo has a docstring""" + + ip.user_ns['foo'] = foo + + with AssertPrints('Docstring:'): + ip._inspect('pinfo', 'foo', detail_level=0) + with AssertPrints('Source:'): + ip._inspect('pinfo', 'foo', detail_level=1) + with AssertNotPrints('Docstring:'): + ip._inspect('pinfo', 'foo', detail_level=1) + + +def test_pinfo_docstring_if_detail_and_no_source(): + """ Docstring should be displayed if source info not available """ + obj_def = '''class Foo(object): + """ This is a docstring for Foo """ + def bar(self): + """ This is a docstring for Foo.bar """ + pass + ''' + + ip.run_cell(obj_def) + ip.run_cell('foo = Foo()') + + with AssertNotPrints("Source:"): + with AssertPrints('Docstring:'): + ip._inspect('pinfo', 'foo', detail_level=0) + with AssertPrints('Docstring:'): + ip._inspect('pinfo', 'foo', detail_level=1) + with AssertPrints('Docstring:'): + ip._inspect('pinfo', 'foo.bar', detail_level=0) + + with AssertNotPrints('Docstring:'): + with AssertPrints('Source:'): + ip._inspect('pinfo', 'foo.bar', detail_level=1) + def test_pinfo_magic(): with AssertPrints('Docstring:'): @@ -445,8 +395,53 @@ def test_init_colors(): def test_builtin_init(): info = inspector.info(list) init_def = info['init_definition'] - # Python < 3.4 can't get init definition from builtins, - # but still exercise the inspection in case of error-raising bugs. - if sys.version_info >= (3,4): - nt.assert_is_not_none(init_def) - + nt.assert_is_not_none(init_def) + + +def test_render_signature_short(): + def short_fun(a=1): pass + sig = oinspect._render_signature( + signature(short_fun), + short_fun.__name__, + ) + nt.assert_equal(sig, 'short_fun(a=1)') + + +def test_render_signature_long(): + from typing import Optional + + def long_function( + a_really_long_parameter: int, + and_another_long_one: bool = False, + let_us_make_sure_this_is_looong: Optional[str] = None, + ) -> bool: pass + + sig = oinspect._render_signature( + signature(long_function), + long_function.__name__, + ) + nt.assert_in(sig, [ + # Python >=3.9 + '''\ +long_function( + a_really_long_parameter: int, + and_another_long_one: bool = False, + let_us_make_sure_this_is_looong: Optional[str] = None, +) -> bool\ +''', + # Python >=3.7 + '''\ +long_function( + a_really_long_parameter: int, + and_another_long_one: bool = False, + let_us_make_sure_this_is_looong: Union[str, NoneType] = None, +) -> bool\ +''', # Python <=3.6 + '''\ +long_function( + a_really_long_parameter:int, + and_another_long_one:bool=False, + let_us_make_sure_this_is_looong:Union[str, NoneType]=None, +) -> bool\ +''', + ]) diff --git a/IPython/core/tests/test_paths.py b/IPython/core/tests/test_paths.py index d0a87681d3b..ab1c4132a8e 100644 --- a/IPython/core/tests/test_paths.py +++ b/IPython/core/tests/test_paths.py @@ -4,11 +4,7 @@ import sys import tempfile import warnings - -try: # Python 3 - from unittest.mock import patch -except ImportError: # Python 2 - from mock import patch +from unittest.mock import patch import nose.tools as nt from testpath import modified_env, assert_isdir, assert_isfile @@ -17,13 +13,13 @@ from IPython.testing.decorators import skip_win32 from IPython.utils.tempdir import TemporaryDirectory -TMP_TEST_DIR = tempfile.mkdtemp() +TMP_TEST_DIR = os.path.realpath(tempfile.mkdtemp()) HOME_TEST_DIR = os.path.join(TMP_TEST_DIR, "home_test_dir") XDG_TEST_DIR = os.path.join(HOME_TEST_DIR, "xdg_test_dir") XDG_CACHE_DIR = os.path.join(HOME_TEST_DIR, "xdg_cache_dir") IP_TEST_DIR = os.path.join(HOME_TEST_DIR,'.ipython') -def setup(): +def setup_module(): """Setup testenvironment for the module: - Adds dummy home dir tree @@ -35,7 +31,7 @@ def setup(): os.makedirs(os.path.join(XDG_CACHE_DIR, 'ipython')) -def teardown(): +def teardown_module(): """Teardown testenvironment for the module: - Remove dummy home dir tree diff --git a/IPython/core/tests/test_prefilter.py b/IPython/core/tests/test_prefilter.py index 83d8e908427..ca447b3d0b7 100644 --- a/IPython/core/tests/test_prefilter.py +++ b/IPython/core/tests/test_prefilter.py @@ -6,12 +6,10 @@ import nose.tools as nt from IPython.core.prefilter import AutocallChecker -from IPython.testing.globalipapp import get_ipython #----------------------------------------------------------------------------- # Tests #----------------------------------------------------------------------------- -ip = get_ipython() def test_prefilter(): """Test user input conversions""" @@ -117,3 +115,13 @@ def __call__(self, x): finally: del ip.user_ns['x'] ip.magic('autocall 0') + + +def test_autocall_should_support_unicode(): + ip.magic('autocall 2') + ip.user_ns['π'] = lambda x: x + try: + nt.assert_equal(ip.prefilter('π 3'),'π(3)') + finally: + ip.magic('autocall 0') + del ip.user_ns['π'] diff --git a/IPython/core/tests/test_profile.py b/IPython/core/tests/test_profile.py index 4c938ed2d2f..e63fb3ef047 100644 --- a/IPython/core/tests/test_profile.py +++ b/IPython/core/tests/test_profile.py @@ -15,7 +15,6 @@ * MinRK """ -from __future__ import absolute_import #----------------------------------------------------------------------------- # Imports @@ -35,7 +34,6 @@ from IPython.testing import decorators as dec from IPython.testing import tools as tt -from IPython.utils import py3compat from IPython.utils.process import getoutput from IPython.utils.tempdir import TemporaryDirectory @@ -50,7 +48,7 @@ # Setup/teardown functions/decorators # -def setup(): +def setup_module(): """Setup test environment for the module: - Adds dummy home dir tree @@ -60,7 +58,7 @@ def setup(): os.makedirs(IP_TEST_DIR) -def teardown(): +def teardown_module(): """Teardown test environment for the module: - Remove dummy home dir tree @@ -101,15 +99,14 @@ def init(self, startup_file, startup, test): f.write(startup) # write simple test file, to check that the startup file was run with open(self.fname, 'w') as f: - f.write(py3compat.doctest_refactor_print(test)) + f.write(test) def validate(self, output): tt.ipexec_validate(self.fname, output, '', options=self.options) @dec.skipif(win32_without_pywin32(), "Test requires pywin32 on Windows") def test_startup_py(self): - self.init('00-start.py', 'zzz=123\n', - py3compat.doctest_refactor_print('print zzz\n')) + self.init('00-start.py', 'zzz=123\n', 'print(zzz)\n') self.validate('123') @dec.skipif(win32_without_pywin32(), "Test requires pywin32 on Windows") @@ -122,7 +119,6 @@ def test_list_profiles_in(): # No need to remove these directories and files, as they will get nuked in # the module-level teardown. td = tempfile.mkdtemp(dir=TMP_TEST_DIR) - td = py3compat.str_to_unicode(td) for name in ('profile_foo', 'profile_hello', 'not_a_profile'): os.mkdir(os.path.join(td, name)) if dec.unicode_paths: @@ -162,4 +158,4 @@ def test_profile_create_ipython_dir(): assert os.path.exists(profile_dir) ipython_config = os.path.join(profile_dir, 'ipython_config.py') assert os.path.exists(ipython_config) - \ No newline at end of file + diff --git a/IPython/core/tests/test_prompts.py b/IPython/core/tests/test_prompts.py index 6595daba6c3..95e6163b213 100644 --- a/IPython/core/tests/test_prompts.py +++ b/IPython/core/tests/test_prompts.py @@ -4,18 +4,12 @@ import unittest from IPython.core.prompts import LazyEvaluate -from IPython.testing.globalipapp import get_ipython -from IPython.utils.py3compat import unicode_type - -ip = get_ipython() - class PromptTests(unittest.TestCase): def test_lazy_eval_unicode(self): u = u'ünicødé' lz = LazyEvaluate(lambda : u) - # str(lz) would fail - self.assertEqual(unicode_type(lz), u) + self.assertEqual(str(lz), u) self.assertEqual(format(lz), u) def test_lazy_eval_nonascii_bytes(self): @@ -31,7 +25,6 @@ def test_lazy_eval_float(self): lz = LazyEvaluate(lambda : f) self.assertEqual(str(lz), str(f)) - self.assertEqual(unicode_type(lz), unicode_type(f)) self.assertEqual(format(lz), str(f)) self.assertEqual(format(lz, '.1'), '0.5') diff --git a/IPython/core/tests/test_pylabtools.py b/IPython/core/tests/test_pylabtools.py index aa9d16b90c9..7b64aab111a 100644 --- a/IPython/core/tests/test_pylabtools.py +++ b/IPython/core/tests/test_pylabtools.py @@ -4,7 +4,6 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. -from __future__ import print_function from io import UnsupportedOperation, BytesIO @@ -62,7 +61,7 @@ def test_figure_to_jpeg(): ax = fig.add_subplot(1,1,1) ax.plot([1,2,3]) plt.draw() - jpeg = pt.print_figure(fig, 'jpeg', quality=50)[:100].lower() + jpeg = pt.print_figure(fig, 'jpeg', pil_kwargs={'optimize': 50})[:100].lower() assert jpeg.startswith(_JPEG) def test_retina_figure(): @@ -106,7 +105,7 @@ def test_select_figure_formats_kwargs(): f = formatter.lookup_by_type(Figure) cell = f.__closure__[0].cell_contents nt.assert_equal(cell, kwargs) - + # check that the formatter doesn't raise fig = plt.figure() ax = fig.add_subplot(1,1,1) @@ -151,7 +150,7 @@ class TestPylabSwitch(object): class Shell(InteractiveShell): def enable_gui(self, gui): pass - + def setup(self): import matplotlib def act_mpl(backend): @@ -245,3 +244,13 @@ def test_qt_gtk(self): nt.assert_equal(gui, 'qt') nt.assert_equal(s.pylab_gui_select, 'qt') + +def test_no_gui_backends(): + for k in ['agg', 'svg', 'pdf', 'ps']: + assert k not in pt.backend2gui + + +def test_figure_no_canvas(): + fig = Figure() + fig.canvas = None + pt.print_figure(fig) diff --git a/IPython/core/tests/test_run.py b/IPython/core/tests/test_run.py index e3ade579548..eff832b3fc0 100644 --- a/IPython/core/tests/test_run.py +++ b/IPython/core/tests/test_run.py @@ -6,39 +6,35 @@ will be kept in this separate file. This makes it easier to aggregate in one place the tricks needed to handle it; most other magics are much easier to test and we do so in a common test_magic file. + +Note that any test using `run -i` should make sure to do a `reset` afterwards, +as otherwise it may influence later tests. """ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. -from __future__ import absolute_import import functools import os from os.path import join as pjoin import random +import string import sys -import tempfile import textwrap import unittest - -try: - from unittest.mock import patch -except ImportError: - from mock import patch +from unittest.mock import patch import nose.tools as nt from nose import SkipTest from IPython.testing import decorators as dec from IPython.testing import tools as tt -from IPython.utils import py3compat from IPython.utils.io import capture_output from IPython.utils.tempdir import TemporaryDirectory from IPython.core import debugger - def doctest_refbug(): """Very nasty problem with references held by multiple runs of a script. See: https://github.com/ipython/ipython/issues/141 @@ -147,13 +143,12 @@ def doctest_run_option_parser_for_windows(): """ -@py3compat.doctest_refactor_print def doctest_reset_del(): """Test that resetting doesn't cause errors in __del__ methods. In [2]: class A(object): ...: def __del__(self): - ...: print str("Hi") + ...: print(str("Hi")) ...: In [3]: a = A() @@ -170,9 +165,9 @@ def doctest_reset_del(): class TestMagicRunPass(tt.TempFileMixin): - def setup(self): - """Make a valid python temp file.""" - self.mktmp('pass\n') + def setUp(self): + content = "a = [1,2,3]\nb = 1" + self.mktmp(content) def run_tmpfile(self): _ip = get_ipython() @@ -209,9 +204,26 @@ def test_builtins_type(self): def test_run_profile( self ): """Test that the option -p, which invokes the profiler, do not crash by invoking execfile""" - _ip = get_ipython() self.run_tmpfile_p() + def test_run_debug_twice(self): + # https://github.com/ipython/ipython/issues/10028 + _ip = get_ipython() + with tt.fake_input(['c']): + _ip.magic('run -d %s' % self.fname) + with tt.fake_input(['c']): + _ip.magic('run -d %s' % self.fname) + + def test_run_debug_twice_with_breakpoint(self): + """Make a valid python temp file.""" + _ip = get_ipython() + with tt.fake_input(['b 2', 'c', 'c']): + _ip.magic('run -d %s' % self.fname) + + with tt.fake_input(['c']): + with tt.AssertNotPrints('KeyError'): + _ip.magic('run -d %s' % self.fname) + class TestMagicRunSimple(tt.TempFileMixin): @@ -233,9 +245,9 @@ def test_obj_del(self): raise SkipTest("Test requires pywin32") src = ("class A(object):\n" " def __del__(self):\n" - " print 'object A deleted'\n" + " print('object A deleted')\n" "a = A()\n") - self.mktmp(py3compat.doctest_refactor_print(src)) + self.mktmp(src) if dec.module_not_available('sqlite3'): err = 'WARNING: IPython History requires SQLite, your history will not be saved\n' else: @@ -305,13 +317,19 @@ def test_run_i_after_reset(self): src = "yy = zz\n" self.mktmp(src) _ip.run_cell("zz = 23") - _ip.magic('run -i %s' % self.fname) - nt.assert_equal(_ip.user_ns['yy'], 23) - _ip.magic('reset -f') + try: + _ip.magic('run -i %s' % self.fname) + nt.assert_equal(_ip.user_ns['yy'], 23) + finally: + _ip.magic('reset -f') + _ip.run_cell("zz = 23") - _ip.magic('run -i %s' % self.fname) - nt.assert_equal(_ip.user_ns['yy'], 23) - + try: + _ip.magic('run -i %s' % self.fname) + nt.assert_equal(_ip.user_ns['yy'], 23) + finally: + _ip.magic('reset -f') + def test_unicode(self): """Check that files in odd encodings are accepted.""" mydir = os.path.dirname(__file__) @@ -368,7 +386,6 @@ def test_ignore_sys_exit(self): with tt.AssertNotPrints('SystemExit'): _ip.magic('run -e %s' % self.fname) - @dec.skip_without('nbformat') # Requires jsonschema def test_run_nb(self): """Test %run notebook.ipynb""" from nbformat import v4, writes @@ -384,7 +401,33 @@ def test_run_nb(self): _ip.magic("run %s" % self.fname) nt.assert_equal(_ip.user_ns['answer'], 42) - + + def test_run_nb_error(self): + """Test %run notebook.ipynb error""" + from nbformat import v4, writes + # %run when a file name isn't provided + nt.assert_raises(Exception, _ip.magic, "run") + + # %run when a file doesn't exist + nt.assert_raises(Exception, _ip.magic, "run foobar.ipynb") + + # %run on a notebook with an error + nb = v4.new_notebook( + cells=[ + v4.new_code_cell("0/0") + ] + ) + src = writes(nb, version=4) + self.mktmp(src, ext='.ipynb') + nt.assert_raises(Exception, _ip.magic, "run %s" % self.fname) + + def test_file_options(self): + src = ('import sys\n' + 'a = " ".join(sys.argv[1:])\n') + self.mktmp(src) + test_opts = '-x 3 --verbose' + _ip.run_line_magic("run", '{0} {1}'.format(self.fname, test_opts)) + nt.assert_equal(_ip.user_ns['a'], test_opts) class TestMagicRunWithPackage(unittest.TestCase): @@ -398,13 +441,13 @@ def writefile(self, name, content): f.write(textwrap.dedent(content)) def setUp(self): - self.package = package = 'tmp{0}'.format(repr(random.random())[2:]) - """Temporary valid python package name.""" + self.package = package = 'tmp{0}'.format(''.join([random.choice(string.ascii_letters) for i in range(10)])) + """Temporary (probably) valid python package name.""" self.value = int(random.random() * 10000) self.tempdir = TemporaryDirectory() - self.__orig_cwd = py3compat.getcwd() + self.__orig_cwd = os.getcwd() sys.path.insert(0, self.tempdir.name) self.writefile(os.path.join(package, '__init__.py'), '') @@ -417,6 +460,10 @@ def setUp(self): self.writefile(os.path.join(package, 'absolute.py'), """ from {0}.sub import x """.format(package)) + self.writefile(os.path.join(package, 'args.py'), """ + import sys + a = " ".join(sys.argv[1:]) + """.format(package)) def tearDown(self): os.chdir(self.__orig_cwd) @@ -458,6 +505,18 @@ def test_debug_run_submodule_with_absolute_import(self): def test_debug_run_submodule_with_relative_import(self): self.check_run_submodule('relative', '-d') + def test_module_options(self): + _ip.user_ns.pop('a', None) + test_opts = '-x abc -m test' + _ip.run_line_magic('run', '-m {0}.args {1}'.format(self.package, test_opts)) + nt.assert_equal(_ip.user_ns['a'], test_opts) + + def test_module_options_with_separator(self): + _ip.user_ns.pop('a', None) + test_opts = '-x abc -m test' + _ip.run_line_magic('run', '-m {0}.args -- {1}'.format(self.package, test_opts)) + nt.assert_equal(_ip.user_ns['a'], test_opts) + def test_run__name__(): with TemporaryDirectory() as td: path = pjoin(td, 'foo.py') @@ -471,6 +530,13 @@ def test_run__name__(): _ip.magic('run -n {}'.format(path)) nt.assert_equal(_ip.user_ns.pop('q'), 'foo') + try: + _ip.magic('run -i -n {}'.format(path)) + nt.assert_equal(_ip.user_ns.pop('q'), 'foo') + finally: + _ip.magic('reset -f') + + def test_run_tb(): """Test traceback offset in %run""" with TemporaryDirectory() as td: @@ -489,6 +555,37 @@ def test_run_tb(): nt.assert_not_in("execfile", out) nt.assert_in("RuntimeError", out) nt.assert_equal(out.count("---->"), 3) + del ip.user_ns['bar'] + del ip.user_ns['foo'] + + +def test_multiprocessing_run(): + """Set we can run mutiprocesgin without messing up up main namespace + + Note that import `nose.tools as nt` mdify the value s + sys.module['__mp_main__'] so wee need to temporarily set it to None to test + the issue. + """ + with TemporaryDirectory() as td: + mpm = sys.modules.get('__mp_main__') + assert mpm is not None + sys.modules['__mp_main__'] = None + try: + path = pjoin(td, 'test.py') + with open(path, 'w') as f: + f.write("import multiprocessing\nprint('hoy')") + with capture_output() as io: + _ip.run_line_magic('run', path) + _ip.run_cell("i_m_undefined") + out = io.stdout + nt.assert_in("hoy", out) + nt.assert_not_in("AttributeError", out) + nt.assert_in("NameError", out) + nt.assert_equal(out.count("---->"), 1) + except: + raise + finally: + sys.modules['__mp_main__'] = mpm @dec.knownfailureif(sys.platform == 'win32', "writes to io.stdout aren't captured on Windows") def test_script_tb(): diff --git a/IPython/core/tests/test_shellapp.py b/IPython/core/tests/test_shellapp.py index 8e15743a1c4..6808114b82e 100644 --- a/IPython/core/tests/test_shellapp.py +++ b/IPython/core/tests/test_shellapp.py @@ -19,13 +19,12 @@ from IPython.testing import decorators as dec from IPython.testing import tools as tt -from IPython.utils.py3compat import PY3 sqlite_err_maybe = dec.module_not_available('sqlite3') SQLITE_NOT_AVAILABLE_ERROR = ('WARNING: IPython History requires SQLite,' ' your history will not be saved\n') -class TestFileToRun(unittest.TestCase, tt.TempFileMixin): +class TestFileToRun(tt.TempFileMixin, unittest.TestCase): """Test the behavior of the file_to_run parameter.""" def test_py_script_file_attribute(self): @@ -53,16 +52,9 @@ def test_py_script_file_attribute_interactively(self): self.mktmp(src) out, err = tt.ipexec(self.fname, options=['-i'], - commands=['"__file__" in globals()', 'exit()']) - self.assertIn("False", out) - - @dec.skip_win32 - @dec.skipif(PY3) - def test_py_script_file_compiler_directive(self): - """Test `__future__` compiler directives with `ipython -i file.py`""" - src = "from __future__ import division\n" - self.mktmp(src) - - out, err = tt.ipexec(self.fname, options=['-i'], - commands=['type(1/2)', 'exit()']) - self.assertIn('float', out) + commands=['"__file__" in globals()', 'print(123)', 'exit()']) + if 'False' not in out: + print("Subprocess stderr:") + print(err) + print('-----') + raise AssertionError("'False' not found in %r" % out) diff --git a/IPython/core/tests/test_splitinput.py b/IPython/core/tests/test_splitinput.py index c789e103c76..98b4189e5b4 100644 --- a/IPython/core/tests/test_splitinput.py +++ b/IPython/core/tests/test_splitinput.py @@ -3,7 +3,6 @@ from IPython.core.splitinput import split_user_input, LineInfo from IPython.testing import tools as tt -from IPython.utils import py3compat tests = [ ('x=1', ('', '', 'x', '=1')), @@ -28,10 +27,7 @@ ('??%%hist4', ('', '??', '%%hist4', '')), ('?x*', ('', '?', 'x*', '')), ] -if py3compat.PY3: - tests.append((u"Pérez Fernando", (u'', u'', u'Pérez', u'Fernando'))) -else: - tests.append((u"Pérez Fernando", (u'', u'', u'P', u'érez Fernando'))) +tests.append((u"Pérez Fernando", (u'', u'', u'Pérez', u'Fernando'))) def test_split_user_input(): return tt.check_pairs(split_user_input, tests) diff --git a/IPython/core/tests/test_ultratb.py b/IPython/core/tests/test_ultratb.py index 27b6fc43fd9..3751117b692 100644 --- a/IPython/core/tests/test_ultratb.py +++ b/IPython/core/tests/test_ultratb.py @@ -2,27 +2,22 @@ """Tests for IPython.core.ultratb """ import io +import logging import sys import os.path from textwrap import dedent import traceback import unittest +from unittest import mock -try: - from unittest import mock -except ImportError: - import mock # Python 2 - -from ..ultratb import ColorTB, VerboseTB, find_recursion +import IPython.core.ultratb as ultratb +from IPython.core.ultratb import ColorTB, VerboseTB, find_recursion from IPython.testing import tools as tt from IPython.testing.decorators import onlyif_unicode_paths from IPython.utils.syspathcontext import prepended_to_syspath from IPython.utils.tempdir import TemporaryDirectory -from IPython.utils.py3compat import PY3 - -ip = get_ipython() file_1 = """1 2 @@ -35,6 +30,30 @@ def f(): 1/0 """ + +def recursionlimit(frames): + """ + decorator to set the recursion limit temporarily + """ + + def inner(test_function): + def wrapper(*args, **kwargs): + _orig_rec_limit = ultratb._FRAME_RECURSION_LIMIT + ultratb._FRAME_RECURSION_LIMIT = 50 + + rl = sys.getrecursionlimit() + sys.setrecursionlimit(frames) + try: + return test_function(*args, **kwargs) + finally: + sys.setrecursionlimit(rl) + ultratb._FRAME_RECURSION_LIMIT = _orig_rec_limit + + return wrapper + + return inner + + class ChangedPyFileTest(unittest.TestCase): def test_changing_py_file(self): """Traceback produced if the line where the error occurred is missing? @@ -115,6 +134,12 @@ def test_nonascii_msg(self): with tt.AssertPrints(expected): ip.run_cell(cell) + ip.run_cell("%xmode minimal") + with tt.AssertPrints(u"Exception: é"): + ip.run_cell(cell) + + # Put this back into Context mode for later tests. + ip.run_cell("%xmode context") class NestedGenExprTestCase(unittest.TestCase): """ @@ -172,6 +197,35 @@ def test_syntaxerror_without_lineno(self): with tt.AssertPrints("line unknown"): ip.run_cell("raise SyntaxError()") + def test_syntaxerror_no_stacktrace_at_compile_time(self): + syntax_error_at_compile_time = """ +def foo(): + .. +""" + with tt.AssertPrints("SyntaxError"): + ip.run_cell(syntax_error_at_compile_time) + + with tt.AssertNotPrints("foo()"): + ip.run_cell(syntax_error_at_compile_time) + + def test_syntaxerror_stacktrace_when_running_compiled_code(self): + syntax_error_at_runtime = """ +def foo(): + eval("..") + +def bar(): + foo() + +bar() +""" + with tt.AssertPrints("SyntaxError"): + ip.run_cell(syntax_error_at_runtime) + # Assert syntax error during runtime generate stacktrace + with tt.AssertPrints(["foo()", "bar()"]): + ip.run_cell(syntax_error_at_runtime) + del ip.user_ns['bar'] + del ip.user_ns['foo'] + def test_changing_py_file(self): with TemporaryDirectory() as td: fname = os.path.join(td, "foo.py") @@ -198,6 +252,17 @@ def test_non_syntaxerror(self): with tt.AssertPrints('QWERTY'): ip.showsyntaxerror() +import sys +if sys.version_info < (3,9): + """ + New 3.9 Pgen Parser does not raise Memory error, except on failed malloc. + """ + class MemoryErrorTest(unittest.TestCase): + def test_memoryerror(self): + memoryerror_code = "(" * 200 + ")" * 200 + with tt.AssertPrints("MemoryError"): + ip.run_cell(memoryerror_code) + class Python3ChainedExceptionsTest(unittest.TestCase): DIRECT_CAUSE_ERROR_CODE = """ @@ -230,20 +295,36 @@ class Python3ChainedExceptionsTest(unittest.TestCase): """ def test_direct_cause_error(self): - if PY3: - with tt.AssertPrints(["KeyError", "NameError", "direct cause"]): - ip.run_cell(self.DIRECT_CAUSE_ERROR_CODE) + with tt.AssertPrints(["KeyError", "NameError", "direct cause"]): + ip.run_cell(self.DIRECT_CAUSE_ERROR_CODE) def test_exception_during_handling_error(self): - if PY3: - with tt.AssertPrints(["KeyError", "NameError", "During handling"]): - ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE) + with tt.AssertPrints(["KeyError", "NameError", "During handling"]): + ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE) def test_suppress_exception_chaining(self): - if PY3: - with tt.AssertNotPrints("ZeroDivisionError"), \ - tt.AssertPrints("ValueError", suppress=False): - ip.run_cell(self.SUPPRESS_CHAINING_CODE) + with tt.AssertNotPrints("ZeroDivisionError"), \ + tt.AssertPrints("ValueError", suppress=False): + ip.run_cell(self.SUPPRESS_CHAINING_CODE) + + def test_plain_direct_cause_error(self): + with tt.AssertPrints(["KeyError", "NameError", "direct cause"]): + ip.run_cell("%xmode Plain") + ip.run_cell(self.DIRECT_CAUSE_ERROR_CODE) + ip.run_cell("%xmode Verbose") + + def test_plain_exception_during_handling_error(self): + with tt.AssertPrints(["KeyError", "NameError", "During handling"]): + ip.run_cell("%xmode Plain") + ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE) + ip.run_cell("%xmode Verbose") + + def test_plain_suppress_exception_chaining(self): + with tt.AssertNotPrints("ZeroDivisionError"), \ + tt.AssertPrints("ValueError", suppress=False): + ip.run_cell("%xmode Plain") + ip.run_cell(self.SUPPRESS_CHAINING_CODE) + ip.run_cell("%xmode Verbose") class RecursionTest(unittest.TestCase): @@ -276,14 +357,17 @@ def test_no_recursion(self): with tt.AssertNotPrints("frames repeated"): ip.run_cell("non_recurs()") + @recursionlimit(150) def test_recursion_one_frame(self): with tt.AssertPrints("1 frames repeated"): ip.run_cell("r1()") + @recursionlimit(150) def test_recursion_three_frames(self): with tt.AssertPrints("3 frames repeated"): ip.run_cell("r3o2()") + @recursionlimit(150) def test_find_recursion(self): captured = [] def capture_exc(*args, **kwargs): @@ -313,44 +397,74 @@ def capture_exc(*args, **kwargs): #---------------------------------------------------------------------------- # module testing (minimal) -if sys.version_info > (3,): - def test_handlers(): - def spam(c, d_e): - (d, e) = d_e - x = c + d - y = c * d - foo(x, y) - - def foo(a, b, bar=1): - eggs(a, b + bar) - - def eggs(f, g, z=globals()): - h = f + g - i = f - g - return h / i - - buff = io.StringIO() - - buff.write('') - buff.write('*** Before ***') - try: - buff.write(spam(1, (2, 3))) - except: - traceback.print_exc(file=buff) - - handler = ColorTB(ostream=buff) - buff.write('*** ColorTB ***') - try: - buff.write(spam(1, (2, 3))) - except: - handler(*sys.exc_info()) - buff.write('') - - handler = VerboseTB(ostream=buff) - buff.write('*** VerboseTB ***') - try: - buff.write(spam(1, (2, 3))) - except: - handler(*sys.exc_info()) - buff.write('') +def test_handlers(): + def spam(c, d_e): + (d, e) = d_e + x = c + d + y = c * d + foo(x, y) + + def foo(a, b, bar=1): + eggs(a, b + bar) + + def eggs(f, g, z=globals()): + h = f + g + i = f - g + return h / i + + buff = io.StringIO() + + buff.write('') + buff.write('*** Before ***') + try: + buff.write(spam(1, (2, 3))) + except: + traceback.print_exc(file=buff) + + handler = ColorTB(ostream=buff) + buff.write('*** ColorTB ***') + try: + buff.write(spam(1, (2, 3))) + except: + handler(*sys.exc_info()) + buff.write('') + + handler = VerboseTB(ostream=buff) + buff.write('*** VerboseTB ***') + try: + buff.write(spam(1, (2, 3))) + except: + handler(*sys.exc_info()) + buff.write('') + +from IPython.testing.decorators import skipif + +class TokenizeFailureTest(unittest.TestCase): + """Tests related to https://github.com/ipython/ipython/issues/6864.""" + + # that appear to test that we are handling an exception that can be thrown + # by the tokenizer due to a bug that seem to have been fixed in 3.8, though + # I'm unsure if other sequences can make it raise this error. Let's just + # skip in 3.8 for now + @skipif(sys.version_info > (3,8)) + def testLogging(self): + message = "An unexpected error occurred while tokenizing input" + cell = 'raise ValueError("""a\nb""")' + + stream = io.StringIO() + handler = logging.StreamHandler(stream) + logger = logging.getLogger() + loglevel = logger.level + logger.addHandler(handler) + self.addCleanup(lambda: logger.removeHandler(handler)) + self.addCleanup(lambda: logger.setLevel(loglevel)) + + logger.setLevel(logging.INFO) + with tt.AssertNotPrints(message): + ip.run_cell(cell) + self.assertNotIn(message, stream.getvalue()) + logger.setLevel(logging.DEBUG) + with tt.AssertNotPrints(message): + ip.run_cell(cell) + self.assertIn(message, stream.getvalue()) diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index 2e4268f8955..45e22bd7b94 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -41,7 +41,7 @@ .. note:: The verbose mode print all variables in the stack, which means it can - potentially leak sensitive information like access keys, or unencryted + potentially leak sensitive information like access keys, or unencrypted password. Installation instructions for VerboseTB:: @@ -88,9 +88,6 @@ # the file COPYING, distributed as part of this software. #***************************************************************************** -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function import dis import inspect @@ -103,12 +100,8 @@ import time import tokenize import traceback -import types -try: # Python 2 - generate_tokens = tokenize.generate_tokens -except AttributeError: # Python 3 - generate_tokens = tokenize.tokenize +from tokenize import generate_tokens # For purposes of monkeypatching inspect to fix a bug in it. from inspect import getsourcefile, getfile, getmodule, \ @@ -120,13 +113,14 @@ from IPython.core.display_trap import DisplayTrap from IPython.core.excolors import exception_colors from IPython.utils import PyColorize -from IPython.utils import openpy from IPython.utils import path as util_path from IPython.utils import py3compat -from IPython.utils import ulinecache from IPython.utils.data import uniq_stable from IPython.utils.terminal import get_terminal_size -from logging import info, error + +from logging import info, error, debug + +from importlib.util import source_from_cache import IPython.utils.colorable as colorable @@ -140,6 +134,12 @@ # to users of ultratb who are NOT running inside ipython. DEFAULT_SCHEME = 'NoColor' + +# Number of frame above which we are likely to have a recursion and will +# **attempt** to detect it. Made modifiable mostly to speedup test suite +# as detecting recursion is one of our slowest test +_FRAME_RECURSION_LIMIT = 500 + # --------------------------------------------------------------------------- # Code begins @@ -193,11 +193,11 @@ def findsource(object): # use the one with the least indentation, which is the one # that's most probably not inside a function definition. candidates = [] - for i in range(len(lines)): - match = pat.match(lines[i]) + for i, line in enumerate(lines): + match = pat.match(line) if match: # if it's at toplevel, it's already the best one - if lines[i][0] == 'c': + if line[0] == 'c': return lines, i # else add whitespace to candidate list candidates.append((match.group(1), i)) @@ -302,7 +302,10 @@ def getargs(co): # Monkeypatch inspect to apply our bugfix. def with_patch_inspect(f): - """decorator for monkeypatching inspect.findsource""" + """ + Deprecated since IPython 6.0 + decorator for monkeypatching inspect.findsource + """ def wrapped(*args, **kwargs): save_findsource = inspect.findsource @@ -318,16 +321,6 @@ def wrapped(*args, **kwargs): return wrapped -if py3compat.PY3: - fixed_getargvalues = inspect.getargvalues -else: - # Fixes for https://github.com/ipython/ipython/issues/8293 - # and https://github.com/ipython/ipython/issues/8205. - # The relevant bug is caused by failure to correctly handle anonymous tuple - # unpacking, which only exists in Python 2. - fixed_getargvalues = with_patch_inspect(inspect.getargvalues) - - def fix_frame_records_filenames(records): """Try to fix the filenames in each record from inspect.getinnerframes(). @@ -339,7 +332,6 @@ def fix_frame_records_filenames(records): # Look inside the frame's globals dictionary for __file__, # which should be better. However, keep Cython filenames since # we prefer the source filenames over the compiled .so file. - filename = py3compat.cast_unicode_py2(filename, "utf-8") if not filename.endswith(('.pyx', '.pxd', '.pxi')): better_fn = frame.f_globals.get('__file__', None) if isinstance(better_fn, str): @@ -369,11 +361,11 @@ def _fixed_getinnerframes(etb, context=1, tb_offset=0): aux = traceback.extract_tb(etb) assert len(records) == len(aux) - for i, (file, lnum, _, _) in zip(range(len(records)), aux): + for i, (file, lnum, _, _) in enumerate(aux): maybeStart = lnum - 1 - context // 2 start = max(maybeStart, 0) end = start + context - lines = ulinecache.getlines(file)[start:end] + lines = linecache.getlines(file)[start:end] buf = list(records[i]) buf[LNUM_POS] = lnum buf[INDEX_POS] = lnum - 1 - start @@ -386,29 +378,33 @@ def _fixed_getinnerframes(etb, context=1, tb_offset=0): # can be recognized properly by ipython.el's py-traceback-line-re # (SyntaxErrors have to be treated specially because they have no traceback) -_parser = PyColorize.Parser() - -def _format_traceback_lines(lnum, index, lines, Colors, lvals=None, scheme=None): +def _format_traceback_lines(lnum, index, lines, Colors, lvals, _line_format): + """ + Format tracebacks lines with pointing arrow, leading numbers... + + Parameters + ========== + + lnum: int + index: int + lines: list[string] + Colors: + ColorScheme used. + lvals: bytes + Values of local variables, already colored, to inject just after the error line. + _line_format: f (str) -> (str, bool) + return (colorized version of str, failure to do so) + """ numbers_width = INDENT_SIZE - 1 res = [] - i = lnum - index - - # This lets us get fully syntax-highlighted tracebacks. - if scheme is None: - ipinst = get_ipython() - if ipinst is not None: - scheme = ipinst.colors - else: - scheme = DEFAULT_SCHEME - _line_format = _parser.format2 - - for line in lines: + for i,line in enumerate(lines, lnum-index): line = py3compat.cast_unicode(line) - new_line, err = _line_format(line, 'str', scheme) - if not err: line = new_line + new_line, err = _line_format(line, 'str') + if not err: + line = new_line if i == lnum: # This is the line with the error @@ -424,7 +420,6 @@ def _format_traceback_lines(lnum, index, lines, Colors, lvals=None, scheme=None) res.append(line) if lvals and i == lnum: res.append(lvals + '\n') - i = i + 1 return res def is_recursion_error(etype, value, records): @@ -439,7 +434,7 @@ def is_recursion_error(etype, value, records): # a recursion error. return (etype is recursion_error_type) \ and "recursion" in str(value).lower() \ - and len(records) > 500 + and len(records) > _FRAME_RECURSION_LIMIT def find_recursion(etype, value, records): """Identify the repeating stack frames from a RecursionError traceback @@ -532,6 +527,30 @@ def _set_ostream(self, val): ostream = property(_get_ostream, _set_ostream) + def get_parts_of_chained_exception(self, evalue): + def get_chained_exception(exception_value): + cause = getattr(exception_value, '__cause__', None) + if cause: + return cause + if getattr(exception_value, '__suppress_context__', False): + return None + return getattr(exception_value, '__context__', None) + + chained_evalue = get_chained_exception(evalue) + + if chained_evalue: + return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__ + + def prepare_chained_exception_message(self, cause): + direct_cause = "\nThe above exception was the direct cause of the following exception:\n" + exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n" + + if cause: + message = [[direct_cause]] + else: + message = [[exception_during_handling]] + return message + def set_colors(self, *args, **kw): """Shorthand access to the color table scheme selector method.""" @@ -596,16 +615,22 @@ class ListTB(TBTools): Because they are meant to be called without a full traceback (only a list), instances of this class can't call the interactive pdb debugger.""" - def __init__(self, color_scheme='NoColor', call_pdb=False, ostream=None, parent=None): + def __init__(self, color_scheme='NoColor', call_pdb=False, ostream=None, parent=None, config=None): TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb, - ostream=ostream, parent=parent) + ostream=ostream, parent=parent,config=config) def __call__(self, etype, value, elist): self.ostream.flush() self.ostream.write(self.text(etype, value, elist)) self.ostream.write('\n') - def structured_traceback(self, etype, value, elist, tb_offset=None, + def _extract_tb(self, tb): + if tb: + return traceback.extract_tb(tb) + else: + return None + + def structured_traceback(self, etype, evalue, etb=None, tb_offset=None, context=5): """Return a color formatted string with the traceback info. @@ -614,15 +639,16 @@ def structured_traceback(self, etype, value, elist, tb_offset=None, etype : exception type Type of the exception raised. - value : object + evalue : object Data stored in the exception - elist : list - List of frames, see class docstring for details. + etb : object + If list: List of frames, see class docstring for details. + If Traceback: Traceback of the exception. tb_offset : int, optional Number of frames in the traceback to skip. If not given, the - instance value is used (set in constructor). + instance evalue is used (set in constructor). context : int, optional Number of lines of context information to print. @@ -631,6 +657,19 @@ def structured_traceback(self, etype, value, elist, tb_offset=None, ------- String with formatted exception. """ + # This is a workaround to get chained_exc_ids in recursive calls + # etb should not be a tuple if structured_traceback is not recursive + if isinstance(etb, tuple): + etb, chained_exc_ids = etb + else: + chained_exc_ids = set() + + if isinstance(etb, list): + elist = etb + elif etb is not None: + elist = self._extract_tb(etb) + else: + elist = [] tb_offset = self.tb_offset if tb_offset is None else tb_offset Colors = self.Colors out_list = [] @@ -643,18 +682,24 @@ def structured_traceback(self, etype, value, elist, tb_offset=None, (Colors.normalEm, Colors.Normal) + '\n') out_list.extend(self._format_list(elist)) # The exception info should be a single entry in the list. - lines = ''.join(self._format_exception_only(etype, value)) + lines = ''.join(self._format_exception_only(etype, evalue)) out_list.append(lines) - # Note: this code originally read: - - ## for line in lines[:-1]: - ## out_list.append(" "+line) - ## out_list.append(lines[-1]) + exception = self.get_parts_of_chained_exception(evalue) - # This means it was indenting everything but the last line by a little - # bit. I've disabled this for now, but if we see ugliness somewhere we - # can restore it. + if exception and not id(exception[1]) in chained_exc_ids: + chained_exception_message = self.prepare_chained_exception_message( + evalue.__cause__)[0] + etype, evalue, etb = exception + # Trace exception to avoid infinite 'cause' loop + chained_exc_ids.add(id(exception[1])) + chained_exceptions_tb_offset = 0 + out_list = ( + self.structured_traceback( + etype, evalue, (etb, chained_exc_ids), + chained_exceptions_tb_offset, context) + + chained_exception_message + + out_list) return out_list @@ -675,9 +720,9 @@ def _format_list(self, extracted_list): list = [] for filename, lineno, name, line in extracted_list[:-1]: item = ' File %s"%s"%s, line %s%d%s, in %s%s%s\n' % \ - (Colors.filename, py3compat.cast_unicode_py2(filename, "utf-8"), Colors.Normal, + (Colors.filename, filename, Colors.Normal, Colors.lineno, lineno, Colors.Normal, - Colors.name, py3compat.cast_unicode_py2(name, "utf-8"), Colors.Normal) + Colors.name, name, Colors.Normal) if line: item += ' %s\n' % line.strip() list.append(item) @@ -685,9 +730,9 @@ def _format_list(self, extracted_list): filename, lineno, name, line = extracted_list[-1] item = '%s File %s"%s"%s, line %s%d%s, in %s%s%s%s\n' % \ (Colors.normalEm, - Colors.filenameEm, py3compat.cast_unicode_py2(filename, "utf-8"), Colors.normalEm, + Colors.filenameEm, filename, Colors.normalEm, Colors.linenoEm, lineno, Colors.normalEm, - Colors.nameEm, py3compat.cast_unicode_py2(name, "utf-8"), Colors.normalEm, + Colors.nameEm, name, Colors.normalEm, Colors.Normal) if line: item += '%s %s%s\n' % (Colors.line, line.strip(), @@ -721,7 +766,7 @@ def _format_exception_only(self, etype, value): if not value.filename: value.filename = "" if value.lineno: lineno = value.lineno - textline = ulinecache.getline(value.filename, value.lineno) + textline = linecache.getline(value.filename, value.lineno) else: lineno = 'unknown' textline = '' @@ -775,7 +820,7 @@ def get_exception_only(self, etype, value): etype : exception type value : exception value """ - return ListTB.structured_traceback(self, etype, value, []) + return ListTB.structured_traceback(self, etype, value) def show_exception_only(self, etype, evalue): """Only print the exception type and message, without a traceback. @@ -811,7 +856,8 @@ class VerboseTB(TBTools): def __init__(self, color_scheme='Linux', call_pdb=False, ostream=None, tb_offset=0, long_header=False, include_vars=True, - check_cache=None, debugger_cls = None): + check_cache=None, debugger_cls = None, + parent=None, config=None): """Specify traceback offset, headers and color scheme. Define how many frames to drop from the tracebacks. Calling it with @@ -819,7 +865,7 @@ def __init__(self, color_scheme='Linux', call_pdb=False, ostream=None, their own code at the top of the traceback (VerboseTB will first remove that frame before printing the traceback info).""" TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb, - ostream=ostream) + ostream=ostream, parent=parent, config=config) self.tb_offset = tb_offset self.long_header = long_header self.include_vars = include_vars @@ -833,13 +879,36 @@ def __init__(self, color_scheme='Linux', call_pdb=False, ostream=None, self.check_cache = check_cache self.debugger_cls = debugger_cls or debugger.Pdb + self.skip_hidden = True def format_records(self, records, last_unique, recursion_repeat): """Format the stack frames of the traceback""" frames = [] + + skipped = 0 for r in records[:last_unique+recursion_repeat+1]: - #print '*** record:',file,lnum,func,lines,index # dbg + if self.skip_hidden: + if r[0].f_locals.get("__tracebackhide__", 0): + skipped += 1 + continue + if skipped: + Colors = self.Colors # just a shorthand + quicker name lookup + ColorsNormal = Colors.Normal # used a lot + frames.append( + " %s[... skipping hidden %s frame]%s\n" + % (Colors.excName, skipped, ColorsNormal) + ) + skipped = 0 + frames.append(self.format_record(*r)) + + if skipped: + Colors = self.Colors # just a shorthand + quicker name lookup + ColorsNormal = Colors.Normal # used a lot + frames.append( + " %s[... skipping hidden %s frame]%s\n" + % (Colors.excName, skipped, ColorsNormal) + ) if recursion_repeat: frames.append('... last %d frames repeated, from the frame below ...\n' % recursion_repeat) @@ -865,13 +934,6 @@ def format_record(self, frame, file, lnum, func, lines, index): Colors.vName, ColorsNormal) tpl_name_val = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal) - tpl_line = '%s%%s%s %%s' % (Colors.lineno, ColorsNormal) - tpl_line_em = '%s%%s%s %%s%s' % (Colors.linenoEm, Colors.line, - ColorsNormal) - - abspath = os.path.abspath - - if not file: file = '?' elif file.startswith(str("<")) and file.endswith(str(">")): @@ -892,18 +954,20 @@ def format_record(self, frame, file, lnum, func, lines, index): pass file = py3compat.cast_unicode(file, util_path.fs_encoding) - link = tpl_link % file - args, varargs, varkw, locals = fixed_getargvalues(frame) + link = tpl_link % util_path.compress_user(file) + args, varargs, varkw, locals_ = inspect.getargvalues(frame) if func == '?': call = '' + elif func == '': + call = tpl_call % (func, '') else: # Decide whether to include variable details or not - var_repr = self.include_vars and eqrepr or nullrepr + var_repr = eqrepr if self.include_vars else nullrepr try: call = tpl_call % (func, inspect.formatargvalues(args, varargs, varkw, - locals, formatvalue=var_repr)) + locals_, formatvalue=var_repr)) except KeyError: # This happens in situations like errors inside generator # expressions, where local variables are listed in the @@ -930,13 +994,13 @@ def format_record(self, frame, file, lnum, func, lines, index): elif file.endswith(('.pyc', '.pyo')): # Look up the corresponding source file. try: - file = openpy.source_from_cache(file) + file = source_from_cache(file) except ValueError: # Failed to get the source file for some reason # E.g. https://github.com/ipython/ipython/issues/9486 return '%s %s\n' % (link, call) - def linereader(file=file, lnum=[lnum], getline=ulinecache.getline): + def linereader(file=file, lnum=[lnum], getline=linecache.getline): line = getline(file, lnum[0]) lnum[0] += 1 return line @@ -976,10 +1040,15 @@ def linereader(file=file, lnum=[lnum], getline=ulinecache.getline): # - see gh-6300 pass except tokenize.TokenError as msg: + # Tokenizing may fail for various reasons, many of which are + # harmless. (A good example is when the line in question is the + # close of a triple-quoted string, cf gh-6864). We don't want to + # show this to users, but want make it available for debugging + # purposes. _m = ("An unexpected error occurred while tokenizing input\n" "The following traceback may be corrupted or invalid\n" "The error message is: %s\n" % msg) - error(_m) + debug(_m) # Join composite names (e.g. "dict.fromkeys") names = ['.'.join(n) for n in names] @@ -987,14 +1056,15 @@ def linereader(file=file, lnum=[lnum], getline=ulinecache.getline): unique_names = uniq_stable(names) # Start loop over vars - lvals = [] + lvals = '' + lvals_list = [] if self.include_vars: for name_full in unique_names: name_base = name_full.split('.', 1)[0] if name_base in frame.f_code.co_varnames: - if name_base in locals: + if name_base in locals_: try: - value = repr(eval(name_full, locals)) + value = repr(eval(name_full, locals_)) except: value = undefined else: @@ -1009,30 +1079,19 @@ def linereader(file=file, lnum=[lnum], getline=ulinecache.getline): else: value = undefined name = tpl_global_var % name_full - lvals.append(tpl_name_val % (name, value)) - if lvals: - lvals = '%s%s' % (indent, em_normal.join(lvals)) - else: - lvals = '' + lvals_list.append(tpl_name_val % (name, value)) + if lvals_list: + lvals = '%s%s' % (indent, em_normal.join(lvals_list)) level = '%s %s\n' % (link, call) if index is None: return level else: + _line_format = PyColorize.Parser(style=col_scheme, parent=self).format2 return '%s%s' % (level, ''.join( _format_traceback_lines(lnum, index, lines, Colors, lvals, - col_scheme))) - - def prepare_chained_exception_message(self, cause): - direct_cause = "\nThe above exception was the direct cause of the following exception:\n" - exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n" - - if cause: - message = [[direct_cause]] - else: - message = [[exception_during_handling]] - return message + _line_format))) def prepare_header(self, etype, long_version=False): colors = self.Colors # just a shorthand + quicker name lookup @@ -1059,7 +1118,6 @@ def prepare_header(self, etype, long_version=False): def format_exception(self, etype, evalue): colors = self.Colors # just a shorthand + quicker name lookup colorsnormal = colors.Normal # used a lot - indent = ' ' * INDENT_SIZE # Get (safely) a string form of the exception info try: etype_str, evalue_str = map(str, (etype, evalue)) @@ -1068,27 +1126,8 @@ def format_exception(self, etype, evalue): etype, evalue = str, sys.exc_info()[:2] etype_str, evalue_str = map(str, (etype, evalue)) # ... and format it - exception = ['%s%s%s: %s' % (colors.excName, etype_str, - colorsnormal, py3compat.cast_unicode(evalue_str))] - - if (not py3compat.PY3) and type(evalue) is types.InstanceType: - try: - names = [w for w in dir(evalue) if isinstance(w, py3compat.string_types)] - except: - # Every now and then, an object with funny internals blows up - # when dir() is called on it. We do the best we can to report - # the problem and continue - _m = '%sException reporting error (object with broken dir())%s:' - exception.append(_m % (colors.excName, colorsnormal)) - etype_str, evalue_str = map(str, sys.exc_info()[:2]) - exception.append('%s%s%s: %s' % (colors.excName, etype_str, - colorsnormal, py3compat.cast_unicode(evalue_str))) - names = [] - for name in names: - value = text_repr(getattr(evalue, name)) - exception.append('\n%s%s = %s' % (indent, name, value)) - - return exception + return ['%s%s%s: %s' % (colors.excName, etype_str, + colorsnormal, py3compat.cast_unicode(evalue_str))] def format_exception_as_a_whole(self, etype, evalue, etb, number_of_lines_of_context, tb_offset): """Formats the header, traceback and exception message for a single exception. @@ -1107,8 +1146,6 @@ def format_exception_as_a_whole(self, etype, evalue, etb, number_of_lines_of_con head = self.prepare_header(etype, self.long_header) records = self.get_records(etb, number_of_lines_of_context, tb_offset) - if records is None: - return "" last_unique, recursion_repeat = find_recursion(orig_etype, evalue, records) @@ -1130,6 +1167,12 @@ def get_records(self, etb, number_of_lines_of_context, tb_offset): # problems, but it generates empty tracebacks for console errors # (5 blanks lines) where none should be returned. return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset) + except UnicodeDecodeError: + # This can occur if a file's encoding magic comment is wrong. + # I can't see a way to recover without duplicating a bunch of code + # from the stdlib traceback module. --TK + error('\nUnicodeDecodeError while processing traceback.\n') + return None except: # FIXME: I've been getting many crash reports from python 2.3 # users, traceable to inspect.py. If I can find a small test-case @@ -1142,20 +1185,6 @@ def get_records(self, etb, number_of_lines_of_context, tb_offset): info('\nUnfortunately, your original traceback can not be constructed.\n') return None - def get_parts_of_chained_exception(self, evalue): - def get_chained_exception(exception_value): - cause = getattr(exception_value, '__cause__', None) - if cause: - return cause - if getattr(exception_value, '__suppress_context__', False): - return None - return getattr(exception_value, '__context__', None) - - chained_evalue = get_chained_exception(evalue) - - if chained_evalue: - return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__ - def structured_traceback(self, etype, evalue, etb, tb_offset=None, number_of_lines_of_context=5): """Return a nice text document describing the traceback.""" @@ -1167,35 +1196,32 @@ def structured_traceback(self, etype, evalue, etb, tb_offset=None, colorsnormal = colors.Normal # used a lot head = '%s%s%s' % (colors.topline, '-' * min(75, get_terminal_size()[0]), colorsnormal) structured_traceback_parts = [head] - if py3compat.PY3: - chained_exceptions_tb_offset = 0 - lines_of_context = 3 - formatted_exceptions = formatted_exception + chained_exceptions_tb_offset = 0 + lines_of_context = 3 + formatted_exceptions = formatted_exception + exception = self.get_parts_of_chained_exception(evalue) + if exception: + formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__) + etype, evalue, etb = exception + else: + evalue = None + chained_exc_ids = set() + while evalue: + formatted_exceptions += self.format_exception_as_a_whole(etype, evalue, etb, lines_of_context, + chained_exceptions_tb_offset) exception = self.get_parts_of_chained_exception(evalue) - if exception: + + if exception and not id(exception[1]) in chained_exc_ids: + chained_exc_ids.add(id(exception[1])) # trace exception to avoid infinite 'cause' loop formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__) etype, evalue, etb = exception else: evalue = None - chained_exc_ids = set() - while evalue: - formatted_exceptions += self.format_exception_as_a_whole(etype, evalue, etb, lines_of_context, - chained_exceptions_tb_offset) - exception = self.get_parts_of_chained_exception(evalue) - - if exception and not id(exception[1]) in chained_exc_ids: - chained_exc_ids.add(id(exception[1])) # trace exception to avoid infinite 'cause' loop - formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__) - etype, evalue, etb = exception - else: - evalue = None - # we want to see exceptions in a reversed order: - # the first exception should be on top - for formatted_exception in reversed(formatted_exceptions): - structured_traceback_parts += formatted_exception - else: - structured_traceback_parts += formatted_exception[0] + # we want to see exceptions in a reversed order: + # the first exception should be on top + for formatted_exception in reversed(formatted_exceptions): + structured_traceback_parts += formatted_exception return structured_traceback_parts @@ -1221,8 +1247,7 @@ def debugger(self, force=False): if force or self.call_pdb: if self.pdb is None: - self.pdb = self.debugger_cls( - self.color_scheme_table.active_scheme_name) + self.pdb = self.debugger_cls() # the system displayhook may have changed, restore the original # for pdb display_trap = DisplayTrap(hook=sys.__displayhook__) @@ -1238,7 +1263,7 @@ def debugger(self, force=False): if etb and etb.tb_next: etb = etb.tb_next self.pdb.botframe = etb.tb_frame - self.pdb.interaction(self.tb.tb_frame, self.tb) + self.pdb.interaction(None, etb) if hasattr(self, 'tb'): del self.tb @@ -1282,29 +1307,26 @@ class FormattedTB(VerboseTB, ListTB): def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False, ostream=None, tb_offset=0, long_header=False, include_vars=False, - check_cache=None, debugger_cls=None): + check_cache=None, debugger_cls=None, + parent=None, config=None): # NEVER change the order of this list. Put new modes at the end: - self.valid_modes = ['Plain', 'Context', 'Verbose'] + self.valid_modes = ['Plain', 'Context', 'Verbose', 'Minimal'] self.verbose_modes = self.valid_modes[1:3] VerboseTB.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb, ostream=ostream, tb_offset=tb_offset, long_header=long_header, include_vars=include_vars, - check_cache=check_cache, debugger_cls=debugger_cls) + check_cache=check_cache, debugger_cls=debugger_cls, + parent=parent, config=config) # Different types of tracebacks are joined with different separators to # form a single string. They are taken from this dict - self._join_chars = dict(Plain='', Context='\n', Verbose='\n') + self._join_chars = dict(Plain='', Context='\n', Verbose='\n', + Minimal='') # set_mode also sets the tb_join_char attribute self.set_mode(mode) - def _extract_tb(self, tb): - if tb: - return traceback.extract_tb(tb) - else: - return None - def structured_traceback(self, etype, value, tb, tb_offset=None, number_of_lines_of_context=5): tb_offset = self.tb_offset if tb_offset is None else tb_offset mode = self.mode @@ -1313,14 +1335,15 @@ def structured_traceback(self, etype, value, tb, tb_offset=None, number_of_lines return VerboseTB.structured_traceback( self, etype, value, tb, tb_offset, number_of_lines_of_context ) + elif mode == 'Minimal': + return ListTB.get_exception_only(self, etype, value) else: # We must check the source cache because otherwise we can print # out-of-date source code. self.check_cache() # Now we can extract and format the exception - elist = self._extract_tb(tb) return ListTB.structured_traceback( - self, etype, value, elist, tb_offset, number_of_lines_of_context + self, etype, value, tb, tb_offset, number_of_lines_of_context ) def stb2text(self, stb): @@ -1357,6 +1380,9 @@ def context(self): def verbose(self): self.set_mode(self.valid_modes[2]) + def minimal(self): + self.set_mode(self.valid_modes[3]) + #---------------------------------------------------------------------------- class AutoFormattedTB(FormattedTB): @@ -1401,7 +1427,11 @@ def structured_traceback(self, etype=None, value=None, tb=None, tb_offset=None, number_of_lines_of_context=5): if etype is None: etype, value, tb = sys.exc_info() - self.tb = tb + if isinstance(tb, tuple): + # tb is a tuple if this is a chained exception. + self.tb = tb[0] + else: + self.tb = tb return FormattedTB.structured_traceback( self, etype, value, tb, tb_offset, number_of_lines_of_context) @@ -1420,8 +1450,8 @@ def __init__(self, color_scheme='Linux', call_pdb=0, **kwargs): class SyntaxTB(ListTB): """Extension which holds some state: the last exception value""" - def __init__(self, color_scheme='NoColor'): - ListTB.__init__(self, color_scheme) + def __init__(self, color_scheme='NoColor', parent=None, config=None): + ListTB.__init__(self, color_scheme, parent=parent, config=config) self.last_syntax_error = None def __call__(self, etype, value, elist): @@ -1435,10 +1465,10 @@ def structured_traceback(self, etype, value, elist, tb_offset=None, # be wrong (retrieved from an outdated cache). This replaces it with # the current value. if isinstance(value, SyntaxError) \ - and isinstance(value.filename, py3compat.string_types) \ + and isinstance(value.filename, str) \ and isinstance(value.lineno, int): linecache.checkcache(value.filename) - newtext = ulinecache.getline(value.filename, value.lineno) + newtext = linecache.getline(value.filename, value.lineno) if newtext: value.text = newtext self.last_syntax_error = value diff --git a/IPython/core/usage.py b/IPython/core/usage.py index a3d17ddda04..37024c44567 100644 --- a/IPython/core/usage.py +++ b/IPython/core/usage.py @@ -60,28 +60,39 @@ environment variable with this name and setting it to the desired path. For more information, see the manual available in HTML and PDF in your - installation, or online at http://ipython.org/documentation.html. + installation, or online at https://ipython.org/documentation.html. """ interactive_usage = """ IPython -- An enhanced Interactive Python ========================================= -IPython offers a combination of convenient shell features, special commands -and a history mechanism for both input (command history) and output (results -caching, similar to Mathematica). It is intended to be a fully compatible -replacement for the standard Python interpreter, while offering vastly -improved functionality and flexibility. +IPython offers a fully compatible replacement for the standard Python +interpreter, with convenient shell features, special commands, command +history mechanism and output results caching. At your system command line, type 'ipython -h' to see the command line options available. This document only describes interactive features. +GETTING HELP +------------ + +Within IPython you have various way to access help: + + ? -> Introduction and overview of IPython's features (this screen). + object? -> Details about 'object'. + object?? -> More detailed, verbose information about 'object'. + %quickref -> Quick reference of all IPython specific syntax and magics. + help -> Access Python's own help system. + +If you are in terminal IPython you can quit this screen by pressing `q`. + + MAIN FEATURES ------------- -* Access to the standard Python help. As of Python 2.1, a help system is - available with access to object docstrings and the Python manuals. Simply - type 'help' (no quotes) to access it. +* Access to the standard Python help with object docstrings and the Python + manuals. Simply type 'help' (no quotes) to invoke it. * Magic commands: type %magic for information on the magic subsystem. @@ -89,13 +100,12 @@ * Dynamic object information: - Typing ?word or word? prints detailed information about an object. If - certain strings in the object are too long (docstrings, code, etc.) they get - snipped in the center for brevity. + Typing ?word or word? prints detailed information about an object. Certain + long strings (code, etc.) get snipped in the center for brevity. Typing ??word or word?? gives access to the full information without - snipping long strings. Long strings are sent to the screen through the less - pager if longer than the screen, printed otherwise. + snipping long strings. Strings that are longer than the screen are printed + through the less pager. The ?/?? system gives access to the full source code for any object (if available), shows function prototypes and other useful information. @@ -103,18 +113,16 @@ If you just want to see an object's docstring, type '%pdoc object' (without quotes, and without % if you have automagic on). -* Completion in the local namespace, by typing TAB at the prompt. +* Tab completion in the local namespace: At any time, hitting tab will complete any available python commands or variable names, and show you a list of the possible completions if there's no unambiguous one. It will also complete filenames in the current directory. -* Search previous command history in two ways: +* Search previous command history in multiple ways: - - Start typing, and then use Ctrl-p (previous, up) and Ctrl-n (next,down) to - search through only the history items that match what you've typed so - far. If you use Ctrl-p/Ctrl-n at a blank prompt, they just behave like - normal arrow keys. + - Start typing, and then use arrow keys up/down or (Ctrl-p/Ctrl-n) to search + through the history items that match what you've typed so far. - Hit Ctrl-r: opens a search prompt. Begin typing and the system searches your history for lines that match what you've typed so far, completing as @@ -126,7 +134,7 @@ * Logging of input with the ability to save and restore a working session. -* System escape with !. Typing !ls will run 'ls' in the current directory. +* System shell with !. Typing !ls will run 'ls' in the current directory. * The reload command does a 'deep' reload of a module: changes made to the module since you imported will actually be available without having to exit. @@ -290,7 +298,7 @@ !cp a.txt b/ : System command escape, calls os.system() cp a.txt b/ : after %rehashx, most system commands work without ! cp ${f}.txt $bar : Variable expansion in magics and system commands -files = !ls /usr : Capture sytem command output +files = !ls /usr : Capture system command output files.s, files.l, files.n: "a b c", ['a','b','c'], 'a\nb\nc' History: @@ -325,27 +333,9 @@ """ -quick_guide = """\ -? -> Introduction and overview of IPython's features. -%quickref -> Quick reference. -help -> Python's own help system. -object? -> Details about 'object', use 'object??' for extra details. -""" - -default_banner_parts = [ - 'Python %s\n' % (sys.version.split('\n')[0],), - 'Type "copyright", "credits" or "license" for more information.\n\n', - 'IPython {version} -- An enhanced Interactive Python.\n'.format( - version=release.version, - ), - quick_guide +default_banner_parts = ["Python %s\n"%sys.version.split("\n")[0], + "Type 'copyright', 'credits' or 'license' for more information\n" , + "IPython {version} -- An enhanced Interactive Python. Type '?' for help.\n".format(version=release.version), ] default_banner = ''.join(default_banner_parts) - -# deprecated GUI banner - -default_gui_banner = '\n'.join([ - 'DEPRECATED: IPython.core.usage.default_gui_banner is deprecated and will be removed', - default_banner, -]) diff --git a/IPython/extensions/autoreload.py b/IPython/extensions/autoreload.py index 60d95725791..ada680fcf08 100644 --- a/IPython/extensions/autoreload.py +++ b/IPython/extensions/autoreload.py @@ -56,6 +56,10 @@ Import module 'foo' and mark it to be autoreloaded for ``%autoreload 1`` +``%aimport foo, bar`` + + Import modules 'foo', 'bar' and mark them to be autoreloaded for ``%autoreload 1`` + ``%aimport -foo`` Mark module 'foo' to not be autoreloaded. @@ -87,7 +91,6 @@ - C extension modules cannot be reloaded, and so cannot be autoreloaded. """ -from __future__ import print_function skip_doctest = True @@ -112,15 +115,10 @@ import traceback import types import weakref - -try: - # Reload is not defined by default in Python3. - reload -except NameError: - from imp import reload - -from IPython.utils import openpy -from IPython.utils.py3compat import PY3 +import gc +from importlib import import_module +from importlib.util import source_from_cache +from imp import reload #------------------------------------------------------------------------------ # Autoreload functionality @@ -177,7 +175,7 @@ def aimport_module(self, module_name): """ self.mark_module_reloadable(module_name) - __import__(module_name) + import_module(module_name) top_name = module_name.split('.')[0] top_module = sys.modules[top_name] return top_module, top_name @@ -186,8 +184,8 @@ def filename_and_mtime(self, module): if not hasattr(module, '__file__') or module.__file__ is None: return None, None - if getattr(module, '__name__', None) == '__main__': - # we cannot reload(__main__) + if getattr(module, '__name__', None) in [None, '__mp_main__', '__main__']: + # we cannot reload(__main__) or reload(__mp_main__) return None, None filename = module.__file__ @@ -197,7 +195,7 @@ def filename_and_mtime(self, module): py_filename = filename else: try: - py_filename = openpy.source_from_cache(filename) + py_filename = source_from_cache(filename) except ValueError: return None, None @@ -249,19 +247,16 @@ def check(self, check_all=False, do_reload=True): del self.failed[py_filename] except: print("[autoreload of %s failed: %s]" % ( - modname, traceback.format_exc(1)), file=sys.stderr) + modname, traceback.format_exc(10)), file=sys.stderr) self.failed[py_filename] = pymtime #------------------------------------------------------------------------------ # superreload #------------------------------------------------------------------------------ -if PY3: - func_attrs = ['__code__', '__defaults__', '__doc__', - '__closure__', '__globals__', '__dict__'] -else: - func_attrs = ['func_code', 'func_defaults', 'func_doc', - 'func_closure', 'func_globals', 'func_dict'] + +func_attrs = ['__code__', '__defaults__', '__doc__', + '__closure__', '__globals__', '__dict__'] def update_function(old, new): @@ -273,14 +268,29 @@ def update_function(old, new): pass +def update_instances(old, new): + """Use garbage collector to find all instances that refer to the old + class definition and update their __class__ to point to the new class + definition""" + + refs = gc.get_referrers(old) + + for ref in refs: + if type(ref) is old: + ref.__class__ = new + + def update_class(old, new): """Replace stuff in the __dict__ of a class, and upgrade - method code objects""" + method code objects, and add new methods, if any""" for key in list(old.__dict__.keys()): old_obj = getattr(old, key) - try: new_obj = getattr(new, key) + # explicitly checking that comparison returns True to handle + # cases where `==` doesn't return a boolean. + if (old_obj == new_obj) is True: + continue except AttributeError: # obsolete attribute: remove it try: @@ -296,6 +306,16 @@ def update_class(old, new): except (AttributeError, TypeError): pass # skip non-writable attributes + for key in list(new.__dict__.keys()): + if key not in list(old.__dict__.keys()): + try: + setattr(old, key, getattr(new, key)) + except (AttributeError, TypeError): + pass # skip non-writable attributes + + # update all instances of class + update_instances(old, new) + def update_property(old, new): """Replace get/set/del functions of a property""" @@ -316,18 +336,9 @@ def isinstance2(a, b, typ): (lambda a, b: isinstance2(a, b, property), update_property), ] - - -if PY3: - UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType), - lambda a, b: update_function(a.__func__, b.__func__)), - ]) -else: - UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.ClassType), - update_class), - (lambda a, b: isinstance2(a, b, types.MethodType), - lambda a, b: update_function(a.__func__, b.__func__)), - ]) +UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType), + lambda a, b: update_function(a.__func__, b.__func__)), +]) def update_generic(a, b): @@ -345,7 +356,7 @@ def __call__(self): return self.obj -def superreload(module, reload=reload, old_objects={}): +def superreload(module, reload=reload, old_objects=None): """Enhanced version of the builtin reload function. superreload remembers objects previously in the module, and @@ -355,6 +366,8 @@ def superreload(module, reload=reload, old_objects={}): - clears the module's namespace before reloading """ + if old_objects is None: + old_objects = {} # collect old objects in the module for name, obj in list(module.__dict__.items()): @@ -364,10 +377,7 @@ def superreload(module, reload=reload, old_objects={}): try: old_objects.setdefault(key, []).append(weakref.ref(obj)) except TypeError: - # weakref doesn't work for all types; - # create strong references for 'important' cases - if not PY3 and isinstance(obj, types.ClassType): - old_objects.setdefault(key, []).append(StrongRef(obj)) + pass # reload module try: @@ -486,6 +496,9 @@ def aimport(self, parameter_s='', stream=None): %aimport foo Import module 'foo' and mark it to be autoreloaded for %autoreload 1 + %aimport foo, bar + Import modules 'foo', 'bar' and mark them to be autoreloaded for %autoreload 1 + %aimport -foo Mark module 'foo' to not be autoreloaded for %autoreload 1 """ @@ -504,10 +517,11 @@ def aimport(self, parameter_s='', stream=None): modname = modname[1:] self._reloader.mark_module_skipped(modname) else: - top_module, top_name = self._reloader.aimport_module(modname) + for _module in ([_.strip() for _ in modname.split(',')]): + top_module, top_name = self._reloader.aimport_module(_module) - # Inject module to user namespace - self.shell.push({top_name: top_module}) + # Inject module to user namespace + self.shell.push({top_name: top_module}) def pre_run_cell(self): if self._reloader.enabled: diff --git a/IPython/extensions/storemagic.py b/IPython/extensions/storemagic.py index 2fd1abf993b..51b79ad314e 100644 --- a/IPython/extensions/storemagic.py +++ b/IPython/extensions/storemagic.py @@ -9,7 +9,6 @@ c.StoreMagics.autorestore = True """ -from __future__ import print_function # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. @@ -19,15 +18,17 @@ from IPython.core.error import UsageError from IPython.core.magic import Magics, magics_class, line_magic from traitlets import Bool -from IPython.utils.py3compat import string_types -def restore_aliases(ip): +def restore_aliases(ip, alias=None): staliases = ip.db.get('stored_aliases', {}) - for k,v in staliases.items(): - #print "restore alias",k,v # dbg - #self.alias_table[k] = v - ip.alias_manager.define_alias(k,v) + if alias is None: + for k,v in staliases.items(): + #print "restore alias",k,v # dbg + #self.alias_table[k] = v + ip.alias_manager.define_alias(k,v) + else: + ip.alias_manager.define_alias(alias, staliases[alias]) def refresh_variables(ip): @@ -60,13 +61,13 @@ class StoreMagics(Magics): """Lightweight persistence for python variables. Provides the %store magic.""" - + autorestore = Bool(False, help= """If True, any %store-d variables will be automatically restored when IPython starts. """ ).tag(config=True) - + def __init__(self, shell): super(StoreMagics, self).__init__(shell=shell) self.shell.configurables.append(self) @@ -96,13 +97,13 @@ def store(self, parameter_s=''): * ``%store`` - Show list of all variables and their current values - * ``%store spam`` - Store the *current* value of the variable spam - to disk + * ``%store spam bar`` - Store the *current* value of the variables spam + and bar to disk * ``%store -d spam`` - Remove the variable and its value from storage * ``%store -z`` - Remove all variables from storage - * ``%store -r`` - Refresh all variables from store (overwrite - current vals) - * ``%store -r spam bar`` - Refresh specified variables from store + * ``%store -r`` - Refresh all variables, aliases and directory history + from store (overwrite current vals) + * ``%store -r spam bar`` - Refresh specified variables and aliases from store (delete current val) * ``%store foo >a.txt`` - Store value of foo to new file a.txt * ``%store foo >>a.txt`` - Append value of foo to file a.txt @@ -114,10 +115,11 @@ def store(self, parameter_s=''): python types can be safely %store'd. Also aliases can be %store'd across sessions. + To remove an alias from the storage, use the %unalias magic. """ opts,argsl = self.parse_options(parameter_s,'drz',mode='string') - args = argsl.split(None,1) + args = argsl.split() ip = self.shell db = ip.db # delete @@ -142,7 +144,10 @@ def store(self, parameter_s=''): try: obj = db['autorestore/' + arg] except KeyError: - print("no stored variable %s" % arg) + try: + restore_aliases(ip, alias=arg) + except KeyError: + print("no stored variable or alias %s" % arg) else: ip.user_ns[arg] = obj else: @@ -174,55 +179,55 @@ def store(self, parameter_s=''): fil = open(fnam, 'a') else: fil = open(fnam, 'w') - obj = ip.ev(args[0]) - print("Writing '%s' (%s) to file '%s'." % (args[0], - obj.__class__.__name__, fnam)) - - - if not isinstance (obj, string_types): - from pprint import pprint - pprint(obj, fil) - else: - fil.write(obj) - if not obj.endswith('\n'): - fil.write('\n') + with fil: + obj = ip.ev(args[0]) + print("Writing '%s' (%s) to file '%s'." % (args[0], + obj.__class__.__name__, fnam)) + + if not isinstance (obj, str): + from pprint import pprint + pprint(obj, fil) + else: + fil.write(obj) + if not obj.endswith('\n'): + fil.write('\n') - fil.close() return # %store foo - try: - obj = ip.user_ns[args[0]] - except KeyError: - # it might be an alias - name = args[0] + for arg in args: try: - cmd = ip.alias_manager.retrieve_alias(name) - except ValueError: - raise UsageError("Unknown variable '%s'" % name) - - staliases = db.get('stored_aliases',{}) - staliases[name] = cmd - db['stored_aliases'] = staliases - print("Alias stored: %s (%s)" % (name, cmd)) - return - - else: - modname = getattr(inspect.getmodule(obj), '__name__', '') - if modname == '__main__': - print(textwrap.dedent("""\ - Warning:%s is %s - Proper storage of interactively declared classes (or instances - of those classes) is not possible! Only instances - of classes in real modules on file system can be %%store'd. - """ % (args[0], obj) )) + obj = ip.user_ns[arg] + except KeyError: + # it might be an alias + name = arg + try: + cmd = ip.alias_manager.retrieve_alias(name) + except ValueError: + raise UsageError("Unknown variable '%s'" % name) + + staliases = db.get('stored_aliases',{}) + staliases[name] = cmd + db['stored_aliases'] = staliases + print("Alias stored: %s (%s)" % (name, cmd)) return - #pickled = pickle.dumps(obj) - db[ 'autorestore/' + args[0] ] = obj - print("Stored '%s' (%s)" % (args[0], obj.__class__.__name__)) + + else: + modname = getattr(inspect.getmodule(obj), '__name__', '') + if modname == '__main__': + print(textwrap.dedent("""\ + Warning:%s is %s + Proper storage of interactively declared classes (or instances + of those classes) is not possible! Only instances + of classes in real modules on file system can be %%store'd. + """ % (arg, obj) )) + return + #pickled = pickle.dumps(obj) + db[ 'autorestore/' + arg ] = obj + print("Stored '%s' (%s)" % (arg, obj.__class__.__name__)) def load_ipython_extension(ip): """Load the extension in IPython.""" ip.register_magics(StoreMagics) - + diff --git a/IPython/extensions/sympyprinting.py b/IPython/extensions/sympyprinting.py index 7f9fb2ef98a..e6a83cd34b6 100644 --- a/IPython/extensions/sympyprinting.py +++ b/IPython/extensions/sympyprinting.py @@ -14,7 +14,7 @@ As of SymPy 0.7.2, maintenance of this extension has moved to SymPy under sympy.interactive.ipythonprinting, any modifications to account for changes to SymPy should be submitted to SymPy rather than changed here. This module is -maintained here for backwards compatablitiy with old SymPy versions. +maintained here for backwards compatibility with old SymPy versions. """ #----------------------------------------------------------------------------- diff --git a/IPython/extensions/tests/test_autoreload.py b/IPython/extensions/tests/test_autoreload.py index 968a19fd971..e81bf221515 100644 --- a/IPython/extensions/tests/test_autoreload.py +++ b/IPython/extensions/tests/test_autoreload.py @@ -15,21 +15,19 @@ import os import sys import tempfile +import textwrap import shutil import random import time +from io import StringIO import nose.tools as nt import IPython.testing.tools as tt +from unittest import TestCase + from IPython.extensions.autoreload import AutoreloadMagics from IPython.core.events import EventManager, pre_run_cell -from IPython.utils.py3compat import PY3 - -if PY3: - from io import StringIO -else: - from StringIO import StringIO #----------------------------------------------------------------------------- # Test fixture @@ -37,10 +35,12 @@ noop = lambda *a, **kw: None -class FakeShell(object): +class FakeShell: def __init__(self): self.ns = {} + self.user_ns = self.ns + self.user_ns_hidden = {} self.events = EventManager(self, {'pre_run_cell', pre_run_cell}) self.auto_magics = AutoreloadMagics(shell=self) self.events.register('pre_run_cell', self.auto_magics.pre_run_cell) @@ -49,7 +49,7 @@ def __init__(self): def run_code(self, code): self.events.trigger('pre_run_cell') - exec(code, self.ns) + exec(code, self.user_ns) self.auto_magics.post_execute_hook() def push(self, items): @@ -63,7 +63,7 @@ def magic_aimport(self, parameter, stream=None): self.auto_magics.post_execute_hook() -class Fixture(object): +class Fixture(TestCase): """Fixture for creating test module files""" test_dir = None @@ -106,31 +106,152 @@ def write_file(self, filename, content): (because that is stored in the file). The only reliable way to achieve this seems to be to sleep. """ - + content = textwrap.dedent(content) # Sleep one second + eps time.sleep(1.05) # Write - f = open(filename, 'w') - try: + with open(filename, 'w') as f: f.write(content) - finally: - f.close() def new_module(self, code): + code = textwrap.dedent(code) mod_name, mod_fn = self.get_module() - f = open(mod_fn, 'w') - try: + with open(mod_fn, 'w') as f: f.write(code) - finally: - f.close() return mod_name, mod_fn #----------------------------------------------------------------------------- # Test automatic reloading #----------------------------------------------------------------------------- +def pickle_get_current_class(obj): + """ + Original issue comes from pickle; hence the name. + """ + name = obj.__class__.__name__ + module_name = getattr(obj, "__module__", None) + obj2 = sys.modules[module_name] + for subpath in name.split("."): + obj2 = getattr(obj2, subpath) + return obj2 + class TestAutoreload(Fixture): + + def test_reload_enums(self): + mod_name, mod_fn = self.new_module(textwrap.dedent(""" + from enum import Enum + class MyEnum(Enum): + A = 'A' + B = 'B' + """)) + self.shell.magic_autoreload("2") + self.shell.magic_aimport(mod_name) + self.write_file(mod_fn, textwrap.dedent(""" + from enum import Enum + class MyEnum(Enum): + A = 'A' + B = 'B' + C = 'C' + """)) + with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'): + self.shell.run_code("pass") # trigger another reload + + def test_reload_class_type(self): + self.shell.magic_autoreload("2") + mod_name, mod_fn = self.new_module( + """ + class Test(): + def meth(self): + return "old" + """ + ) + assert "test" not in self.shell.ns + assert "result" not in self.shell.ns + + self.shell.run_code("from %s import Test" % mod_name) + self.shell.run_code("test = Test()") + + self.write_file( + mod_fn, + """ + class Test(): + def meth(self): + return "new" + """, + ) + + test_object = self.shell.ns["test"] + + # important to trigger autoreload logic ! + self.shell.run_code("pass") + + test_class = pickle_get_current_class(test_object) + assert isinstance(test_object, test_class) + + # extra check. + self.shell.run_code("import pickle") + self.shell.run_code("p = pickle.dumps(test)") + + def test_reload_class_attributes(self): + self.shell.magic_autoreload("2") + mod_name, mod_fn = self.new_module(textwrap.dedent(""" + class MyClass: + + def __init__(self, a=10): + self.a = a + self.b = 22 + # self.toto = 33 + + def square(self): + print('compute square') + return self.a*self.a + """ + ) + ) + self.shell.run_code("from %s import MyClass" % mod_name) + self.shell.run_code("first = MyClass(5)") + self.shell.run_code("first.square()") + with nt.assert_raises(AttributeError): + self.shell.run_code("first.cube()") + with nt.assert_raises(AttributeError): + self.shell.run_code("first.power(5)") + self.shell.run_code("first.b") + with nt.assert_raises(AttributeError): + self.shell.run_code("first.toto") + + # remove square, add power + + self.write_file( + mod_fn, + textwrap.dedent( + """ + class MyClass: + + def __init__(self, a=10): + self.a = a + self.b = 11 + + def power(self, p): + print('compute power '+str(p)) + return self.a**p + """ + ), + ) + + self.shell.run_code("second = MyClass(5)") + + for object_name in {'first', 'second'}: + self.shell.run_code("{object_name}.power(5)".format(object_name=object_name)) + with nt.assert_raises(AttributeError): + self.shell.run_code("{object_name}.cube()".format(object_name=object_name)) + with nt.assert_raises(AttributeError): + self.shell.run_code("{object_name}.square()".format(object_name=object_name)) + self.shell.run_code("{object_name}.b".format(object_name=object_name)) + self.shell.run_code("{object_name}.a".format(object_name=object_name)) + with nt.assert_raises(AttributeError): + self.shell.run_code("{object_name}.toto".format(object_name=object_name)) + def _check_smoketest(self, use_aimport=True): """ Functional test for the automatic reloader using either @@ -319,3 +440,8 @@ def test_smoketest_aimport(self): def test_smoketest_autoreload(self): self._check_smoketest(use_aimport=False) + + + + + diff --git a/IPython/extensions/tests/test_storemagic.py b/IPython/extensions/tests/test_storemagic.py index 373a7169261..6f8371d336f 100644 --- a/IPython/extensions/tests/test_storemagic.py +++ b/IPython/extensions/tests/test_storemagic.py @@ -3,33 +3,49 @@ from traitlets.config.loader import Config import nose.tools as nt -ip = get_ipython() -ip.magic('load_ext storemagic') + +def setup_module(): + ip.magic('load_ext storemagic') def test_store_restore(): + assert 'bar' not in ip.user_ns, "Error: some other test leaked `bar` in user_ns" + assert 'foo' not in ip.user_ns, "Error: some other test leaked `foo` in user_ns" + assert 'foobar' not in ip.user_ns, "Error: some other test leaked `foobar` in user_ns" + assert 'foobaz' not in ip.user_ns, "Error: some other test leaked `foobaz` in user_ns" ip.user_ns['foo'] = 78 ip.magic('alias bar echo "hello"') + ip.user_ns['foobar'] = 79 + ip.user_ns['foobaz'] = '80' tmpd = tempfile.mkdtemp() ip.magic('cd ' + tmpd) ip.magic('store foo') ip.magic('store bar') - + ip.magic('store foobar foobaz') + # Check storing nt.assert_equal(ip.db['autorestore/foo'], 78) nt.assert_in('bar', ip.db['stored_aliases']) - + nt.assert_equal(ip.db['autorestore/foobar'], 79) + nt.assert_equal(ip.db['autorestore/foobaz'], '80') + # Remove those items ip.user_ns.pop('foo', None) + ip.user_ns.pop('foobar', None) + ip.user_ns.pop('foobaz', None) ip.alias_manager.undefine_alias('bar') ip.magic('cd -') ip.user_ns['_dh'][:] = [] - + # Check restoring - ip.magic('store -r') + ip.magic('store -r foo bar foobar foobaz') nt.assert_equal(ip.user_ns['foo'], 78) assert ip.alias_manager.is_alias('bar') + nt.assert_equal(ip.user_ns['foobar'], 79) + nt.assert_equal(ip.user_ns['foobaz'], '80') + + ip.magic('store -r') # restores _dh too nt.assert_in(os.path.realpath(tmpd), ip.user_ns['_dh']) - + os.rmdir(tmpd) def test_autorestore(): diff --git a/IPython/external/__init__.py b/IPython/external/__init__.py index 3104c194622..1c8c546f118 100644 --- a/IPython/external/__init__.py +++ b/IPython/external/__init__.py @@ -2,4 +2,4 @@ This package contains all third-party modules bundled with IPython. """ -__all__ = ["simplegeneric"] +__all__ = [] diff --git a/IPython/external/decorators/__init__.py b/IPython/external/decorators/__init__.py index dd8f52b711a..1db80edd357 100644 --- a/IPython/external/decorators/__init__.py +++ b/IPython/external/decorators/__init__.py @@ -1,8 +1,7 @@ try: - from numpy.testing.decorators import * - from numpy.testing.noseclasses import KnownFailure + from numpy.testing import KnownFailure, knownfailureif except ImportError: - from ._decorators import * + from ._decorators import knownfailureif try: from ._numpy_testing_noseclasses import KnownFailure except ImportError: diff --git a/IPython/external/decorators/_decorators.py b/IPython/external/decorators/_decorators.py index 19de5e5cded..18f847adadd 100644 --- a/IPython/external/decorators/_decorators.py +++ b/IPython/external/decorators/_decorators.py @@ -13,14 +13,9 @@ ``nose.tools`` for more information. """ -import warnings # IPython changes: make this work if numpy not available # Original code: -#from numpy.testing.utils import \ -# WarningManager, WarningMessage -# Our version: -from ._numpy_testing_utils import WarningManager try: from ._numpy_testing_noseclasses import KnownFailureTest except: @@ -28,73 +23,6 @@ # End IPython changes -def slow(t): - """ - Label a test as 'slow'. - - The exact definition of a slow test is obviously both subjective and - hardware-dependent, but in general any individual test that requires more - than a second or two should be labeled as slow (the whole suite consists of - thousands of tests, so even a second is significant). - - Parameters - ---------- - t : callable - The test to label as slow. - - Returns - ------- - t : callable - The decorated test `t`. - - Examples - -------- - The `numpy.testing` module includes ``import decorators as dec``. - A test can be decorated as slow like this:: - - from numpy.testing import * - - @dec.slow - def test_big(self): - print 'Big, slow test' - - """ - - t.slow = True - return t - -def setastest(tf=True): - """ - Signals to nose that this function is or is not a test. - - Parameters - ---------- - tf : bool - If True, specifies that the decorated callable is a test. - If False, specifies that the decorated callable is not a test. - Default is True. - - Notes - ----- - This decorator can't use the nose namespace, because it can be - called from a non-test module. See also ``istest`` and ``nottest`` in - ``nose.tools``. - - Examples - -------- - `setastest` can be used in the following way:: - - from numpy.testing.decorators import setastest - - @setastest(False) - def func_with_test_in_name(arg1, arg2): - pass - - """ - def set_test(t): - t.__test__ = tf - return t - return set_test def skipif(skip_condition, msg=None): """ @@ -176,13 +104,9 @@ def knownfailureif(fail_condition, msg=None): """ Make function raise KnownFailureTest exception if given condition is true. - If the condition is a callable, it is used at runtime to dynamically - make the decision. This is useful for tests that may require costly - imports, to delay the cost until the test suite is actually executed. - Parameters ---------- - fail_condition : bool or callable + fail_condition : bool Flag to determine whether to mark the decorated test as a known failure (if True) or not (if False). msg : str, optional @@ -192,9 +116,8 @@ def knownfailureif(fail_condition, msg=None): Returns ------- decorator : function - Decorator, which, when applied to a function, causes SkipTest - to be raised when `skip_condition` is True, and the function - to be called normally otherwise. + Decorator, which, when applied to a function, causes KnownFailureTest to + be raised when `fail_condition` is True and the test fails. Notes ----- @@ -205,77 +128,16 @@ def knownfailureif(fail_condition, msg=None): if msg is None: msg = 'Test skipped due to known failure' - # Allow for both boolean or callable known failure conditions. - if callable(fail_condition): - fail_val = lambda : fail_condition() - else: - fail_val = lambda : fail_condition - def knownfail_decorator(f): # Local import to avoid a hard nose dependency and only incur the # import time overhead at actual test-time. import nose + def knownfailer(*args, **kwargs): - if fail_val(): + if fail_condition: raise KnownFailureTest(msg) else: return f(*args, **kwargs) return nose.tools.make_decorator(f)(knownfailer) return knownfail_decorator - -def deprecated(conditional=True): - """ - Filter deprecation warnings while running the test suite. - - This decorator can be used to filter DeprecationWarning's, to avoid - printing them during the test suite run, while checking that the test - actually raises a DeprecationWarning. - - Parameters - ---------- - conditional : bool or callable, optional - Flag to determine whether to mark test as deprecated or not. If the - condition is a callable, it is used at runtime to dynamically make the - decision. Default is True. - - Returns - ------- - decorator : function - The `deprecated` decorator itself. - - Notes - ----- - .. versionadded:: 1.4.0 - - """ - def deprecate_decorator(f): - # Local import to avoid a hard nose dependency and only incur the - # import time overhead at actual test-time. - import nose - - def _deprecated_imp(*args, **kwargs): - # Poor man's replacement for the with statement - ctx = WarningManager(record=True) - l = ctx.__enter__() - warnings.simplefilter('always') - try: - f(*args, **kwargs) - if not len(l) > 0: - raise AssertionError("No warning raised when calling %s" - % f.__name__) - if not l[0].category is DeprecationWarning: - raise AssertionError("First warning for %s is not a " \ - "DeprecationWarning( is %s)" % (f.__name__, l[0])) - finally: - ctx.__exit__() - - if callable(conditional): - cond = conditional() - else: - cond = conditional - if cond: - return nose.tools.make_decorator(f)(_deprecated_imp) - else: - return f - return deprecate_decorator diff --git a/IPython/external/decorators/_numpy_testing_utils.py b/IPython/external/decorators/_numpy_testing_utils.py deleted file mode 100644 index ad7bd0f9817..00000000000 --- a/IPython/external/decorators/_numpy_testing_utils.py +++ /dev/null @@ -1,112 +0,0 @@ -# IPython: modified copy of numpy.testing.utils, so -# IPython.external._decorators works without numpy being installed. -""" -Utility function to facilitate testing. -""" - -import sys -import warnings - -# The following two classes are copied from python 2.6 warnings module (context -# manager) -class WarningMessage(object): - - """ - Holds the result of a single showwarning() call. - - Notes - ----- - `WarningMessage` is copied from the Python 2.6 warnings module, - so it can be used in NumPy with older Python versions. - - """ - - _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", - "line") - - def __init__(self, message, category, filename, lineno, file=None, - line=None): - local_values = locals() - for attr in self._WARNING_DETAILS: - setattr(self, attr, local_values[attr]) - if category: - self._category_name = category.__name__ - else: - self._category_name = None - - def __str__(self): - return ("{message : %r, category : %r, filename : %r, lineno : %s, " - "line : %r}" % (self.message, self._category_name, - self.filename, self.lineno, self.line)) - -class WarningManager: - """ - A context manager that copies and restores the warnings filter upon - exiting the context. - - The 'record' argument specifies whether warnings should be captured by a - custom implementation of ``warnings.showwarning()`` and be appended to a - list returned by the context manager. Otherwise None is returned by the - context manager. The objects appended to the list are arguments whose - attributes mirror the arguments to ``showwarning()``. - - The 'module' argument is to specify an alternative module to the module - named 'warnings' and imported under that name. This argument is only useful - when testing the warnings module itself. - - Notes - ----- - `WarningManager` is a copy of the ``catch_warnings`` context manager - from the Python 2.6 warnings module, with slight modifications. - It is copied so it can be used in NumPy with older Python versions. - - """ - def __init__(self, record=False, module=None): - self._record = record - if module is None: - self._module = sys.modules['warnings'] - else: - self._module = module - self._entered = False - - def __enter__(self): - if self._entered: - raise RuntimeError("Cannot enter %r twice" % self) - self._entered = True - self._filters = self._module.filters - self._module.filters = self._filters[:] - self._showwarning = self._module.showwarning - if self._record: - log = [] - def showwarning(*args, **kwargs): - log.append(WarningMessage(*args, **kwargs)) - self._module.showwarning = showwarning - return log - else: - return None - - def __exit__(self, type_, value, traceback): - if not self._entered: - raise RuntimeError("Cannot exit %r without entering first" % self) - self._module.filters = self._filters - self._module.showwarning = self._showwarning - -def assert_warns(warning_class, func, *args, **kw): - """Fail unless a warning of class warning_class is thrown by callable when - invoked with arguments args and keyword arguments kwargs. - - If a different type of warning is thrown, it will not be caught, and the - test case will be deemed to have suffered an error. - """ - - # XXX: once we may depend on python >= 2.6, this can be replaced by the - # warnings module context manager. - with WarningManager(record=True) as l: - warnings.simplefilter('always') - func(*args, **kw) - if not len(l) > 0: - raise AssertionError("No warning raised when calling %s" - % func.__name__) - if not l[0].category is warning_class: - raise AssertionError("First warning for %s is not a " \ - "%s( is %s)" % (func.__name__, warning_class, l[0])) diff --git a/IPython/external/qt_for_kernel.py b/IPython/external/qt_for_kernel.py index 33cf6225d81..1a94e7e0a2e 100644 --- a/IPython/external/qt_for_kernel.py +++ b/IPython/external/qt_for_kernel.py @@ -33,10 +33,10 @@ from IPython.utils.version import check_version from IPython.external.qt_loaders import (load_qt, loaded_api, QT_API_PYSIDE, - QT_API_PYQT, QT_API_PYQT5, + QT_API_PYSIDE2, QT_API_PYQT, QT_API_PYQT5, QT_API_PYQTv1, QT_API_PYQT_DEFAULT) -_qt_apis = (QT_API_PYSIDE, QT_API_PYQT, QT_API_PYQT5, QT_API_PYQTv1, +_qt_apis = (QT_API_PYSIDE, QT_API_PYSIDE2, QT_API_PYQT, QT_API_PYQT5, QT_API_PYQTv1, QT_API_PYQT_DEFAULT) #Constraints placed on an imported matplotlib @@ -83,7 +83,8 @@ def get_options(): qt_api = os.environ.get('QT_API', None) if qt_api is None: #no ETS variable. Ask mpl, then use default fallback path - return matplotlib_options(mpl) or [QT_API_PYQT_DEFAULT, QT_API_PYSIDE, QT_API_PYQT5] + return matplotlib_options(mpl) or [QT_API_PYQT_DEFAULT, QT_API_PYSIDE, + QT_API_PYQT5, QT_API_PYSIDE2] elif qt_api not in _qt_apis: raise RuntimeError("Invalid Qt API %r, valid values are: %r" % (qt_api, ', '.join(_qt_apis))) diff --git a/IPython/external/qt_loaders.py b/IPython/external/qt_loaders.py index fd695661a9e..46cd9c35cb9 100644 --- a/IPython/external/qt_loaders.py +++ b/IPython/external/qt_loaders.py @@ -11,6 +11,7 @@ import sys import types from functools import partial +from importlib import import_module from IPython.utils.version import check_version @@ -20,6 +21,15 @@ QT_API_PYQTv1 = 'pyqtv1' # Force version 2 QT_API_PYQT_DEFAULT = 'pyqtdefault' # use system default for version 1 vs. 2 QT_API_PYSIDE = 'pyside' +QT_API_PYSIDE2 = 'pyside2' + +api_to_module = {QT_API_PYSIDE2: 'PySide2', + QT_API_PYSIDE: 'PySide', + QT_API_PYQT: 'PyQt4', + QT_API_PYQTv1: 'PyQt4', + QT_API_PYQT5: 'PyQt5', + QT_API_PYQT_DEFAULT: 'PyQt4', + } class ImportDenier(object): @@ -47,21 +57,28 @@ def load_module(self, fullname): """ % (fullname, loaded_api())) ID = ImportDenier() -sys.meta_path.append(ID) +sys.meta_path.insert(0, ID) def commit_api(api): """Commit to a particular API, and trigger ImportErrors on subsequent dangerous imports""" - if api == QT_API_PYSIDE: + if api == QT_API_PYSIDE2: + ID.forbid('PySide') + ID.forbid('PyQt4') + ID.forbid('PyQt5') + elif api == QT_API_PYSIDE: + ID.forbid('PySide2') ID.forbid('PyQt4') ID.forbid('PyQt5') elif api == QT_API_PYQT5: + ID.forbid('PySide2') ID.forbid('PySide') ID.forbid('PyQt4') else: # There are three other possibilities, all representing PyQt4 ID.forbid('PyQt5') + ID.forbid('PySide2') ID.forbid('PySide') @@ -73,7 +90,7 @@ def loaded_api(): Returns ------- - None, 'pyside', 'pyqt', 'pyqt5', or 'pyqtv1' + None, 'pyside2', 'pyside', 'pyqt', 'pyqt5', or 'pyqtv1' """ if 'PyQt4.QtCore' in sys.modules: if qtapi_version() == 2: @@ -82,53 +99,50 @@ def loaded_api(): return QT_API_PYQTv1 elif 'PySide.QtCore' in sys.modules: return QT_API_PYSIDE + elif 'PySide2.QtCore' in sys.modules: + return QT_API_PYSIDE2 elif 'PyQt5.QtCore' in sys.modules: return QT_API_PYQT5 return None def has_binding(api): - """Safely check for PyQt4/5 or PySide, without importing - submodules + """Safely check for PyQt4/5, PySide or PySide2, without importing submodules + + Parameters + ---------- + api : str [ 'pyqtv1' | 'pyqt' | 'pyqt5' | 'pyside' | 'pyside2' | 'pyqtdefault'] + Which module to check for + + Returns + ------- + True if the relevant module appears to be importable + """ + module_name = api_to_module[api] + from importlib.util import find_spec + + required = ['QtCore', 'QtGui', 'QtSvg'] + if api in (QT_API_PYQT5, QT_API_PYSIDE2): + # QT5 requires QtWidgets too + required.append('QtWidgets') + + for submod in required: + try: + spec = find_spec('%s.%s' % (module_name, submod)) + except ImportError: + # Package (e.g. PyQt5) not found + return False + else: + if spec is None: + # Submodule (e.g. PyQt5.QtCore) not found + return False - Parameters - ---------- - api : str [ 'pyqtv1' | 'pyqt' | 'pyqt5' | 'pyside' | 'pyqtdefault'] - Which module to check for + if api == QT_API_PYSIDE: + # We can also safely check PySide version + import PySide + return check_version(PySide.__version__, '1.0.3') - Returns - ------- - True if the relevant module appears to be importable - """ - # we can't import an incomplete pyside and pyqt4 - # this will cause a crash in sip (#1431) - # check for complete presence before importing - module_name = {QT_API_PYSIDE: 'PySide', - QT_API_PYQT: 'PyQt4', - QT_API_PYQTv1: 'PyQt4', - QT_API_PYQT5: 'PyQt5', - QT_API_PYQT_DEFAULT: 'PyQt4'} - module_name = module_name[api] - - import imp - try: - #importing top level PyQt4/PySide module is ok... - mod = __import__(module_name) - #...importing submodules is not - imp.find_module('QtCore', mod.__path__) - imp.find_module('QtGui', mod.__path__) - imp.find_module('QtSvg', mod.__path__) - if api == QT_API_PYQT5: - # QT5 requires QtWidgets too - imp.find_module('QtWidgets', mod.__path__) - - #we can also safely check PySide version - if api == QT_API_PYSIDE: - return check_version(mod.__version__, '1.0.3') - else: - return True - except ImportError: - return False + return True def qtapi_version(): @@ -203,10 +217,9 @@ def import_pyqt5(): ImportErrors rasied within this function are non-recoverable """ - import sip from PyQt5 import QtCore, QtSvg, QtWidgets, QtGui - + # Alias PyQt-specific functions for PySide compatibility. QtCore.Signal = QtCore.pyqtSignal QtCore.Slot = QtCore.pyqtSlot @@ -229,6 +242,22 @@ def import_pyside(): from PySide import QtGui, QtCore, QtSvg return QtCore, QtGui, QtSvg, QT_API_PYSIDE +def import_pyside2(): + """ + Import PySide2 + + ImportErrors raised within this function are non-recoverable + """ + from PySide2 import QtGui, QtCore, QtSvg, QtWidgets, QtPrintSupport + + # Join QtGui and QtWidgets for Qt4 compatibility. + QtGuiCompat = types.ModuleType('QtGuiCompat') + QtGuiCompat.__dict__.update(QtGui.__dict__) + QtGuiCompat.__dict__.update(QtWidgets.__dict__) + QtGuiCompat.__dict__.update(QtPrintSupport.__dict__) + + return QtCore, QtGuiCompat, QtSvg, QT_API_PYSIDE2 + def load_qt(api_options): """ @@ -240,7 +269,7 @@ def load_qt(api_options): Parameters ---------- api_options: List of strings - The order of APIs to try. Valid items are 'pyside', + The order of APIs to try. Valid items are 'pyside', 'pyside2', 'pyqt', 'pyqt5', 'pyqtv1' and 'pyqtdefault' Returns @@ -253,15 +282,17 @@ def load_qt(api_options): Raises ------ ImportError, if it isn't possible to import any requested - bindings (either becaues they aren't installed, or because + bindings (either because they aren't installed, or because an incompatible library has already been installed) """ - loaders = {QT_API_PYSIDE: import_pyside, + loaders = { + QT_API_PYSIDE2: import_pyside2, + QT_API_PYSIDE: import_pyside, QT_API_PYQT: import_pyqt4, QT_API_PYQT5: import_pyqt5, QT_API_PYQTv1: partial(import_pyqt4, version=1), QT_API_PYQT_DEFAULT: partial(import_pyqt4, version=None) - } + } for api in api_options: @@ -281,16 +312,18 @@ def load_qt(api_options): else: raise ImportError(""" Could not load requested Qt binding. Please ensure that - PyQt4 >= 4.7, PyQt5 or PySide >= 1.0.3 is available, + PyQt4 >= 4.7, PyQt5, PySide >= 1.0.3 or PySide2 is available, and only one is imported per session. Currently-imported Qt library: %r PyQt4 available (requires QtCore, QtGui, QtSvg): %s PyQt5 available (requires QtCore, QtGui, QtSvg, QtWidgets): %s PySide >= 1.0.3 installed: %s + PySide2 installed: %s Tried to load: %r """ % (loaded_api(), has_binding(QT_API_PYQT), has_binding(QT_API_PYQT5), has_binding(QT_API_PYSIDE), + has_binding(QT_API_PYSIDE2), api_options)) diff --git a/IPython/frontend.py b/IPython/frontend.py index 49cf8b236a6..9cc3eaff2f0 100644 --- a/IPython/frontend.py +++ b/IPython/frontend.py @@ -17,7 +17,7 @@ from IPython.utils.shimmodule import ShimModule, ShimWarning -warn("The top-level `frontend` package has been deprecated. " +warn("The top-level `frontend` package has been deprecated since IPython 1.0. " "All its subpackages have been moved to the top `IPython` level.", ShimWarning) # Unconditionally insert the shim into sys.modules so that further import calls diff --git a/IPython/html.py b/IPython/html.py index 5a9a0aee907..050be5c5991 100644 --- a/IPython/html.py +++ b/IPython/html.py @@ -9,7 +9,7 @@ from IPython.utils.shimmodule import ShimModule, ShimWarning -warn("The `IPython.html` package has been deprecated. " +warn("The `IPython.html` package has been deprecated since IPython 4.0. " "You should import from `notebook` instead. " "`IPython.html.widgets` has moved to `ipywidgets`.", ShimWarning) diff --git a/IPython/kernel/__init__.py b/IPython/kernel/__init__.py index 2500a07b135..70a05ed4aa5 100644 --- a/IPython/kernel/__init__.py +++ b/IPython/kernel/__init__.py @@ -9,7 +9,7 @@ from IPython.utils.shimmodule import ShimModule, ShimWarning -warn("The `IPython.kernel` package has been deprecated. " +warn("The `IPython.kernel` package has been deprecated since IPython 4.0." "You should import from ipykernel or jupyter_client instead.", ShimWarning) diff --git a/IPython/lib/backgroundjobs.py b/IPython/lib/backgroundjobs.py index b724126bbb3..31997e13f28 100644 --- a/IPython/lib/backgroundjobs.py +++ b/IPython/lib/backgroundjobs.py @@ -21,7 +21,6 @@ An example notebook is provided in our documentation illustrating interactive use of the system. """ -from __future__ import print_function #***************************************************************************** # Copyright (C) 2005-2006 Fernando Perez @@ -36,8 +35,7 @@ from IPython import get_ipython from IPython.core.ultratb import AutoFormattedTB -from logging import error -from IPython.utils.py3compat import string_types +from logging import error, debug class BackgroundJobManager(object): @@ -88,6 +86,7 @@ def __init__(self): self._s_running = BackgroundJobBase.stat_running_c self._s_completed = BackgroundJobBase.stat_completed_c self._s_dead = BackgroundJobBase.stat_dead_c + self._current_job_id = 0 @property def running(self): @@ -172,7 +171,7 @@ def new(self, func_or_exp, *args, **kwargs): if callable(func_or_exp): kw = kwargs.get('kw',{}) job = BackgroundJobFunc(func_or_exp,*args,**kw) - elif isinstance(func_or_exp, string_types): + elif isinstance(func_or_exp, str): if not args: frame = sys._getframe(1) glob, loc = frame.f_globals, frame.f_locals @@ -189,10 +188,11 @@ def new(self, func_or_exp, *args, **kwargs): if kwargs.get('daemon', False): job.daemon = True - job.num = len(self.all)+1 if self.all else 0 + job.num = self._current_job_id + self._current_job_id += 1 self.running.append(job) self.all[job.num] = job - print('Starting job # %s in a separate thread.' % job.num) + debug('Starting job # %s in a separate thread.' % job.num) job.start() return job diff --git a/IPython/lib/clipboard.py b/IPython/lib/clipboard.py index ac9b685c7df..316a8ab1f8a 100644 --- a/IPython/lib/clipboard.py +++ b/IPython/lib/clipboard.py @@ -32,15 +32,15 @@ def win32_clipboard_get(): win32clipboard.CloseClipboard() return text -def osx_clipboard_get(): +def osx_clipboard_get() -> str: """ Get the clipboard's text on OS X. """ p = subprocess.Popen(['pbpaste', '-Prefer', 'ascii'], stdout=subprocess.PIPE) - text, stderr = p.communicate() + bytes_, stderr = p.communicate() # Text comes in with old Mac \r line endings. Change them to \n. - text = text.replace(b'\r', b'\n') - text = py3compat.cast_unicode(text, py3compat.DEFAULT_ENCODING) + bytes_ = bytes_.replace(b'\r', b'\n') + text = py3compat.decode(bytes_) return text def tkinter_clipboard_get(): @@ -51,13 +51,10 @@ def tkinter_clipboard_get(): implementation that uses that toolkit. """ try: - from tkinter import Tk, TclError # Py 3 + from tkinter import Tk, TclError except ImportError: - try: - from Tkinter import Tk, TclError # Py 2 - except ImportError: - raise TryNext("Getting text from the clipboard on this platform " - "requires Tkinter.") + raise TryNext("Getting text from the clipboard on this platform requires tkinter.") + root = Tk() root.withdraw() try: diff --git a/IPython/lib/deepreload.py b/IPython/lib/deepreload.py index 521acf352b0..bd8c01b2a75 100644 --- a/IPython/lib/deepreload.py +++ b/IPython/lib/deepreload.py @@ -7,13 +7,7 @@ imported from that module, which is useful when you're changing files deep inside a package. -To use this as your default reload function, type this for Python 2:: - - import __builtin__ - from IPython.lib import deepreload - __builtin__.reload = deepreload.reload - -Or this for Python 3:: +To use this as your default reload function, type this:: import builtins from IPython.lib import deepreload @@ -25,7 +19,6 @@ This code is almost entirely based on knee.py, which is a Python re-implementation of hierarchical module import. """ -from __future__ import print_function #***************************************************************************** # Copyright (C) 2001 Nathaniel Gray # @@ -33,14 +26,14 @@ # the file COPYING, distributed as part of this software. #***************************************************************************** +import builtins as builtin_mod from contextlib import contextmanager import imp import sys from types import ModuleType from warnings import warn - -from IPython.utils.py3compat import builtin_mod, builtin_mod_name +import types original_import = builtin_mod.__import__ @@ -164,6 +157,7 @@ def load_next(mod, altmod, name, buf): return result, next, buf + # Need to keep track of what we've already reloaded to prevent cyclic evil found_now = {} @@ -271,6 +265,12 @@ def deep_import_hook(name, globals=None, locals=None, fromlist=None, level=-1): def deep_reload_hook(m): """Replacement for reload().""" + # Hardcode this one as it would raise a NotImplementedError from the + # bowels of Python and screw up the import machinery after. + # unlike other imports the `exclude` list already in place is not enough. + + if m is types: + return m if not isinstance(m, ModuleType): raise TypeError("reload() argument must be module") @@ -321,13 +321,11 @@ def deep_reload_hook(m): return newm # Save the original hooks -try: - original_reload = builtin_mod.reload -except AttributeError: - original_reload = imp.reload # Python 3 +original_reload = imp.reload # Replacement for reload() -def reload(module, exclude=('sys', 'os.path', builtin_mod_name, '__main__')): +def reload(module, exclude=('sys', 'os.path', 'builtins', '__main__', + 'numpy', 'numpy._globals')): """Recursively reload all modules used in the given module. Optionally takes a list of modules to exclude from reloading. The default exclude list contains sys, __main__, and __builtin__, to prevent, e.g., resetting @@ -341,21 +339,3 @@ def reload(module, exclude=('sys', 'os.path', builtin_mod_name, '__main__')): return deep_reload_hook(module) finally: found_now = {} - - -def _dreload(module, **kwargs): - """ - **deprecated** - - import reload explicitly from `IPython.lib.deepreload` to use it - - """ - # this was marked as deprecated and for 5.0 removal, but - # IPython.core_builtin_trap have a Deprecation warning for 6.0, so cannot - # remove that now. - warn(""" -injecting `dreload` in interactive namespace is deprecated since IPython 4.0. -Please import `reload` explicitly from `IPython.lib.deepreload`. -""", DeprecationWarning, stacklevel=2) - reload(module, **kwargs) - diff --git a/IPython/lib/demo.py b/IPython/lib/demo.py index 4db31aab358..0b19c413c37 100644 --- a/IPython/lib/demo.py +++ b/IPython/lib/demo.py @@ -41,6 +41,9 @@ The classes here all include a few methods meant to make customization by subclassing more convenient. Their docstrings below have some more details: + - highlight(): format every block and optionally highlight comments and + docstring content. + - marquee(): generates a marquee to provide visible on-screen markers at each block start and end. @@ -106,7 +109,7 @@ This is probably best explained with the simple example file below. You can copy this into a file named ex_demo.py, and try running it via:: - from IPython.demo import Demo + from IPython.lib.demo import Demo d = Demo('ex_demo.py') d() @@ -167,7 +170,6 @@ ################### END EXAMPLE DEMO ############################ """ -from __future__ import unicode_literals #***************************************************************************** # Copyright (C) 2005-2006 Fernando Perez. @@ -176,14 +178,13 @@ # the file COPYING, distributed as part of this software. # #***************************************************************************** -from __future__ import print_function import os import re import shlex import sys +import pygments -from IPython.utils import io from IPython.utils.text import marquee from IPython.utils import openpy from IPython.utils import py3compat @@ -196,12 +197,13 @@ def re_mark(mark): class Demo(object): - re_stop = re_mark('-*\s?stop\s?-*') + re_stop = re_mark(r'-*\s?stop\s?-*') re_silent = re_mark('silent') re_auto = re_mark('auto') re_auto_all = re_mark('auto_all') - def __init__(self,src,title='',arg_str='',auto_all=None): + def __init__(self,src,title='',arg_str='',auto_all=None, format_rst=False, + formatter='terminal', style='default'): """Make a new demo object. To run the demo, simply call the object. See the module docstring for full details and an example (you can use @@ -227,6 +229,15 @@ def __init__(self,src,title='',arg_str='',auto_all=None): applies to the whole demo. It is an attribute of the object, and can be changed at runtime simply by reassigning it to a boolean value. + + - format_rst(False): a bool to enable comments and doc strings + formatting with pygments rst lexer + + - formatter('terminal'): a string of pygments formatter name to be + used. Useful values for terminals: terminal, terminal256, + terminal16m + + - style('default'): a string of pygments style name to be used. """ if hasattr(src, "read"): # It seems to be a file or a file-like object @@ -247,16 +258,29 @@ def __init__(self,src,title='',arg_str='',auto_all=None): self.auto_all = auto_all self.src = src - # get a few things from ipython. While it's a bit ugly design-wise, - # it ensures that things like color scheme and the like are always in - # sync with the ipython mode being used. This class is only meant to - # be used inside ipython anyways, so it's OK. - ip = get_ipython() # this is in builtins whenever IPython is running - self.ip_ns = ip.user_ns - self.ip_colorize = ip.pycolorize - self.ip_showtb = ip.showtraceback - self.ip_run_cell = ip.run_cell - self.shell = ip + try: + ip = get_ipython() # this is in builtins whenever IPython is running + self.inside_ipython = True + except NameError: + self.inside_ipython = False + + if self.inside_ipython: + # get a few things from ipython. While it's a bit ugly design-wise, + # it ensures that things like color scheme and the like are always in + # sync with the ipython mode being used. This class is only meant to + # be used inside ipython anyways, so it's OK. + self.ip_ns = ip.user_ns + self.ip_colorize = ip.pycolorize + self.ip_showtb = ip.showtraceback + self.ip_run_cell = ip.run_cell + self.shell = ip + + self.formatter = pygments.formatters.get_formatter_by_name(formatter, + style=style) + self.python_lexer = pygments.lexers.get_lexer_by_name("py3") + self.format_rst = format_rst + if format_rst: + self.rst_lexer = pygments.lexers.get_lexer_by_name("rst") # load user data and initialize data structures self.reload() @@ -304,7 +328,7 @@ def reload(self): self.src_blocks = src_blocks # also build syntax-highlighted source - self.src_blocks_colored = list(map(self.ip_colorize,self.src_blocks)) + self.src_blocks_colored = list(map(self.highlight,self.src_blocks)) # ensure clean namespace and seek offset self.reset() @@ -384,7 +408,7 @@ def edit(self,index=None): new_block = f.read() # update the source and colored block self.src_blocks[index] = new_block - self.src_blocks_colored[index] = self.ip_colorize(new_block) + self.src_blocks_colored[index] = self.highlight(new_block) self.block_index = index # call to run with the newly edited index self() @@ -463,9 +487,11 @@ def __call__(self,index=None): sys.argv = save_argv except: - self.ip_showtb(filename=self.fname) + if self.inside_ipython: + self.ip_showtb(filename=self.fname) else: - self.ip_ns.update(self.user_ns) + if self.inside_ipython: + self.ip_ns.update(self.user_ns) if self.block_index == self.nblocks: mq1 = self.marquee('END OF DEMO') @@ -490,6 +516,28 @@ def post_cmd(self): """Method called after executing each block.""" pass + def highlight(self, block): + """Method called on each block to highlight it content""" + tokens = pygments.lex(block, self.python_lexer) + if self.format_rst: + from pygments.token import Token + toks = [] + for token in tokens: + if token[0] == Token.String.Doc and len(token[1]) > 6: + toks += pygments.lex(token[1][:3], self.python_lexer) + # parse doc string content by rst lexer + toks += pygments.lex(token[1][3:-3], self.rst_lexer) + toks += pygments.lex(token[1][-3:], self.python_lexer) + elif token[0] == Token.Comment.Single: + toks.append((Token.Comment.Single, token[1][0])) + # parse comment content by rst lexer + # remove the extrat newline added by rst lexer + toks += list(pygments.lex(token[1][1:], self.rst_lexer))[:-1] + else: + toks.append(token) + tokens = toks + return pygments.format(tokens, self.formatter) + class IPythonDemo(Demo): """Class for interactive demos with IPython's input processing applied. @@ -538,7 +586,7 @@ def reload(self): self.src_blocks = src_b # also build syntax-highlighted source - self.src_blocks_colored = map(self.ip_colorize,self.src_blocks) + self.src_blocks_colored = list(map(self.highlight,self.src_blocks)) # ensure clean namespace and seek offset self.reset() @@ -572,8 +620,8 @@ def pre_cmd(self): """Method called before executing each block. This one simply clears the screen.""" - from IPython.utils.terminal import term_clear - term_clear() + from IPython.utils.terminal import _term_clear + _term_clear() class ClearDemo(ClearMixin,Demo): pass @@ -581,3 +629,43 @@ class ClearDemo(ClearMixin,Demo): class ClearIPDemo(ClearMixin,IPythonDemo): pass + + +def slide(file_path, noclear=False, format_rst=True, formatter="terminal", + style="native", auto_all=False, delimiter='...'): + if noclear: + demo_class = Demo + else: + demo_class = ClearDemo + demo = demo_class(file_path, format_rst=format_rst, formatter=formatter, + style=style, auto_all=auto_all) + while not demo.finished: + demo() + try: + py3compat.input('\n' + delimiter) + except KeyboardInterrupt: + exit(1) + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser(description='Run python demos') + parser.add_argument('--noclear', '-C', action='store_true', + help='Do not clear terminal on each slide') + parser.add_argument('--rst', '-r', action='store_true', + help='Highlight comments and dostrings as rst') + parser.add_argument('--formatter', '-f', default='terminal', + help='pygments formatter name could be: terminal, ' + 'terminal256, terminal16m') + parser.add_argument('--style', '-s', default='default', + help='pygments style name') + parser.add_argument('--auto', '-a', action='store_true', + help='Run all blocks automatically without' + 'confirmation') + parser.add_argument('--delimiter', '-d', default='...', + help='slides delimiter added after each slide run') + parser.add_argument('file', nargs=1, + help='python demo file') + args = parser.parse_args() + slide(args.file[0], noclear=args.noclear, format_rst=args.rst, + formatter=args.formatter, style=args.style, auto_all=args.auto, + delimiter=args.delimiter) diff --git a/IPython/lib/display.py b/IPython/lib/display.py index 63da80fb916..de31788ab97 100644 --- a/IPython/lib/display.py +++ b/IPython/lib/display.py @@ -2,13 +2,14 @@ Authors : MinRK, gregcaporaso, dannystaple """ +from html import escape as html_escape from os.path import exists, isfile, splitext, abspath, join, isdir -from os import walk, sep +from os import walk, sep, fsdecode -from IPython.core.display import DisplayObject +from IPython.core.display import DisplayObject, TextDisplayObject __all__ = ['Audio', 'IFrame', 'YouTubeVideo', 'VimeoVideo', 'ScribdDocument', - 'FileLink', 'FileLinks'] + 'FileLink', 'FileLinks', 'Code'] class Audio(DisplayObject): @@ -32,9 +33,9 @@ class Audio(DisplayObject): * Bytestring containing raw PCM data or * URL pointing to a file on the web. - If the array option is used the waveform will be normalized. + If the array option is used, the waveform will be normalized. - If a filename or url is used the format support will be browser + If a filename or url is used, the format support will be browser dependent. url : unicode A URL to download the data from. @@ -53,6 +54,12 @@ class Audio(DisplayObject): autoplay : bool Set to True if the audio should immediately start playing. Default is `False`. + normalize : bool + Whether audio should be normalized (rescaled) to the maximum possible + range. Default is `True`. When set to `False`, `data` must be between + -1 and 1 (inclusive), otherwise an error is raised. + Applies only when `data` is a list or array of samples; other types of + audio are never normalized. Examples -------- @@ -62,7 +69,7 @@ class Audio(DisplayObject): import numpy as np framerate = 44100 t = np.linspace(0,5,framerate*5) - data = np.sin(2*np.pi*220*t) + np.sin(2*np.pi*224*t)) + data = np.sin(2*np.pi*220*t) + np.sin(2*np.pi*224*t) Audio(data,rate=framerate) # Can also do stereo or more channels @@ -79,12 +86,18 @@ class Audio(DisplayObject): Audio(b'RAW_WAV_DATA..) # From bytes Audio(data=b'RAW_WAV_DATA..) + See Also + -------- + + See also the ``Audio`` widgets form the ``ipywidget`` package for more flexibility and options. + """ _read_flags = 'rb' - def __init__(self, data=None, filename=None, url=None, embed=None, rate=None, autoplay=False): + def __init__(self, data=None, filename=None, url=None, embed=None, rate=None, autoplay=False, normalize=True, *, + element_id=None): if filename is None and url is None and data is None: - raise ValueError("No image data found. Expecting filename, url, or data.") + raise ValueError("No audio data found. Expecting filename, url, or data.") if embed is False and url is None: raise ValueError("No url found. Expecting url when embed=False") @@ -93,10 +106,13 @@ def __init__(self, data=None, filename=None, url=None, embed=None, rate=None, au else: self.embed = True self.autoplay = autoplay + self.element_id = element_id super(Audio, self).__init__(data=data, url=url, filename=filename) if self.data is not None and not isinstance(self.data, bytes): - self.data = self._make_wav(data,rate) + if rate is None: + raise ValueError("rate must be specified when data is a numpy array or list of audio samples.") + self.data = Audio._make_wav(data, rate, normalize) def reload(self): """Reload the raw data from file or URL.""" @@ -111,41 +127,16 @@ def reload(self): else: self.mimetype = "audio/wav" - def _make_wav(self, data, rate): + @staticmethod + def _make_wav(data, rate, normalize): """ Transform a numpy array to a PCM bytestring """ - import struct from io import BytesIO import wave try: - import numpy as np - - data = np.array(data, dtype=float) - if len(data.shape) == 1: - nchan = 1 - elif len(data.shape) == 2: - # In wave files,channels are interleaved. E.g., - # "L1R1L2R2..." for stereo. See - # http://msdn.microsoft.com/en-us/library/windows/hardware/dn653308(v=vs.85).aspx - # for channel ordering - nchan = data.shape[0] - data = data.T.ravel() - else: - raise ValueError('Array audio input must be a 1D or 2D array') - scaled = np.int16(data/np.max(np.abs(data))*32767).tolist() + scaled, nchan = Audio._validate_and_normalize_with_numpy(data, normalize) except ImportError: - # check that it is a "1D" list - idata = iter(data) # fails if not an iterable - try: - iter(idata.next()) - raise TypeError('Only lists of mono audio are ' - 'supported if numpy is not installed') - except TypeError: - # this means it's not a nested list, which is what we want - pass - maxabsvalue = float(max([abs(x) for x in data])) - scaled = [int(x/maxabsvalue*32767) for x in data] - nchan = 1 + scaled, nchan = Audio._validate_and_normalize_without_numpy(data, normalize) fp = BytesIO() waveobj = wave.open(fp,mode='wb') @@ -153,12 +144,61 @@ def _make_wav(self, data, rate): waveobj.setframerate(rate) waveobj.setsampwidth(2) waveobj.setcomptype('NONE','NONE') - waveobj.writeframes(b''.join([struct.pack(' 1: + raise ValueError('Audio data must be between -1 and 1 when normalize=False.') + return max_abs_value if normalize else 1 + def _data_and_metadata(self): """shortcut for returning metadata with url information, if defined""" md = {} @@ -171,12 +211,13 @@ def _data_and_metadata(self): def _repr_html_(self): src = """ -