diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000000..ea217df9307d --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,106 @@ +version: 2.1 + +jobs: + + # The following job is to run any image comparison test, and runs on any branch + # or in any pull request. It will generate a summary page for each tox environment + # being run, and giles will report the URL of the summary page back to the pull + # request (alternatively you can find the summary page in the artifacts in the + # CircleCI UI). + figure: + parameters: + jobname: + type: string + docker: + - image: cimg/python:3.11 + environment: + TOXENV: << parameters.jobname >> + PY_COLORS: "1" + steps: + - checkout + - run: + name: Install dependencies + command: | + sudo apt update + sudo apt install texlive texlive-latex-extra texlive-fonts-recommended dvipng cm-super + pip install pip tox --upgrade + - run: + name: Run tests + command: tox -v + - run: + name: Upload coverage results to codecov + command: | + curl -Os https://uploader.codecov.io/latest/linux/codecov + chmod +x codecov + ./codecov -t ${CODECOV_TOKEN} -f coverage.xml + - store_artifacts: + path: results + - run: + name: "Image comparison page is available at: " + command: echo "${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/results/fig_comparison.html" + + # The following job runs only on main - and its main purpose is to update the reference + # images in the astropy-figure-tests repository. This job needs a deploy key. To produce + # this, go to the astropy-figure-tests repository settings and go to SSH keys, then add + # your public SSH key. + deploy-reference-images: + parameters: + jobname: + type: string + docker: + - image: cimg/python:3.11 + environment: + TOXENV: << parameters.jobname >> + PY_COLORS: "1" + GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa_bfaaefe38d95110b75c79252bafbe0fc + steps: + - checkout + - run: + name: Install dependencies + command: | + sudo apt update + sudo apt install texlive texlive-latex-extra texlive-fonts-recommended dvipng cm-super + pip install pip tox --upgrade + - run: ssh-add -D + - add_ssh_keys: + fingerprints: "bf:aa:ef:e3:8d:95:11:0b:75:c7:92:52:ba:fb:e0:fc" + - run: ssh-keyscan github.com >> ~/.ssh/known_hosts + - run: git config --global user.email "astropy@circleci" && git config --global user.name "Astropy Circle CI" + - run: git clone git@github.com:astropy/astropy-figure-tests.git --depth 1 -b astropy-${CIRCLE_BRANCH} ~/astropy-figure-tests/ + - run: + name: Generate reference images + command: tox -v -- --mpl-generate-path=/home/circleci/astropy-figure-tests/figures/$TOXENV + - run: | + cd ~/astropy-figure-tests/ + git pull + git status + git add . + git commit -m "Update reference figures from ${CIRCLE_BRANCH}" || echo "No changes to reference images to deploy" + git push + +workflows: + version: 2 + + figure-tests: + jobs: + - figure: + name: << matrix.jobname >> + matrix: + parameters: + jobname: + - "py311-test-image-mpl380-cov" + - "py311-test-image-mpldev-cov" + + - deploy-reference-images: + name: baseline-<< matrix.jobname >> + matrix: + parameters: + jobname: + - "py311-test-image-mpl380-cov" + - "py311-test-image-mpldev-cov" + requires: + - << matrix.jobname >> + filters: + branches: + only: + - main diff --git a/.clang-format b/.clang-format new file mode 100644 index 000000000000..0ae810b7e9dc --- /dev/null +++ b/.clang-format @@ -0,0 +1,32 @@ +# clang-format configuration for astropy C code + +BasedOnStyle: Google +IndentWidth: 4 +TabWidth: 4 +UseTab: Never +ContinuationIndentWidth: 4 +ColumnLimit: 100 +AlignAfterOpenBracket: BlockIndent +AlwaysBreakAfterReturnType: None +BreakBeforeBraces: Stroustrup +InsertBraces: true +BinPackArguments: false +BinPackParameters: false +PointerAlignment: Right +SpaceAfterCStyleCast: false +IncludeBlocks: Preserve +SortIncludes: false +ReflowComments: false +MaxEmptyLinesToKeep: 2 +KeepEmptyLinesAtTheStartOfBlocks: true +AllowShortEnumsOnASingleLine: false +AllowShortIfStatementsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Empty +IndentPPDirectives: None +IndentGotoLabels: false +SpaceBeforeParens: ControlStatements +SpacesBeforeTrailingComments: 1 +BreakStringLiterals: false +DerivePointerAlignment: false +AlignEscapedNewlines: DontAlign +StatementMacros: [PyObject_HEAD, PyObject_VAR_HEAD, PyObject_HEAD_EXTRA] diff --git a/.continuous-integration/README.md b/.continuous-integration/README.md deleted file mode 100644 index b6f29cf8d6e6..000000000000 --- a/.continuous-integration/README.md +++ /dev/null @@ -1,15 +0,0 @@ -About -===== - -This directory contains a set of scripts that are used by the -``.travis.yml`` and ``appveyor.yml`` files for the -[Travis](http://travis-ci.org) and [AppVeyor](http://www.appveyor.com/) -services respectively. - -The scripts include: - -* ``appveyor/install-miniconda.ps1`` - set up conda on Windows -* ``appveyor/windows_sdk.cmd`` - set up the compiler environment on Windows -* ``travis/setup_dependencies_common.sh`` - set up conda packages on Linux and MacOS X -* ``travis/setup_environment_linux.sh`` - set up conda and non-Python dependencies on Linux -* ``travis/setup_environment_osx.sh`` - set up conda and non-Python dependencies on MacOS X diff --git a/.continuous-integration/appveyor/install-miniconda.ps1 b/.continuous-integration/appveyor/install-miniconda.ps1 deleted file mode 100644 index aeccc3b1c196..000000000000 --- a/.continuous-integration/appveyor/install-miniconda.ps1 +++ /dev/null @@ -1,71 +0,0 @@ -īģŋ# Sample script to install anaconda under windows -# Authors: Stuart Mumford -# Borrwed from: Olivier Grisel and Kyle Kastner -# License: BSD 3 clause - -$MINICONDA_URL = "http://repo.continuum.io/miniconda/" - -function DownloadMiniconda ($version, $platform_suffix) { - $webclient = New-Object System.Net.WebClient - $filename = "Miniconda-" + $version + "-Windows-" + $platform_suffix + ".exe" - - $url = $MINICONDA_URL + $filename - - $basedir = $pwd.Path + "\" - $filepath = $basedir + $filename - if (Test-Path $filename) { - Write-Host "Reusing" $filepath - return $filepath - } - - # Download and retry up to 3 times in case of network transient errors. - Write-Host "Downloading" $filename "from" $url - $retry_attempts = 2 - for($i=0; $i -lt $retry_attempts; $i++){ - try { - $webclient.DownloadFile($url, $filepath) - break - } - Catch [Exception]{ - Start-Sleep 1 - } - } - if (Test-Path $filepath) { - Write-Host "File saved at" $filepath - } else { - # Retry once to get the error message if any at the last try - $webclient.DownloadFile($url, $filepath) - } - return $filepath -} - -function InstallMiniconda ($python_version, $architecture, $python_home) { - Write-Host "Installing miniconda" $python_version "for" $architecture "bit architecture to" $python_home - if (Test-Path $python_home) { - Write-Host $python_home "already exists, skipping." - return $false - } - if ($architecture -eq "x86") { - $platform_suffix = "x86" - } else { - $platform_suffix = "x86_64" - } - $filepath = DownloadMiniconda $python_version $platform_suffix - Write-Host "Installing" $filepath "to" $python_home - $args = "/InstallationType=AllUsers /S /AddToPath=1 /RegisterPython=1 /D=" + $python_home - Write-Host $filepath $args - Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru - #Start-Sleep -s 15 - if (Test-Path C:\conda) { - Write-Host "Miniconda $python_version ($architecture) installation complete" - } else { - Write-Host "Failed to install Python in $python_home" - Exit 1 - } -} - -function main () { - InstallMiniconda $env:MINICONDA_VERSION $env:PLATFORM $env:PYTHON -} - -main diff --git a/.continuous-integration/appveyor/windows_sdk.cmd b/.continuous-integration/appveyor/windows_sdk.cmd deleted file mode 100644 index 3a472bc836c3..000000000000 --- a/.continuous-integration/appveyor/windows_sdk.cmd +++ /dev/null @@ -1,47 +0,0 @@ -:: To build extensions for 64 bit Python 3, we need to configure environment -:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) -:: -:: To build extensions for 64 bit Python 2, we need to configure environment -:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) -:: -:: 32 bit builds do not require specific environment configurations. -:: -:: Note: this script needs to be run with the /E:ON and /V:ON flags for the -:: cmd interpreter, at least for (SDK v7.0) -:: -:: More details at: -:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows -:: http://stackoverflow.com/a/13751649/163740 -:: -:: Author: Olivier Grisel -:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ -@ECHO OFF - -SET COMMAND_TO_RUN=%* -SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows - -SET MAJOR_PYTHON_VERSION="%PYTHON_VERSION:~0,1%" -IF %MAJOR_PYTHON_VERSION% == "2" ( - SET WINDOWS_SDK_VERSION="v7.0" -) ELSE IF %MAJOR_PYTHON_VERSION% == "3" ( - SET WINDOWS_SDK_VERSION="v7.1" -) ELSE ( - ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" - EXIT 1 -) - -IF "%PYTHON_ARCH%"=="64" ( - ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture - SET DISTUTILS_USE_SDK=1 - SET MSSdk=1 - "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% - "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 -) ELSE ( - ECHO Using default MSVC build environment for 32 bit architecture - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 -) diff --git a/.continuous-integration/travis/setup_dependencies_common.sh b/.continuous-integration/travis/setup_dependencies_common.sh deleted file mode 100755 index 61b417a530bf..000000000000 --- a/.continuous-integration/travis/setup_dependencies_common.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -x - -# CONDA -conda create --yes -n test -c astropy-ci-extras python=$PYTHON_VERSION pip -source activate test - -# EGG_INFO -if [[ $SETUP_CMD == egg_info ]] -then - exit # no more dependencies needed -fi - -# PEP8 -if [[ $MAIN_CMD == pep8* ]] -then - pip install pep8 - exit # no more dependencies needed -fi - -# CORE DEPENDENCIES -conda install --yes pytest Cython jinja2 psutil - -# NUMPY -if [[ $NUMPY_VERSION == dev ]] -then - pip install git+http://github.com/numpy/numpy.git - export CONDA_INSTALL="conda install --yes python=$PYTHON_VERSION" -else - conda install --yes numpy=$NUMPY_VERSION - export CONDA_INSTALL="conda install --yes python=$PYTHON_VERSION numpy=$NUMPY_VERSION" -fi - -# Now set up shortcut to conda install command to make sure the Python and Numpy -# versions are always explicitly specified. - -# OPTIONAL DEPENDENCIES -if $OPTIONAL_DEPS -then - $CONDA_INSTALL scipy h5py matplotlib pyyaml scikit-image pandas - pip install beautifulsoup4 -fi - -# DOCUMENTATION DEPENDENCIES -# build_sphinx needs sphinx and matplotlib (for plot_directive). Note that -# this matplotlib will *not* work with py 3.x, but our sphinx build is -# currently 2.7, so that's fine -if [[ $SETUP_CMD == build_sphinx* ]] -then - $CONDA_INSTALL Sphinx=1.2.2 Pygments matplotlib -fi - -# COVERAGE DEPENDENCIES -if [[ $SETUP_CMD == 'test --coverage' ]] -then - pip install coverage coveralls -fi - - diff --git a/.continuous-integration/travis/setup_environment_linux.sh b/.continuous-integration/travis/setup_environment_linux.sh deleted file mode 100755 index 154142b87b47..000000000000 --- a/.continuous-integration/travis/setup_environment_linux.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Install conda -wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh -chmod +x miniconda.sh -./miniconda.sh -b -export PATH=/home/travis/miniconda/bin:$PATH -conda update --yes conda - -# Install non-Python dependencies for documentation -if [[ $SETUP_CMD == build_sphinx* ]] -then - sudo apt-get update - sudo apt-get install graphviz texlive-latex-extra dvipng -fi - -# Install Python dependencies -source "$( dirname "${BASH_SOURCE[0]}" )"/setup_dependencies_common.sh diff --git a/.continuous-integration/travis/setup_environment_osx.sh b/.continuous-integration/travis/setup_environment_osx.sh deleted file mode 100755 index 102f01321ac7..000000000000 --- a/.continuous-integration/travis/setup_environment_osx.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Install conda -wget http://repo.continuum.io/miniconda/Miniconda-3.7.3-MacOSX-x86_64.sh -O miniconda.sh -chmod +x miniconda.sh -./miniconda.sh -b -export PATH=/Users/travis/miniconda/bin:$PATH -conda update --yes conda - -# Install Python dependencies -source "$( dirname "${BASH_SOURCE[0]}" )"/setup_dependencies_common.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000000..5830b977771d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,17 @@ +{ + "name": "AstroPy", + "image": "mcr.microsoft.com/devcontainers/miniconda:0-3", + "onCreateCommand": "conda init bash && sudo cp .devcontainer/welcome-message.txt /usr/local/etc/vscode-dev-containers/first-run-notice.txt", + "postCreateCommand": "git fetch --tags && pip install tox", + "waitFor": "postCreateCommand", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.black-formatter", + "charliermarsh.ruff", + "stkb.rewrap" + ] + } + } +} diff --git a/.devcontainer/welcome-message.txt b/.devcontainer/welcome-message.txt new file mode 100644 index 000000000000..08e7cea12d05 --- /dev/null +++ b/.devcontainer/welcome-message.txt @@ -0,0 +1,12 @@ +👋 Welcome to "AstroPy" in GitHub Codespaces! + +🔍 To explore VS Code to its fullest, search using the Command Palette (Cmd/Ctrl + Shift + P or F1). + +ℹī¸ Look at https://docs.astropy.org/en/latest/development/quickstart.html + for more contribution details. + +⭐⭐ ================================= IMPORTANT!! ================================== ⭐⭐ + To complete setup of your development environment run the following in the terminal: + + pip install -e .[all,test_all,docs] +⭐⭐ ================================================================================== ⭐⭐ diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000000..6a3eb2a50ff5 --- /dev/null +++ b/.flake8 @@ -0,0 +1,31 @@ +[flake8] +max-line-length = 88 +select = E,F,W +exclude = extern,*parsetab.py,*lextab.py +extend-ignore = E203,E501,E711,E721,E731,E741,F403,F841,W5 +per-file-ignores = + __init__.py:F401,F403,E402 + test_*.py:E402 + astropy/io/fits/card.py:E131 + astropy/io/registry/compat.py:F822 + astropy/convolution/convolve.py:E241 + astropy/modeling/functional_models.py:E226,E241 + astropy/modeling/models.py:F401,F403,F405 + astropy/modeling/tests/test_constraints.py:E241 + astropy/stats/tests/test_histogram.py:E241 + astropy/stats/tests/test_sigma_clipping.py:E126,E131,E241 + astropy/units/__init__.py:F401,F821 + astropy/units/astrophys.py:F821 + astropy/units/cgs.py:F821 + astropy/units/imperial.py:F821 + astropy/units/misc.py:F821 + astropy/units/photometric.py:F821 + astropy/units/si.py:F821 + astropy/units/tests/test_quantity_decorator.py:F821 + astropy/units/tests/test_quantity_typing.py:F821 + astropy/wcs/wcs.py:F821 + astropy/wcs/wcsapi/fitswcs.py:F821 + astropy/wcs/wcsapi/utils.py:E127,E128 + docs/conf.py:F401 + examples/*.py:E1,E2,E402 + */examples/*.py:E1,E2,E402 diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 000000000000..c0f3b1a2fa57 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,55 @@ +# Commits to ignore in blame: +# black for astropy.config +5a4c5fcb00bf917c4bbe3c342b66177cf6cb3c75 +# black for astropy.constants +e789729540438eb891ae83ef1a39171f7a4c007a +# black for astropy.convolution +9d4adb8a5ab74a8c85e0d4b41e65bb78ee19aedb +# black for astropy.coordinates +fbbc0cb370d033722fef7f501b6aed4b500812d0 +# black for astropy.cosmology +420686db541271e8a20ad15e3a2b68bf30655889 +# black for astropy.io.ascii +6e80e68e817fd629e04e80b45478052b6a4a3c4a +# black for astropy.io.fits +6750c7f86129858b92d83b0086c3b99b9470bf8b +# black for astropy.io.misc +128fe2873f25581feda0c2d726181b093bdc306a +# black for atropy.io.votable +7ca4c9e44e762e258b3c455bdfca793b83a93802 +# black for other astropy.io +2e948e310fa2882ef3fd483b130297e3c692481f +# black for astropy.modeling +d8f7f53c7342e4e40c1a76351a3befa1f9b2f2db +# black for astropy.nddata +a7cb990203a9d2093fa81bbbbd4ac0f763f6da6b +# black for astropy.samp +3c9c91ae593cad2f669d65bf3e67accc5c49333c +# black for astropy.stats +ecfab5a9f56d5f5724f30a6dabe580960136e4c6 +# black for astropy.table +1403108bcd07341657e41d47c99abad373f27644 +# black for astropy.tests +369402b47317f4752e3dadaa136a1f8ccce74a2e +# black for astropy.time +fa2df9f32e5c041d74f7d1ef3d5a188c521cbc79 +# black for astropy.timeseries +059495d362003d3f88c1e7456bfe56693c958e10 +# black for astropy.uncertainty +58470fa36befa7efd1e37bbc7374859d7c22223a +# black for astropy.units +53ba687f90131a7b5a578af32b376b8fac04863a +# black for astropy.utils +0630d6acf23aab2ef338b35f9a5130ee592e7aa5 +# black for astropy.visualization +1265c1a9970d467c9372f9143b8cb0a1d37facf1 +# black for astropy.wcs +5bf365bd9638f3c335963745b46246fabe428a0f +# black for other parts of astropy +d2165fa02c1a2d79f4a6cb69a5cebeee0b073033 +# DOC: Moved dev docs around +f5a1738f32d4920ac853c7c0ec6c941ccec47555 +# clang-format for most sub-packages except astropy.wcs +933e3e6eadff215165c838fd91c21e8615d0cf88 +# clang-format for astropy.wcs +616797e13c0b926ad48ea9e9aac22a0421b74d73 diff --git a/.gitattributes b/.gitattributes index 054f238026f2..9461903e124f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,2 @@ *.fits -text -astropy.0.3.windows.cfg eol=crlf -CHANGES.rst merge=union +astropy/io/fits/tiled_compression/tests/data/*fits binary diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000000..c94945f28da7 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,66 @@ +# Despite the name of this file, the people listed here DO NOT OWN the code in astropy subpackages, +# the copyright is held by the Astropy Project as a whole - see the license file for details. +# +# This file exists because as of 2022, GitHub does not offer a mechanism to subscribe to only +# a fraction of new PRs in astropy. Either one chooses to "watch" the entire repository and +# receives a large number of notifications or one needs to manually check for new PRs for +# sub-package(s) of interest. +# +# All names listed here by sub-packages will be requested to review any new PR +# and thus get notified. Only people with maintainer-level permission can be added as +# PR reviewers. +# +# Instructions for core maintainers -- Add your GitHub username to opt-in to automatic +# review requests for specific sub-packages below: + +# astropy/constants +astropy/convolution @larrybradley +# astropy/coordinates +astropy/cosmology @astropy/cosmology +astropy/io/ascii @taldcroft @dhomeier +astropy/io/fits @saimn +astropy/io/misc @WilliamJamieson @matteobachetti @neutrinoceros +astropy/io/registry @nstarman @neutrinoceros +# astropy/io/votable +astropy/modeling @astropy/modeling +# astropy/nddata +# astropy/samp +astropy/stats @larrybradley +astropy/table @taldcroft @neutrinoceros +astropy/time @taldcroft +# astropy/timeseries +# astropy/uncertainty +# astropy/units +# astropy/utils +astropy/visualization @astrofrog @larrybradley +astropy/wcs @mcara +astropy/wcs/wcsapi @astrofrog + +# docs/constants +docs/convolution @larrybradley +# docs/coordinates +docs/cosmology @astropy/cosmology +docs/io/ascii @taldcroft @dhomeier +docs/io/fits @saimn +# docs/io/votable +docs/modeling @astropy/modeling +# docs/nddata +# docs/samp +docs/stats @larrybradley +docs/table @taldcroft @neutrinoceros +docs/time @taldcroft +# docs/timeseries +# docs/uncertainty +# docs/units +# docs/utils +docs/visualization @astrofrog @larrybradley +docs/wcs @mcara + +scripts/* @neutrinoceros +pyproject.toml @neutrinoceros +setup.py @neutrinoceros +MANIFEST.in @neutrinoceros +tox.ini @neutrinoceros +*.git* @neutrinoceros # git files and .github dir +.pre-commit-config.yaml @WilliamJamieson @nstarman @neutrinoceros +.ruff.toml @WilliamJamieson @nstarman @neutrinoceros diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml new file mode 100644 index 000000000000..8e4612ef90a7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -0,0 +1,51 @@ +name: Bug report +description: Create a report describing unexpected or incorrect behavior in astropy. +labels: Bug +body: + - type: markdown + attributes: + value: >- + Thanks for taking the time to fill out this bug report! + Please have a search on our GitHub repository to see if a similar + issue has already been posted. If a similar issue is closed, have a + quick look to see if you are satisfied by the resolution. + If not please go ahead and open an issue! + Please check that the + [development version](https://docs.astropy.org/en/latest/development/quickstart.html#install-the-development-version-of-astropy) + still produces the same bug. + - type: textarea + attributes: + label: Description + description: >- + A clear and concise description of what the bug is. + - type: textarea + attributes: + label: Expected behavior + description: >- + A clear and concise description of what you expected to happen. + - type: textarea + attributes: + label: How to Reproduce + description: >- + A clear and concise description of what actually happened instead. + Was the output confusing or poorly described? Please provide steps to reproduce this bug. + value: | + 1. Get package from '...' + 2. Then run '...' + 3. An error occurs. + + ```python + # Put your Python code snippet here. + ``` + - type: textarea + attributes: + label: Versions + description: Please run the following script and paste the output + value: | + ```python + import astropy + astropy.system_info() + ``` + ``` + # Paste the result here + ``` diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000000..5c70a42877ce --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,4 @@ +contact_links: + - name: Question/Help/Support + url: https://www.astropy.org/help + about: "If you have a question, please look at the listed resources available on the website." diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml new file mode 100644 index 000000000000..546cc83995ed --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -0,0 +1,32 @@ +name: Feature request +description: Suggest an idea to improve astropy. +labels: "Feature Request" +body: + - type: markdown + attributes: + value: >- + Thanks for taking the time to fill out this feature request! + Please have a search on our GitHub repository to see if a similar + issue has already been posted. If a similar issue is closed, have a + quick look to see if you are satisfied by the resolution. + If not please go ahead and open an issue! + - type: textarea + attributes: + label: What is the problem this feature will solve? + description: >- + What are you trying to do, that you are unable to achieve with astropy + and its affiliated packages as it currently stands? + - type: textarea + attributes: + label: Describe the desired outcome + description: >- + Clear and concise description of what you want to happen. Please use examples + of real world use cases that this would help with, and how it solves the + problem described above. If you want to, you can suggest a draft design or API + so we can have a deeper discussion on the feature. + - type: textarea + attributes: + label: Additional context + description: >- + Add any other context, links, etc. relevant to the feature request. + You may also include screenshots if necessary. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000000..3f2b6b3dd23a --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,47 @@ + + + + + + + + + + +### Description + + + + + + +This pull request is to address ... + + + +Fixes # + + + +- [ ] By checking this box, the PR author has requested that maintainers do **NOT** use the "Squash and Merge" button. Maintainers should respect this when possible; however, the final decision is at the discretion of the maintainer that merges the PR. + + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000000..2266623596cc --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,22 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" # See documentation for possible values + directory: ".github/workflows" # Location of package manifests + labels: + - dependencies + - dev-automation + - github_actions + - no-changelog-entry-needed + schedule: + interval: "monthly" + groups: + actions: + patterns: + - "*" + cooldown: + default-days: 7 diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 000000000000..be4183f0c795 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,247 @@ +dependencies: +- any: + - head-branch: + - '^update-astropy-iers-data-pin-\d+$' + +Docs: +- changed-files: + - any-glob-to-any-file: + - docs/* + - docs/_static/* + - docs/_templates/* + - docs/development/**/* + - docs/whatsnew/* + - examples/**/* + - licenses/* + - CITATION + - CITATION.cff + - .mailmap + - .readthedocs.yaml + - '*.md' + - all-globs-to-any-file: + - '**/*.rst' + - '!CHANGES.rst' + - '!docs/changes/**/*' + +testing: +- changed-files: + - any-glob-to-any-file: + - astropy/tests/**/* + - codecov.yml + - conftest.py + - '**/conftest.py' + - tox.ini + - .circleci/* + - .github/workflows/CFF-test.yml + - .github/workflows/check_changelog.yml + - .github/workflows/ci*.yml + - .github/workflows/codeql-analysis.yml + - .pyinstaller/**/* + - .flake8 + - .pycodestyle + - .pre-commit-config.yaml + - .ruff.toml + +dev-automation: +- changed-files: + - any-glob-to-any-file: + - .pre-commit-config.yaml + - .ruff.toml + - .devcontainer/* + - .github/ISSUE_TEMPLATE/* + - .github/* + - .github/workflows/open_actions.yml + - .github/workflows/stalebot.yml + - .github/workflows/update_astropy_iers_data_pin.* + +skip-changelog-checks: +- changed-files: + - any-glob-to-any-file: + - .pre-commit-config.yaml + +external: +- changed-files: + - any-glob-to-any-file: + - astropy/extern/**/* + - cextern/**/* + +installation: +- changed-files: + - any-glob-to-any-file: + - docs/install.rst + - MANIFEST.in + - setup.py + +Release: +- changed-files: + - any-glob-to-any-file: + - docs/development/maintainers/releasing.rst + - .github/workflows/publish.yml + +config: +- changed-files: + - any-glob-to-any-file: + - '**/config/**/*' + - astropy/extern/configobj/**/* + +constants: +- changed-files: + - any-glob-to-any-file: + - '**/constants/**/*' + +convolution: +- changed-files: + - any-glob-to-any-file: + - '**/convolution/**/*' + +coordinates: +- changed-files: + - any-glob-to-any-file: + - '**/coordinates/**/*' + +cosmology: +- changed-files: + - any-glob-to-any-file: + - '**/cosmology/**/*' + +io.ascii: +- changed-files: + - any-glob-to-any-file: + - '**/io/ascii/**/*' + +io.fits: +- changed-files: + - any-glob-to-any-file: + - '**/io/fits/**/*' + - cextern/cfitsio/**/* + +io.misc: +- changed-files: + - any-glob-to-any-file: + - astropy/io/misc/* + - astropy/io/misc/pandas/**/* + - astropy/io/misc/tests/**/* + - docs/io/misc*.rst + +io.registry: +- changed-files: + - any-glob-to-any-file: + - astropy/io/* + - astropy/io/registry/**/* + - astropy/io/tests/* + - docs/io/registry*.rst + +io.votable: +- changed-files: + - any-glob-to-any-file: + - '**/io/votable/**/*' + +logging: +- changed-files: + - any-glob-to-any-file: + - astropy/logger.py + - astropy/tests/test_logger.py + - docs/logging.rst + +modeling: +- changed-files: + - any-glob-to-any-file: + - '**/modeling/**/*' + +nddata: +- changed-files: + - any-glob-to-any-file: + - '**/nddata/**/*' + +samp: +- changed-files: + - any-glob-to-any-file: + - '**/samp/**/*' + +stats: +- changed-files: + - any-glob-to-any-file: + - '**/stats/**/*' + +table: +- changed-files: + - any-glob-to-any-file: + - '**/table/**/*' + +time: +- changed-files: + - any-glob-to-any-file: + - '**/time/**/*' + +timeseries: +- changed-files: + - any-glob-to-any-file: + - '**/timeseries/**/*' + +uncertainty: +- changed-files: + - any-glob-to-any-file: + - '**/uncertainty/**/*' + +unified-io: +- changed-files: + - any-glob-to-any-file: + - astropy/table/connect.py + - astropy/io/**/connect.py + - docs/io/unified.rst + +units: +- changed-files: + - any-glob-to-any-file: + - '**/units/**/*' + - astropy/extern/ply/**/* + +utils: +- changed-files: + - any-glob-to-any-file: + - cextern/expat/**/* + - all-globs-to-any-file: + - '**/utils/**/*' + - '!astropy/utils/iers/**/*' + - '!docs/utils/iers.rst' + - '!astropy/utils/masked/**/*' + - '!docs/utils/masked/**/*' + +utils.iers: +- changed-files: + - any-glob-to-any-file: + - astropy/utils/iers/**/* + - docs/utils/iers.rst + - .github/workflows/update_astropy_iers_data_pin.* + +utils.masked: +- changed-files: + - any-glob-to-any-file: + - astropy/utils/masked/**/* + - docs/utils/masked/**/* + +visualization: +- changed-files: + - all-globs-to-any-file: + - '**/visualization/**/*' + - '!**/visualization/wcsaxes/**/*' + +visualization.wcsaxes: +- changed-files: + - any-glob-to-any-file: + - '**/visualization/wcsaxes/**/*' + +wcs: +- changed-files: + - any-glob-to-any-file: + - cextern/wcslib/**/* + - all-globs-to-any-file: + - '**/wcs/**/*' + - '!astropy/wcs/wcsapi/**/*' + - '!docs/wcs/wcsapi.rst' + +wcs.wcsapi: +- changed-files: + - any-glob-to-any-file: + - astropy/wcs/wcsapi/**/* + - docs/wcs/wcsapi.rst diff --git a/.github/workflows/CFF-test.yml b/.github/workflows/CFF-test.yml new file mode 100644 index 000000000000..723a27e46cd6 --- /dev/null +++ b/.github/workflows/CFF-test.yml @@ -0,0 +1,27 @@ +name: Checking CITATION.cff + +on: + push: + paths: + - "CITATION.cff" + pull_request: + paths: + - "CITATION.cff" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + cffconvert: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: citation-file-format/cffconvert-github-action@4cf11baa70a673bfdf9dad0acc7ee33b3f4b6084 # 2.0.0 + with: + args: --validate diff --git a/.github/workflows/check_changelog.yml b/.github/workflows/check_changelog.yml new file mode 100644 index 000000000000..4cf23b18fe2b --- /dev/null +++ b/.github/workflows/check_changelog.yml @@ -0,0 +1,26 @@ +name: Check PR change log + +on: + pull_request: + types: [opened, synchronize, labeled, unlabeled] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + pull-requests: read + +jobs: + changelog_checker: + name: Check if towncrier change log entry is correct + runs-on: ubuntu-latest + if: github.repository == 'astropy/astropy' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: scientific-python/action-towncrier-changelog@f9c7df9a9f8b55cb0c12d94d14b281e9bcd101c0 # v2.0.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BOT_USERNAME: gilesbot diff --git a/.github/workflows/check_milestone.yml b/.github/workflows/check_milestone.yml new file mode 100644 index 000000000000..9c852f41d393 --- /dev/null +++ b/.github/workflows/check_milestone.yml @@ -0,0 +1,33 @@ +name: Check PR milestone + +on: + pull_request: + types: [synchronize, milestoned, demilestoned] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + pull-requests: read + +jobs: + # https://stackoverflow.com/questions/69434370/how-can-i-get-the-latest-pr-data-specifically-milestones-when-running-yaml-jobs + milestone_checker: + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + if: github.repository == 'astropy/astropy' + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { data } = await github.request("GET /repos/{owner}/{repo}/pulls/{pr}", { + owner: context.repo.owner, + repo: context.repo.repo, + pr: context.payload.pull_request.number + }); + if (data.milestone) { + core.info(`This pull request has a milestone set: ${data.milestone.title}`); + } else { + core.setFailed(`A maintainer needs to set the milestone for this pull request.`); + } diff --git a/.github/workflows/ci_benchmark.yml b/.github/workflows/ci_benchmark.yml new file mode 100644 index 000000000000..11af0b70e681 --- /dev/null +++ b/.github/workflows/ci_benchmark.yml @@ -0,0 +1,101 @@ +### Inspired from https://github.com/scikit-image/scikit-image/blob/main/.github/workflows/benchmarks.yml + +name: Benchmark + +on: + pull_request: + types: [labeled, synchronize] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + benchmark: + if: (github.repository == 'astropy/astropy' && contains(github.event.pull_request.labels.*.name, 'benchmark')) + name: "Compare asv with astropy main" + runs-on: ubuntu-latest + env: + CCACHE_BASEDIR: "${{ github.workspace }}" + CCACHE_DIR: "${{ github.workspace }}/.ccache" + CCACHE_COMPRESS: true + CCACHE_COMPRESSLEVEL: 6 + CCACHE_MAXSIZE: 400M + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + fetch-depth: 0 + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + name: Install Python + with: + python-version: "3.11" + + - name: Setup some dependencies + shell: bash -l {0} + run: | + sudo apt-get update -y && sudo apt-get install -y ccache + # Make gcc/gxx symlinks first in path + sudo /usr/sbin/update-ccache-symlinks + echo "/usr/lib/ccache" >> $GITHUB_PATH + + - name: "Prepare ccache" + id: prepare-ccache + shell: bash -l {0} + run: | + echo "key=benchmark-$RUNNER_OS" >> $GITHUB_OUTPUT + echo "timestamp=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_OUTPUT + ccache -p + ccache -z + + - name: "Restore ccache" + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: .ccache + key: ccache-${{ secrets.CACHE_VERSION }}-${{ steps.prepare-ccache.outputs.key }}-${{ steps.prepare-ccache.outputs.timestamp }} + restore-keys: | + ccache-${{ secrets.CACHE_VERSION }}-${{ steps.prepare-ccache.outputs.key }}- + + - name: Run benchmarks + shell: bash -l {0} + id: benchmark + env: + OPENBLAS_NUM_THREADS: 1 + MKL_NUM_THREADS: 1 + OMP_NUM_THREADS: 1 + ASV_FACTOR: 1.3 + ASV_SKIP_SLOW: 1 + BASE_SHA: ${{ github.event.pull_request.base.sha }} + BASE_LABEL: ${{ github.event.pull_request.base.label }} + run: | + set -x + python -m pip install asv virtualenv packaging + + git clone -b main https://github.com/astropy/astropy-benchmarks.git --single-branch + + # ID this runner + python -m asv machine --yes --conf asv.ci.conf.json + + echo "Baseline: ${BASE_SHA} (${BASE_LABEL})" + echo "Contender: ${GITHUB_SHA} (${BASE_LABEL})" + + # Run benchmarks for current commit against base + ASV_OPTIONS="--split --show-stderr --factor $ASV_FACTOR --conf asv.ci.conf.json" + python -m asv continuous $ASV_OPTIONS ${BASE_SHA} ${GITHUB_SHA} + + - name: "Check ccache performance" + shell: bash -l {0} + run: ccache -s + if: always() + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: asv-benchmark-results + path: | + results/ diff --git a/.github/workflows/ci_cron_daily.yml b/.github/workflows/ci_cron_daily.yml new file mode 100644 index 000000000000..e00dffaa6406 --- /dev/null +++ b/.github/workflows/ci_cron_daily.yml @@ -0,0 +1,84 @@ +name: Daily cron + +on: + workflow_dispatch: + schedule: + # run every day at 3am UTC + - cron: '0 3 * * *' + pull_request: + # We also want this workflow triggered if the 'Extra CI' label is added + # or present when PR is updated + types: + - synchronize + - labeled + push: + # We want this workflow to always run on release branches as well as + # all tags since we want to be really sure we don't introduce + # regressions on the release branches, and it's also important to run + # this on pre-release and release tags. + branches: + - 'v*' + tags: + - '*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + ARCH_ON_CI: "normal" + IS_CRON: "true" + PY_COLORS: "1" + +permissions: + contents: read + +jobs: + tests: + runs-on: ${{ matrix.os }} + if: | + github.repository == 'astropy/astropy' && ( + github.event_name == 'schedule' || + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'Extra CI') + ) + strategy: + fail-fast: false + matrix: + include: + - name: Bundling with pyinstaller + os: ubuntu-latest + python: '3.11' + toxenv: pyinstaller + + - name: Python 3.12 with all optional dependencies and pre-releases + os: ubuntu-latest + python: '3.12' + toxenv: py312-test-alldeps-predeps + toxargs: -v + toxposargs: --remote-data=any + + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ matrix.python }} + + - name: Install language-pack-de and tzdata + if: ${{ matrix.os == 'ubuntu-latest' }} + run: | + sudo apt-get update + sudo apt-get install language-pack-de tzdata + + - name: Install Python dependencies + run: python -m pip install --upgrade tox + + - name: Run tests + run: tox ${{ matrix.toxargs}} -e ${{ matrix.toxenv}} -- ${{ matrix.toxposargs}} diff --git a/.github/workflows/ci_cron_monthly.yml b/.github/workflows/ci_cron_monthly.yml new file mode 100644 index 000000000000..1ee9939ae692 --- /dev/null +++ b/.github/workflows/ci_cron_monthly.yml @@ -0,0 +1,105 @@ +name: Monthly cron + +on: + workflow_dispatch: + schedule: + # run every first of month at 6am UTC + - cron: '0 6 1 * *' + pull_request: + # We also want this workflow triggered if the 'Extra CI' label is added + # or present when PR is updated + types: + - synchronize + - labeled + push: + # We want this workflow to always run on release branches as well as + # all tags since we want to be really sure we don't introduce + # regressions on the release branches, and it's also important to run + # this on pre-release and release tags. + branches: + - 'v*' + tags: + - '*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + IS_CRON: 'true' + PY_COLORS: "1" + +permissions: + contents: read + +jobs: + + tests_more_arch_bundled_lib: + + # Testing exotic architectures using bundled external libs, i.e. without setting ASTROPY_USE_SYSTEM_ALL. + # This is a variant and subset of weekly cron, currently only including the big endian s390x arch. + + runs-on: ubuntu-24.04 + name: Emulated arch bundled lib + # keep condition in sync with test_arm64 + if: | + github.repository == 'astropy/astropy' && ( + github.event_name == 'schedule' || + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'Extra CI') + ) + env: + ARCH_ON_CI: ${{ matrix.arch }} + + strategy: + fail-fast: false + matrix: + include: + - arch: s390x + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + fetch-depth: 0 + - uses: uraimo/run-on-arch-action@d94c13912ea685de38fccc1109385b83fd79427d # v3.0.1 + name: Run tests + id: build + with: + arch: ${{ matrix.arch }} + distro: ubuntu_rolling + + shell: /bin/bash + env: | + ARCH_ON_CI: ${{ env.ARCH_ON_CI }} + IS_CRON: ${{ env.IS_CRON }} + + install: | + apt-get update -q -y + apt-get install -q -y gnupg2 + apt-get update -q -y + apt-get install -q -y --no-install-recommends \ + git \ + g++ \ + pkg-config \ + cython3 \ + python3 \ + python3-erfa \ + python3-extension-helpers \ + python3-jinja2 \ + python3-numpy \ + python3-pytest-astropy \ + python3-setuptools-scm \ + python3-yaml \ + python3-venv \ + python3-wheel + + run: | + uname -a + echo "LONG_BIT="$(getconf LONG_BIT) + python3 -m venv --system-site-packages tests + source tests/bin/activate + pip install -v --no-build-isolation -e .[test] + pip list + python3 -m pytest --strict-markers --pyargs astropy -m "not hypothesis" diff --git a/.github/workflows/ci_cron_weekly.yml b/.github/workflows/ci_cron_weekly.yml new file mode 100644 index 000000000000..8e2bbea50bff --- /dev/null +++ b/.github/workflows/ci_cron_weekly.yml @@ -0,0 +1,198 @@ +name: Weekly cron + +on: + workflow_dispatch: + schedule: + # run every Monday at 6am UTC + - cron: '0 6 * * 1' + pull_request: + # We also want this workflow triggered if the 'Extra CI' label is added + # or present when PR is updated + types: + - synchronize + - labeled + push: + # We want this workflow to always run on release branches as well as + # all tags since we want to be really sure we don't introduce + # regressions on the release branches, and it's also important to run + # this on pre-release and release tags. + branches: + - 'v*' + tags: + - '*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + IS_CRON: 'true' + PY_COLORS: "1" + +permissions: + contents: read + +jobs: + tests: + runs-on: ${{ matrix.os }} + if: | + github.repository == 'astropy/astropy' && ( + github.event_name == 'schedule' || + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'Extra CI') + ) + env: + ARCH_ON_CI: "normal" + strategy: + fail-fast: false + matrix: + include: + + # We check numpy-dev also in a job that only runs from cron, so that + # we can spot issues sooner. We do not use remote data here, since + # that gives too many false positives due to URL timeouts. We also + # install all dependencies via pip here so we pick up the latest + # releases. + - name: Python 3.11 with dev version of key dependencies + os: ubuntu-latest + python: '3.11' + toxenv: py311-test-devdeps + + # https://github.com/astropy/astropy/issues/15701 + - name: Python 3.11 with dev version of key dependencies without scipy + os: ubuntu-latest + python: '3.11' + toxenv: py311-test-devdeps-noscipy + + - name: Python 3.14 with dev version of infrastructure dependencies + os: ubuntu-latest + python: '3.14' + toxenv: py314-test-devpytest + + - name: Python 3.14t (free-threading) with recommended dependencies + os: ubuntu-latest + python: '3.14t' + toxenv: py314t-test-recdeps + PYTHON_GIL: 0 + # https://github.com/astropy/astropy/issues/19205 + PYTHON_CONTEXT_AWARE_WARNINGS: 0 + + - name: Documentation link check + os: ubuntu-latest + python: '3.x' + toxenv: linkcheck + toxposargs: --color + + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ matrix.python }} + - name: Install language-pack-de and tzdata + if: ${{ matrix.os == 'ubuntu-latest' }} + run: | + sudo apt-get update + sudo apt-get install language-pack-de tzdata + - name: Install graphviz + if: ${{ matrix.toxenv == 'linkcheck' }} + run: sudo apt-get install graphviz + - name: Install Python dependencies + run: python -m pip install --upgrade tox + - name: Run tests + run: tox ${{ matrix.toxargs}} -e ${{ matrix.toxenv}} -- ${{ matrix.toxposargs}} + env: + PYTHON_GIL: ${{ matrix.PYTHON_GIL }} + PYTHON_CONTEXT_AWARE_WARNINGS: ${{ matrix.PYTHON_CONTEXT_AWARE_WARNINGS }} + + + tests_more_architectures: + + # The following architectures are emulated and are therefore slow, so + # we include them just in the weekly cron. These also serve as a test + # of using system libraries and using pytest directly. + # No doctest run here due to architecture differences in some outputs + # (e.g., big-endian or 32-bit OS) with numpy 2. + + runs-on: ubuntu-24.04 + name: Emulated arch + # keep condition in sync with test_arm64 + if: | + github.repository == 'astropy/astropy' && ( + github.event_name == 'schedule' || + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'Extra CI') + ) + env: + ARCH_ON_CI: ${{ matrix.arch }} + + strategy: + fail-fast: false + matrix: + include: + - arch: s390x + - arch: ppc64le + - arch: armv7 + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + fetch-depth: 0 + - uses: uraimo/run-on-arch-action@d94c13912ea685de38fccc1109385b83fd79427d # v3.0.1 + name: Run tests + id: build + with: + arch: ${{ matrix.arch }} + distro: ubuntu_rolling + + shell: /bin/bash + env: | + ARCH_ON_CI: ${{ env.ARCH_ON_CI }} + IS_CRON: ${{ env.IS_CRON }} + + install: | + apt-get update -q -y + apt-get install -q -y gnupg2 + # Add test-support repository for wcslib8 + echo "deb http://ppa.launchpadcontent.net/astropy/test-support/ubuntu lunar main" > /etc/apt/sources.list.d/test-support.list + gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv CC75F07B3EF41EFC + gpg --export --armor CC75F07B3EF41EFC | tee /etc/apt/trusted.gpg.d/test-support.asc + apt-get update -q -y + apt-get install -q -y --no-install-recommends \ + git \ + g++ \ + pkg-config \ + python3 \ + python3-erfa \ + python3-extension-helpers \ + python3-jinja2 \ + python3-numpy \ + python3-pytest-astropy \ + python3-setuptools-scm \ + python3-yaml \ + python3-venv \ + python3-wheel \ + wcslib-dev + + run: | + uname -a + echo "LONG_BIT="$(getconf LONG_BIT) + python3 -m venv --system-site-packages tests + source tests/bin/activate + # cython and pyerfa versions in ubuntu repos are too old currently + pip install -U cython setuptools packaging + pip install -U --no-build-isolation pyerfa + ASTROPY_USE_SYSTEM_ALL=1 pip install -v --no-build-isolation -e .[test] + pip list + # A fresh CI runner has no ~/.astropy directory, but developer + # machines usually do. Create empty cache and config directories + # so we catch failures that only show up when they already exist. + python3 -c "from pathlib import Path; [(Path.home() / '.astropy' / d).mkdir(parents=True, exist_ok=True) for d in ('cache', 'config')]" + python3 -m pytest --strict-markers --pyargs astropy -m "not hypothesis" diff --git a/.github/workflows/ci_workflows.yml b/.github/workflows/ci_workflows.yml new file mode 100644 index 000000000000..5a615f98f421 --- /dev/null +++ b/.github/workflows/ci_workflows.yml @@ -0,0 +1,217 @@ +name: CI + +on: + push: + branches: + - main + - 'v*' + tags: + - '*' + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + ARCH_ON_CI: "normal" + IS_CRON: "false" + +permissions: + contents: read + +jobs: + initial_checks: + name: Mandatory checks before CI + runs-on: ubuntu-latest + steps: + - name: Check base branch + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + if: github.event_name == 'pull_request' && github.repository == 'astropy/astropy' + with: + script: | + const skip_label = 'skip-basebranch-check'; + const { default_branch: allowed_basebranch } = context.payload.repository; + const pr = context.payload.pull_request; + if (pr.user.login === 'meeseeksmachine') { + core.info(`Base branch check is skipped since this is auto-backport by ${pr.user.login}`); + return; + } + if (pr.labels.find(lbl => lbl.name === skip_label)) { + core.info(`Base branch check is skipped due to the presence of ${skip_label} label`); + return; + } + if (pr.base.ref !== allowed_basebranch) { + core.setFailed(`PR opened against ${pr.base.ref}, not ${allowed_basebranch}`); + } else { + core.info(`PR opened correctly against ${allowed_basebranch}`); + } + + lowest-tree-check: + name: Check environment lowest resolution against reference + needs: [initial_checks] + permissions: + contents: none + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + fetch-depth: 0 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + name: Install Python + with: + python-version: "3.14" + - run: pipx run scripts/check-lowest-resolved-tree.py + + - if: failure() + run: pipx run scripts/check-lowest-resolved-tree.py --markdown --quiet >> "$GITHUB_STEP_SUMMARY" || true + + tests: + needs: [initial_checks] + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@2835f0cacddf3f8de198db9afdb5354a5cebe0ef # v2.6.3 + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + setenv: | + ARCH_ON_CI: "normal" + IS_CRON: "false" + PY_COLORS: "1" + submodules: false + coverage: '' + libraries: | + apt: + - language-pack-fr + - tzdata + + envs: | + # NOTE: this coverage test is needed for tests and code that + # run only with minimal dependencies. + - name: Python 3.12 with minimal dependencies and full coverage + linux: py312-test-cov + coverage: codecov + cache-path: .tox + cache-key: mindeps-${{ github.ref_name }} + + - name: Python 3.13 in Parallel with all optional dependencies + linux: py313-test-alldeps-fitsio + libraries: + apt: + - language-pack-fr + - tzdata + - libbz2-dev + - libcfitsio-dev + toxargs: -v --develop + posargs: -n=4 --run-slow + cache-path: .tox + cache-key: alldeps-linux-${{ github.ref_name }} + + - name: Python 3.11 with oldest supported version of all dependencies + linux: py311-test-oldestdeps-alldeps-cov-clocale + posargs: --remote-data=astropy + coverage: codecov + + - name: Python 3.13 with all optional dependencies (Windows) + windows: py313-test-alldeps + posargs: --durations=50 + cache-path: .tox + cache-key: alldeps-windows-${{ github.ref_name }} + + - name: Python 3.13 with all optional dependencies (MacOS X) + macos: py313-test-alldeps + posargs: --durations=50 --run-slow + runs-on: macos-latest + cache-path: .tox + cache-key: alldeps-macos-${{ github.ref_name }} + + - name: Python 3.13 aarch64, Run tests twice + linux: py313-test-double + runs-on: ubuntu-24.04-arm + posargs: -n=4 + setenv: | + ARCH_ON_CI: "arm64" + IS_CRON: "false" + cache-path: .tox + cache-key: doubletest-${{ github.ref_name }} + + allowed_failures: + needs: [initial_checks] + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@2835f0cacddf3f8de198db9afdb5354a5cebe0ef # v2.6.3 + with: + setenv: | + ARCH_ON_CI: "normal" + IS_CRON: "false" + PY_COLORS: "1" + submodules: false + coverage: '' + libraries: | + apt: + - language-pack-de + - tzdata + envs: | + - name: (Allowed Failure) Python 3.14 with remote data and dev version of key dependencies + linux: py314-test-devdeps + posargs: --remote-data=any --verbose + + stub_tests: + needs: [initial_checks] + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@2835f0cacddf3f8de198db9afdb5354a5cebe0ef # v2.6.3 + with: + setenv: | + ARCH_ON_CI: "normal" + IS_CRON: "false" + submodules: false + envs: | + - name: Stubs for unit definition modules + linux: stubtest + cache-path: .tox + cache-key: stub_tests-${{ github.ref_name }} + + test_wheel_building: + needs: + - initial_checks + - lowest-tree-check + # This ensures that a couple of wheel targets work fine in pull requests and pushes + permissions: + contents: none + uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish.yml@2835f0cacddf3f8de198db9afdb5354a5cebe0ef # v2.6.3 + with: + upload_to_pypi: false + upload_to_anaconda: false + test_extras: test + test_command: pytest -Wdefault --astropy-header -m "not hypothesis" -k "not test_data_out_of_range and not test_set_locale and not TestQuantityTyping" --strict-markers --pyargs astropy + targets: | + - cp311-manylinux_x86_64 + + + test_limited_api_build: + # Test to make sure that we can build astropy with the limited API + # This job is also important to check `build` as our build frontend + # see https://github.com/astropy/astropy/pull/18253 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + runs-on: ${{ matrix.os }} + name: Python 3.11 (build only, partial) with limited API (${{ matrix.os }}) + needs: + - initial_checks + - lowest-tree-check + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + fetch-depth: 0 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + name: Install Python + with: + python-version: "3.11" + - run: | + python -m pip install build + python -m build + name: Run build + env: + EXTENSION_HELPERS_PY_LIMITED_API: 'cp311' diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000000..25b5122a9374 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,83 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + schedule: + # run every Wednesday at 6am UTC + - cron: '0 6 * * 3' + +permissions: + contents: read + +jobs: + analyze: + permissions: + actions: read # for github/codeql-action/init to get workflow details + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/autobuild to send a status report + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: ['cpp', 'python'] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + fetch-depth: 0 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@5618c9fc1e675841ca52c1c6b1304f5255a905a0 # codeql-bundle-v2.19.0 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + if: matrix.language != 'cpp' + uses: github/codeql-action/autobuild@5618c9fc1e675841ca52c1c6b1304f5255a905a0 # codeql-bundle-v2.19.0 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + if: matrix.language == 'cpp' + with: + python-version: '3.11' + + - name: Manual build + if: matrix.language == 'cpp' + run: | + pip install -U pip setuptools_scm setuptools packaging wheel + pip install extension-helpers cython numpy pyerfa + python setup.py build_ext --inplace + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@5618c9fc1e675841ca52c1c6b1304f5255a905a0 # codeql-bundle-v2.19.0 diff --git a/.github/workflows/open_actions.yml b/.github/workflows/open_actions.yml new file mode 100644 index 000000000000..ac9c0cc2d12a --- /dev/null +++ b/.github/workflows/open_actions.yml @@ -0,0 +1,54 @@ +name: "When Opened" + +on: + issues: + types: + - opened + # pull_request_target is only dangerous if combined with an explicit checkout + # of the opener's code, which it isn't here. See + # https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/ + pull_request_target: # zizmor: ignore[dangerous-triggers] + types: + - opened + +permissions: + pull-requests: write + +jobs: + triage: + runs-on: ubuntu-latest + steps: + - name: Label PR + uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1 + if: | + github.event_name == 'pull_request_target' && + github.event.pull_request.user.login != 'meeseeksmachine' + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" + - name: 'Reviewer Checklist' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + if: github.event_name == 'pull_request_target' + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `Thank you for your contribution to Astropy! 🌌 This checklist is meant to remind the package maintainers who will review this pull request of some common things to look for. + + - [ ] Do the proposed changes actually accomplish desired goals? + - [ ] Do the proposed changes follow the [Astropy coding guidelines](https://docs.astropy.org/en/latest/development/codeguide.html)? + - [ ] Are tests added/updated as required? If so, do they follow the [Astropy testing guidelines](https://docs.astropy.org/en/latest/development/testguide.html)? + - [ ] Are docs added/updated as required? If so, do they follow the [Astropy documentation guidelines](https://docs.astropy.org/en/latest/development/docguide.html)? + - [ ] Is rebase and/or squash necessary? If so, please provide the author with appropriate instructions. Also see instructions for [rebase](https://docs.astropy.org/en/latest/development/development_details.html#rebase-if-necessary) and [squash](https://docs.astropy.org/en/latest/development/development_details.html#squash-if-necessary). + - [ ] Did the CI pass? If no, are the failures related? If you need to run daily and weekly cron jobs as part of the PR, please apply the "Extra CI" label. Codestyle issues can be fixed by the [bot](https://docs.astropy.org/en/latest/development/development_details.html#pre-commit). + - [ ] Is a change log needed? If yes, did the change log check pass? If no, add the "no-changelog-entry-needed" label. If this is a manual backport, use the "skip-changelog-checks" label unless special changelog handling is necessary. + - [ ] Is this a big PR that makes a "What's new?" entry worthwhile and if so, is (1) a "what's new" entry included in this PR and (2) the "whatsnew-needed" label applied? + - [ ] At the time of adding the milestone, if the milestone set requires a backport to release branch(es), apply the appropriate "backport-X.Y.x" label(s) *before* merge.` + }) + # Special action for a special day. Until next year! + #- name: Special comment + # uses: pllim/action-special_pr_comment@5126c189c02418a55448480b28efd1a00af48d7b # 0.2 + # with: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000000..5d16d356d333 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,94 @@ +name: Wheel building + +on: + schedule: + # run every day at 4am UTC + - cron: '0 4 * * *' + workflow_dispatch: + push: + pull_request: + # We also want this workflow triggered if the 'Build all wheels' label is added + # or present when PR is updated + types: + - synchronize + - labeled + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + # This does the actual wheel building as part of the cron job + # or if triggered manually via the workflow dispatch, or for a tag. + permissions: + contents: none + uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish.yml@2835f0cacddf3f8de198db9afdb5354a5cebe0ef # v2.6.3 + if: | + github.repository == 'astropy/astropy' && ( + startsWith(github.ref, 'refs/tags/v') || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'Build all wheels') + ) + with: + + # We use trusted publishing so the upload is handled in a separate job below + upload_to_pypi: false + save_artifacts: true + upload_to_anaconda: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} + anaconda_user: astropy + anaconda_package: astropy + anaconda_keep_n_latest: 10 + env: | + EXTENSION_HELPERS_PY_LIMITED_API: 'cp311' + + test_extras: test + # FIXME: we exclude the test_data_out_of_range test since it + # currently fails, see https://github.com/astropy/astropy/issues/10409 + # We also exclude test_set_locale as it sometimes relies on the correct locale + # packages being installed, which it isn't always. + test_command: pytest -Wdefault --astropy-header -m "not hypothesis" -k "not test_data_out_of_range and not test_set_locale and not TestQuantityTyping" --strict-markers --pyargs astropy + targets: | + # Linux wheels + - cp3*-manylinux_x86_64 + - target: cp3*-manylinux_aarch64 + runs-on: ubuntu-24.04-arm + + # Note that following wheels are not currently tested: + - cp3*-musllinux_x86_64 + + # MacOS X wheels - as noted in https://github.com/astropy/astropy/pull/12379 we deliberately + # do not build universal2 wheels. + - cp3*-macosx_x86_64 + - cp3*-macosx_arm64 + + # Windows wheels + - cp3*-win32 + - cp3*-win_amd64 + - cp3*-win_arm64 + + secrets: + anaconda_token: ${{ secrets.anaconda_token }} + + upload: + # Upload to PyPI using trusted publishing for all tags starting with v but not ones ending in .dev + if: startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '.dev') && github.event_name == 'push' + name: Upload release to PyPI + runs-on: ubuntu-latest + needs: [build] + environment: pypi + permissions: + id-token: write + steps: + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + merge-multiple: true + pattern: dist-* + path: dist + - name: Upload to PyPI + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 diff --git a/.github/workflows/sphinx_rc_workflows.yml b/.github/workflows/sphinx_rc_workflows.yml new file mode 100644 index 000000000000..662170a467fe --- /dev/null +++ b/.github/workflows/sphinx_rc_workflows.yml @@ -0,0 +1,35 @@ +name: Sphinx RC testing + + # We do not run this from cron as the Sphinx RC period is very short, but do trigger it manually when necessary. +on: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + tests: + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@2835f0cacddf3f8de198db9afdb5354a5cebe0ef # v2.6.3 + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + setenv: | + ARCH_ON_CI: "normal" + IS_CRON: "false" + PY_COLORS: "1" + submodules: false + coverage: '' + libraries: | + apt: + - graphviz + envs: | + - name: CI with upstream RC including docs dependencies if available + linux: py313-test-docdeps-predeps + + - name: Build docs with Sphinx RC if available + linux: build_docs-predeps + python-version: '3.13' diff --git a/.github/workflows/stalebot.yml b/.github/workflows/stalebot.yml new file mode 100644 index 000000000000..87078a8ef621 --- /dev/null +++ b/.github/workflows/stalebot.yml @@ -0,0 +1,23 @@ +name: Astropy stalebot + +on: + schedule: + # * is a special character in YAML so you have to quote this string + # run every day at 5:30 am UTC + - cron: '30 5 * * *' + workflow_dispatch: + +permissions: + pull-requests: write + +jobs: + stalebot: + runs-on: ubuntu-latest + if: github.repository == 'astropy/astropy' + steps: + - uses: pllim/action-astropy-stalebot@8a20289632f0d5068e9c129360b27b0a35e4d256 # main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + STALEBOT_MAX_ISSUES: -1 + STALEBOT_MAX_PRS: -1 + STALEBOT_SLEEP: 10 diff --git a/.github/workflows/update_astropy_iers_data_main.yml b/.github/workflows/update_astropy_iers_data_main.yml new file mode 100644 index 000000000000..d569e90bfd45 --- /dev/null +++ b/.github/workflows/update_astropy_iers_data_main.yml @@ -0,0 +1,37 @@ +# Due to a limitation of Github Actions where the `schedule` trigger +# only applies to the default branch, we need a way of triggering the +# jobs on the other branches. +# +# Therefore, this workflow triggers on main, and dispatches to other branches. +# +# This is adapted from +# https://github.com/sunpy/sunpy/blob/main/.github/workflows/scheduled_builds.yml + +name: Scheduled astropy-iers-data auto-update + +on: + workflow_dispatch: + schedule: + - cron: '0 0 2 * *' # Monthly + +permissions: {} + +jobs: + dispatch_release_branches: + if: github.repository == 'astropy/astropy' + permissions: + actions: write + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - branch: "main" + - branch: "v8.0.x" + - branch: "v7.2.x" + steps: + - name: Trigger workflow dispatch for auto-update + uses: benc-uk/workflow-dispatch@7a027648b88c2413826b6ddd6c76114894dc5ec4 # v1.3.1 + with: + workflow: "Auto-update astropy-iers-data minimum version" + ref: "${{ matrix.branch }}" diff --git a/.github/workflows/update_astropy_iers_data_pin.md b/.github/workflows/update_astropy_iers_data_pin.md new file mode 100644 index 000000000000..9d08b9ae0a17 --- /dev/null +++ b/.github/workflows/update_astropy_iers_data_pin.md @@ -0,0 +1,5 @@ +This is an automated update of the minimum version of astropy-iers-data package. + +One pull request per active branch would be opened separately. Please apply proper milestone to each of them. + +:warning: Please close and re-open this pull request to trigger the CI. :warning: \ No newline at end of file diff --git a/.github/workflows/update_astropy_iers_data_pin.yml b/.github/workflows/update_astropy_iers_data_pin.yml new file mode 100644 index 000000000000..98c9f818a42e --- /dev/null +++ b/.github/workflows/update_astropy_iers_data_pin.yml @@ -0,0 +1,61 @@ +# Regularly update the minimum version of astropy-iers-data in pyproject.toml, +# to ensure that if users update astropy, astropy-iers-data will also get +# updated to a recent version. + +name: Auto-update astropy-iers-data minimum version + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + + update-astropy-iers-data-pin: + permissions: + contents: write # to create a branch + pull-requests: write # to create a PR + name: Auto-update astropy-iers-data minimum version + runs-on: ubuntu-latest + if: github.repository == 'astropy/astropy' + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: true # needed for git push + fetch-depth: 0 + - name: Set up uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + # we take advantage of uv add's ability to set a lower bound on a new dependency + # by first removing it (simulating astropy-iers-data being "new") and re-adding it. + # --bounds=lower is passed in order to make the default behavior extra explicit + - name: upgrade astropy-iers-data + run: | + uv remove astropy-iers-data --no-sync + uv add astropy-iers-data --no-sync --bounds=lower + uv run scripts/check-lowest-resolved-tree.py --overwrite + + - name: Commit changes + run: | + git config user.name github-actions + git config user.email github-actions@github.com + BRANCH_NAME=update-astropy-iers-data-pin-$(date +"%s") + git switch -c "$BRANCH_NAME" + git add --update + if ! git diff --cached --exit-code; then + git commit -m "Update minimum required version of astropy-iers-data" + fi + git push --set-upstream origin "$BRANCH_NAME" + + - name: Create Pull Request + run: | + gh pr create \ + --title "Update minimum required version of astropy-iers-data" \ + --label no-changelog-entry-needed --label utils.iers --label skip-basebranch-check \ + --body-file .github/workflows/update_astropy_iers_data_pin.md \ + --base "${PR_BASE_REF_NAME}" + env: + PR_BASE_REF_NAME: ${{ github.ref_name }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/update_credits.md b/.github/workflows/update_credits.md new file mode 100644 index 000000000000..fcb01c4ea112 --- /dev/null +++ b/.github/workflows/update_credits.md @@ -0,0 +1,13 @@ +This is an automated update of the credits for the core package. + +Checklist: + +* [ ] Apply backport labels to any active backport branches +* [ ] Check the resulting docs/credits.rst, and update the .mailmap file for any duplicates or missing names +* [ ] If you update .mailmap, re-run: + +``` +python scripts/update-credits.py +``` + +and commit and push the changes to this branch. diff --git a/.github/workflows/update_credits.yml b/.github/workflows/update_credits.yml new file mode 100644 index 000000000000..219e3ad41e5d --- /dev/null +++ b/.github/workflows/update_credits.yml @@ -0,0 +1,64 @@ +# Regularly update the credits.rst file, to reduce maintenance burden at the time +# of release. This will allow issues with the .mailmap issues to be spotted early. + +name: Auto-update credits + +on: + schedule: + - cron: '0 6 * * 1' # Weekly + pull_request: + paths: + - '.github/workflows/update_credits.yml' + - '.github/workflows/update_credits.md' + - 'scripts/update-credits.py' + - 'docs/credits.rst' + - '.mailmap' + workflow_dispatch: + +permissions: + contents: read + +jobs: + + update-credits: + permissions: + contents: write # to create a branch + pull-requests: write # to create a PR + name: Auto-update credits + runs-on: ubuntu-latest + if: github.repository == 'astropy/astropy' || github.event_name == 'pull_request' + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: true # needed for git push + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: 3.x + - name: Run update script + run: pipx run scripts/update-credits.py + - name: Show diff + run: git diff docs/credits.rst + - name: Commit changes + if: github.event_name != 'pull_request' + run: | + git config user.name github-actions + git config user.email github-actions@github.com + BRANCH_NAME=update-credits-$(date +"%s") + git switch -c "$BRANCH_NAME" + git add docs/credits.rst + if ! git diff --cached --exit-code; then + git commit -m "Update docs/credits.rst with new contributors" + fi + git push --set-upstream origin "$BRANCH_NAME" + - name: Create Pull Request + if: github.event_name != 'pull_request' + run: | + gh pr create \ + --title "Update credits to add new contributors" \ + --label no-changelog-entry-needed --label Docs \ + --body-file .github/workflows/update_credits.md + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 65c428113475..f6a2c965223f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,25 +4,34 @@ *.o *.so *.pyd +*.dll __pycache__ +# Ignore .pyi files since those are all auto-generated, and there are no plans +# to use them otherwise (as of 2025-08-31). Can be adjusted as needed. +*.pyi + # Ignore .c files by default to avoid including generated code. If you want to -# add a non-generated .c extension, use `git add -f filename.c`. +# add a non-generated .c extension, put that into the src/ subdirectory of a +# package or else use `git add -f filename.c`. *.c +!astropy/*/src/*.c +!astropy/timeseries/periodograms/bls/bls.c +astropy/modeling/src/projections.c # Other generated files MANIFEST -astropy/version.py astropy/cython_version.py astropy/wcs/include/wcsconfig.h -astropy/_erfa/core.py -astropy/_erfa/core.pyx +astropy/_version.py # Sphinx _build _generated docs/api - +docs/generated +docs/visualization/ngc6976*.jpeg +docs/sg_execution_times.rst # Packages/installer info *.egg @@ -38,6 +47,18 @@ sdist develop-eggs .installed.cfg distribute-*.tar.gz +.venv +venv +# we are not currently using pipenv directly, but people who +# install astropy with pipenv will have these files generated +Pipfile +Pipfile.lock + +# pyinstaller files +.pyinstaller/astropy_tests/ +.pyinstaller/run_astropy_tests +.pyinstaller/run_astropy_tests.spec + # Other .cache @@ -51,9 +72,25 @@ distribute-*.tar.gz .coverage cover htmlcov +.hypothesis +.github_cache # Mac OSX .DS_Store # PyCharm .idea + +# Pytest +v +.pytest_cache + +# VSCode +.vscode +.history + +.tmp +pip-wheel-metadata + +# Files generated if figure tests are run +results diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 6a20fa6e5f85..000000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "astropy_helpers"] - path = astropy_helpers - url = https://github.com/astropy/astropy-helpers.git diff --git a/.mailmap b/.mailmap index 79c246e4d6af..e707ced0f6f8 100644 --- a/.mailmap +++ b/.mailmap @@ -1,48 +1,405 @@ -Adam Ginsburg -Adam Ginsburg Adam Ginsburg -Alex Conley Alexander Conley -Alex Hagen -Asish Panda -Axel Donath -Bryce Nordgren -Bogdan Nicula -Christopher Bonnett -Christoph Gohlke -Daniel Bell Daniel -Daniel Bell stampsrule -Daniel Datsev -Daniel Datsev -David PÊrez-SuÃĄrez -Demitri Muna -Demitri Muna -Dylan Gregersen -Emma Hogan -Erik M. Bray -Erik M. Bray Erik Bray -Gerrit Schellenberger -Gustavo Bragança -Hans Moritz GÃŧnther -Hans Moritz GÃŧnther hamogu -James Turner -Jeff Taylor -Kacper Kowalik -Karan Grover -Kirill Tchernyshyov -Kelle Cruz -Kevin Gullikson -Lisa Walter -Marten van Kerkwijk -Matt Davis -Nadia Dencheva -Neil Crighton -Perry Greenfield -Pritish Chakraborty -Ryan Cooke -Shantanu Srivastava -Simon Conseil -Simon Liedtke -Thomas Erben -Thompson Le Blanc -Thompson Le Blanc astrocaribe -Tom Aldcroft -Zach Edwards +Aarya Patil +Aarya Patil +Adam Ginsburg +Adam Ginsburg +Adam Ginsburg +Adele Plunkett +Adrian Price-Whelan +Adrian Price-Whelan +Akshat Dixit +Albert Y. Shih +Aleh Khvalko +Aleksi Suutarinen +Alex Conley +Alex Conley +Alex Fox <156043000+AlexFoxOSU@users.noreply.github.com> +Alex Hagen +Alex Rudy +Alexander Bakanov +Alexandre Beelen +Alexandre Beelen +Amit Kumar +Ana Posses +Anany Shrey Jain <31594632+ananyashreyjain@users.noreply.github.com> +Andreas Faisst +Andreas Michael Hermansen <97125645+AMHermansen@users.noreply.github.com> +Andy Casey +Aniket Kulkarni +Aniket Sanghi +Anirudh Katipally +Anne Archibald +Anne Archibald +Anthony Horton +Arthur Xavier Joao Pedro Maia <90696992+arthurxvtv@users.noreply.github.com> +Aryan Shukla <88445101+Telomelonia@users.noreply.github.com> +Asish Panda +Asra Nizami +Asra Nizami +Austen Groener +Austen Groener +Axel Donath +Axel Donath +Benjamin Alan Weaver +Benjamin Alan Weaver +Benjamin Alan Weaver +Ben Green <120208759+ScatteredComet@users.noreply.github.com> +Benjamin Roulston +Benjamin Winkel +Bhavya Khandelwal +Bogdan Nicula +Brett Graham +Brett Morris +Brett Morris +Brett Morris +Brett Morris +Brian Soto +Brian Svoboda +Brigitta Sipőcz +Brigitta Sipőcz +Brigitta Sipőcz +Bruce Merry +Bruce Merry +Bryce Nordgren +Chiara Marmo +Chiara Marmo +Chiara Marmo +Chiara Marmo +Chris Osborne <2087801o@student.gla.ac.uk> +Chris Simpson +Christian Clauss +Christoph Gohlke +Christopher Bonnett +Clara Brasseur +Clara Brasseur +ClÊment Robert +ClÊment Robert +Craig Jones +Craig Jones +Curtis McCully +Dan Foreman-Mackey +Dan Foreman-Mackey +Dan P. Cunningham +Dan Taranu +Daniel Bell +Daniel Bell +Daniel D'Avella +Daniel D'Avella +Daniel Datsev +Daniel Datsev +Daniel Giles +Daniel Lenz +Daniel Ryan +Daria Cara +Daria Cara <36781821+daria-cara@users.noreply.github.com> +David Collom +David Collom +David Collom +David Collom +David Grant <33813984+DavoGrant@users.noreply.github.com> +David Kirkby +David PÊrez-SuÃĄrez +David Shupe +Deen-Dot +Deen-Dot <80238871+Deen-dot@users.noreply.github.com> +Demitri Muna +Demitri Muna +Demitri Muna +Dhruv Yadav +Derek Homeier +Derek Homeier +Derek Homeier <709020+dhomeier@users.noreply.github.com> +Diego Asterio de Zaballa +Douglas Burke +Dylan Gregersen +Edward Gomez +Edward Slavich +Eduardo Olinto <90293761+olintoeduardo@users.noreply.github.com> +Eero Vaher +Elijah Bernstein-Cooper +Emily Deibert +Emma Hogan +Eric Depagne +Eric Koch +E. Madison Bray +E. Madison Bray +E. Madison Bray +E. Rykoff +Emir Karamehmetoglu +Emir Karamehmetoglu +Evan Jones <60061381+E-W-Jones@users.noreply.github.com> +Everett Schlawin +Everett Schlawin +Esteban Pardo SÃĄnchez +Francesco Montesano +Gabriel Brammer +Gabriel Brammer +Gabriel Perren +Gabriel Perren +Geert Barentsen +George Galvin +Gerrit Schellenberger +Gilles Landais +Giorgio Calderone +Graham Kanarek +Guillaume Pernot +Guillaume Pernot +Gustavo Bragança +Henrike F. +Hannes Breytenbach +Hans Moritz GÃŧnther +Hans Moritz GÃŧnther +Harry Ferguson +Harshada Raut +Henrik Norman +Henrik Norman +Henry Schreiner +HÊlvio Peixoto +Himanshu Pathak +Humna Awan +Igor Lemos +Ivo Busko +Ivo Busko +J. Berg +Jackson Hayward +Jaime AndrÊs +Jake VanderPlas +Jake VanderPlas +Jake VanderPlas +James Davies +James McCormac +James Tocknell +James Tocknell +James Turner +James Turner +Jane Rigby +Jani Å umak +Jason Segnini <47617351+JasonS09@users.noreply.github.com> +Javier Blasco +Javier Duran +Javier Duran +Javier Duran +Javier Duran +Javier Duran +Javier Duran +Javier Duran +Javier Pascual Granado +Jeff Jennings +Jeff Taylor +Jennifer Karr +Jero Bado +Jero Bado <10357742+jerobado@users.noreply.github.com> +Jo Bovy +Joe Hunkeler +Johannes Zeman +John Parejko +John Parejko +Johnny Greco +Johnny Greco +Jon Carifio +Joren Hammudoglu +Jonathan Foster +Jonathan Foster +Jonathan Gagne +Jordan Mirocha +Joseph Long +Joseph Long +Joseph Schlitz +Juan Carlos Segovia +Juan Luis Cano Rodríguez +Juan Luis Cano Rodríguez +Juan Luis Cano Rodríguez +Juan Luis Cano Rodríguez +Julien Woillez +Julien Woillez +Jurien Huisman +Kacper Kowalik +Kacper Kowalik +Kacper Rutkowski +Kacper Rutkowski +Kang Wang +Karan Grover +Kartavay Verma +Karl Gordon +Karl Vyhmeister +Kelle Cruz +Kevin Gullikson +Kirill Tchernyshyov +Kris Stern +Kris Stern +Kunam Balaram Reddy +Kyle Barbary +Kyle Oman +Kyle Oman +Larry Bradley +Larry Bradley +Laura Watkins +Lauren Glattly <44421608+lglattly@users.noreply.github.com> +Laurent Michel +Lennard Kiehl +Leo Singer +Leo Singer +Leonardo Ferreira <[leonardo.ferreira.furg@gmail.com]> +Lia Corrales +Lingyi Hu +Lisa Martin <48742903+lisamartin72@users.noreply.github.com> +Lisa Walter +Loïc SÊguin-C +Luke G. Bouma +Luke Kelley +Luz Paz +Maximilian Linhoff +Maximilian Linhoff +Maximilian Linhoff +M. Atakan GÃŧrkan +M. S. R. Dinesh <39215691+msrdinesh@users.noreply.github.com> +Madhura Parikh +Magali Mebsout +Magnus Persson +Maneesh Yadav +Mangala Gowri Krishnamoorthy +Manon Marchand +Marcello Nascif <118627858+marcellonascif@users.noreply.github.com> +Marten van Kerkwijk +Marten van Kerkwijk +Marten van Kerkwijk +Marten van Kerkwijk +Marten van Kerkwijk +Matt Davis +Matteo Bachetti +Matthew Craig +Matthias Stein +Matthieu Baumann +Matthieu Bec +Matthieu Bec +Mavani Bhautik +Michael Belfrage <216956+mikez@users.noreply.github.com> +Michael Brewer +Michael Brewer +Michael Hirsch +Michael Lindner-D'Addario <38199062+MDAddario@users.noreply.github.com> +Michael Mommert +Michael Mommert +Michael Seifert +Michele Costa +Michele Costa +Miguel de Val-Borro +Mihai Cara +Mihai Cara +Mike Alexandersen +Mikhail Minin +Moataz Hisham +Mridul Seth +Mubin Manasia <48038715+Mubin17@users.noreply.github.com> +Nabil Freij +Nadia Dencheva +Nadia Dencheva +Nathaniel Starkman +Nathaniel Starkman +Nathaniel Starkman +Naveen Selvadurai <172697+naveensrinivasan@users.noreply.github.com> +Neil Crighton +Neal McBurnett +Nicholas Earl +Nicholas Earl +Nicholas Earl +Nick Lloyd +Nora Luetzgendorf +Ole Streicher +Ole Streicher +Parikshit Sakurikar +Patricio Rojo +Pauline Barmby +Perry Greenfield +P. L. Lim <2090236+pllim@users.noreply.github.com> +P. L. Lim <2090236+pllim@users.noreply.github.com> +Porter Averett <46609497+paverett@users.noreply.github.com> +Prajwel Joseph +Pratik Patel +Pritish Chakraborty +Rachel Guo +Ricardo Fonseca +Ricardo Fonseca +Richard R. +Richard R. <58728519+rrjbca@users.noreply.github.com> +R. Virinchi +Reem Hamraz +Ricky O'Steen <39831871+rosteen@users.noreply.github.com> +Ritiek Malhotra +Ritwick DSouza +Robel Geda +Robel Geda +Rohan Rajpal +Rohit Kapoor +Rohit Patil +Rohit Patil +Roman Tolesnikov +Ryan Cooke +Sam Bianco <70121323+snbianco@users.noreply.github.com> +Sam Lee +Sam Van Kooten +Sam Verstocken +Sanjeev Dubey +Sara Ogaz +Sarah Graves +Sashank Mishra +Sebastian Meßlinger <39328484+krachyon@users.noreply.github.com> +Sebastian Meßlinger +Sergio Pascual +Shane Maloney +Shane Maloney +Shantanu Srivastava +Sharath Ramkumar <29162020+tnfssc@users.noreply.github.com> +Shilpi Jain +Shivansh Mishra +Shivansh Mishra +Simon Alinder <92031780+AlinderS@users.noreply.github.com> +Simon Alinder +Simon Conseil +Simon Conseil +Simon Conseil +Simon Conseil +Simon Conseil +Simon Liedtke +Somia Floret +Somia Floret <57394764+somilia@users.noreply.github.com> +Sourabh Cheedella +Stelios Voutsinas +Steve Crawford +Steve Crawford +Steve Crawford +Stuart Littlefair +Stuart Mumford +Sudheesh Singanamalla +Sudheesh Singanamalla +Surya K. +Sushobhana Patra +Tanvi Pooranmal Meena <96572616+mtanvi19@users.noreply.github.com> <96572616+TanviPooranmal@users.noreply.github.com> +Thomas Erben +Thompson Le Blanc +Thompson Le Blanc +Tiago Gomes +Tim Jenness +Tim Jenness +Timothy P. Ellsworth Bowers +Tom Aldcroft +Tom Donaldson +Tom J Wilson +Tyler Finethy +VSN Reddy Janga +Vishnunarayan K. I. +Varun Kasyap Pentamaraju +Vishwas +Vital FernÃĄndez +William Jamieson +William Jamieson +Yannick Copin +Yaocheng Chen +Yaocheng Chen <51320658+yaochengchen@users.noreply.github.com> +Yash Kumar +Yash Nandwana +Yash Sharma +Yingqi Ying <33911276+dyq0811@users.noreply.github.com> +Zach Edwards +Zac Hatfield-Dodds +ZÊ Vinicius +Zhiyuan Ma diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000000..ba4b50e1a340 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,113 @@ +ci: + autofix_commit_msg: "chore: pre-commit fixes" + autofix_prs: false + autoupdate_schedule: 'monthly' + +exclude: "^cextern" + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: check-added-large-files + args: ["--enforce-all", "--maxkb=300"] + exclude: "^(\ + cextern/expat/lib/xmlparse.c|\ + cextern/wcslib/C/flexed/.*|\ + CHANGES.rst|\ + )$" + # Prevent giant files from being committed. + - id: check-case-conflict + # Check for files with names that would conflict on a case-insensitive + # filesystem like MacOS HFS+ or Windows FAT. + - id: check-json + # Attempts to load all json files to verify syntax. + - id: check-merge-conflict + # Check for files that contain merge conflict strings. + - id: check-symlinks + # Checks for symlinks which do not point to anything. + - id: check-toml + # Attempts to load all TOML files to verify syntax. + - id: check-xml + # Attempts to load all xml files to verify syntax. + - id: check-yaml + # Attempts to load all yaml files to verify syntax. + exclude: ".*(.github.*)$" + - id: detect-private-key + # Checks for the existence of private keys. + - id: end-of-file-fixer + # Makes sure files end in a newline and only a newline. + exclude: ".*(data.*|extern.*|licenses.*|_static.*|_parsetab.py)$" + # - id: fix-encoding-pragma # covered by pyupgrade + - id: trailing-whitespace + # Trims trailing whitespace. + exclude_types: [python] # Covered by Ruff W291. + exclude: ".*(data.*|extern.*|licenses.*|_static.*)$" + + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: rst-directive-colons + # Detect mistake of rst directive not ending with double colon. + - id: rst-inline-touching-normal + # Detect mistake of inline code touching normal text in rst. + - id: text-unicode-replacement-char + # Forbid files which have a UTF-8 Unicode replacement character. + + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.24.1 + hooks: + - id: zizmor + + - repo: https://github.com/codespell-project/codespell + rev: v2.4.2 + hooks: + - id: codespell + args: ["--write-changes"] + additional_dependencies: + - tomli + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.12 + hooks: + - id: ruff-check + args: ["--fix", "--show-fixes"] + - id: ruff-format + + - repo: https://github.com/scientific-python/cookie + rev: 2026.04.04 + hooks: + - id: sp-repo-review + + - repo: local + hooks: + - id: changelogs-rst + name: changelog filenames + language: fail + entry: >- + changelog files must be named /####.(bugfix|feature|api|perf).rst + or ####.other.rst (in the root directory only) + exclude: >- + ^docs/changes/[\w\.]+/(\d+\.(bugfix|feature|api|perf)(\.\d)?.rst|.gitkeep) + files: ^docs/changes/[\w\.]+/ + - id: changelogs-rst-other + name: changelog filenames for other category + language: fail + entry: >- + only "other" changelog files must be placed in the root directory + exclude: >- + ^docs/changes/(\d+\.other.rst|README.rst|template.rst) + files: ^docs/changes/\d+.\w+.rst + + # Linting hook for stylistic issues in rst doc files. + - repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v1.0.2 + hooks: + - id: sphinx-lint + exclude: "(licenses/|docs/_build/|.*parsetab.py|astropy/extern/)" + + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: 'v22.1.4' + hooks: + - id: clang-format + files: \.(c|h)$ diff --git a/.pycodestyle b/.pycodestyle new file mode 100644 index 000000000000..3d53d33c438a --- /dev/null +++ b/.pycodestyle @@ -0,0 +1,3 @@ +[pycodestyle] +max-line-length = 88 +exclude = extern,*parsetab.py,*lextab.py diff --git a/.pyinstaller/hooks/hook-astropy_iers_data.py b/.pyinstaller/hooks/hook-astropy_iers_data.py new file mode 100644 index 000000000000..af591ea6d83c --- /dev/null +++ b/.pyinstaller/hooks/hook-astropy_iers_data.py @@ -0,0 +1,3 @@ +from PyInstaller.utils.hooks import collect_data_files + +datas = collect_data_files("astropy_iers_data") diff --git a/.pyinstaller/hooks/hook-skyfield.py b/.pyinstaller/hooks/hook-skyfield.py new file mode 100644 index 000000000000..caefbfcb6fae --- /dev/null +++ b/.pyinstaller/hooks/hook-skyfield.py @@ -0,0 +1,6 @@ +# NOTE: this hook should be added to +# https://github.com/pyinstaller/pyinstaller-hooks-contrib +# once that repository is ready for pull requests +from PyInstaller.utils.hooks import collect_data_files + +datas = collect_data_files("skyfield") diff --git a/.pyinstaller/run_astropy_tests.py b/.pyinstaller/run_astropy_tests.py new file mode 100644 index 000000000000..74fbf1a32305 --- /dev/null +++ b/.pyinstaller/run_astropy_tests.py @@ -0,0 +1,128 @@ +import os +import shutil +import sys + +import erfa # noqa: F401 +import matplotlib as mpl +import pytest + +import astropy # noqa: F401 + +if len(sys.argv) == 3 and sys.argv[1] == "--astropy-root": + ROOT = sys.argv[2] +else: + # Make sure we don't allow any arguments to be passed - some tests call + # sys.executable which becomes this script when producing a pyinstaller + # bundle, but we should just error in this case since this is not the + # regular Python interpreter. + if len(sys.argv) > 1: + print("Extra arguments passed, exiting early") + sys.exit(1) + +for root, dirnames, files in os.walk(os.path.join(ROOT, "astropy")): + # NOTE: we can't simply use + # test_root = root.replace('astropy', 'astropy_tests') + # as we only want to change the one which is for the module, so instead + # we search for the last occurrence and replace that. + pos = root.rfind("astropy") + test_root = root[:pos] + "astropy_tests" + root[pos + 7 :] + + # Copy over the astropy 'tests' directories and their contents + for dirname in dirnames: + final_dir = os.path.relpath(os.path.join(test_root, dirname), ROOT) + # We only copy over 'tests' directories, but not astropy/tests (only + # astropy/tests/tests) since that is not just a directory with tests. + if dirname == "tests" and not root.endswith("astropy"): + shutil.copytree(os.path.join(root, dirname), final_dir, dirs_exist_ok=True) + else: + # Create empty __init__.py files so that 'astropy_tests' still + # behaves like a single package, otherwise pytest gets confused + # by the different conftest.py files. + init_filename = os.path.join(final_dir, "__init__.py") + if not os.path.exists(os.path.join(final_dir, "__init__.py")): + os.makedirs(final_dir, exist_ok=True) + with open(os.path.join(final_dir, "__init__.py"), "w") as f: + f.write("#") + # Copy over all conftest.py files + for file in files: + if file == "conftest.py": + final_file = os.path.relpath(os.path.join(test_root, file), ROOT) + shutil.copy2(os.path.join(root, file), final_file) + +# Add the top-level __init__.py file +with open(os.path.join("astropy_tests", "__init__.py"), "w") as f: + f.write("#") + +# Remove test file that tries to import all sub-packages at collection time +os.remove( + os.path.join("astropy_tests", "utils", "iers", "tests", "test_leap_second.py") +) + +# Remove convolution tests for now as there are issues with the loading of the C extension. +# FIXME: one way to fix this would be to migrate the convolution C extension away from using +# ctypes and using the regular extension mechanism instead. +shutil.rmtree(os.path.join("astropy_tests", "convolution")) +os.remove(os.path.join("astropy_tests", "modeling", "tests", "test_convolution.py")) +os.remove(os.path.join("astropy_tests", "modeling", "tests", "test_core.py")) +os.remove(os.path.join("astropy_tests", "visualization", "tests", "test_lupton_rgb.py")) + +# FIXME: PIL minversion check does not work +os.remove( + os.path.join("astropy_tests", "visualization", "wcsaxes", "tests", "test_misc.py") +) +os.remove( + os.path.join("astropy_tests", "visualization", "wcsaxes", "tests", "test_wcsapi.py") +) + +# FIXME: The following tests rely on the fully qualified name of classes which +# don't seem to be the same. +os.remove(os.path.join("astropy_tests", "table", "mixins", "tests", "test_registry.py")) + +# Copy the top-level conftest.py +shutil.copy2( + os.path.join(ROOT, "astropy", "conftest.py"), + os.path.join("astropy_tests", "conftest.py"), +) + +# matplotlib hook in pyinstaller 5.0 and later no longer collects every backend, see +# https://github.com/pyinstaller/pyinstaller/issues/6760 +mpl.use("svg") + +# We skip a few tests, which are generally ones that rely on explicitly +# checking the name of the current module (which ends up starting with +# astropy_tests rather than astropy). + +SKIP_TESTS = [ + "test_exception_logging_origin", + "test_log", + "test_configitem", + "test_config_noastropy_fallback", + "test_no_home", + "test_path", + "test_rename_path", + "test_data_name_third_party_package", + "test_pkg_finder", + "test_wcsapi_extension", + "test_find_current_module_bundle", + "test_minversion", + "test_imports", + "test_generate_config", + "test_generate_config2", + "test_create_config_file", + "test_download_parallel_fills_cache", + "test_defined_constants_do_not_change", + "test_physical_constants_versions", +] + +# Run the tests! +sys.exit( + pytest.main( + ["astropy_tests", "-k " + " and ".join("not " + test for test in SKIP_TESTS)], + plugins=[ + "pytest_astropy.plugin", + "pytest_doctestplus.plugin", + "pytest_remotedata.plugin", + "pytest_astropy_header.display", + ], + ) +) diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000000..322fa423119f --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,32 @@ +version: 2 + +build: + os: "ubuntu-22.04" + tools: + python: "miniforge3-25.11" + jobs: + post_checkout: + - git fetch --unshallow || true + pre_install: + - git update-index --assume-unchanged docs/conf.py docs/rtd_environment.yaml + +conda: + environment: docs/rtd_environment.yaml + +sphinx: + builder: html + configuration: docs/conf.py + fail_on_warning: true + +# Install regular dependencies. +# Then, install special pinning for RTD. +python: + install: + - method: pip + path: . + extra_requirements: + - docs + - all + +# Don't build any extra formats +formats: [] diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 000000000000..ec8b1b572e6b --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,367 @@ +extend = "pyproject.toml" +lint.ignore = [ + # NOTE: to find a good code to fix, run: + # ruff check --select="ALL" --statistics astropy/ + + # flake8-annotations (ANN) : static typing + "ANN001", # Function argument without type annotation + "ANN003", # `**kwargs` without type annotation + "ANN202", # Private function without return type annotation + "ANN401", # Use of `Any` type + + # flake8-unused-arguments (ARG) + "ARG001", # unused-function-argument + "ARG002", # unused-method-argument + "ARG003", # unused-class-method-argument + "ARG004", # unused-static-method-argument + "ARG005", # unused-lambda-argument + + # flake8-bugbear (B) + "B006", # MutableArgumentDefault + "B023", # FunctionUsesLoopVariable + "B028", # No-explicit-stacklevel + "B904", # RaiseWithoutFromInsideExcept + "B905", # ZipWithoutExplicitStrict + + # flake8-blind-except (BLE) + "BLE001", # blind-except + + # mccabe (C90) : code complexity + # TODO: configure maximum allowed complexity. + "C901", # McCabeComplexity + + # pydocstyle (D) + # Missing Docstrings + "D100", # undocumented-public-module + "D101", # undocumented-public-class + "D103", # undocumented-public-function + "D104", # undocumented-public-package + "D205", # blank-line-after-summary + # Quotes Issues + "D301", # escape-sequence-in-docstring + # Docstring Content Issues + "D403", # first-line-capitalized + "D404", # docstring-starts-with-this + "D401", # non-imperative-mood. + "D414", # empty-docstring-section + "D419", # docstring is empty + + # flake8-datetimez (DTZ) + "DTZ001", # call-datetime-without-tzinfo + "DTZ005", # call-datetime-now-without-tzinfo + + # pycodestyle (E, W) + "E501", # line-too-long + "E721", # type-comparison + "E731", # lambda-assignment + + # flake8-errmsg (EM) : nicer error tracebacks + "EM101", # raw-string-in-exception + "EM102", # f-string-in-exception + "EM103", # dot-format-in-exception + + # eradicate (ERA) + # NOTE: be careful that developer notes are kept. + "ERA001", # commented-out-code + + # Pyflakes (F) + "F841", # unused-variable + + # flake8-boolean-trap (FBT) : boolean flags should be kwargs, not args + # NOTE: a good thing to fix, but changes API. + "FBT001", # boolean-positional-arg-in-function-definition + "FBT002", # boolean-default-value-in-function-definition + "FBT003", # boolean-positional-value-in-function-call + + # flake8-fixme (FIX) + "FIX001", # Line contains FIXME. this should be fixed or at least FIXME replaced with TODO + "FIX004", # Line contains HACK. replace HACK with NOTE. + + "FURB166", # int-on-sliced-str + + # pep8-naming (N) + # NOTE: some of these can/should be fixed, but this changes the API. + "N801", # invalid-class-name + "N802", # invalid-function-name + "N803", # invalid-argument-name + "N805", # invalid-first-argument-name-for-method + "N807", # dunder-function-name + "N813", # camelcase-imported-as-lowercase + "N815", # mixed-case-variable-in-class-scope + "N816", # mixed-case-variable-in-global-scope + "N818", # error-suffix-on-exception-name + + # NumPy-specific rules (NPY) + "NPY002", # Replace legacy `np.random.rand` call with `np.random.Generator` (2023-05-03) + + # Perflint (PERF) + "PERF203", # `try`-`except` within a loop incurs performance overhead + "PERF401", # Use a list comprehension to create a transformed list + + # Pylint (PLC, PLE, PLR, PLW) + "PLR0124", # Name compared with itself + "PLR0402", # ConsiderUsingFromImport + "PLR0912", # too-many-branches + "PLR0913", # too-many-args + "PLR0915", # too-many-statements + "PLR1714", # Consider merging multiple comparisons + "PLR2004", # MagicValueComparison + "PLR5501", # collapsible-else-if + "PLW0603", # global-statement + "PLW1641", # Object does not implement `__hash__` method + "PLW2901", # redefined-loop-name + + # flake8-pytest-style (PT) + "PT003", # pytest-extraneous-scope-function + "PT006", # pytest-parametrize-names-wrong-type + "PT007", # pytest-parametrize-values-wrong-type + "PT011", # pytest-raises-too-broad + "PT012", # pytest-raises-with-multiple-statements + "PT017", # pytest-assert-in-exceptinstead + "PT018", # pytest-composite-assertion + "PT028", # pytest-parameter-with-default-argument + "PT030", # pytest.warns({warning}) is too broad + + # flake8-return (RET) + "RET501", # unnecessary-return-none + "RET502", # implicit-return-value + "RET503", # implicit-return + "RET507", # superfluous-else-continue + + # flake8-raise (RSE) + "RSE102", # unnecessary-paren-on-raise-exception + + # Ruff-specific rules (RUF) + "RUF001", # ambiguous-unicode-character-string + "RUF002", # ambiguous-unicode-character-docstring + "RUF010", # use conversion in f-string + "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar` + "RUF043", # pytest-raises-ambiguous-pattern + "RUF059", # unused-unpacked-variable + + # flake8-bandit (S) + "S101", # Use of `assert` detected + "S105", # hardcoded-password-string + "S110", # try-except-pass + "S112", # try-except-continue + "S301", # suspicious-pickle-usage + "S307", # Use of possibly insecure function; consider using `ast.literal_eval` + "S311", # suspicious-non-cryptographic-randomness + "S324", # hashlib-insecure-hash-function + "S506", # UnsafeYAMLLoad + "S310", # Suspicious-url-open-usage + "S603", # `subprocess` call: check for execution of untrusted input + "S607", # Starting a process with a partial executable path + + # flake8-simplify (SIM) + "SIM102", # NestedIfStatements + "SIM105", # UseContextlibSuppress + "SIM108", # UseTernaryOperator + "SIM114", # if-with-same-arms + "SIM115", # OpenFileWithContextHandler + "SIM117", # MultipleWithStatements + "SIM118", # KeyInDict + "SIM201", # NegateEqualOp + "SIM300", # yoda condition + + # flake8-print (T20) + "T201", # PrintUsed + + # flake8-todos (TD) + "TD001", # Invalid TODO tag + "TD003", # Missing issue link on the line following this TODO + "TD004", # Missing colon in TODO + "TD007", # Missing space after colon in TODO + + # tryceratops (TRY) + "TRY002", # raise-vanilla-class + "TRY003", # raise-vanilla-args + "TRY004", # prefer-type-error + "TRY201", # verbose-raise + "TRY301", # raise-within-try +] +lint.unfixable = [ + "E711" # NoneComparison. Hard to fix b/c numpy has it's own None. +] + +[lint.extend-per-file-ignores] +"__init__.py" = [ + "F401", + "PLC0415", # import-outside-top-level +] +"conftest.py" = [ + "PLC0415", # import-outside-top-level +] +"test_*.py" = [ + "PTH", # all flake8-use-pathlib + "RUF015", # Prefer next({iterable}) over single element slice +] +# TODO: fix these, on a per-subpackage basis. +# When a general exclusion is being fixed, but it affects many subpackages, it +# is better to fix for subpackages individually. The general exclusion should be +# copied to these subpackage sections and fixed there. +"astropy/**/setup_package.py" = [ + # all flake8-use-pathlib + # reason: subtle differences between pathlib.Path.relative_to and os.path.relpath + "PTH", +] +"astropy/config/*" = [ + "PLC0415", # import-outside-top-level + "PT031", # pytest-warns-with-multiple-statements +] +"astropy/constants/*" = [ + "PLC0415", # import-outside-top-level +] +"astropy/convolution/*" = [ + "PLC0415", # import-outside-top-level + "PT031", # pytest-warns-with-multiple-statements +] +"astropy/coordinates/*" = [ + "PLC0415", # import-outside-top-level + "PT031", # pytest-warns-with-multiple-statements + "PTH", # all flake8-use-pathlib + "DTZ007", # call-datetime-strptime-without-zone +] +"astropy/cosmology/*" = [ + "PLC0415", # import-outside-top-level + "PT019", # pytest-fixture-param-without-value + "PT031", # pytest-warns-with-multiple-statements +] +"astropy/io/*" = [ + "G001", # logging-string-format + "G004", # logging-f-string + "PLR0911", # too-many-return-statements + "PTH116", # os-stat + "PTH117", # os-path-isabs + "S314", # defusedxml + "SLOT000", # Subclasses of `str` should define `__slots__` + "TD005", # Missing issue description after `TODO` + "TRY400", # error-instead-of-exception + "TRY300", # Consider `else` block +] +"astropy/io/ascii/*" = [ + "B007", # UnusedLoopControlVariable + "PLC0415", # import-outside-top-level + "PT031", # pytest-warns-with-multiple-statements +] +"astropy/io/fits/*" = [ + "B007", # UnusedLoopControlVariable + "PLC0415", # import-outside-top-level + "PT031", # pytest-warns-with-multiple-statements + "PTH", +] +"astropy/io/misc/*" = [ + "B007", # UnusedLoopControlVariable + "PLC0415", # import-outside-top-level + "PT031", # pytest-warns-with-multiple-statements +] +"astropy/io/registry/*" = [ + "PLC0415", # import-outside-top-level + "PTH", +] +"astropy/io/votable/*" = [ + "B007", # UnusedLoopControlVariable + "PLC0415", # import-outside-top-level + "PT031", # pytest-warns-with-multiple-statements + "PTH", +] +"astropy/logger.py" = [ + "PLC0415", # import-outside-top-level +] +"astropy/modeling/*" = [ + "PLC0415", # import-outside-top-level + "PLR0911", # too-many-return-statements + "RET504", # unnecessary-assign + "SLOT001", # Subclasses of `tuple` should define `__slots__` + "TRY300", # Consider `else` block + "F811", # see https://github.com/astropy/astropy/pull/16633 +] +"astropy/nddata/*" = [ + "PLC0415", # import-outside-top-level +] +"astropy/samp/*" = [ + "PLC0415", # import-outside-top-level + "PTH", # all flake8-use-pathlib + "RET504", # unnecessary-assign +] +"astropy/stats/*" = [ + "B007", # UnusedLoopControlVariable + "PLC0415", # import-outside-top-level + "PLR0911", # too-many-return-statements + "PT031", # pytest-warns-with-multiple-statements + "RET504", # unnecessary-assign +] +"astropy/table/*" = [ + "PLC0415", # import-outside-top-level + "PT031", # pytest-warns-with-multiple-statements + "RET504", # unnecessary-assign + "S605", # Starting a process with a shell, possible injection detected + "TRY300", # Consider `else` block +] +"astropy/tests/*" = [ + "PLC0415", # import-outside-top-level + "PT031", # pytest-warns-with-multiple-statements + "PTH", # all flake8-use-pathlib +] +"astropy/time/*" = [ + "FIX003", # Line contains XXX. replace XXX with TODO + "PIE794", # duplicate-class-field-definition + "PLC0415", # import-outside-top-level + "PT031", # pytest-warns-with-multiple-statements + "RET504", # unnecessary-assign +] +"astropy/timeseries/*" = [ + "PLC0415", # import-outside-top-level + "PLR0911", # too-many-return-statements + "PT031", # pytest-warns-with-multiple-statements + "RET504", # unnecessary-assign +] +"astropy/units/*" = [ + "N812", # lowercase-imported-as-non-lowercase + "PLC0415", # import-outside-top-level + "PLR0911", # too-many-return-statements + "PT031", # pytest-warns-with-multiple-statements + "RET504", # unnecessary-assign + "TRY300", # Consider `else` block +] +"astropy/uncertainty/*" = [ + "PLC0415", # import-outside-top-level +] +"astropy/utils/*" = [ + "B007", # UnusedLoopControlVariable + "N811", # constant-imported-as-non-constant + "PLC0415", # import-outside-top-level + "PLR0911", # too-many-return-statements + "PT031", # pytest-warns-with-multiple-statements + "PTH", # all flake8-use-pathlib + "RET504", # unnecessary-assign + "S321", # Suspicious-ftp-lib-usage + "TRY300", # Consider `else` block +] +"astropy/visualization/*" = [ + "B015", + "PLC0415", # import-outside-top-level + "PLR0911", # too-many-return-statements +] +"astropy/wcs/*" = [ + "F821", # undefined-name + "PLC0415", # import-outside-top-level + "PT031", # pytest-warns-with-multiple-statements + "PTH", # all flake8-use-pathlib + "RET504", # unnecessary-assign + "S608", # Posslibe SQL injection + "SIM202", # NegateNotEqualOp + "TRY300", # Consider `else` block +] +"docs/*" = [] + +".pyinstaller/*.py" = ["PTH"] + + +[lint.flake8-import-conventions.aliases] +# xml is hardly ever used thus the alias should not be mandated +# There is no way to remove from the default list, only to override +# the default thus we list the things here that we actually should use. +"numpy" = "np" +"matplotlib" = "mpl" +"matplotlib.pyplot" = "plt" diff --git a/.stubtest.ini b/.stubtest.ini new file mode 100644 index 000000000000..3ce063d3c3d2 --- /dev/null +++ b/.stubtest.ini @@ -0,0 +1,2 @@ +[mypy] +follow_imports = silent diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 22f026cc4601..000000000000 --- a/.travis.yml +++ /dev/null @@ -1,82 +0,0 @@ -# We set the language to c because python isn't supported on the MacOS X nodes -# on Travis. However, the language ends up being irrelevant anyway, since we -# install Python ourselves using conda. -language: c - -os: - - linux - -env: - global: - # Set defaults to avoid repeating in most cases - - NUMPY_VERSION=1.9 - - OPTIONAL_DEPS=false - - MAIN_CMD='python setup.py' - matrix: - - PYTHON_VERSION=2.6 SETUP_CMD='egg_info' - - PYTHON_VERSION=2.7 SETUP_CMD='egg_info' - - PYTHON_VERSION=3.3 SETUP_CMD='egg_info' - - PYTHON_VERSION=3.4 SETUP_CMD='egg_info' - -matrix: - - # Don't wait for allowed failures - fast_finish: true - - include: - - # Try MacOS X - - os: osx - env: PYTHON_VERSION=2.7 SETUP_CMD='test' OPTIONAL_DEPS=true - - # Check for sphinx doc build warnings - we do this first because it - # runs for a long time - - os: linux - env: PYTHON_VERSION=2.7 SETUP_CMD='build_sphinx -w' OPTIONAL_DEPS=true - # OPTIONAL_DEPS needed because the plot_directive in sphinx needs them - - # Try all python versions with the latest numpy - - os: linux - env: PYTHON_VERSION=2.6 SETUP_CMD='test --open-files' - - os: linux - env: PYTHON_VERSION=2.7 SETUP_CMD='test --open-files' - - os: linux - env: PYTHON_VERSION=3.3 SETUP_CMD='test --open-files' - - os: linux - env: PYTHON_VERSION=3.4 SETUP_CMD='test --open-files' - - # Now try with all optional dependencies on 2.7 and an appropriate 3.x - # build (with latest numpy). We also note the code coverage on Python - # 2.7. - - os: linux - env: PYTHON_VERSION=2.7 SETUP_CMD='test --coverage' OPTIONAL_DEPS=true LC_CTYPE=C.ascii LC_ALL=C.ascii - - os: linux - env: PYTHON_VERSION=3.4 SETUP_CMD='test' OPTIONAL_DEPS=true LC_CTYPE=C.ascii LC_ALL=C.ascii - - # Try older numpy versions - - os: linux - env: PYTHON_VERSION=2.7 NUMPY_VERSION=1.8 SETUP_CMD='test' - - os: linux - env: PYTHON_VERSION=2.7 NUMPY_VERSION=1.7 SETUP_CMD='test' - - os: linux - env: PYTHON_VERSION=2.7 NUMPY_VERSION=1.6 SETUP_CMD='test' - - # Try developer version of Numpy - - os: linux - env: PYTHON_VERSION=2.7 NUMPY_VERSION=dev SETUP_CMD='test' - - # Do a PEP8 test - - os: linux - env: PYTHON_VERSION=2.7 MAIN_CMD='pep8 astropy --count' SETUP_CMD='' - - allow_failures: - - env: PYTHON_VERSION=2.7 NUMPY_VERSION=dev SETUP_CMD='test' - -install: - - source .continuous-integration/travis/setup_environment_$TRAVIS_OS_NAME.sh - -script: - - $MAIN_CMD $SETUP_CMD - -after_success: - - if [[ $SETUP_CMD == 'test --coverage' ]]; then coveralls --rcfile='astropy/tests/coveragerc'; fi diff --git a/CHANGES.rst b/CHANGES.rst index 23306b093994..f0fdc870ad8f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,1239 +1,15996 @@ -1.1 (unreleased) ----------------- +Version 7.2.0 (2025-11-25) +========================== + + +New Features +------------ + +astropy.constants +^^^^^^^^^^^^^^^^^ + +- Added CODATA 2022 support in ``astropy.constants``. + + This update affects the following constants while the rest are unchanged from CODATA 2018: + + - ``m_p`` (Proton mass) + - ``m_n`` (Neutron mass) + - ``m_e`` (Electron mass) + - ``u`` (Atomic mass) + - ``eps0`` (Vacuum electric permittivity) + - ``Ryd`` (Rydberg constant) + - ``a0`` (Bohr radius) + - ``muB`` (Bohr magneton) + - ``alpha`` (Fine-structure constant) + - ``mu0`` (Vacuum magnetic permeability) + - ``sigma_T`` (Thomson scattering cross-section) [#18118] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Allow ``np.concatenate``, ``np.stack`` and similar numpy functions to + be applied on representations and differentials. + + They can also be applied to coordinate frames and ``SkyCoord``, though + with the same limitation as for setting elements of frames and + coordinates: all frame attributes have to be scalars (or arrays with + only identical elements). [#18193] + +- The results of ``match_coordinates_3d()``, ``match_coordinates_sky()``, + ``search_around_3d()`` and ``search_around_sky()`` and the corresponding + ``SkyCoord`` methods now have named attributes. [#18459] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- The trait ``astropy.cosmology.traits.CurvatureComponent`` has been added to work with + objects that have attributes and methods related to the global curvature. [#18232] + +- The trait ``astropy.cosmology.traits.HubbleParameter`` has been added to work with objects that have attributes and methods related to the Hubble parameter. [#18271] + +- The trait ``astropy.cosmology.traits.DarkEnergyComponent`` has been added to work with objects that have attributes and methods related to the Dark Energy component. [#18447] + +- Cosmology methods now exclusively return arrays, not floats or other scalars. [#18632] + +- The trait ``astropy.cosmology.traits.DarkMatterComponent`` has been added to work with + objects that have attributes and methods related to dark matter. [#18760] + +- The trait ``astropy.cosmology.traits.MatterComponent`` has been added to work with + objects that have attributes and methods related to matter density. + The trait ``astropy.cosmology.traits.BaryonComponent`` has been added to work with + objects that have attributes and methods related to baryonic matter. + The trait ``astropy.cosmology.traits.CriticalDensity`` has been added to work with + objects that have attributes and methods related to the critical density. [#18769] + +- The trait ``astropy.cosmology.traits.PhotonComponent`` has been added to work with objects that have attributes and methods related to photons. [#18787] + +- The trait ``astropy.cosmology.traits.TotalComponent`` has been added to work with objects that have attributes and methods related to the total density component of the universe. [#18794] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- CDS table reader will find the metadata for gzipped tables in the accompanying ReadMe file. [#18506] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Enable color and suggestion-on-typos in all ``argparse`` CLIs for Python 3.14 + (``fitscheck``, ``fitsdiff``, ``fitsheader`` and ``fitsinfo``). [#18151] + +- Allow reading a FITS file hosted on a cloud resource like Amazon S3 via + ``Table.read()``. This is done with a new ``fsspec_kwargs`` dict argument + that gets passed through to ``fsspec`` to access cloud resources. [#18379] + +- It is now possible to check the existence of ``Columns`` in ``ColDefs`` by using the membership operator. [#18717] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Added a new ECSV table reading module that supports different backend engines for the + CSV data parsing. In addition to the default "io.ascii" engine, this includes engines + that use the PyArrow and Pandas CSV readers. These can be up to 16 times faster and are + more memory efficient than the native astropy ECSV reader. To get help with this + interface run ``Table.read.help(format="ecsv")``. [#18267] + +- Improve support for compressed file formats in the ECSV and the pyarrow CSV + Table readers. All formats supported by ``astropy.utils.data.get_readable_fileobj()`` + (currently gzip, bzip2, lzma (xz) or lzw (Z)) will now work with these readers. [#18712] + +astropy.io.registry +^^^^^^^^^^^^^^^^^^^ + +- Allow setting EXTNAME when writing a ``Table`` to a FITS file, e.g. + ``tbl.write("filename.fits", name="CAT", append=True)``. [#18470] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Enable color and suggestion-on-typos in ``volint`` CLI for Python 3.14 [#18151] + +- Modified the constructor for ``astropy.io.votable.tree.TableElement`` to use the version configuration information from the parent ``VOTableFile`` instance. This allows for better handling of version-specific features and ensures that the table element is created with the correct context regarding the VOTable version. [#18366] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Add support for unit change propagation through the ``|`` (model composition) operator, + using either `~astropy.modeling.compose_models_with_units` or by setting the + ``unit_change_composition`` attribute on the model after composition. [#17304] + +astropy.nddata +^^^^^^^^^^^^^^ + +- The ``interpret_bit_flags`` function now strips whitespace from flag names. [#18205] + +astropy.samp +^^^^^^^^^^^^ + +- Enable color and suggestion-on-typos in ``samp_hub`` CLI for Python 3.14 [#18151] + +astropy.table +^^^^^^^^^^^^^ + +- Enable color and suggestion-on-typos in ``showtable`` CLI for Python 3.14 [#18151] + +- Added generic ``from_df`` and ``to_df`` methods to ``astropy.Table`` using + ``narwhals``. These methods provide a unified interface for converting between + Astropy Tables and various DataFrame formats (pandas, polars, pyarrow, etc.) + through the narwhals library. The ``to_df`` method converts an Astropy Table + to any supported DataFrame format, while ``from_df`` creates an Astropy Table + from any narwhals-compatible DataFrame. Narwhals is a lightweight compatibility + layer that provides a unified API across different DataFrame libraries, allowing + seamless interoperability without requiring all DataFrame libraries as dependencies. [#18435] + +- Setting the ``units`` or ``descriptions`` of ``QTable`` and ``Table`` + has been made more flexible for tables with optional columns that may + or may not appear in the data. This applies to directly creating a table + as well as reading formatted data with the ``read()`` method. + + In both cases you can supply ``units`` and ``description`` arguments as a + ``dict`` that specifies the units and descriptions for column names in + the table. Previously, if the input table did not contain a column that + was specified in the ``units`` or ``description`` dict, a ``ValueError`` + was raised. Now, such columns are simply ignored. [#18641] + +- A new method has been added for accessing a table index for tables with multiple + indices. You can now select the index with the ``with_index(index_id)`` method of the + ``.loc``, ``.iloc``, and ``.loc_indices`` properties. For example, for a table ``t`` + which has two indices on columns ``"a"`` and ``"b"`` respectively, + ``t.loc.with_index("b")[2]`` will use index ``"b"`` to find all the table rows where + ``t["b"] == 2``. Doing this query using the previous syntax ``t.loc["b", 2]`` is + deprecated and this functionality is planned for removal in astropy 9.0. + + In addition, support has been added for using ``.loc``, ``.iloc``, and ``.loc_indices`` + with an index based on two or more key columns. Previously this raised a ``ValueError``. [#18680] + +astropy.time +^^^^^^^^^^^^ + +- Allow ``np.concatenate``, ``np.stack`` and similar numpy functions to + be applied on ``Time`` and ``TimeDelta`` instances. [#18193] + +- Add a new time format ``galex`` for the GALEX satellite. + + In GALEX data, due to uncertainty in the spacecraft clock, the absolute time is only accurate to + about 1-10 seconds while the relative time within an observation is better than 0.005 s or so, + except on days with leap seconds, where relative times can be wrong by up to 1 s. + See question 101.2 in https://www.galex.caltech.edu/researcher/faq.html [#18330] + +astropy.units +^^^^^^^^^^^^^ + +- Some unit formats have deprecated units and converting such units to strings + emits a warning. + The new ``deprecations`` parameter of the unit ``to_string()`` methods allows + automatically converting deprecated units (if possible), silencing the warnings + or raising them as errors instead. [#18586] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Enable color and suggestion-on-typos in ``fits2bitmap`` CLI for Python 3.14 [#18151] + +- Added ``show_decimal_unit`` to ``set_major_formatter`` to control whether + or not units are shown in decimal mode. [#18312] + +- Added the methods ``set_visible()`` and ``set_position()`` to control the visibility and position of ticks, tick labels, and axis labels in a single call. + + Also added ``get_ticks_visible()``, ``get_ticklabel_visible()``, and ``get_axislabel_visible()`` methods to get the visibility state of each coordinate element. [#18443] + +- Added an image interval option (``SymmetricInterval``) for specifying a + symmetric extent about a midpoint, and the extent that contains both the image + minimum and maximum can be automatically determined. [#18602] + +astropy.wcs +^^^^^^^^^^^ + +- Enable color and suggestion-on-typos in all ``wcslint`` CLI for Python 3.14 [#18151] + +- Added a ``perserve_units`` keyword argument to ``WCS`` to optionally request + that units are not converted to SI (the default behavior is for celestial axes + to have units converted to degrees, and spectral axes to m or Hz). [#18338] + + +API Changes +----------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- The functionality of ``astropy.coordinates.concatenate`` and + ``astropy.coordinates.concatenate_representations`` is now available using + ``np.concatenate``. Hence, these functions are being deprecated, emitting an + ``AstropyPendingDeprecationWarning`` starting with astropy 7.2. This will be + followed by a regular deprecation warning in astropy 8.0, and removal in 9.0. [#18193] + +- The ``matrix_utilities`` module was not included in the ``astropy`` API + documentation, but it was nonetheless explicitly referred to in some of the + other documentation. + This made it unclear if the functions in the module are public or private. + The public matrix utilities ``is_rotation_or_reflection()`` and + ``rotation_matrix()`` have been made available from the ``astropy.coordinates`` + namespace and should be imported from there. + Functions not available from the ``astropy.coordinate`` namespace are private + and may be changed or removed without warning. + However, three functions have been explicitly deprecated, despite being + private, as a courtesy to existing users. + ``matrix_utilites.angle_axis()`` and ``matrix_utilites.is_rotation()`` are + deprecated without replacement. + ``matrix_utilities.is_O3()`` is deprecated and the public + ``is_rotation_or_reflection()`` function can be used as a replacement. [#18418] + +- The undocumented ``earth_orientation`` module has been removed. [#18638] + +- ``astropy`` prefers reading data required for ``EarthLocation.of_site()`` from + a local cache and tries downloading (and caching) the data from the Internet if + the cache is empty. + As a last resort ``astropy`` has so far read a small bundled data file that + provided data for Greenwich as the single entry, but now ``astropy`` will raise + an error. [#18649] + +astropy.io.registry +^^^^^^^^^^^^^^^^^^^ + +- ``UnifiedInputRegistry`` and ``UnifiedOutputRegistry``'s ``delay_doc_updates`` + method's effect is disabled under Python's optimized mode (``-OO`` flag). [#17572] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Added a ``config`` property to ``astropy.io.votable.tree.VOTableFile``. + This property can be passed to the ``config`` parameter of constructors that need to know the associated VOTable version, such as ``TimeSys`` and ``CooSys``. [#18366] + +astropy.table +^^^^^^^^^^^^^ + +- Add additional detail to the text of the ``ValueError`` that is raised when + ``pprint`` cannot parse a column format string. [#17631] + +- Selecting a table index in the ``.loc``, ``.iloc``, or ``.loc_indices`` properties by + passing the index identifier as the first element of the item is deprecated and is + planned for removal in astropy 9.0. For example, if a table ``t`` has two indices on + columns ``"a"`` and ``"b"`` respectively, then ``t.loc["b", 2]`` (to find table rows + where ``t["b"] == 2``) is deprecated. This is replaced by ``t.loc.with_index("b")[2]``. [#18680] + +astropy.tests +^^^^^^^^^^^^^ + +- API changes towards a future deprecation of astropy test runner: + + * ``astropy.tests.runner.keyword`` is removed from public API. + It is used internally as a decorator within astropy test runner and + its exposure as public API was a mistake. In the future, it will be + removed without any deprecation. + * ``astropy.test``, ``astropy.tests.runner.TestRunnerBase``, and ``astropy.tests.runner.TestRunner`` + are now pending deprecation (``AstropyPendingDeprecationWarning``). + This will also affect downstream ``packagename.test`` generated using ``TestRunner``. + They may start to emit ``AstropyDeprecationWarning`` in v8.0 (but no earlier). [#17874] + +astropy.utils +^^^^^^^^^^^^^ + +- The ``isiterable()`` utility is deprecated. + ``numpy.iterable()`` can be used as a drop-in replacement. [#18053] + +- ``astropy.utils.metadata.MergeStrategy`` no longer modifies the ``merge()`` + methods of its subclasses at runtime to re-raise all exceptions as + ``MergeConflictError``. + This does not affect the functionality of ``MergeStrategy`` subclasses within + the ``astropy`` metadata merging machinery. [#18518] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- A warning is now emitted for each axis name which is + invalid in ``set_ticklabel_position``, ``set_axislabel_position``, + and ``set_ticks_position``. This is a deprecation warning, + and in future invalid axis names will result in an error. [#18324] + +- A warning is now emitted if arguments are given to the getter method + ``get_axislabel_visibility_rule``. This is a deprecation warning, and in + future, giving arguments to this method will result in an error. [#18792] + + +Bug Fixes +--------- + +astropy.config +^^^^^^^^^^^^^^ + +- ``get_config_dir()`` and ``get_cache_dir()`` now emit warnings in all cases + where the ``XDG_CACHE_HOME`` (``XDG_CONFIG_HOME``, respectively) environment + variable doesn't meet internal assumptions and is ignored as a result. [#17934] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- The ``angle`` argument of the ``rotation_matrix()`` function can now be any + angle-like value, like its docstring states. + Previously some angle-like values (e.g. angle-like strings) were erroneously + rejected. [#18504] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix bug with heap which was not updated after a VLA column is modified. [#18487] + +- Make ``fitscheck`` verify all HDUs before listing errors. [#18574] + +- Fix calculation of DATASUM/CHECKSUM for heap data in ``BinTableHDU``. [#18681] + +- Fixed a bug in ``fitsdiff`` script where failing to read a single file could + crash the entire program. A warning is now printed instead, and such files + are simply ignored. [#18882] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Fixed a bug where writing a table to ECSV fails if meta + contains a value that is a numpy string. [#18677] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Updated IVOA UCD1+ controlled vocabulary from version 1.5 to 1.6. This adds + support for new atmospheric observation terms including ``obs.atmos.wind``, + ``obs.atmos.humidity``, ``obs.atmos.rain``, ``obs.atmos.turbulence``, + ``obs.atmos.turbulence.isoplanatic``, ``obs.atmos.water``, and + ``phys.temperature.dew`` which are now recognized when parsing UCDs with + ``check_controlled_vocabulary=True``. [#18483] + +- Fixed a bug in ``add_data_origin_info()`` where ``content`` is ignored for some INFO names. [#18771] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Fixed a bug in ``modeling.tabular`` models when the ``lookup_table`` is a Quantity, where the result might lose its unit in some cases. [#18958] + +astropy.nddata +^^^^^^^^^^^^^^ + +- Fixed unexpected upcasting to 64 bits when doing arithmetic with Python scalars + on ``numpy`` 2. + + Don't upcast ``NDData`` unnecessarily when doing arithmetic involving a single + unit (consistent with the behaviour when there are no units). Upcasting still + occurs if an operand's unit gets converted to match the other, or where + required by the other operand's dtype. [#18392] + +astropy.samp +^^^^^^^^^^^^ + +- ``SAMPHubServer._call_and_wait`` raises a new ``SAMPProxyTimeoutError`` (derived from ``SAMPProxyError``) exception on timeout. + This allows client code to more easily distinguish timeouts from other kind of exceptions. [#18169] + +astropy.stats +^^^^^^^^^^^^^ + +- ``poisson_conf_interval`` ``kraft-burrows-nousek`` no longer fails for large N. [#18676] + +astropy.table +^^^^^^^^^^^^^ + +- Fix a bug when slicing a table that has a multi-column index. Previously, after slicing + the table then ``tbl.indices`` would show duplicates of the multi-column index, one for + each column in the index. The underlying indices on the index columns were incorrectly + distinct objects instead of the expected reference to a single index object. [#18694] + +- Fix bugs when indexing a ``QTable`` with a ``Quantity`` column. Previously, after adding + the index then indexed item access via with a ``Quantity`` or slicing was failing. [#18725] + +- Fix a bug where the ECSV writer was not correctly quoting column names if the first name + starts with the "#" character or any names contain leading/trailing whitespace. In this + situation, all column names are now surrounded by double quotes per the ECSV standard. + Likewise the ECSV reader was incorrectly stripping surrounding whitespace from column + names, leading to a consistency check failure when reading. [#18752] + +- Fixed a bug when writing ``Table`` to FITS files, if the table contained masked arrays of integers. [#18818] + +astropy.units +^^^^^^^^^^^^^ + +- The string representations of the liter with the different ``astropy`` unit + formatters are now more consistent with each other. + This change only affects converting units to strings, it has no effect on + parsing strings to units. [#18500] + +- So far only the ``"cds"`` unit format has been capable of parsing the string + ``"as"`` as the attosecond, but now the other unit formats recognize that + string too. [#18723] + +- The ``"ogip"`` unit formatter can now parse strings that include signed + fractions in the exponent, e.g. ``u.Unit("m**(-1/2)", format="ogip")``. [#18776] + +astropy.utils +^^^^^^^^^^^^^ + +- If ``numpy.msort()`` is called with a ``Masked`` array then ``astropy`` no + longer erroneously hides the deprecation warning (with ``numpy`` versions + 1.24-1.26). [#18173] + +- For ``numpy < 2.0``, applying ``np.atleast_*d`` to iterables of most astropy + classes will now return a list of instances instead of a tuple, to match the + behaviour for arrays. For numpy >= 2.0, tuples continue to be returned. [#18193] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Fixed an image-normalization bug where the interval on a ``ImageNormalize`` + instance could be ignored when plotting. [#18590] + +astropy.wcs +^^^^^^^^^^^ + +- Fixed a bug that caused world_to_array_index to return lists instead of Numpy arrays. [#18730] + +Other Changes and Additions +--------------------------- + +- The minimum required NumPy version is now 1.24. [#18160] + +- The minimum required Matplotlib version is now 3.8.0. [#18164] + +- Bundled ``expat`` is updated to version 2.7.3. [#18657] + +- Updated the bundled CFITSIO library to 4.6.3. [#18689] + +- Wheels are now provided for Windows arm64. [#18786] + +Version 7.1.1 (2025-10-10) +========================== + +Bug Fixes +--------- + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Writing zero-row BinTableHDU with string columns no longer raises a broadcast error in _ascii_encode() [#18230] + +- Compute maximum absolute and relative differences reported by ``ImageDataDiff`` + on the full arrays instead of only a few values. [#18451] + +- Fix slicing FITS compressed file with ``.section`` when data is scaled. [#18640] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Fixed a bug where coordinate frame objects could not be serialized to YAML. This caused + an exception when saving a ``SkyCoord`` object in particular frames like + ``galactocentric`` in which frame attributes are themselves a frame. [#18526] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Fix handling of bounded variable-length char arrays in BINARY2 format which were previously treated as fixed length. [#18105] + +astropy.nddata +^^^^^^^^^^^^^^ + +- Fixed key error with numpy functions ``np.min``, ``np.max``, ``np.mean``, and ``np.sum``. [#18424] + +- Fix partial cutouts with FITS compressed file and scaled data. [#18640] + +astropy.table +^^^^^^^^^^^^^ + +- Fixed a bug in ``table.table_helpers.ArrayWrapper`` where byteorder of the + underlying data was not necessarily preserved through roundtrips. [#18139] + +- Fix bug #10732 where removing rows on an indexed table that was subsequently sliced + (e.g. ``t.add_index("a"); ts = t[1:5]; ts.remove_row(2)``) was giving incorrect results + or failing. [#18511] + +astropy.time +^^^^^^^^^^^^ + +- Ensure that the fast C parser for ``Time`` works also with numpy 2.3.0, fixing + a bug in our implementation which had no effect in previous numpy versions. [#18265] + +astropy.timeseries +^^^^^^^^^^^^^^^^^^ + +- Fixed the ``aggregate_downsample`` performance degradation when + non-default ``aggregate_func`` is used. [#18188] + +astropy.units +^^^^^^^^^^^^^ + +- Fixed the LaTeX representation of ``DexUnit`` in ``astropy.units``, + and thus also how it is represented in, e.g., jupyter notebooks. [#18627] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Fix a bug that caused ``WCSAxes.get_transform`` to not return the correct + transform when using WCS instances with celestial axes that were not in + degrees. [#18311] + +- Fixed a bug that under certain conditions could lead to ticks being incorrectly + labelled with a single "$" dollar sign in WCSAxes. [#18313] + +- Fixed WCSAxes.get_transform() in the case of 1D WCS [#18327] + +- Fixed a bug that caused the default format unit to be incorrect for RA/Dec WCSes with non-degree units [#18346] + +- Fix a bug where the units of the ``values=`` keyword argument to ``set_ticks`` was not respected. [#18577] + +astropy.wcs +^^^^^^^^^^^ + +- Fixed an issue which caused calls to WCS coordinate conversion routines to not be thread-safe due to calls to WCS.wcs.set() from multiple threads. [#16411] + +- Fix a bug that caused the output of ``WCS.wcs.print_contents()`` to be truncated + and to then cause the output of subsequent ``print_contents()`` calls (on + ``WCS.wcs`` or other wcs objects such as ``WCS.wcs.wtb``) to be corrupted. [#18350] + +- Fixed a bug in ``WCS.pixel_to_world`` for spectral WCS where ``restfrq`` was + defined but CTYPE was ``VOPT``, and likewise where ``restwav`` was defined but + CTYPE was ``VRAD``. [#18352] + +- Fixed a bug where world->pixel conversions did not work correctly on a 1D WCS + sliced via ``SlicedLowLevelWCS``. [#18394] + +- Fixed a bug that caused slicing of WCS objects with an ellipsis to not return a WCS + object but instead a SlicedLowLevelWCS object. [#18417] + +- Fixed a bug in ``wcs.py`` that caused the WCS object to not properly initialize + the `_naxis` attribute when the header was empty or did not contain any WCS + information. This could lead to crashes when attempting to take a slice of a 3D + WCS object or it could lead unexpected behavior when accessing pixel shape + or other properties that depend on the number of axes. [#18419] + +- Fixed a race condition when using the APE-14 API for the ``WCS`` class in a multi-threaded environment. [#18692] + + +Performance Improvements +------------------------ + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Improved performance of ``modeling.rotations.spherical2cartesian()`` by 11-18% depending on the size of the input data arrays. [#18238] + + +Other Changes and Additions +--------------------------- + +- Fixed errors with building the package from source on Windows via + ``python -m build`` and similar commands. [#18253] + +- Pre-built binaries (wheels) for Linux are now built using the ``manylinux_2_28`` + image (previously, ``manylinux2014`` was used). [#18374] + +Version 7.1.0 (2025-05-20) +========================== + + +New Features +------------ + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- ``search_around_sky`` and ``search_around_3D`` now accept separations/distlimits + broadcastable to the same shape as ``coords1``. [#17824] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Add functionality to read and write to a Table from the TDAT format as part of + the Unified File Read/Write Interface. [#16780] + +- ``io.ascii`` now supports on-the-fly decompression of LZW-compressed files + (typically ".Z" extension) via the optional package uncompresspy. [#17960] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Astropy can now not only read but also write headers that have ``HIERARCH`` + keys with long values, by allowing the use of ``CONTINUE`` cards for those + (as was already the case for regular FITS keys). [#17748] + +- Add ``strip_spaces`` option to ``Table.read`` to strip trailing whitespaces in + string columns. This will be activated by default in the next major release. [#17777] + +- ``io.fits`` now supports on-the-fly decompression of LZW-compressed files + (typically ".Z" extension) via the optional package uncompresspy. [#17960] + +- ``io.fits`` now supports on-the-fly decompression of LZMA-compressed files + (typically ".xz" extension) if the lzma module is provided by the Python + installation. [#17968] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Add a fast ``Table`` CSV reader that uses the PyArrow ``read_csv()`` function. This can + be significantly faster and more memory-efficient than the ``astropy.io.ascii`` fast + reader. This new reader can be used with ``Table.read()`` by setting + ``format="pyarrow.csv"``. [#17706] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- New module ``astropy.io.votable.dataorigin`` to extract Data Origin information from INFO in VOTable. [#17839] + +- ``CooSys`` VOTable elements now have a method ``to_astropy_frame`` that returns the + corresponding astropy built-in frame, when possible. [#17999] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Added a ``fit_info=`` keyword argument to ``parallel_fit_dask`` to allow users to preserve fit information from each individual fit. [#17538] + +astropy.nddata +^^^^^^^^^^^^^^ + +- Adds a utility class, ``astropy.nddata.Covariance``, used to construct, access, + and store covariance matrices. The class depends on use of the ``scipy.sparse`` + module. [#16690] + +- Add the ``limit_rounding_method`` parameter to `~astropy.nddata.Cutout2D`, + `~astropy.nddata.overlap_slices`, `~astropy.nddata.extract_array`, and + `~astropy.nddata.add_array` to allow users to specify the rounding method + used when calculating the pixel limits of the cutout. The default method + is to use `~numpy.ceil`. [#17876] + +astropy.table +^^^^^^^^^^^^^ + +- Document that ``Table.group_by``'s underlying sorting algorithm is guaranteed + to be stable. This reflects behavior that was already present but undocumented, + at least since astropy 6.0 . [#17676] + +astropy.timeseries +^^^^^^^^^^^^^^^^^^ + +- Downsampling now works correctly also on ``MaskedColumn`` and + ``MaskedQuantity`` with possibly masked elements. Furthermore, the type of + (Masked) column will now be properly preserved in downsampling. [#18023] + +astropy.units +^^^^^^^^^^^^^ + +- Units with the "micro" prefix can now be imported using ``"Îŧ"`` in the name. + For example, the microgram can now be imported with + ``from astropy.units import Îŧg``. [#17651] + +- It is now possible to import angstrÃļm, litre and ohm from ``astropy.units`` + using the ``Å``, ``ℓ`` and ``Ί`` symbols. [#17829] + +- Unit conversions between kelvins and degrees Rankine no longer require the + ``temperature`` equivalency. [#17985] + +astropy.utils +^^^^^^^^^^^^^ + +- Make commonly used Masked subclasses importable for ASDF support. + + Registered types associated with ASDF converters must be importable by + their fully qualified name. Masked classes are dynamically created and have + apparent names like ``astropy.utils.masked.core.MaskedQuantity`` although + they aren't actually attributes of this module. Customize module attribute + lookup so that certain commonly used Masked classes are importable. + + See: + + - https://asdf.readthedocs.io/en/latest/asdf/extending/converters.html#entry-point-performance-considerations + - https://github.com/astropy/asdf-astropy/pull/253 [#17685] + +- ``astropy.utils.data.download_file`` can now recover from a ``TimeoutError`` + when given a list of alternative source URLs. Previously, only ``URLError`` + exceptions were recoverable. An exception is still being raised after trying all + URLs provided if none of them could be reached. [#17691] + +- ``utils.data`` now supports on-the-fly decompression of LZW-compressed files + (typically ".Z" extension) via the optional package uncompresspy. [#17960] + + +API Changes +----------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- On representations the method ``get_name`` has been deprecated in favor of the class-level + attribute ``name``. The method will be removed in a future release. [#17503] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- A new public module, ``astropy.cosmology.io``, has been added to provide support + for reading, writing, and converting cosmology instances. + + The private modules ``astropy.cosmology.funcs``, + ``astropy.cosmology.parameter``, ``astropy.cosmology.connect``, + ``astropy.cosmology.core``, and ``astropy.cosmology.flrw`` have been deprecated. + Their functionality remains accessible in the `astropy.cosmology` module or in + the new ``astropy.cosmology.io`` module. [#17543] + +- Comoving distances now accept an optional 2nd argument, where the two-argument form is + the comoving distance between two redshifts. The one-argument form is the comoving + distance from redshift 0 to the input redshift. [#17701] + +- A new public module, ``astropy.cosmology.traits``, has been added to provide building + blocks for custom cosmologies. The currently available traits are: + - ``astropy.cosmology.traits.ScaleFactor`` + - ``astropy.cosmology.traits.TemperatureCMB`` [#17702] + +astropy.extern +^^^^^^^^^^^^^^ + +- Astropy used to bundle the javascript libraries jQuery and DataTables for + interactive (e.g. sorting by column values) tables using the ``show_in_browser()`` + method. + This bundling requires relatively large files in astropy itself, for a relatively minor feature. + Furthermore, the astropy developers are not experts in javascript development, and + javascript libraries many need updates to improve on security vulnerabilities. + This change removes the bundled versions of jQuery and DataTables from astropy, + updates the default version of the remote URLs to version 2.1.8 of DataTables, and + sets the default for ``show_in_browser(use_local_files=False)`` to use the remote versions + in all cases. If the method is called with ``use_local_files=True``, a warning is + displayed and remote version are used anyway. + This may break the use of the method when working offline, unless the javascript + files are cached by the browser from a previous online session. [#17521] + +astropy.table +^^^^^^^^^^^^^ + +- ``showtable`` CLI is now deprecated to avoid a name clash on Debian; use ``showtable-astropy`` instead. [#18047] + +- Fix issues in the handling of a call like ``tbl.loc[item]`` or ``tbl.loc_indices[item]`` + and make the behavior consistent with pandas. Here ``tbl`` is a ``Table`` or ``QTable`` + with an index defined. + + If ``item`` is an empty list or zero-length ``np.ndarray`` or an empty slice, then + previously ``tbl.loc[item]`` would raise a ``KeyError`` exception. Now it returns the + zero-length table ``tbl[[]]``. + + If ``item`` is a one-element list like ``["foo"]``, then previously + ``tbl.loc[item]`` would return either a ``Row`` or a ``Table`` with multiple row, + depending on whether the index was unique. Now it always returns a ``Table``, consistent + with behavior for ``tbl.loc[[]]`` and ``tbl.loc[["foo", "bar"]]``. + + See https://github.com/astropy/astropy/pull/18051 for more details. [#18051] + +astropy.units +^^^^^^^^^^^^^ + +- Passing ``fraction='multiline'`` to ``unit.to_string()`` will no longer raise + an exception if the given format does not support multiline fractions, but + rather give a warning and use an inline fraction. [#17374] + +- Automatic conversion of a ``str`` or ``bytes`` instance to a unit when it is + multiplied or divided with an existing unit or quantity is deprecated. [#17586] + +- Accessing the contents of the ``units.deprecated`` module now emits deprecation + warnings. + The module may be removed in a future version. [#17929] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- All arguments from ``simple_norm`` are marked as future keyword-only, with the + exception of the first two (``data`` and ``stretch``). + A warning is now displayed if any other arguments are passed positionally. [#17489] + + +Bug Fixes +--------- + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix possible int overflow in the tile compression C code. [#17995] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- In ``CooSys`` elements, the system was not checked for votable version 1.5 [#17999] + +astropy.samp +^^^^^^^^^^^^ + +- Fix setting logging level from the ``samp_hub`` command line. + Previously, ``samp_hub --log-level OFF`` was documented as supported but actually caused an exception to be raised. + The patch infers valid choices from the standard library's ``logging`` module. + A ``CRITICAL`` level will closely emulate the intended ``OFF`` setting. [#17673] + +astropy.table +^^^^^^^^^^^^^ + +- Initializing a Table with ``rows`` or ``data`` set to ``[]`` or a numpy array with + zero size (e.g., ``np.array([[], []])``) is now equivalent to + ``Table(data=None, ...)`` and creates a table with no data values. This allows + defining the table names and/or dtype when creating the table, for instance: + ``Table(rows=[], names=["a", "b"], dtype=[int, float])``. Previously this + raised an exception. [#17717] + +- Fix issues in the handling of a call like ``tbl.loc[item]`` or ``tbl.loc_indices[item]`` + and make the behavior consistent with pandas. Here ``tbl`` is a ``Table`` or ``QTable`` + with an index defined. + + If ``item`` is an empty list or zero-length ``np.ndarray`` or an empty slice, then + previously ``tbl.loc[item]`` would raise a ``KeyError`` exception. Now it returns the + zero-length table ``tbl[[]]``. + + If ``item`` is a one-element list like ``["foo"]``, then previously + ``tbl.loc[item]`` would return either a ``Row`` or a ``Table`` with multiple row, + depending on whether the index was unique. Now it always returns a ``Table``, consistent + with behavior for ``tbl.loc[[]]`` and ``tbl.loc[["foo", "bar"]]``. + + See https://github.com/astropy/astropy/pull/18051 for more details. [#18051] + +astropy.timeseries +^^^^^^^^^^^^^^^^^^ + +- Made ``TimeSeries.from_pandas`` and ``BinnedTimeSeries.read`` more robust to + subclassing. [#17351] + +astropy.units +^^^^^^^^^^^^^ + +- For the Angstrom unit in the CDS module, ``u.cds.Angstrom``, the string + representation is now "Angstrom" (instead of "AA"), consistent with what was + always the case for ``u.Angstrom``, and conformant with the CDS standard. [#17536] + +- Previously the string representation of the ``solMass`` unit in the ``"cds"`` + format depended on whether the unit was imported directly from ``units`` or + from ``units.cds``. + Although both representations were valid according to the CDS standard, the + inconsistency was nonetheless needlessly surprising. + The representation of ``units.cds.solMass`` has been changed to match the + representation of ``units.solMass``. [#17560] + +- The degrees Rankine is now represented as "$\mathrm{{}^{\circ}R}$" in the + ``"latex"`` and ``"latex_inline"`` formats and as "°R" in the ``"unicode"`` + format. [#18049] + +astropy.utils +^^^^^^^^^^^^^ + +- Properly detect invalid LZMA files in ``utils.data``. [#17984] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Fixed an issue when using ``plot_coord`` after slicing the ``WCS`` object coordinates. [#18005] + + +Performance Improvements +------------------------ + +astropy.timeseries +^^^^^^^^^^^^^^^^^^ + +- Improved the ``aggregate_downsample`` performance using a new default ``aggregate_func``. [#17574] + +astropy.units +^^^^^^^^^^^^^ + +- Converting strings to units with ``Unit()`` is now up to 225% faster. [#17399] + +- ``UnitBase.compose()`` is now 20% faster. [#17425] + + +Other Changes and Additions +--------------------------- + +- After ``import astropy``, ``dir(astropy)`` will now list all subpackages, + including those that have not yet been loaded. This also means tab + completion will work as expected (e.g., ``from astropy.coo`` will + expand to ``from astropy.coordinates``). [#17598] + +- Updated bundled WCSLIB version to 8.4, fixing issues in ``wcs_chksum`` + and ``wcs_fletcher32``. For a full list of changes - see + ``astropy/cextern/wcslib/CHANGES``. [#17886] + +Version 7.0.2 (2025-05-12) +========================== + +Bug Fixes +--------- + +astropy.config +^^^^^^^^^^^^^^ + +- Fix a bug where config file generation did not parse nested subclasses of ``astropy.config.ConfigNamespace``. [#18107] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix a bug in ``nddata.Cutout2D`` when creating partial cutouts of ``Section`` objects by adding a ``dtype`` property to the ``Section`` class. [#17611] + +- Fixed a bug so that now the scaling state from the source HDU to the new appended HDU is copied on the + destination file, when the HDU is read with ``do_not_scale_image_data=True``. [#17642] + +- Fix setting a slice on table rows (``FITS_record``). [#17737] + +- Fix checksum computation for tables with VLA columns, when table is loaded in + memory. [#17806] + +- Fix ``.fileinfo()`` for compressed HDUs. [#17815] + +- Fix FITS_rec repr when a column has scaling factors, leading to a crash with + numpy>=2.0. [#17933] + +- Fixed a bug that caused THEAP, ZBLANK, ZSCALE, and ZZERO to not be correctly + removed during decompression of tile-compressed FITS files. [#18072] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- ``astropy`` v7.0.0 erroneously refused to write a VOTable if it contained units that + could not be represented in the CDS format. + Now ``astropy`` correctly chooses the unit format based on the VOTable version. + The bug in question did not cause any corruption in tables that were successfully + written because the newer VOUnit format is backwards compatible with the CDS format. + Furthermore, any unit that is in neither formats would still be written out + but would issue a warning. [#17570] + +- ``unicodeChar`` fields can now be of bounded variable size (``arraysize="10*``). [#18075] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Fixed an issue where the ``filter_non_finite`` option was not working + for 2D models. An error is raised when the ``filter_non_finite`` option + is set to ``True`` and all values are non-finite. [#17869] + +astropy.stats +^^^^^^^^^^^^^ + +- Now ``bayesian_blocks(t, x, fitness="events")`` correctly handles the case + when the input data ``x`` contains zeros. [#17800] + +astropy.table +^^^^^^^^^^^^^ + +- Prevent corrupting a column by mutating its name to an invalid type. + A ``TypeError`` is now raised when a name is set to anything other than a + string. [#17450] + +- Fix a bug in creating a ``Table`` from a list of rows that dropped the units + of non-scalar Quantity, e.g., ``Table(rows=[([1] * u.m,), ([2] * u.m,)])``. [#17936] + +astropy.units +^^^^^^^^^^^^^ + +- Ensured that the units of ``yp``, ``refa`` and ``refb`` are properly + taken into account when calling ``erfa.apio`` (previously, the + conversion required for ``xp`` was applied to those inputs too). [#17742] + +- The machinery that injects units into a namespace (used e.g. by ``def_unit()``) + now applies NFKC normalization to unit names when checking for name collisions. + This prevents name collisions if the namespace belongs to a module and the unit + is accessed as an attribute of that module. [#17853] + +- The string representations of the prefixed versions of ``solLum``, ``solMass`` + and ``solRad`` units can now be parsed by default. + Previously they could only be parsed if the ``required_by_vounit`` module had + been imported, possibly indirectly by using the ``"vounit"`` format. [#17868] + +astropy.utils +^^^^^^^^^^^^^ + +- Prevent corrupting a mixin column's ``info`` attribute by mutating its name to + an invalid type. A ``TypeError`` is now raised when a name is set to anything + other than a string. [#17450] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Ensure that the ``astropy.visualization.wcsaxes.custom_ucd_coord_meta_mapping`` + context manager performs a (correct) cleanup. [#17749] + +- Fixed interval classes for masked input (``MaskedArray`` and ``MaskedNDArray``). [#17927] + +- Fixed the limits of ``a`` parameter in the ``PowerDistStretch`` + and ``InvertedPowerDistStretch`` classes so that a value of + 0 in no longer allowed. That value gives infinity values in + ``InvertedPowerDistStretch`` and it makes the ``PowerDistStretch`` + results independent of the input data. [#17941] + +- Fixed an issue where LinearStretch values were not being clipped to + [0:1] when ``clip=True``. [#17943] + +astropy.wcs +^^^^^^^^^^^ + +- Fix UCD for air wavelengths, following the IVOA recommendation that ``'em.wl'`` + be reserved for vacuum wavelengths. ``'em.wl;obs.atmos'`` is now used to + represent air wavelengths instead. [#17769] + + +Other Changes and Additions +--------------------------- + +- Updated the bundled CFITSIO library to 4.6.0. [#17904] + +Version 7.0.1 (2025-02-06) +========================== + +API Changes +----------- + +astropy.table +^^^^^^^^^^^^^ + +- The use of the keyword ``use_local_files`` for the js viewer in + ``astropy.table.Table.show_in_browser`` is now deprecated. Starting in Astropy + 7.1 this keyword will be ignored and use of it will issue a warning. The + default behavior will be to use the remote versions of jQuery and DataTables + from a CDN. [#17480] + +Bug Fixes +--------- + +astropy.config +^^^^^^^^^^^^^^ + +- With ``astropy`` v7.0.0 the cache directory cannot be customized with the + ``XDG_CACHE_HOME`` environment variable. + Instead, ``XDG_CONFIG_HOME`` erroneously controls both configuration and cache + directories. + The correct pre-v7.0.0 behaviour has been restored, but it is possible that + ``astropy`` v7.0.0 has written cache files to surprising locations. + Concerned users can use the ``get_cache_dir_path()`` function to check where + the cache files are written. + + The bug in question does not affect systems where the ``XDG_CACHE_HOME`` and + ``XDG_CONFIG_HOME`` environment variables are unset. [#17514] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Fixed a numerical-precision bug with the calculation of the ``theta`` + component when converting from ``CylindricalRepresentation`` to + ``PhysicsSphericalRepresentation`` for vectors very close to the Z axis (within + milliarcseconds). [#17693] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Fixed parsing ASCII table with data that starts with a tilda. [#17565] + +- Find and read ASCII tables even if there is white space before + ``\begin{tabular}``, ``\tablehead``, and similar markers. [#17624] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix memory leak in ```BinTableHDU.copy()``` [#16143] + +- Fix overflow error with Numpy 2 and VLA columns using P format. [#17328] + +- Fix ``ImageHDU.scale`` with float. [#17458] + +- Fixed ``Table.write(..., format="fits", overwrite=True)`` when filename is + provided as ``pathlib.Path``. [#17552] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Updated XML writer for ``VOTableFile`` element to include or drop + ``coordinate_systems`` regardless of version. [#17356] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Fix fitting of compound models when inputs has more than one dimension, and + specifically fix fitting of compound models with Polynomial2D components + [#17618] + +astropy.table +^^^^^^^^^^^^^ + +- Ensure that representations and differentials, like SkyCoord, can be used in + table join operations, by making use of the fact that they can now be masked. [#17381] + +- Fix a crash in ``Table.show_in_browser`` due to an internal type inconsistency. [#17513] + +- Fix incorrect description of the ``unique`` parameter in ``Table.add_index``'s + docstring. Add missing Raises section. [#17677] + +astropy.units +^^^^^^^^^^^^^ + +- Ensure that ``Unit.to`` allows as ``value`` argument all array types that + follow the array API standard and define ``__array_namespace__``. Furthermore, + for backwards compatibility, generally pass through arguments that define a + ``.dtype``, independent of whether that is a numpy data type. [#17469] + +- The zebi (Zi, 2^70) and yobi (Yi, 2^80) binary prefixes are now supported. [#17692] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Fix ``CoordinateHelper.ticklabels``. The getter was incorrectly returning + the helper's ticks rather than the labels. [#17444] + +- The following private classes from ``astropy.visualization.lupton_rgb``, that + were dropped without deprecation in astropy 7.0.0, were re-introduced following + a report that they were used downstream. The following classes are now + considered public: + + - ``Mapping`` + - ``AsinhMapping`` + - ``LinearMapping`` + - ``AsinhZScaleMapping`` [#17531] + +Other Changes and Additions +--------------------------- + +- Update bundled js library datatables to version 2.1.8, which is current at the time of this PR. [#17480] + +Version 7.0.0 (2024-11-21) +========================== + + +New Features +------------ + +astropy.config +^^^^^^^^^^^^^^ + +- Added ``get_config_dir_path`` (and ``get_cache_dir_path``) which is equivalent + to ``get_config_dir`` (respectively ``get_cache_dir``) except that it returns a + ``pathlib.Path`` object instead of ``str``. [#17118] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- ``BaseCoordinateFrame`` instances such as ``ICRS``, ``SkyOffsetFrame``, etc., + can now be stored directly in tables (previously, they were stored as + ``object`` type columns). Furthermore, storage in tables is now also possible + for frames that have no data (but which have attributes with the correct shape + to fit in the table). [#16831] + +- ``BaseCoordinateFrame`` now has a ``to_table()`` method, which converts the + frame to a ``QTable``, analogously to the ``SkyCoord.to_table()`` method. [#17009] + +- ``SkyCoord``, coordinate frames, and representations have all have gained the + ability to deal with ``Masked`` data. In general, the procedure is similar to + that of ``Time``, except that different representation components do not share + the mask, to enable holding, e.g., a catalogue of objects in which only some + have associated distances. [#17016] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Add support for ``pathlib.Path`` objects in + ``astropy.io.ascii.core.BaseInputter.get_lines``. [#16930] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Expanded ``FITSDiff`` output for ``PrimaryHDU`` and ``ImageHDU`` to include the + maximum relative and absolute differences in the data. [#17097] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- The HDF5 writer, ``write_table_hdf5()``, now accepts ``os.PathLike`` objects + as ``output``. [#16955] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Support reading and writing of VOTable version 1.5, including the new + ``refposition`` attribute of ``COOSYS``. [#16856] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Added ``Model.has_tied``, ``Model.has_fixed``, and ``Model.has_bounds`` attributes to make + it easy to check whether models have various kinds of constraints set without having to + inspect ``Model.tied``, ``Model.fixed``, and ``Model.bounds`` in detail. [#16677] + +- Added a new ``parallel_fit_dask`` function that can be used to fit models to + many sections (e.g. spectra, image slices) on an N-dimensional array in + parallel. [#16696] + +- Added a ``Lorentz2D`` model. [#16800] + +- Added ``inplace=False/True`` keyword argument to the ``__call__`` method of most fitters, + to optionally allow the original model passed to the fitter to be modified with the fitted + values of the parameters, rather than return a copy. This can improve performance if users + don't need to keep hold of the initial parameter values. [#17033] + +astropy.stats +^^^^^^^^^^^^^ + +- Added a ``SigmaClippedStats`` convenience class for computing sigma-clipped + statistics. [#17221] + +astropy.table +^^^^^^^^^^^^^ + +- Changed a number of dict-like containers in ``io.ascii`` from ``OrderedDict`` to + ``dict``. The ``dict`` class maintains key order since Python 3.8 so ``OrderedDict`` is + no longer needed. The changes are largely internal and should not affect users in any + way. See also the API change log entry for this PR. [#16250] + +- Add a ``keep_order`` argument to the ``astropy.table.join`` function which specifies to + maintain the original order of the key table in the joined output. This applies for + inner, left, and right joins. The default is ``False`` in which case the output is + ordered by the join keys, consistent with prior behavior. [#16361] + +astropy.units +^^^^^^^^^^^^^ + +- Add a ``formatter`` argument to the ``to_string`` method of the ``Quantity`` + class. Enables custom number formatting with a callable formatter or + format_spec, especially useful for consistent notation. [#16087] + +- Add the unit foe (or Bethe, equivalent to 1e51 erg), which is often used to + express the energy emitted by a supernova explosion. [#16441] + +- Add ``magnetic_flux_field`` equivalency to convert magnetic field between + magnetic field strength (H) and magnetic flux density (B). [#16516] + +- Added SI-units ``sievert``, ``gray``, ``katal``, and ``hectare`` in ``astropy.units.si``. [#16729] + +- When parsing invalid unit strings with ``u.Unit(..., parse_strict="warn")`` or + ``u.Unit(..., parse_strict="silent")``, a normal unit may be returned if the + problem is not too serious. + If parsing the string fails completely then an ``UnrecognizedUnit`` instance is + returned, just as before. [#16892] + +- Added a ``np.arange`` dispatch for ``Quantity`` (requires one to use + ``like=``). [#17059] + +- Added support for calling numpy array constructors (``np.empty``, ``np.ones``, + ``np.zeros`` and ``np.full``) with ``like=Quantity(...)`` . [#17120] + +- Added support for calling numpy array constructors (``np.array``, + ``np.asarray``, ``np.asanyarray``, ``np.ascontiguousarray`` and + ``np.asfortranarray``) with ``like=Quantity(...)`` . [#17125] + +- Added support for calling numpy array constructors (``np.frombuffer``, + ``np.fromfile``, ``np.fromiter``, ``np.fromstring`` and ``np.fromfunction``) + with ``like=Quantity(...))`` . [#17128] + +- Added support for calling numpy array constructors (``np.require``, + ``np.identity``, ``np.eye``, ``np.tri``, ``np.genfromtxt`` and ``np.loadtxt``) + with ``like=Quantity(...))`` . [#17130] + +astropy.utils +^^^^^^^^^^^^^ + +- Added the ``astropy.system_info`` function to help document runtime systems in + bug reports. [#16335] + +- Add support for specifying files as ``pathlib.Path`` objects in ``IERS_A.read`` + and ``IERS_B.read``. [#16931] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Add ``make_rgb()``, a convenience + function for creating RGB images with independent scaling on each filter. + Refactors ``make_lupton_rgb()`` to work with instances of subclasses of + ``BaseStretch``, including the new Lupton-specific classes + ``LuptonAsinhStretch`` and ``LuptonAsinhZscaleStretch``. [#15081] + +- Add support for custom coordinate frames for ``WCSAxes`` through a context + manager ``astropy.visualization.wcsaxes.custom_ucd_coord_meta_mapping``. [#16347] + +- Added ``get_ticks_position``, ``get_ticklabel_position``, and + ``get_axislabel_position`` methods on ``CoordinateHelper`` in WCSAxes. [#16686] + +- Added the ability to disable the automatic simplification of WCSAxes tick labels + by specifying ``simplify=False`` to ``set_ticklabel()`` for a coordinate axis. [#16938] + +- Added the ability to specify that WCSAxes tick labels always include the sign + (namely for positive values) by starting the format string with a ``+`` + character. [#16985] + +- Allow ``astropy.visualization.units.quantity_support`` to be used as a + decorator in addition to the already supported use as a context manager. [#17006] + +- Added the ability to specify a callable function in ``CoordinateHelper.set_major_formatter`` [#17020] + +- Added a ``SimpleNorm`` class to create a matplotlib normalization object. [#17217] + +- WCSAxes will now select which axis to draw which tick labels and axis labels on based on the number of drawn tick labels, rather than picking them in the order they are listed in the WCS. This means that axes may be swapped in comparison with previous versions of Astropy by default. [#17243] + + +API Changes +----------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- For non-scalar frames without data, ``len(frame)`` will now return the first + element of its ``shape``, just like for frames with data (or arrays more + generally). For scalar frames, a ``TypeError`` will be raised. Both these + instead of raising a ``ValueError`` stating the frame has no data. [#16833] + +- The deprecated ``coordinates.get_moon()`` function has been removed. Use + ``coordinates.get_body("moon")`` instead. [#17046] + +- The deprecated ``BaseCoordinateFrame.get_frame_attr_names()`` is removed. + Use ``get_frame_attr_defaults()`` instead. [#17252] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- Passing redshift arguments as keywords is deprecated in many methods. [#16597] + +- Deprecated ``cosmology.utils`` module has been removed. Any public API may + be imported directly from the ``cosmology`` module instead. [#16730] + +- Setting ``Ob0 = None`` in FLRW cosmologies has been deprecated in favor of ``Ob0 = + 0.0``. Conceptually this is a change in that baryons are now always a component of the + cosmology. Practically, the only change (besides that ``Ob0`` is never ``None``) is that + methods relying on ``Ob0`` always work, rather than sometimes raising an exception, + instead by default taking the contribution of the baryons to be negligible. [#16847] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Remove all deprecated arguments from functions within ``astropy.io.ascii``. + + ``read()``: + - ``Reader`` is removed. Instead supply the equivalent ``format`` argument. + - Use ``inputter_cls`` instead of ``Inputter``. + - Use ``outputter_cls`` instead of ``Outputter``. + + ``get_reader()``: + - Use ``reader_cls`` instead of ``Reader``. + - Use ``inputter_cls`` instead of ``Inputter``. + - Use ``outputter_cls`` instead of ``Outputter``. + + ``write()``: + - ``Writer`` is removed. Instead supply the equivalent ``format`` argument. + + ``get_writer()``: + - Use ``writer_cls`` instead of ``Writer``. [#15758] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- The ``CompImageHDU`` class has been refactored to inherit from ``ImageHDU`` + instead of ``BinTableHDU``. This change should be for the most part preserve the + API, but any calls to ``isinstance(hdu, BinTableHDU)`` will now return ``False`` + if ``hdu`` is a ``CompImageHDU`` whereas before it would have returned ``True``. + In addition, the ``uint`` keyword argument to ``CompImageHDU`` now defaults to + ``True`` for consistency with ``ImageHDU``. [#15474] + +- Remove many unintended exports from ``astropy.io.fits.hdu.compressed``. + The low-level functions ``compress_image_data`` and ``decompress_image_data_section`` + are now only available at the qualified names + ``astropy.io.fits.hdu.compressed._tiled_compression.compress_image_data`` + and ``astropy.io.fits.hdu.compressed._tiled_compression.decompress_image_data_section``. + The rest of the removed exports are external modules or properly exported + elsewhere in astropy. May break imports in rare cases that relied + on these exports. [#15781] + +- The ``CompImageHeader`` class is now deprecated, and headers on ``CompImageHDU`` + instances are now plain ``Header`` instances. If a reserved keyword is set on + ``CompImageHDU.header``, a warning will now be emitted at the point where the + file is written rather than at the point where the keyword is set. [#17100] + +- - Remove code that was deprecated in previous versions: ``_ExtensionHDU`` and + ``_NonstandardExtHDU``, ``(Bin)Table.update``, ``tile_size`` argument for + ``CompImageHDU``. Also specifying an invalid ``tile_shape`` now raises an + error. [#17155] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- New format ``"parquet.votable"`` is added to read and write a parquet file + with a votable metadata included. [#16375] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- ``Table.read(..., format='votable')``, ``votable.parse`` and + ``votable.parse_single_table`` now respect the ``columns`` argument and will only output + selected columns. Previously, unselected columns would just be masked (and unallocated). + ``astropy.io.votable.tree.TableElement.create_arrays`` also gained a ``colnumbers`` + keyword argument to allow column selection. [#15959] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Subclasses of ``_NonLinearLSQFitter``, so any subclasses of the public ``LevMarLSQFitter``, ``TRFLSQFitter``, ``LMLSQFitter`` or ``DogBoxLSQFitter``, should now accept an additional ``fit_param_indices`` kwarg in the function signature of their ``objective_function`` methods. + Nothing is needed to be done with this kwarg, and it might not be set, but it can optionally be passed through to ``fitter_to_model_params_array`` for a performance improvement. + We also recommended accepting all kwargs (with ``**kwargs``) in this method so that future additional kwargs do not cause breakage. [#16673] + +- Exception message for when broadcast shapes mismatch has changed. + Previously, it used complicated regex to maintain backward compatibility. + To ease maintenance, this regex has been removed and now directly + passes exception from ``numpy.broadcast_shapes`` function. [#16770] + +- Using the ``LMLSQFitter`` fitter with models that have bounds is now deprecated, + as support for bounds was very basic. Instead, non-linear fitters with more + sophisticated support for bounds should be used instead. [#16994] + +- The optional ``use_min_max_bounds`` keyword argument in ``TRFLSQFitter`` and + ``DogBoxLSQFitter`` has now been deprecated and should not be used. These + fitters handle bounds correctly by default and this keyword argument was only + provided to opt-in to a more basic form of bounds handling. [#16995] + +- The deprecated ``comb()`` function has been removed. + Use ``math.comb()`` from the Python standard library instead. [#17248] + +astropy.stats +^^^^^^^^^^^^^ + +- Integer inputs to ``sigma_clip`` and ``SigmaClip`` are not converted to + ``np.float32`` instead of ``float`` if necessary. [#17116] + +astropy.table +^^^^^^^^^^^^^ + +- Change the default type for the ``meta`` attribute in ``Table`` and ``Column`` (and + subclasses) from ``OrderedDict`` to ``dict``. Since Python 3.8 the ``dict`` class is + ordered by default, so there is no need to use ``OrderedDict``. + + In addition the ECSV table writer in ``astropy.io.ascii`` was updated to consistently + write the ``meta`` attribute as an ordered map using the ``!!omap`` tag. This + convention conforms to the ECSV specification and is supported by existing ECSV readers. + Previously the ``meta`` attribute could be written as an ordinary YAML map, which is not + guaranteed to preserve the order of the keys. [#16250] + +- An exception is now raised when trying to add a multi-dimensional column as an + index via ``Table.add_index``. [#16360] + +- Aggregating table groups for ``MaskedColumn`` no longer converts + fully masked groups to ``NaN``, but instead returns a masked element. [#16498] + +- Always use ``MaskedQuantity`` in ``QTable`` to represent masked ``Quantity`` + data or when the ``QTable`` is created with ``masked=True``. Previously the + default was to use a normal ``Quantity`` with a ``mask`` attribute of type + ``FalseArray`` as a stub to allow a minimal level of compatibility for certain + operations. This update brings more consistent behavior and fixes functions + like reading of table data from a list of dict that includes quantities with + missing entries, and aggregation of ``MaskedQuantity`` in table groups. [#16500] + +- Setting an empty table to a scalar no longer raises an exception, but + creates an empty column. This is to support cases where the number of + elements in a table is not known in advance, and could be zero. [#17102] + +- ``show_in_notebook`` method for Astropy tables has been un-deprecated and the API has + been updated to accept a ``backend`` keyword and require only keyword arguments. The new + default ``backend="ipydatagrid"`` relies on an optional dependency, ``ipydatagrid``. The + previous default table viewer (prior to v7.0) is still available as + ``backend="classic"``, but it has been deprecated since v6.1 and will be removed in a future release. [#17165] + +- The default behavior of ``Table.pformat`` was changed to include all rows and columns + instead of truncating the outputs to fit the current terminal. The new default + keyword arguments ``max_width=-1`` and ``max_lines=-1`` now match those in + ``Table.pformat_all``. Since the ``Table.pformat_all`` method is now redundant, it is + pending deprecation. Similarly, the default behavior of ``Column.pformat`` was changed + to include all rows instead of truncating the outputs to fit the current terminal. [#17184] + +astropy.time +^^^^^^^^^^^^ + +- ``Time.ptp`` now properly emits a deprecation warning independently of NumPy's + version. This method was previously deprecated in astropy 6.1, but the warning + was not visible for users that had NumPy 1.x installed. Because of this, the + warning message was updated to state that ``Time.ptp`` is deprecated since + version 7.0 instead. [#17212] + +astropy.units +^^^^^^^^^^^^^ + +- The deprecated ``Quantity.nansum()`` method has been removed. Use + ``np.nansum`` instead. [#15642] + +- The ``factor`` parameter of the ``spectral_density`` equivalency, the use of + which has been discouraged in the documentation since version 0.3, is now + deprecated. + Use the ``wav`` parameter as a ``Quantity``, not as a bare unit. [#16343] + +- The ``format.Fits`` formatter class has been renamed to ``format.FITS`` and the + old name is deprecated. + Specifying the FITS format for converting ``Quantity`` and ``UnitBase`` + instances to and from strings is not affected by this change. [#16455] + +- Conversion from one unit to another using ``old_unit.to(new_unit, value)`` no longer + converts ``value`` automatically to a numpy array, but passes through array duck types + such as ``dask`` arrays, with equivalencies properly accounted for. [#16613] + +- The ``format_exponential_notation()`` method of the ``Base`` unit formatter has + changed. + Any unit formatters that inherit directly from ``Base`` but have not + implemented their own ``format_exponential_notation()`` and wish to retain + previous behavior should implement it as: + + .. code-block:: python + + def format_exponential_notation(cls, val, format_spec): + return format(val, format_spec) + + Any formatters that inherit directly from ``Base`` and call + ``super().format_exponential_notation(val, format_spec)`` should instead call + ``format(val, format_spec)`` + The specific unit formatters in ``astropy.units`` and custom formatters that + inherit from any of them are not affected. [#16676] + +- The deprecated ``units.format.Unscaled`` has been removed. Use ``units.format.Generic`` + instead. [#16707] + +- Added a __round__() dunder method to ``Quantity`` + in order to support the built-in round() function. [#16784] + +- For ``Masked`` initialization in which a mask is passed in, ensure that that + mask is combined with any mask present on the input. [#16875] + +- The ``get_format_name()`` method of ``NamedUnit`` and its subclasses is + deprecated. + The ``to_string()`` method can be used instead. [#16958] + +- The ``UnitBase.in_units()`` method is deprecated. + The ``to()`` method can be used as a drop-in replacement. [#17121] + +- Unit conversions to a given system with ``unit.to_system()``, + ``unit.si``, and ``unit.cgs``, will now prefer the simplest unit if it + is in the given system, rather than prioritizing more complicated + units if those had a base unit component. E.g., ``u.Pa.si`` will now + simply return ``Unit("Pa")`` rather than ``Unit("N / m2")``. However, + the case where a unit can be simply described in base units remains + unchanged: ``u.Gal.cgs`` will still give ``Unit("cm / s2")``. [#17122] + +- The ``CDS``, ``OGIP`` and ``VOUnit`` unit formatters are now subclasses of the + ``FITS`` unit formatter. [#17178] + +- The ``eV`` and ``rydberg`` units were moved to ``astropy.units.misc`` (from + ``astropy.units.si`` and ``astropy.units.astrophys``, respectively). + Practically, this means that ``Unit.to_system(u.si)`` no longer includes + ``eV`` as a SI-compatible unit. [#17246] + +astropy.utils +^^^^^^^^^^^^^ + +- ``IERS_Auto.open()`` now always returns a table of type ``IERS_Auto`` that + contains the combination of IERS-A and IERS-B data, even if automatic + updating of the IERS-A file is disabled or if downloading the new file fails. + Previously, under those conditions, it would return a table of a different type + (``IERS_B``) with only IERS-B data. [#16187] + +- ``astropy.utils.check_broadcast`` is now deprecated in favor of + ``numpy.broadcast_shapes`` [#16346] + +- Added a new keyword ``pending_warning_type`` to ``deprecated`` decorator so downstream developers could customize the type of warning for pending deprecation state. [#16463] + +- The ``introspection.resolve_name()`` function is deprecated. + It is better to use the standard library ``importlib`` instead. [#16479] + +- ``format_exception()`` is deprecated because it provides little benefit, if + any, over normal Python tracebacks. [#16807] + +- The ``utils.masked`` module has gained a mixin class, ``MaskableShapedLikeNDArray``, + as well as two utility functions, ``get_data_and_mask`` and ``combine_masks``, + that can help make a container classes carry masked data. Within astropy, these + are now used in the implementation of masks for ``Time``. [#16844] + +- The deprecated ``compat.override__dir__()`` utility has been removed. [#17190] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Removed deprecated ``exp`` attribute in the ``LogStretch``, + ``InvertedLogStretch``, ``PowerDistStretch``, and + ``InvertedPowerDistStretch`` stretch classes, and the ``power`` + attribute in the ``PowerStretch``. Instead, use the ``a`` attribute, + which matches the input keyword. [#15751] + +- Removes the unintended NumPy export previously at ``astropy.visualization.np``. [#15781] + +- Accessing or setting the following attributes on ``CoordinateHelper`` has been deprecated: + + * ``ticks`` + * ``ticklabels`` + * ``axislabels`` + + Setting the following attributes on ``CoordinateHelper`` directly has been deprecated: + + * ``parent_axes`` + * ``parent_map`` + * ``transform`` + * ``coord_index`` + * ``coord_unit`` + * ``coord_type`` (use ``set_coord_type`` instead) + * ``coord_wrap`` (use ``set_coord_type`` instead) + * ``frame`` + * ``default_label`` + + Accessing or setting the following attributes on ``CoordinateHelper`` has been + removed (without deprecation, as these were clearly internal variables): + + * ``grid_lines_kwargs`` + * ``grid_lines`` + * ``lblinfo`` + * ``lbl_world`` + * ``minor_frequency`` (there were already public methods to set/get this) [#16685] + +- The deprecated ``nsamples`` parameter of ``ZScaleInterval`` is removed. [#17186] + +astropy.wcs +^^^^^^^^^^^ + +- Errors may now occur if a ``BaseLowLevelWCS`` class defines + ``world_axis_object_components`` which returns values that are not scalars or + plain Numpy arrays as per APE 14. [#16287] + +- ``WCS.pixel_to_world_values``, ``WCS.world_to_pixel_values``, + ``WCS.pixel_to_world`` and ``WCS.world_to_pixel`` now correctly return NaN values for + pixel positions that are outside of ``pixel_bounds``. [#16328] + + +Bug Fixes +--------- + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Fix the broken behavior of reading an ASCII table and filling values using column names. + This PR addresses the issue and improves the functionality. [#15774] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix a number of bugs in ``CompImageHDU``: + + * Fix the ability to pickle ``CompImageHDU`` objects + * Ensure that compression settings are not lost if initializing ``CompImageHDU`` + without data but with compression settings and setting the data later + * Make sure that keywords are properly updated when setting the header of a + ``CompImageHDU`` to an existing image header. + * Fix the ability to use ``CompImageHDU.section`` on instances that have not yet + been written to disk + * Fix the image checksum/datasum in ``CompImageHDU.header`` to be those for the + image HDU instead of for the underlying binary table. [#15474] + +- Fix a spurious exception when reading integer compressed images with blanks. [#17099] + +- Fix creating ``CompImageHDU`` from header with BSCALE/BZERO: keywords are now + ignored, as done in ``ImageHDU``. [#17237] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Making the "votable.parquet" format available as a reader format to ensure + consistency with the writer formats, even though the format it recognised + automatically by "votable". [#16488] + +- Explicitly set ``usedforsecurity=False`` when using ``hashlib.md5``. Without this, ``hashlib.md5`` will be blocked in FIPS mode. + FIPS (Federal Information Processing Standards) is a set of standards created by NIST (National Institute of Standards and Technology) for US government agencies regarding computer security and interoperability. + This affects validation results ingestion. [#17156] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Fixed the output representation of models with parameters that have + units of ``dimensionless_unscaled``. [#16829] + +astropy.stats +^^^^^^^^^^^^^ + +- Fixed accuracy of sigma clipping for large ``float32`` arrays when + ``bottleneck`` is installed. Performance may be impacted for computations + involving arrays with dtype other than ``float64``. This change has no impact + for environments that do not have ``bottleneck`` installed. [#17204] + +- Fix an issue in sigma-clipping where the use of ``np.copy()`` was causing + the input data mask to be discarded in cases where ``grow`` was set. [#17402] + +astropy.table +^^^^^^^^^^^^^ + +- Fix a bug where column names would be lost when instantiating ``Table`` from a list of ``Row`` objects. [#15735] + +- Aggregating table groups for ``MaskedColumn`` now ensures that fully-masked + groups result in masked elements rather than ``NaN``. [#16498] + +- Ensure that tables holding coordinates or representations can also be stacked + if they have zero length. This fix also ensures that the ``insert`` method + works correctly with a zero-length table holding a coordinate object. [#17380] + +- Fixed table aggregate with empty columns when float is present. [#17385] + +astropy.units +^^^^^^^^^^^^^ + +- Allow SI-prefixes for radioactivity units ``becquerel`` and ``curie`` in ``astropy.units.si``, conforming to BIPM's guidelines for SI units. [#16529] + +- The OGIP unit parser no longer accepts strings where a component unit is + followed by a parenthesized unit without a separator in between, such as + ``'m(s)'`` or ``'m(s)**2'``. + Such strings are not allowed by the OGIP standard. [#16749] + +- A few edge cases that could result in a power of a unit to be a numerical value + from ``numpy``, instead of the intended Python ``int``, ``float`` or + ``fractions.Fraction`` instance, have been fixed. [#16779] + +- The OGIP unit parser now detects negative powers that are not enclosed in + parenthesis. + For example, ``u.Unit("s**-1", format="ogip")`` now raises an error because the + OGIP standard expects the string to be written as ``"s**(-1)"`` instead, but it + is still possible to parse the unit with + ``u.Unit("s**-1", format="ogip", parse_strict="warn")`` or + ``parse_strict="silent"``. [#16788] + +- ``UnitScaleError`` can now be imported from the ``astropy.units`` namespace. [#16861] + +- Parsing custom units with ``u.Unit()`` using the ``"vounit"`` format now obeys + the ``parse_strict`` parameter, unless the custom units are made explicit with + quotation marks. + For example, ``u.Unit("custom_unit", format="vounit")`` now raises an error, + but ``u.Unit("custom_unit", format="vounit", parse_strict="silent")`` or + ``u.Unit("'custom_unit'", format="vounit")`` do not. [#17232] + +- It is now possible to use ``Unit`` to create dimensionless units with a scale + factor that is a complex number or a ``fractions.Fraction`` instance. + It was already possible to create such units directly with ``CompositeUnit``. [#17355] + +astropy.utils +^^^^^^^^^^^^^ + +- Fixed the unintended behavior where the IERS-A file bundled in ``astropy-iers-data`` would be ignored if automatic updating of the IERS-A file were disabled or if downloading the new file failed. [#16187] + +- Ensure ``MaskedQuantity`` can be initialized with a list of masked + quantities (as long as their shapes match), just like regular + ``Quantity`` and ``ndarray``. [#16503] + +- For ``Masked`` instances, ``np.put``, ``np.putmask``, ``np.place`` and + ``np.copyto`` can now handle putting/copying not just ``np.ma.masked`` but + also ``np.ma.nomask``; for both cases, only the mask of the relevant entries + will be set. [#17014] + +- Explicitly set ``usedforsecurity=False`` when using ``hashlib.md5``. Without this, ``hashlib.md5`` will be blocked in FIPS mode. + FIPS (Federal Information Processing Standards) is a set of standards created by NIST (National Institute of Standards and Technology) for US government agencies regarding computer security and interoperability. + This affects download caching. [#17156] + +- Fixed a bug where an old IERS-A table with stale predictive values could trigger + the download of a new IERS-A table even if automatic downloading was disabled. [#17387] + +astropy.wcs +^^^^^^^^^^^ + +- Avoid a ``RuntimeWarning`` in ``WCS.world_to_array_index`` by converting + NaN inputs to int. [#17236] + + +Performance Improvements +------------------------ + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- The performance of guessing the table format when reading large files with + ``astropy.io.ascii`` has been improved. Now the process uses at most + 10000 lines of the file to check if it matches the format. This behavior can + be configured using the ``astropy.io.ascii.conf.guess_limit_lines`` + configuration item, including disabling the limit entirely. [#16840] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Optimize checksum computation. [#17209] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Improved the performance of 1D models, models with scalar parameters, and models + without units, when evaluating them with scalar or small arrays of inputs. For + models that satisfy all of the conditions above, the improvement can be on the + order of 30-40% in execution time. [#16670] + +- Performance of most non-linear fitters has been significantly improved by reducing the overhead in evaluating models inside the objective function. [#16673] + +- Improved the performance of ``parallel_fit_dask`` by avoiding unnecessary copies of the + model inside the fitter. [#17033] + +- ``CompoundModel`` now implements numerical derivatives of parameters when using the +, -, * or / operators. This improves the speed of fitting these models because numerical derivatives of the parameters are not calculated. [#17034] + +astropy.stats +^^^^^^^^^^^^^ + +- The performance of biweight_location, biweight_scale, + biweight_midvariance, and median_absolute_deviation has been improved by + using the bottleneck nan* functions when available. This requires the + bottleneck optional dependency to be installed. [#16967] + +astropy.units +^^^^^^^^^^^^^ + +- The ``units.quantity_input`` decorator has been optimized, especially in the case that no equivalencies are provided to the decorator, and the speed-up is very noticeable when wrapping short functions. [#16742] + +- Parsing composite units with the OGIP formatter is now up to 25% faster. [#16761] + +- Parsing units with scale factors is now up to 50% faster. [#16813] + +- Parsing strings representing non-composite units with ``Unit`` is now up to 25% + faster. [#17004] + +- Converting composite units to strings with the ``"cds"``, ``"fits"``, + ``"ogip"`` and ``"vounit"`` formatters is now at least twice as fast. [#17043] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Removed redundant transformations when WCSAxes determines the coordinate ranges + for ticks/gridlines, which speeds up typical plot generation by ~10%, and by + much more if ``astropy.visualization.wcsaxes.conf.coordinate_range_samples`` is + set to a large value [#16366] + + +Other Changes and Additions +--------------------------- + +- Updated minimum supported Python version to 3.11. As a result, minimum + requirements were updated to compatible versions. + Astropy now requires + - ``numpy>=1.23.2`` + - ``PyYAML>=6.0.0`` + - ``packaging>=22.0.0`` [#16903] + +- The minimum supported version of Pandas is now v2.0. + This is in line with https://scientific-python.org/specs/spec-0000/. [#16308] + +- Update minimal recommendation for matplotlib from version 3.3.4 to 3.6.0 [#16557] + +- The Contributor documentation has been significantly improved. It now includes a + Quickstart Guide with concise instructions on setting up a development environment and + making a pull request. In addition, the developer documentation was reorganized and + simplified where possible to improve readability and accessibility. [#16561] + +Version 6.1.7 (2024-11-22) +========================== + +Bug Fixes +--------- + +astropy.stats +^^^^^^^^^^^^^ + +- Fix an issue in sigma-clipping where the use of ``np.copy()`` was causing + the input data mask to be discarded in cases where ``grow`` was set. [#17402] + +Version 6.1.6 (2024-11-11) +========================== + +Bug Fixes +--------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Fixed instantiating ``Angle`` from a ``pandas`` ``Series`` object. [#17358] + +astropy.units +^^^^^^^^^^^^^ + +- Fixed calling ``np.nanvar`` and ``np.nanstd`` with ``Quantity`` ``out`` argument. [#17354] + + +Version 6.1.5 (2024-11-07) +========================== + +Bug Fixes +--------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Ensure that coordinates can be transformed to other coordinate frames + also if they have size zero (i.e., hold empty data arrays). [#17013] + +- ``Longitude`` and ``Latitude`` can no longer be initialized with strings + ending in "N" or "S", and "E" or "W", respectively, since those suggest + the other type. [#17132] + +- ``np.nanvar(angle)`` now produces a ``Quantity`` with the correct + unit, rather than raising an exception. [#17239] + +- Fix a crash when instantiating ``Angle`` (or ``Latitude``, or ``Longitude``) + from a non-numpy array (for instance pyarrow arrays). [#17263] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix access to VLA columns after slicing ``.data``. [#16996] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Updated xml writer for VOTable Resource elements to include groups. [#17344] + +astropy.nddata +^^^^^^^^^^^^^^ + +- Add support for positional only and keyword only arguments when using the ``support_nddata`` decorator. [#17281] + +astropy.stats +^^^^^^^^^^^^^ + +- Fixed a bug where float32 inputs to sigma_clip and SigmaClip were + changed to float. [#17086] + +astropy.table +^^^^^^^^^^^^^ + +- Fix a crash when calling ``Column.pprint`` on a scalar column. [#15749] + +- Ensure that setting an existing column to a scalar always properly fills it + (rather than breaking the table if there was only one column in it). [#17105] + +astropy.units +^^^^^^^^^^^^^ + +- The unit parsers are now better at recognizing unusual composite + units: + + - units involving special unicode symbols, like "L☉/pc²"; + - units that include CDS units ending in a 0, like "eps0/s"; + - units including the degree symbol, "°". For example, "°C/s" is no + longer incorrectly interpreted as "°C/s^2". [#17011] + +- Converting the ohm to a string with the OGIP unit formatter (e.g. + ``f"{u.ohm:ogip}"``) previously produced the string ``'V / A'``, but now + produces ``'ohm'`` as expected. [#17200] + +- The ``OGIP`` unit formatter now handles the unit ``day`` and the corresponding + string ``"d"`` in full compliance with the standard. [#17216] + +- The ``"ogip"`` unit format now represents the unit angstrom as ``"angstrom"`` + instead of ``"0.1 nm"``. [#17241] + +astropy.utils +^^^^^^^^^^^^^ + +- Ensure that queries of ``.ut1_utc()`` and ``.pm_xy()`` return the correct + results also when passing in an empty array of times. [#17013] + +- Fixed a bug where astropy's logger wouldn't perform lazy string interpolation. [#17196] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Fixed a bug that caused ``CoordinateHelper.get_axislabel()`` to return an + empty string instead of the default label if no label has been explicitly + provided. [#17175] + +astropy.wcs +^^^^^^^^^^^ + +- Fixed a bug that caused ``WCS.slice`` to ignore ``numpy_order`` and always + interpret the slices as if ``numpy_order`` was ``True``, in the specific case + where the slices were such that dimensions in the WCS would be dropped. [#17147] + +Version 6.1.4 (2024-09-26) +========================== + +Bug Fixes +--------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Keep ``Latitude`` from printing long input arrays in their entirety when failing + limits check in ``_validate_angles``, indicating their range instead. [#13997] + +- Avoid some components not being included in table output of coordinates if + the representation type was ``"unitspherical"``. + + In the process, also ensured that one can pass in the ``radial_velocity`` + keyword argument if one uses ``differential_type="radial"``. [#16999] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Ensure proper handling of null values during BINARY2 serialization. Previously, masks were handled in two different ways for BINARY2 serialization, resulting in incorrect handling of null values and errors. [#16091] + +astropy.stats +^^^^^^^^^^^^^ + +- Fixed a bug in biweight_location, biweight_scale, and + biweight_midvariance where the returned array shape would be wrong if + the input array had an axis length of 1 along any axis that was not + included in the axis keyword. Also fixed a bug in these same functions + where for constant data and axis set to a tuple containing all axes, the + returned value would be NaN instead of the constant value. [#16964] + +astropy.table +^^^^^^^^^^^^^ + +- Ensure that initializing a ``QTable`` with explicit units` also succeeds if + one of the units is ``u.one``. [#17048] + +astropy.units +^^^^^^^^^^^^^ + +- An exception is now raised if it is attempted to create a unit with a + scale of zero, avoiding bugs further downstream (including surprising + ones, such as a comparison of ``np.ma.masked == u.one`` leading to + a ``ZeroDivisionError``). [#17048] + +astropy.wcs +^^^^^^^^^^^ + +- Fix a bug that caused the results from local_partial_pixel_derivative to be incorrect when using normalize_by_world=True (the matrix was previously normalized along the wrong axis) [#17003] + + +Other Changes and Additions +--------------------------- + +- Minimal requirement for (optional dependency) matplotlib was bumped + to 3.5.0, which is the oldest version with support for Python 3.10 [#16993] + +Version 6.1.3 (2024-08-30) +========================== + +Bug Fixes +--------- + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix reading zero-width columns such as 0A fields. [#16894] + +- Ensure that ``QTable``, like ``Table``, can read zero-length string columns, + and not convert them to length 1 strings. In the process, avoid a needless + copy of all the data for ``QTable``. [#16898] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Fix KeyError when parsing certain VOTables. [#16830] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Fixed the ``fit_deriv`` calculations in the ``Lorentz1D`` model. [#16794] + +astropy.table +^^^^^^^^^^^^^ + +- Pretty-printing of Tables now also works in the presence of zero-length string + columns (which sometimes are present in FITS tables). [#16898] + +astropy.utils +^^^^^^^^^^^^^ + +- Fix the return type for ``np.broadcast_arrays`` on a single ``Masked`` + instance: it now correctly returns a 1-element sequence instead of a single + array, just like would be the case with a regular array. [#16842] + +astropy.wcs +^^^^^^^^^^^ + +- Fix a bug where ``wcs_info_str``'s results would look different in numpy 2 VS + numpy 1. [#16586] + + +Other Changes and Additions +--------------------------- + +- The minimum required version of PyArrow is now v7.0.0. [#16785] + +Version 6.1.2 (2024-07-23) +========================== + +Bug Fixes +--------- + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- When reading CDS and MRT files, only interpret a line as a section delimiter if + it contains exclusively dashes or equal signs. This enables rows starting with dashes. [#16735] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix a spurious exception when reading integer compressed images with blanks. [#16550] + +- Fixed a crash that occurred for files opened via + ``fits.open(..., mode='update')``, on Windows, and with numpy 2.0 installed. + A warning is now emitted in cases most likely to escalate into + undefined behavior (e.g., segfaults), i.e., when a closed memory map object is + still referenced by external code. Please report any regression found. [#16581] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Fixed a bug that caused models returned by non-linear fitters to have + ``sync_constraints`` set to `False`, which caused constraints accessed through, e.g., + ``Model.fixed`` to not be in sync with the ``fixed`` attribute of the parameters. [#16664] + +- Fixed a bug that caused ``CompoundModel.without_units_for_data`` to return an + incorrectly constructed model when the compound model contained a * or / + operation, and which also caused fitting to not work correctly with compound + models that contained * or / operations. [#16678] + +astropy.units +^^^^^^^^^^^^^ + +- The OGIP parser is now less restrictive with strings that represent a unit that + includes the ``sqrt`` function. + For example, ``u.Unit("sqrt(m)**3", format="ogip")`` no longer causes a + ``ValueError``. [#16743] + +astropy.utils +^^^^^^^^^^^^^ + +- Fixed an edge-case bug in ``overlap_slices`` where the function could + return an empty slice for non-overlapping slices. [#16544] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Fixed a WCSAxes bug when overlaying a frame with default units that are not degrees. [#16662] + + +Version 6.1.1 (2024-06-14) +========================== + + +Bug Fixes +--------- + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Let fitsdiff compare files with lower case HIERARCH keywords [#16357] + +- Fix writing a ``HDUList`` to file when numpy 2 is installed and at least some of + the data is represented as dask arrays. [#16384] + +- Fix display of diff reports with numpy 2. [#16426] + +- Ensure that also zero-length tables preserve whether integer data are + signed or unsigned. [#16505] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Fix YAML table serialization compatibility with numpy 2. [#16416] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Fix bugs in io.votable related to numpy 2's representation of scalars. [#16442] + +astropy.stats +^^^^^^^^^^^^^ + +- Ensure that return types from ``sigma_clip`` ``cenfunc`` and ``stdfunc`` + are np.float64 for scalar values. [#16431] + +astropy.table +^^^^^^^^^^^^^ + +- Ensure structured ``MaskedColumn`` are serialized correctly, including + the mask. [#16380] + +- Fix problems converting Pandas Series to ``Table`` with numpy >=2.0. [#16439] + +astropy.time +^^^^^^^^^^^^ + +- Ensure Time in ymdhms format can also be serialized to files as part of a + table if it is masked. [#16380] + +astropy.utils +^^^^^^^^^^^^^ + +- Ensure Masked versions of ``np.recarray`` will show the correct class + name of ``MaskedRecarray`` in their ``repr``, and that they will be + serialized correctly if part of a table. [#16380] + +- Fix bugs with how masked structured arrays were represented with numpy 2. [#16443] + +- ``MaskedQuantity`` now works properly with ``np.block``. [#16499] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Fix a bug where ``WCSAxes`` could be missing negative signs on axis labels when using matplotlib's ``usetex`` mode. [#16406] + +astropy.wcs +^^^^^^^^^^^ + +- Fix compilation with gcc 14, avoid implicit pointer conversions. [#16450] + + +Other Changes and Additions +--------------------------- + +- Updated bundled WCSLIB version to 8.3. This update changes the behavior of + various ``*set`` functions in order to improve stability of WCSLIB in threaded + applications. For a full list of changes - see ``astropy/cextern/wcslib/CHANGES``. [#16451] + +Version 6.1.0 (2024-05-03) +========================== + +New Features +------------ + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- ``BaseCoordinateFrame`` now has a ``position_angle()`` method, which is the + same as the ``position_angle`` method of ``SkyCoord`` instances. [#15737] + +- By default the ``SkyCoord`` and ``BaseCoordinateFrame`` ``separation()`` + methods now emit a warning if they have to perform a coordinate transformation + that is not a pure rotation to inform the user that the angular separation can + depend on the direction of the transformation. + It is possible to modify this behaviour with the new optional keyword-only + ``origin_mismatch`` argument. + Specifying ``origin_mismatch="ignore"`` allows any transformation to + succeed without warning, which has been the behaviour so far. + ``origin_mismatch="error"`` forbids all transformations that are not + pure rotations. [#16246] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Clearer error message in reading ASCII tables when there is + a mismatch between converter type and column type. [#15991] + +astropy.io.registry +^^^^^^^^^^^^^^^^^^^ + +- The module ``astropy.io.typing`` has been added to provide type annotations for + I/O-related functionality. [#15916] + +astropy.samp +^^^^^^^^^^^^ + +- SAMP web profile CORS HTTP server implements `Private Network Access proposal `_. [#16193] + +astropy.table +^^^^^^^^^^^^^ + +- ``Table`` now has a ``setdefault()`` method, analogous to + ``dict.setdefault()``. [#16188] + +astropy.units +^^^^^^^^^^^^^ + +- Added a new module ``astropy.units.typing`` that provides support for type annotations related to + ``astropy.units``. [#15860] + +- Added a new CGS unit Oersted. [#15962] + +- Added "surface brightness", "surface brightness wav", "photon surface brightness", and "photon surface brightness wav" to recognized physical types. [#16032] + +- Added magnetic helicity as a physical type. [#16101] + +astropy.utils +^^^^^^^^^^^^^ + +- For gufuncs on ``Masked`` instances, add support for the ``axes`` argument. [#16121] + +- ``Masked`` instances now support the various numpy array set operations, such + as ``np.unique`` and ``np.isin``. [#16224] + +astropy.wcs +^^^^^^^^^^^ + +- Added support for slicing WCS objects containing ``cpdis`` or ``det2im`` distortions, which previously were ignored. [#16163] + + +API Changes +----------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- The ``astropy.coordinates.transformations`` module has been refactored into a module. + There should be no user-visible changes, but if you notice any, please open an + Issue. [#15895] + +- Changed the default value of the ``copy`` argument in + ``astropy.coordinates.representation.CylindricalDifferential.__init__`` from + ``False`` to ``True``, which is the intended behaviour for all subclasses of + ``astropy.coordinates.representation.BaseDifferential``. [#16198] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- ``Cosmology`` and its subclasses are now frozen ``dataclass`` objects. [#15484] + +- The argument ``verbose`` in the function ``z_at_value`` is now keyword-only. [#15855] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- The ``io.ascii`` Python and C table readers were updated to use a 64-bit integer field by + default when reading a column of integer numeric data. This changes the default behavior + on Windows and potentially 32-bit architectures. Previously on those platforms, table + columns with any long integers which overflowed the 32-bit integer would be returned + as string columns. The new default behavior is consistent with ``numpy`` v2 and ``pandas``. [#16005] + +- The parallel fast-reader parser for reading ASCII files has been removed. + Since astropy v4.0.4 requesting this option has issued a warning that + this option is broken and that the serial parser will be used. + The ``parallel`` key in the ``fast_reader`` argument for reading + ASCII tables is no longer available. [#16103] + +astropy.table +^^^^^^^^^^^^^ + +- ``show_in_notebook`` is deprecated and it is recommended to use dedicated + tools in the Jupyter ecosystem to create interactive plots in notebooks. [#15905] + +- A warning is now emitted when ``Quantity`` values are inserted into empty ``Column`` objects + via ``Table.insert_row`` or ``Table.add_row``. [#16038] + +- ``show_in_browser`` is deprecated (pending feedback from the community). + Please use https://github.com/astropy/astropy/issues/16067 if you are + actively using the function. [#16068] + +- ``TableColumns.setdefault()`` and ``TableColumns.update()`` methods (which + would typically be called as ``Table.columns.setdefault()`` and + ``Table.columns.update()``) have been deprecated because they can easily + corrupt the ``Table`` instance the ``TableColumns`` instance is attached to. + The ``Table.setdefault()`` and ``Table.update()`` methods are safe. [#16154] + +astropy.time +^^^^^^^^^^^^ + +- ``TIME_FORMATS`` and ``TIME_DELTA_FORMATS`` in ``astropy.time.formats`` + are changed from ``OrderedDict`` to Python ``dict``. [#15491] + +- A ``FutureWarning`` is now emitted when mutating ``Time.location`` post-initialization. [#16063] + +- Following the removal of ``np.ndarray.ptp`` in Numpy v2, ``Time.ptp`` is now + deprecated in favor of ``np.ptp``. [#16212] + +astropy.units +^^^^^^^^^^^^^ + +- If any iterable such as a list of tuple was input to ``Quantity``, a check was + done to see if they contained only quantities, and, if so, the quantities were + concatenated. This makes sense for list and tuple, but is not necessarily + logical for all iterables and indeed was broken for those that do not have a + length (such as ``array_api`` array instances). Hence, the check will now be + done only for values where it makes sense, i.e., instances of list and tuple. [#15752] + +- Units now exposes ``get_converter`` which returns a function that + will convert a scalar or array from one unit to another. This can be + useful to speed up code that converts many quantities with the same + unit to another one, especially if the quantity has not many elements, + so that the overhead of creating a conversion function is relatively large. [#16139] + +astropy.utils +^^^^^^^^^^^^^ + +- Deprecate importing ``ErfaError`` and ``ErfaWarning`` from ``astropy.utils.exceptions``. + They should be imported directly from ``erfa`` instead. [#15777] + +- ``introspection.isinstancemethod()`` and ``introspection.find_mod_objs()`` are + deprecated. [#15934] + +- ``astropy.utils.console.terminal_size`` is now deprecated in favour of + ``shutil.get_terminal_size`` from the standard library. [#16045] + +- ``indent()`` is deprecated. + Use ``textwrap.indent()`` from Python standard library instead. [#16223] + +- Unmasked ``Masked`` scalar instances are now considered hashable, to match the + implicit behaviour of regular arrays, where if an operation leads to a scalar, + a hashable array scalar is returned. [#16224] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Renamed the ``min_cut`` and ``max_cut`` keywords in ``simple_norm`` and + ``fits2bitmap`` to ``vmin`` and ``vmax``. The old names are deprecated. [#15621] + +- If ``vmin == vmax``, the ``ImageNormalize`` class now maps the input + data to 0. If ``vmin > vmax``, the ``ImageNormalize`` class now raises a + ``ValueError``. [#15622] + + +Bug Fixes +--------- + +astropy.convolution +^^^^^^^^^^^^^^^^^^^ + +- Avoid a segfault when calling ``astropy.convolution.convolve`` on an empty array. + An exception is now raised instead. [#15840] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Previously passing a ``SkyCoord`` instance to the ``BaseCoordinateFrame`` + ``separation()`` or ``separation_3d()`` methods could produce wrong results, + depending on what additional frame attributes were defined on the ``SkyCoord``, + but now ``SkyCoord`` input can be used safely. [#15659] + +- ``Distance`` now accepts as ``parallax`` any angle-like value. + This includes types like ``Column`` which have a unit but are not ``Quantity`` subclasses. [#15712] + +- The new default for the class method ``SkyCoord.from_name()`` + is to look for coordinates first in SIMBAD, then in NED, and then in VizieR, + instead of having no specific order. [#16046] + +- Fix ``Angle.to_string()`` for angles in degrees represented in 'hms' and angles in hours represented in 'dms'. [#16085] + +- Fix a bug where ``SkyCoord.spherical_offsets_by`` would crash when a wrap + was needed. [#16241] + +- ``search_around_3d()`` now always raises a ``UnitConversionError`` if the units + of the distances in ``coord1`` and ``coord2`` and the unit of ``distlimit`` do + not agree. + Previously the error was not raised if at least one of the coordinates was + empty. [#16280] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- Fixed a bug where the attribute ``ParametersAttribute.attr_name`` could be None + instead of a string. [#15882] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Reading of CDS header files with multi-line descriptions where the continued line started with a number was broken. This is now fixed. [#15617] + +- Ensure that the names of mixin columns are properly propagated as + labels for the MRT format. [#15848] + +- Fixed reading IPAC tables for ``long`` column type on some platforms, e.g., Windows. [#16005] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Avoid ``WinError 1455`` in opening some large files with memory + mapping on windows. [#15388] + +- Fix TDISP parsing for floating numbers. [#16007] + +- Fix a crash when calling FITS ``writeto`` methods with stdout as the output stream. [#16008] + +- Fix TDISP parsing for floating numbers in formats ES / EN. [#16015] + +- Fix conversion of ``Table`` to ``BinTableHDU`` with ``character_as_bytes=True``. [#16358] + +- Improved error message when instantiating a fits table with an ill-formed array. [#16363] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Reading an empty table stored in parquet format now creates an empty + table instead of raising an unexpected error. [#16237] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- When reading a VOTable, if some user-requested columns were not present then the + resulting error message previously listed all the requested column names. + Now only columns that are actually missing are shown. [#15956] + +astropy.stats +^^^^^^^^^^^^^ + +- Fix a spurious warning when calling ``sigma_clipped_stats`` on a ``MaskedColumn``. [#15844] + +astropy.table +^^^^^^^^^^^^^ + +- Fix a Table bug when setting items (via slice or index list) in a ``bytes`` type + ``MaskedColumn`` would cause the column mask to be set to all ``False``. A common way to + trigger this bug was reading a FITS file with masked string data and then sorting the + table. [#15669] + +- Fix slicing logic for Row. + Previously, slicing a ``astropy.table.row.Row`` object would incorrectly return a column, + now it correctly returns a list of values from that row. [#15733] + +- Fix a ``ValueError`` raised by ``table.join`` when fed with large tables. + This would typically happen in situations when the result joined table would be + too large to fit in memory. In those situations, the error message is now much more + clearly about the necessary memory size. [#15734] + +- Fix an unintended exception being raised when attempting to compare two unequal ``Table`` instances. [#15845] + +- Ensure that if a ``Column`` is initialized with a ``Quantity`` it will use by + default a possible name defined on the quantity's ``.info``. [#15848] + +- Fix a bug where columns with ``dtype=object`` wouldn't be properly deep-copied using ``copy.deepcopy``. [#15871] + +- Fix ``hasattr(Table, "iloc")`` raising an exception, preventing use of tables e.g. with scikit-learn. [#15913] + +- Calling ``Table.group_by`` on an empty table no longer raises an exception. [#16093] + +- The unit conversion ``convert_unit_to`` with MaskedColumn was + broken as it was storing the old unit in a dictionary attached + to underlying np.ma.MaskedArray. This fixes it by overwriting + the old unit after unit conversion. [#16118] + +- ``astropy.table.vstack`` will no longer modify the input list even when it + contains non-Table objects like ``astropy.table.Row``. [#16130] + +- Update old dataTables.js version. + This should not affect the end user. [#16315] + +astropy.time +^^^^^^^^^^^^ + +- Fix comparing NaN ``Quantity`` with ``TimeDelta`` object. [#15830] + +- Scalar ``Time`` instances are now hashable if they are not masked, also if one + uses ``Masked`` internally, matching the behaviour prior to astropy 6.0 (and + the current behaviour when masking using ``np.ma.MaskedArray``). [#16224] + +astropy.units +^^^^^^^^^^^^^ + +- Fix rare signature incompatibilities between helper and helped array functions. + Most involve cases where the corresponding numpy function has had its + arguments renamed between numpy versions. Since all those generally changed + the first arguments, which are typically passed as positional arguments, + this should not affect user code. + Affected functions: + - ``numpy.array_str`` + - ``numpy.choose`` + - ``numpy.convolve`` + - ``numpy.correlate`` + - ``numpy.histogram`` + - ``numpy.histogramdd`` + - ``numpy.histogram2d`` + - ``numpy.isin`` + - ``numpy.inner`` + - ``numpy.nanmedian`` + - ``numpy.unique`` + - ``numpy.matrix_rank`` + - ``numpy.unwrap`` + - ``numpy.vdot`` + - ``numpy.lib.recfunctions.unstructured_to_structured`` [#15710] + +- Fix an issue with unicode string representations of units shown as + superscripts (like degree) when raised to some power. Like for + LaTeX representations, now the superscript unicode character is + replaced by the literal short name before adding the power. [#15755] + +- Fix a missing ``Sun`` unit in the list of VOUnits simple_units. [#15832] + +- Fix an unhelpful ``TypeError`` when attempting truediv, ``lshift`` (``<<``) or ``mul`` (``*``) or ``truediv`` (``/``) with a ``Unit`` for right operand and a numpy array with non-numerical dtype for left operand. [#15883] + +- Fix write/read roundtrips with empty ``Table`` dumped to ECSV. [#15885] + +- Fix a bug where LaTeX formatter would return empty strings for unity (1) input. [#15923] + +- Fix extraneous space in LaTeX repr for ``Quantity`` objects with superscript + units (e.g. angles or temperatures in degree Celsius). [#16043] + +- Ensure powers of units are consistently as simple as possible. So, an + integer if possible, otherwise a float, or a fraction if the float is + really close to that. This also ensures the hash of a unit is unique + for any given unit (previously, the same power could be represented as + float, int or fraction, which made the hash different). [#16058] + +- Ensure that ``find_equivalent_units`` only returns actual units, not units + that raised to some power match the requested one. With this fix, + ``(u.m**-3).find_equivalent_units()`` properly finds nothing, rather than all + units of length. [#16127] + +- Using a dimensionless ``Quantity`` as an exponent works anew. + In astropy 6.0.1 an exception was erroneously raised. [#16261] + +astropy.utils +^^^^^^^^^^^^^ + +- Fix rare signature incompatibilities between helper and helped array functions. + These typically cover corner cases and should not affect user code. + Some arguments weren't being re-exposed correctly or at all, depending on + numpy's version. + Affected functions: + - ``numpy.broadcast_arrays`` + - ``numpy.median`` + - ``numpy.quantile`` + - ``numpy.empty_like`` + - ``numpy.ones_like`` + - ``numpy.zeros_like`` + - ``numpy.full_like`` [#16025] + +- Fix a bug where ``astropy.utils.console.Spinner`` would leak newlines for + messages longer than terminal width. [#16040] + +- Update ``report_diff_values`` so the diff no longer depends on the + console terminal size. [#16065] + +- Fix support in ``Masked`` for generalized ufuncs with more than a + single core dimension (such as ``erfa.rxp``). [#16120] + +- ``Masked`` array instances now deal more properly with structured dtypes, + combining field masks to get element masks for generalized ufuncs, and + allowing ``.view()`` any time the mask can be viewed as well. This allows a + larger number of ``erfa`` routines to work with masked data. [#16125] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- ``WCSAxes`` will correctly set certain defaults when ``wcs.world_axis_physical_types`` contains ``custom:`` prefixes. [#15626] + +- Fix an edge case where ``quantity_support`` would produce duplicate tick labels for small data ranges. [#15841] + +- Fix a bug where ``AngleFormatterLocator`` and ``ScalarFormatterLocator`` wouldn't respect matplotlib.rc's ``axes.unicode_minus`` parameter. [#15902] + +- Fixed a bug in ``CoordinateHelper.grid`` method to properly handle ``draw_grid=False`` and ``draw_grid=None``, + ensuring grid lines are controlled correctly even when not explicitly called. [#15985] + +astropy.wcs +^^^^^^^^^^^ + +- Updated bundled WCSLIB version to 8.2.2. This update fixes character buffer + overflows in the comment string for the longitude and latitude axes triggered + by some projections in ``wcshdo()``, and also the formatting for generic + coordinate systems. For a full list of changes - see + http://www.atnf.csiro.au/people/mcalabre/WCS/CHANGES or + ``astropy/cextern/wcslib/CHANGES`` [#15795] + +- Fixed a bug in ``fit_wcs_from_points`` that does not set the default value of the ``cdelt`` of the returned WCS object. [#16027] + +- Fixed a bug in ``DistortionLookupTable`` (which implements ``cpdis`` and ``det2im`` projection corrections to a WCS) in which image pixels received an incorrect distortion value, from a location in the lookup table incorrectly offset by about 1 table pixel. [#16163] + + +Other Changes and Additions +--------------------------- + +- Update minimum supported Python version to 3.10 [#15603] + +- The minimum required NumPy version is now 1.23 and the minimum required SciPy version is 1.8. [#15706] + +- Fix loading parser tabs on pyc-only installations. + + Fix a bug in the wrappers for the lex and yacc wrappers that are + used for parsing Astropy units so that they work on pyc-only + installations. + + According to the Python module loading + `flow chart `_, when evaluating + ``import foo`` and ``foo.py`` is not found, Python then reads ``foo.pyc``. + + One can take advantage of this fact to strip source files and leave only Python + bytecode files for deployment inspace-constrained execution environments such + as AWS Lambda. Astropy is now compatible with pyc-only deployments. [#16159] + +- Change the default value of ``copy`` arguments in public APIs from ``False`` to + ``None`` if Numpy 2.0 or newer is installed. + For details, see the "Copy semantics" section on the What's New page for Astropy 6.1 . [#16181] + +- astropy is now compiled against NumPy 2.0, enabling runtime compatibility + with this new major release. Compatibility with NumPy 1.23 and newer + versions of NumPy 1.x is preserved through this change. [#16252] + +Version 6.0.1 (2024-03-25) +========================== + +Bug Fixes +--------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Previously passing a ``SkyCoord`` instance to the ``BaseCoordinateFrame`` + ``separation()`` or ``separation_3d()`` methods could produce wrong results, + depending on what additional frame attributes were defined on the ``SkyCoord``, + but now ``SkyCoord`` input can be used safely. [#15659] + +- ``Distance`` now accepts as ``parallax`` any angle-like value. + This includes types like ``Column`` which have a unit but are not ``Quantity`` subclasses. [#15712] + +- The new default for the class method ``SkyCoord.from_name()`` + is to look for coordinates first in SIMBAD, then in NED, and then in VizieR, + instead of having no specific order. [#16046] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Reading of CDS header files with multi-line descriptions where the continued line started with a number was broken. This is now fixed. [#15617] + +- Ensure that the names of mixin columns are properly propagated as + labels for the MRT format. [#15848] + +- Fixed reading IPAC tables for ``long`` column type on some platforms, e.g., Windows. [#15992] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix TDISP parsing for floating numbers. [#16007] + +- Fix a crash when calling FITS ``writeto`` methods with stdout as the output stream. [#16008] + +- Fix TDISP parsing for floating numbers in formats ES / EN. [#16015] + +astropy.stats +^^^^^^^^^^^^^ + +- Fix a spurious warning when calling ``sigma_clipped_stats`` on a ``MaskedColumn``. [#15844] + +astropy.table +^^^^^^^^^^^^^ + +- Fix a Table bug when setting items (via slice or index list) in a ``bytes`` type + ``MaskedColumn`` would cause the column mask to be set to all ``False``. A common way to + trigger this bug was reading a FITS file with masked string data and then sorting the + table. [#15669] + +- Fix slicing logic for Row. + Previously, slicing a ``astropy.table.row.Row`` object would incorrectly return a column, + now it correctly returns a list of values from that row. [#15733] + +- Fix a ``ValueError`` raised by ``table.join`` when fed with large tables. + This would typically happen in situations when the result joined table would be + too large to fit in memory. In those situations, the error message is now much more + clearly about the necessary memory size. [#15734] + +- Fix an unintended exception being raised when attempting to compare two unequal ``Table`` instances. [#15845] + +- Ensure that if a ``Column`` is initialized with a ``Quantity`` it will use by + default a possible name defined on the quantity's ``.info``. [#15848] + +- The unit conversion ``convert_unit_to`` with MaskedColumn was + broken as it was storing the old unit in a dictionary attached + to underlying np.ma.MaskedArray. This fixes it by overwriting + the old unit after unit conversion. [#16118] + +- ``astropy.table.vstack`` will no longer modify the input list even when it + contains non-Table objects like ``astropy.table.Row``. [#16130] + +astropy.units +^^^^^^^^^^^^^ + +- Fix an issue with unicode string representations of units shown as + superscripts (like degree) when raised to some power. Like for + LaTeX representations, now the superscript unicode character is + replaced by the literal short name before adding the power. [#15755] + +- Fix a missing ``Sun`` unit in the list of VOUnits simple_units. [#15832] + +- Fix write/read roundtrips with empty ``Table`` dumped to ECSV. [#15885] + +- Fix a bug where LaTeX formatter would return empty strings for unity (1) input. [#15923] + +- Ensure powers of units are consistently as simple as possible. So, an + integer if possible, otherwise a float, or a fraction if the float is + really close to that. This also ensures the hash of a unit is unique + for any given unit (previously, the same power could be represented as + float, int or fraction, which made the hash different). [#16058] + +- Ensure that ``find_equivalent_units`` only returns actual units, not units + that raised to some power match the requested one. With this fix, + ``(u.m**-3).find_equivalent_units()`` properly finds nothing, rather than all + units of length. [#16127] + +astropy.utils +^^^^^^^^^^^^^ + +- Fix a bug where ``astropy.utils.console.Spinner`` would leak newlines for + messages longer than terminal width. [#16040] + +- Update ``report_diff_values`` so the diff no longer depends on the + console terminal size. [#16065] + +- Fix support in ``Masked`` for generalized ufuncs with more than a + single core dimension (such as ``erfa.rxp``). [#16120] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Fix an edge case where ``quantity_support`` would produce duplicate tick labels for small data ranges. [#15841] + +astropy.wcs +^^^^^^^^^^^ + +- Updated bundled WCSLIB version to 8.2.2. This update fixes character buffer + overflows in the comment string for the longitude and latitude axes triggered + by some projections in ``wcshdo()``, and also the formatting for generic + coordinate systems. For a full list of changes - see + http://www.atnf.csiro.au/people/mcalabre/WCS/CHANGES or + ``astropy/cextern/wcslib/CHANGES`` [#15795] + +- Fixed a bug in ``fit_wcs_from_points`` that does not set the default value of the ``cdelt`` of the returned WCS object. [#16027] + +Other Changes and Additions +--------------------------- + +- Given the potential breaking changes with the upcoming Numpy 2.0 release, + this release pins Numpy<2.0 and support for Numpy 2.0 will be added in the + v6.1.0 release. + +Version 6.0.0 (2023-11-25) +========================== + +New Features +------------ + +astropy.config +^^^^^^^^^^^^^^ + +- The new ``ConfigNamespace.help()`` method provides a convenient way to get + information about configuration items. [#13499] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Support has been added to create geodetic representations not just for existing ellipsoids + from ERFA, but also with explicitly provided values, by defining a subclass of + ``BaseGeodeticRepresentation`` with the equatorial radius and flattening assigned to + ``_equatorial_radius`` and ``_flattening`` attributes. [#14763] + +- Add ``BaseBodycentricRepresentation``, a new spheroidal representation for bodycentric + latitudes and longitudes. [#14851] + +- Support Numpy broadcasting over frame data and attributes. [#15121] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- Registered a ``latex`` writer for exporting a Cosmology object to a LaTex table. [#14701] + +- Added argument ``rename`` to Cosmology's I/O, allowing for input and output symbols to + be renamed. [#14780] + +- All non-abstract Cosmology subclasses are now automatically registered to work with + Astropy's YAML serialization. [#14979] + +- Cosmology I/O now auto-identifies the '.tex' suffix with the 'ascii.latex' format. [#15088] + +- The ``Cosmology`` class now has a new property to access the parameters of the + cosmology: ``.parameters``. This property return a read-only dictionary of all the + non-derived parameter values on the cosmology object. When accessed from the class (not + an instance) the dictionary contains ``Parameter`` instances, not the values. [#15168] + +- The field ``default`` has been added to ``Parameter``. This can be used to introspect + the default value of a parameter on a cosmology class e.g. ``LambdaCDM.H0.default``. [#15400] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Add new option ``decompress_in_memory`` to ``fits.open``, to decompress the + whole file in memory at once, instead of decompressing the file progressively + as data is needed. Default behavior is better for memory usage but sometimes + slow, especially for files with many small HDUs. [#15501] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Add support for Parquet serialization of VOTables. Writing of this + serialization is available with using the new ``'votable.parquet'`` format. [#15281] + +- Added MIVOT feature through the ``MivotBlock`` class + that allows model annotations reading and writing in VOTable. [#15390] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Added a ``GeneralSersic2D`` model that can have "boxy" or "disky" + isophotes. [#15545] + +astropy.nddata +^^^^^^^^^^^^^^ + +- A more flexible and/or compact string representation is now available for + ``NDData`` objects which visually indicates masked entries, and provides for + better for dask array support. [#14438] + +astropy.table +^^^^^^^^^^^^^ + +- The new ``Row.get()`` method, analogous to ``dict.get()``, returns the value of + the specified column from the row if the column present, otherwise it returns a + fallback value, which by default is ``None``. [#14878] + +astropy.time +^^^^^^^^^^^^ + +- Masked ``Time`` instances now use astropy's own ``Masked`` class internally. + This means that ``Masked`` input is now properly recognized, and that masks + get propagated also to ``Quantity`` output (such as from a ``TimeDelta`` + converted to a unit of time), creating ``MaskedQuantity`` instances. [#15231] + +- Added a ``TimeDelta`` format ``quantity_str`` that represents the time delta as a string + with one or more ``Quantity`` components. This format provides a human-readable + multi-scale string representation of a time delta. The default output sub-format is not + considered stable in this release, please see https://github.com/astropy/astropy/issues/15485 + for more information. [#15264] + +astropy.uncertainty +^^^^^^^^^^^^^^^^^^^ + +- Uncertainty ``Distribution`` now support structured data types, and as + a result it now works also with ``EarthLocation``. [#15304] + +- Uncertainty ``Distribution`` can now be used inside representations, which + also allows basic support in ``SkyCoord``. While most calculations work, there + are remaining issues. For instance, the ``repr`` does not show that the + coordinates are distributions. [#15395] + +astropy.units +^^^^^^^^^^^^^ + +- Add support for gc2gde and gd2gce erfa functions to allow geodetic representations + using equatorial radius and flattening. [#14729] + +astropy.utils +^^^^^^^^^^^^^ + +- The ``astropy.utils.metadata.MetaData`` default dictionary can now be + set with the ``default_factory`` keyword argument. [#15265] + +- ``astropy.utils.decorators.deprecated`` now adds the ``__deprecated__`` attribute to + the objects it wraps, following the practice in https://peps.python.org/pep-0702/. [#15310] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Add ``WCSAxes.text_coord`` method to print text using ``SkyCoord`` objects + parallel to plotting data points with ``WCSAxes.plot_coord``. [#14661] + +astropy.wcs +^^^^^^^^^^^ + +- Support WCS descriptions of basic planetary coordinate frames. [#14820] + +- Updated bundled WCSLIB version to 8.1. This update adds support planetary keywords ``A_RADIUS``, ``B_RADIUS``, ``C_RADIUS``, ``BLON_OBS``, ``BLAT_OBS``, and ``BDIS_OBS`` in ``auxprm`` and adds ``wcsprm::time`` to the ``wcsprm`` struct to record the ``TIME`` axis. This update also includes several bug fixes. For a full list of changes - see http://www.atnf.csiro.au/people/mcalabre/WCS/CHANGES [#15035] + + +API Changes +----------- + +astropy.config +^^^^^^^^^^^^^^ + +- Removed deprecated ``ConfigurationMissingWarning`` class and ``update_default_config`` function; + There are no replacements as they should no be used anymore. [#15466] + +astropy.convolution +^^^^^^^^^^^^^^^^^^^ + +- Invalid kernel arithmetic operations now raise a ``KernelArithmeticError`` instead of a + bare ``Exception``. [#14728] + +- Added base ``KernelError`` error class and removed ``DiscretizationError`` error class (a ``ValueError`` will be raised instead). [#14732] + +- ``discretize_model`` will now raise a ``ValueError`` if + ``mode='oversample'`` and ``factor`` does not have an integer value. [#14794] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Removed deprecated angle parsing and formatting utilities from ``angle_utilities``. + Use the functions from ``angle_formats`` instead. [#14675] + +- The deprecated functionality of initializing ``Angle`` or ``Longitude`` from a + ``tuple`` is no longer supported. [#15205] + +- Angle-related classes and functions have been moved within ``astropy.coordinates``. + There is no change to public API as everything moved should still be imported from + ``astropy.coordinates``, not a sub-module. If you are using private API, try importing + from ``astropy.coordinates`` instead. If you need something that has been moved and is + not available in ``astropy.coordinates``, please open an issue on the Astropy issue + tracker. [#15220] + +- It is no longer possible to pass frame classes to the ``transform_to()`` method + of a low-level coordinate-frame class. It is still possible to pass frame + instances. The ``transform_to()`` method of the high-level ``SkyCoord`` class + is unaffected. [#15500] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- Removed support of importing private constants and functions from ``astropy.cosmology.flrw``. [#14672] + +- Removed deprecated Cosmology Parameter argument ``fmt``. [#14673] + +- Removed deprecated ``vectorize_if_needed`` and ``inf_like`` from ``cosmology.utils``. [#14677] + +- Removed deprecated import paths from ``astropy.cosmology.core``. [#14782] + +- Cosmology ``Parameter`` is now a ``dataclass``, and can work with all of Python's dataclasses + machinery, like field introspection and type conversion. [#14874] + +- A new property -- ``scale_factor0`` -- has been added to Cosmology objects. + This is the scale factor at redshift 0, and is defined to be 1.0. [#14931] + +- Added registration label ``ascii.latex`` to Cosmology IO. [#14938] + +- The private module ``astropy.cosmology.utils`` has been deprecated. [#14980] + +- Removed deprecated ``get_cosmology_from_string`` class method in ``default_cosmology``; use ``get`` instead. [#15467] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Several arguments in functions within ``astropy.io.ascii`` have been deprecated and + are either renamed or scheduled to be removed. + + ``read()``: + - ``Reader`` will be removed. Instead supply the equivalent ``format`` argument. + - ``Inputter`` has been renamed to ``inputter_cls``. + - ``Outputter`` has been renamed to ``outputter_cls``. + + ``get_reader()``: + - ``Reader`` has been renamed to ``reader_cls``. + - ``Inputter`` has been renamed to ``inputter_cls``. + - ``Outputter`` has been renamed to ``outputter_cls``. + + ``write()``: + - ``Writer`` will be removed. Instead supply the equivalent ``format`` argument. + + ``get_writer()``: + - ``Writer`` has been renamed to ``writer_cls``. [#14914] + +- Removed deprecated ``astropy.io.ascii.tests.common.raises`` test helper; use ``pytest.raises`` instead. [#15470] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Deprecate ``_ExtensionHDU`` and ``_NonstandardExtHDU`` (use ``ExtensionHDU`` or + ``NonstandardExtHDU`` instead). [#15396] + +- Remove special handling of TCTYP TCUNI TCRPX TCRVL TCDLT TRPOS (#7157). [#15396] + +- Rename and deprecate ``TableHDU.update`` to ``TableHDU.update_header``, for + consistency with ``ImageHDU``. [#15396] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Removed deprecated ``astropy.io.misc.asdf`` subpackage. Use ``asdf-astropy`` package instead. [#14668] + +- ``fnunpickle`` and ``fnpickle`` are deprecated because they are not used anywhere within ``astropy``. + If you must, use the module from Python standard library but be advised that pickle is insecure + so you should only unpickle data that you trust. [#15418] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Removed deprecated ``pedantic`` option from the + ``astropy.io.votable.table.parse()`` function and the corresponding configuration + setting. Use the ``verify`` option instead. [#14669] + +- Class ``astropy.io.votable.tree.Table`` has been renamed to ``TableElement`` + to avoid sharing the name with ``astropy.table.Table``. [#15372] + +- Fully removed support for version = '1.0' on ``VOTableFile__init__()`` and changed its tests to check correctly. + It was raising a ``DeprecationWarning`` and now is raising a ``ValueError``. [#15490] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Removed the ``AliasDict`` class from ``modeling.utils``. [#12943] + +- Creating a model instance with parameters that have incompatible shapes will + now raise a ``ValueError`` rather than an ``IncompatibleShapeError``. [#15209] + +- Removal of deprecated code ``_model_to_fit_params`` and ``_fitter_to_model_params`` from ``fitting.py``. [#15461] + +astropy.stats +^^^^^^^^^^^^^ + +- The ``BoxLeastSquares``, ``BoxLeastSquaresResults`` and ``LombScargle`` classes + are not available from ``astropy.stats`` anymore, they are now available only + from ``astropy.timeseries``. [#15530] + +astropy.tests +^^^^^^^^^^^^^ + +- Removed deprecated deprecation, warning, and exception handling functionality provided by ``astropy.tests.helper``. [#14670] + +- ``astropy.tests.command.FixRemoteDataOption`` and ``astropy.tests.command.AstropyTest`` are deprecated. + They are no longer necessary after sunsetting ``astropy-helpers``. [#15204] + +astropy.time +^^^^^^^^^^^^ + +- ``Time`` has switched to use ``Masked`` arrays internally, instead of + indicating masked values using NaN in the internal ``jd2`` attribute. As a + result, any output from instances, such as one gets with, say, the ``.isot`` + format, will also use ``Masked`` by default. + + For backwards compatibility, a new configuration item, + ``astropy.time.conf.masked_array_type`` is introduced which is set to + "astropy" by default (which indicates one wants to use ``Masked``), but can + also be set to "numpy", in which case ``numpy.ma.MaskedArray`` will be used + where possible (essentially, for all but ``Quantity``). [#15231] + +- Changed the ``TimeDelta`` init signature to be consistent with that of ``Time``. + Previously the argument order was ``val, val2, format, scale, copy``. Now the order is + ``val, val2, format, scale, *, precision, in_subfmt, out_subfmt, copy``, where the + arguments after the ``*`` must be specified by keyword. [#15264] + +astropy.timeseries +^^^^^^^^^^^^^^^^^^ + +- Removed deprecated ``midpoint_epoch`` in ``fold`` function; use ``epoch_time`` instead. [#15462] + +astropy.uncertainty +^^^^^^^^^^^^^^^^^^^ + +- The ``.dtype`` attribute exposed by ``Distribution`` is now that of + the samples, rather than one that has a "samples" entry. This makes + quantities with structured data types and units easier to support, and + generally makes the ``Distribution`` appear more similar to regular + arrays. It should have little effect on code. For instance, + ``distribution["samples"]`` still will return the actual distribution. + + As a consequence of this refactoring, most arrays that are not + C-contiguous can now be viewed and will thus not be copied on input + any more. The only exceptions are arrays for which the strides are + negative. + + Note that the true data type is considered an implementation detail. + But for reference, it now is a structured data type with a single + field, "samples", which itself is an array of "sample" fields, which + contain the actual data. [#15304] + +astropy.units +^^^^^^^^^^^^^ + +- Like ``np.ndarray``, under numpy 2.0 ``Quantity`` and all its subclasses + (``Angle``, ``Masked``, etc.) will no longer support the ``.ptp()`` method. + Use ``np.ptp(...)`` instead. + + Similarly, support for the much less frequently used ``.newbyteorder()`` and + ``.itemset()`` methods has been removed. [#15378] + +- The following deprecated functionality has been removed: + + * ``littleh`` unit and ``with_H0`` equivalency. They are still available from + ``cosmology.units``. + * ``brightness_temperature`` equivalency no longer automatically swaps the + order of its arguments if it does not match the expectation. + * ``PhysicalType`` no longer supports ``str`` methods and attributes. [#15514] + +astropy.utils +^^^^^^^^^^^^^ + +- Removed deprecated ``OrderedDescriptor``, ``OrderedDescriptorContainer``, and ``set_locale`` in ``astropy.utils.misc``. [#14679] + +- ``is_path_hidden()`` and ``walk_skip_hidden()`` are deprecated. [#14759] + +- The structure of ``utils.metadata`` has been refactored, but all the available + functions and classes are still present and should be imported as before. [#15166] + +- The ``astropy.utils.metadata.MetaData`` class, which is used throughout astropy + to carry metadata on tables, columns, etc., can now also be used on dataclasses. + + When accessing the meta attribute on a class ``astropy.utils.metadata.MetaData`` + now returns None instead of itself. [#15237] + +- The ``astropy.utils.metadata.MetaData`` class, which is used throughout astropy + to carry metadata on tables, columns, etc., can now also be used on frozen dataclasses. [#15404] + +- Removed deprecated ``version_path`` in ``minversion`` function; it is no longer used. [#15468] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- The ``bboxes``, ``ticklabels_bbox``, and ``tick_out_size`` arguments to ``astropy.visualization.wcaxes.ticklabels.TickLabels.draw()`` now have no effect and are deprecated. + This is to allow rasterized ticks to be drawn correctly on WCSAxes. [#14760] + +- It is now not possible to pass any keyword arguments to ``astropy.visualization.wcsaxes.WCSAxes.draw()``. + Previously passing any keyword arguments would have errored anyway, as ``matplotlib.axes.Axes.draw()`` does not accept keyword arguments. [#14772] + +- Deprecated the ``exp`` attribute in the ``LogStretch``, + ``InvertedLogStretch``, ``PowerDistStretch``, and + ``InvertedPowerDistStretch`` stretch classes, and the ``power`` + attribute in the ``PowerStretch``. Instead, use the ``a`` attribute, + which matches the input keyword. [#15538] + +- Removed the maximum value of the ``a`` parameter in the ``AsinhStretch`` + and ``SinhStretch`` stretch classes. [#15539] + +astropy.wcs +^^^^^^^^^^^ + +- Removed deprecated ``accuracy`` from ``all_world2pix`` method in ``WCS``; use ``tolerance`` instead. [#15464] + +- ``NoConvergence`` no longer accepts arbitrary keyword arguments. [#15504] + + +Bug Fixes +--------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Fixed minor bug when getting solar system positions of objects from Type 3 SPICE kernel files. [#15612] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- The exponent in ``w0wzCDM.de_density_scale`` has been corrected to 3, from -3. + This correction has also been made to the scalar ``inv_efunc`` cpython functions. [#14991] + +- ``pandas.Series`` are now uniformly converted to their underlying data type when given + as an argument to a Cosmology method. [#15600] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Reading a table from FITS now respects the TNULL property of a column, passing + it into the column's ``fill_value``. [#14723] + +- Fix crash when a PrimaryHDU has a GROUPS keyword with a non-boolean value (i.e. + not a random-groups HDU). [#14998] + +- Fixed a bug that caused ``Cutout2D`` to not work correctly with ``CompImageHDU.section`` [#14999] + +- Fixed a bug that caused compressed images with TFORM missing the optional '1' prefix to not be readable. [#15001] + +- Ensure that tables written to FITS with both masked and unmasked columns + roundtrip properly (previously, all integer columns would become masked + if any column was masked). [#15473] + +- Fix segfault with error report in tile decompression. [#15489] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Output of ``repr`` for VOTable instance now clearly shows it is a VOTable and not generic astropy Table. [#14702] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- All models can be pickled now. [#14902] + +astropy.nddata +^^^^^^^^^^^^^^ + +- Restore bitmask propagation behavior in ``NDData.mask``, plus a fix + for arithmetic between masked and unmasked ``NDData`` objects. [#14995] + +astropy.table +^^^^^^^^^^^^^ + +- ``Table.as_array`` now respects the ``fill_value`` property of masked columns. [#14723] + +- Fix a bug where table indexes were not using a stable sort order. This was causing the + order of rows within groups to not match the original table order when an indexed table + was grouped. [#14907] + +- Fixed issue #14964 that when grouping a Table on a mixin column such as ``Quantity`` or + ``Time``, the grouped table keys did not reflect the original column values. For + ``Quantity`` this meant that the key values were pure float values without the unit, + while for ``Time`` the key values were the pair of ``jd1`` and ``jd2`` float values. [#14966] + +astropy.time +^^^^^^^^^^^^ + +- Ensure that the ``Time`` caches of formats and scales do not get out + of sync with the actual data, even if another instance, holding a view + of the data is written to. E.g., if one does ``t01 = t[:2]``, and + sets ``t[0]`` after, it is now guaranteed that ``t01.value`` will + correctly reflect that change in value. [#15453] + +astropy.units +^^^^^^^^^^^^^ + +- In VOunits, "pix", "au", "a", and "ct" are removed from the list of deprecated units. [#14885] + +astropy.utils +^^^^^^^^^^^^^ + +- Ufuncs with more than 2 operands (such as ``erfa.dtf2d``) now work + also if all inputs are scalars and more than two inputs have masks. [#15450] + +- Ensured that ``str(masked_array)`` looks like ``str(unmasked_array)`` also for + array scalars. Thus, like regular array scalars, the precision is ignored for + float, and strings do not include extra quoting. [#15451] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- The location of ticklabels on a WCSAxes is now correctly calculated when the figure is rasterized. [#14760] + +- Fixed a bug where a ``ValueError`` would be raised in the + ``AsinhStretch`` and ``SinhStretch`` classes for valid ``a`` parameter + values. [#15539] + +astropy.wcs +^^^^^^^^^^^ + +- ``wcs.validate(filename)`` now properly closes the file handler. [#15054] + +- Fix a regression in custom WCS mapping due to the recent introduction of + Solar System frames. [#15630] + + +Other Changes and Additions +--------------------------- + +- The minimum supported version of NumPy is now 1.22. [#15006] + +- Moved International Earth Rotation and Reference Systems (IERS) and Leap Second + files out into standalone astropy-iers-data package, maintaining full + backward-compatibility in the ``astropy.utils.iers`` API. Deprecation + warnings may be issued when certain files are accessed directly. [#14819] + +- Switch from using ``setup.cfg`` for project configuration to using ``pyproject.toml``. [#15247] + +- Update bundled expat to 2.5.0. [#15585] + +Version 5.3.4 (2023-10-03) +========================== + +Bug Fixes +--------- + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Updated ``astropy.io.misc.yaml`` so ``dump()`` with a numpy object array or + ``load()`` with YAML representing a Numpy object array both raise + ``TypeError``. This prevents problems like a segmentation fault. [#15373] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Fixed a bug in ``convert_to_writable_filelike`` where ``GzipFile`` was not + closed properly. [#15359] + +astropy.units +^^^^^^^^^^^^^ + +- In VOUnit, the spaces around the slash were removed in the formatting of + fractions, and fractional powers now also use the "**" operator. [#15282] + +- We now ensure that the unit ``u.cgs.cm`` is just an alias of ``u.si.cm``, + instead of a redefinition. This ensures that ``u.Unit("cm") / u.cm`` + will reliably cancel to dimensionless (instead of some "cm / cm"). [#15368] + +astropy.utils +^^^^^^^^^^^^^ + +- For ``Masked``, ``np.ptp`` and the ``.ptp()`` method now properly account for + the mask, ensuring the result is identical to subtracting the maximum and + minimum (with the same arguments). [#15380] + +Other Changes and Additions +--------------------------- + +- Compatibility with Python 3.12. [#14784] + +- Replaced the URL of ``IETF_LEAP_SECOND_URL`` because the original is now + defunct and IETF now defers to IANA for such look-up. [#15421] + + +Version v5.3.3 (2023-09-07) +=========================== + +Bug Fixes +--------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- ``TransformGraph.to_dot_graph()`` now throws an exception for invalid ``savelayout``. + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- The exponent of ``w0wzCDM`` functions in ``inv_efunc`` has been corrected to 3, from -3. [#15224] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Astropy modeling can filter non-finite data values using the ``filter_non_finite`` + keyword argument in a fitter call. Now when ``filter_non_finite`` is True, + non-finite *weights* will also be filtered to prevent crashes in ``LevMarLSQFitter``. [#15215] + +astropy.units +^^^^^^^^^^^^^ + +- Fixed ``astropy.units.Quantity``'s implementation of ``numpy.nanmedian()``, + where for Numpy >= 1.25 an exception was raised for some array shapes and axis + combinations. [#15228] + + +Other Changes and Additions +--------------------------- + +- v5.3.x will not support NumPy 2.0 or later. [#15234] + + +Version 5.3.2 (2023-08-11) +========================== + +Bug Fixes +--------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Fixed import when called with Python ``-OO`` flag. [#15037] + +astropy.nddata +^^^^^^^^^^^^^^ + +- Fix for collapse operations on ``NDData`` without masks or units. [#15082] + +astropy.units +^^^^^^^^^^^^^ + +- Modified the implementation of ``np.power()`` for instances of ``Quantity`` to + allow any array as the second operand if all its elements have the same value. [#15101] + +Version 5.3.1 (2023-07-06) +========================== + +Bug Fixes +--------- + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- The exponent in ``wowzCDM.de_density_scale`` has been corrected to 3, from -3. [#14991] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix crash when a PrimaryHDU has a GROUPS keyword with a non-boolean value (i.e. + not a random-groups HDU). [#14998] + +- Fixed a bug that caused ``Cutout2D`` to not work correctly with ``CompImageHDU.section`` [#14999] + +- Fixed a bug that caused compressed images with TFORM missing the optional '1' prefix to not be readable. [#15001] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- All models can be pickled now. [#14902] + +astropy.nddata +^^^^^^^^^^^^^^ + +- Restore bitmask propagation behavior in ``NDData.mask``, plus a fix + for arithmetic between masked and unmasked ``NDData`` objects. [#14995] + +astropy.table +^^^^^^^^^^^^^ + +- Fix a bug where table indexes were not using a stable sort order. This was causing the + order of rows within groups to not match the original table order when an indexed table + was grouped. [#14907] + +astropy.units +^^^^^^^^^^^^^ + +- In VOunits, "pix", "au", "a", and "ct" are removed from the list of deprecated units. [#14885] + +Version 5.3 (2023-05-22) +======================== + +New Features +------------ + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Add optional parameter ``refresh_cache`` to ``EarthLocation.of_site()`` and + ``EarthLocation.get_site_names()`` to force the download of the latest site + registry. [#13993] + +- Added ``atol`` argument to function ``is_O3`` and ``is_rotation`` in matrix utilities. [#14371] + +- A new class ``astropy.coordinates.StokesCoord`` has been added to represent world coordinates describing polarization state. + This change introduces a breaking change to the return value of ``astropy.wcs.WCS.pixel_to_world`` where before a ``u.Quantity`` object would be returned containing numerical values representing a Stokes profile now a ``StokesCoord`` object is returned. The previous numerical values can be accessed with ``StokesCoord.value``. [#14482] + +- Add an optional parameter ``location`` to ``EarthLocation.get_itrs()`` + to allow the generation of topocentric ITRS coordinates with respect + to a specific location. [#14628] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- Two new cosmologies have been added, ``FlatwpwaCDM`` and ``Flatw0wzCDM``, which are the + flat variants of ``wpwaCDM`` and ``w0wzCDM``, respectively. [#12353] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Add ability to read and write an RST (reStructuredText) ASCII table that + includes additional header rows specifying any or all of the column dtype, unit, + format, and description. This is available via the new ``header_rows`` keyword + argument. [#14182] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Added support for >3D data in CompImageHDU [#14252] + +- Added a ``CompImageHDU.section`` property which can be used to + efficiently access subsets of the data, similarly to ``ImageHDU.section``. + When using this, only the tiles required to cover the section are + read from disk and decompressed. [#14353] + +- Added support for ``'NOCOMPRESS'`` for the ``compression_type`` option in ``CompImageHDU``. [#14408] + +- Added new properties ``compression_type`` and ``tile_shape`` on + ``CompImageHDU``, giving the name of the compression algorithm + and the shape of the tiles in the tiled compression respectively. [#14428] + +- Do not call ``gc.collect()`` when closing a ``CompImageHDU`` object as it has a + large performance penalty. [#14576] + +- VLA tables can now be written with the unified I/O interface. + When object types are present or the VLA contains different types a `TypeError` + is thrown. [#14578] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Add support for writing/reading fixed-size and variable-length array columns to the parquet formatter. [#14237] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Added a method ``get_infos_by_name`` to make it easier to implement + DALI-compliant protocols [#14212] + +- Updating the built-in UCD list to upstream 1.5 (which requires a minor + update to the parser) [#14554] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Enable check for poorly conditioned fits in ``LinearLSQFitter`` for polynomial + models with fixed inputs. [#14037] + +astropy.nddata +^^^^^^^^^^^^^^ + +- ``astropy.nddata.NDDataArray`` now has collapsing methods like ``sum``, + ``mean``, ``min``, and ``max`` which operate along any axes, and better + support for ``astropy.utils.Masked`` objects. [#14175] + +astropy.stats +^^^^^^^^^^^^^ + +- ``vonmisesmle`` has now functioning "weights" and "axis" parameters that work equivalently + to the rest of the functions in the ``circstats`` module (``circmean``, ``rayleightest``, etc.) [#14533] + +astropy.table +^^^^^^^^^^^^^ + +- ``Table`` and ``QTable`` can now use the ``|`` and ``|=`` operators for + dictionary-style merge and update. [#14187] + +astropy.time +^^^^^^^^^^^^ + +- Add a ``leap_second_strict`` argument to the ``Time.to_datetime()`` method. This + controls the behavior when converting a time within a leap second to the ``datetime`` + format and can take the values ``raise`` (the default), ``warn``, or ``silent``. [#14606] + +astropy.timeseries +^^^^^^^^^^^^^^^^^^ + +- Adds the ``astropy.timeseries.LombScargleMultiband`` class, which is an + extension of the ``astropy.timeseries.LombScargle`` class. It enables the + generation of periodograms for datasets with measurements taken in more than + one photometric band. [#14016] + +- Add ``unit_parse_strict`` parameter to the Kepler reader to control the warnings + emitted when reading files. [#14294] + +astropy.units +^^^^^^^^^^^^^ + +- Add support for degrees Celsius for FITS. Parsing "Celsius" and "deg C" is now + supported and astropy will output "Celsius" into FITS. + + Note that "deg C" is only provided for compatibility with existing FITS files, + as it does not conform to the normal unit standard, where this should be read + as "degree * Coulomb". Indeed, compound units like "deg C kg-1" will still be + parsed as "Coulomb degree per kilogram". [#14042] + +- Enabled the ``equal_nan`` keyword argument for ``np.array_equal()`` when the + arguments are ``astropy.units.Quantity`` instances. [#14135] + +- Allow "console" and "unicode" formats for conversion to string of + function units. [#14407] + +- Add a "fraction" options to all the unit ``format`` classes, which determine + whether, if a unit has bases raised to a negative power, a string + representation should just show the negative powers (``fraction=False``) or + use a fraction, and, in the latter case, whether to use a single-line + representation using a solidus (``fraction='inline'`` or ``fraction=True``) + or, if the format supports it, a multi-line presentation with the numerator + and denominator separated by a horizontal line (``fraction='multiline'``). [#14449] + +astropy.utils +^^^^^^^^^^^^^ + +- The ``mean`` method on ``NDDataArray`` now avoids a division by zero + warning when taking the mean of a fully-masked slice (and still + returns ``np.nan``). [#14341] + +- Ensure we can read the newer ``IERS_B`` files produced by the International + Earth Rotation and Reference Systems Service, and point + ``astropy.utils.iers.IERS_B_URL`` to the new location. [#14382] + + +API Changes +----------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- ``get_moon()`` is deprecated and may be removed in a future version of + ``astropy``. Calling ``get_moon(...)`` should be replaced with + ``get_body("moon", ...)``. [#14354] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Deprecate the auto-fixing of tile sizes for HCOMPRESS_1 tiled + image compression when the tile size could be changed by +1 + to make it acceptable. [#14410] + +- The ``tile_size=`` argument to ``CompImageHDU`` has been deprecated + as it was confusing that it was required to be in the opposite + order to the data shape (it was in header rather than Numpy order). + Instead, users should make use of the ``tile_shape=`` argument which + is in Numpy shape order. [#14428] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Deprecate the ``humlicek2`` method for `~astropy.modeling.functional_models.Voigt1D` in favor + of using the ``wofz`` method using the `scipy.special.wofz` implementation of the + Fadeeva function whenever `scipy` is installed. [#14013] + +- Deprecated ``astropy.modeling.utils.comb()`` function in favor of ``comb()`` + from ``math`` standard library. [#14038] + +- Propagate measurement uncertainties via the ``weights`` keyword argument into the + parameter covariances. [#14519] + +astropy.units +^^^^^^^^^^^^^ + +- The conversion of ``astropy.units.Quantity`` to ``bool`` + that was deprecated since astropy 3.0 now raises a ``ValueError``. + This affects statements like ``if quantity``. + Use explicit comparisons like ``if quantity.value != 0`` + or ``if quantity is not None`` instead. [#14124] + +- Operations on ``Quantity`` in tables are sped up by only copying ``info`` when + it makes sense (i.e., when the object can still logically be thought of as the + same, such as in unit changes or slicing). ``info`` is no longer copied if a + ``Quantity`` is part of an operation. [#14253] + +- The ``Quantity.nansum`` method has been deprecated. It was always weird that it + was present, since ``ndarray`` does not have a similar method, and the other + ``nan*`` functions such as ``nanmean`` did not have a corresponding method. + Use ``np.nansum(quantity)`` instead. [#14267] + +- The unused ``units.format.Unscaled`` format class has been deprecated. [#14417] + +- The order in which unit bases are displayed has been changed to match the + order bases are stored in internally, which is by descending power to which + the base is raised, and alphabetical after. This helps avoid monstrosities + like ``beam^-1 Jy`` for ``format='fits'``. + + Note that this may affect doctests that use quantities with complicated units. [#14439] + +astropy.utils +^^^^^^^^^^^^^ + +- For ``Masked`` instances, the ``where`` argument for any ufunc can now + also be masked (with any masked elements masked in the output as well). + This is not very useful in itself, but avoids problems in conditional + functions (like ``np.add(ma, 1, where=ma>10)``). [#14590] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- The pixel attribute of ``astropy.visualization.wcsaxes.frame.Spine`` is deprecated + and will be removed in a future astropy version. + Because it is (in general) not possible to correctly calculate pixel + coordinates before Matplotlib is drawing a figure, instead set the world or data + coordinates of the ``Spine`` using the appropriate setters. [#13989] + +- Passing a bare number as the ``coord_wrap`` argument to ``CoordinateHelper.set_coord_type`` is deprecated. + Pass a ``Quantity`` with units equivalent to angular degrees instead. + + The ``.coord_wrap`` attribute of ``CoordinateHelper`` is now a ``Quantity`` instead of a bare number. [#14050] + + +Bug Fixes +--------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- ``Angle.to_string()`` was changed to ensure it matches the behaviour of + ``Quantity.to_string()`` in having a space between the value and the unit + for display with non-degree and hourangle units (i.e., the case in which + units are displayed by their name; the sexagesimal case for degrees or + hourangle that uses symbols is not changed). [#14379] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Fix an issue in the ``io.ascii`` QDP format reader to allow lower-case commands in the + table data file. Previously it required all upper case in order to parse QDP files. [#14365] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Compressing/decompressing a floating point dataset containing NaN values will + no longer read in the whole tile as NaNs. + + Fixed segmentation faults that occurred when compressing/decompressing data + with the PLIO_1 algorithm. [#14252] + +- ``Card`` now uses the default Python representation for floating point + values. [#14508] + +- ``ImageHDU`` now properly rejects Numpy scalars, avoiding data corruption. [#14528] + +- Fix issues with double quotes in CONTINUE cards. [#14598] + +- Fixes an issue where FITS_rec was incorrectly raising a ValueError exception when the heapsize was greater than 2**31 + when the Column type was 'Q' instead of 'P'. [#14810] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Columns with big-endian byte ordering (such as those read in from a FITS table) can now be serialized with Parquet. [#14373] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Bugfix for using ``getter/setter`` in properties to adjust the internal (computational) + value of a property vs its external proxy value when the values involve units. [#14512] + +- Fix issue with ``filter_non_finite`` option when fitting with ``weights`` via passing + the ``weights`` through the non-finite-filter alongside the input data. [#14695] + +- Fixed an issue with Parameter where a getter could be input without a + setter (or vice versa). [#14708] + +astropy.time +^^^^^^^^^^^^ + +- Using quantities with units of time for ``Time`` format 'decimalyear' will now + raise an error instead of converting the quantity to days and then + interpreting the value as years. An error is raised instead of attempting to + interpret the unit as years, since the interpretation is ambiguous: in + 'decimaltime' years are equal to 365 or 366 days, while for regular time units + the year is defined as 365.25 days. [#14566] + +astropy.uncertainty +^^^^^^^^^^^^^^^^^^^ + +- Ensure that ``Distribution`` can be compared with ``==`` and ``!=`` + with regular arrays or scalars, and that inplace operations like + ``dist[dist<0] *= -1`` work. [#14421] + +astropy.units +^^^^^^^^^^^^^ + +- Modified ``astropy.units.Quantity.__array_ufunc__()`` to return ``NotImplemented`` instead of raising a ``ValueError`` if the inputs are incompatible. [#13977] + +- Modified the behavior of ``numpy.array_equal()`` and ``numpy.array_equiv()`` to + return ``False`` instead of raising an error if their arguments are + ``astropy.units.Quantity`` instances with incompatible units. [#14163] + +- Spaces have been regularized for the ``unicode`` and ``console`` output + formats: no extraneous spaces in front of the unit, and always a space + between a possible scale factor and the unit. [#14413] + +- Prefixed degrees and arcmin are now typeset without using the symbol in + ``latex`` and ``unicode`` formats (i.e., ``mdeg`` instead of ``m°``), + as was already the case for arcsec. [#14419] + +- Ensure the unit is kept in ``np.median`` even if the result is a scalar ``nan`` + (the unit was lost for numpy < 1.22). [#14635] + +- Ensure that ``Quantity`` with structured dtype can be set using non-structured + ``Quantity`` (if units match), and that structured dtype names are inferred + correctly in the creation of ``StructuredUnit``, thus avoiding mismatches + when setting units. [#14680] + +astropy.utils +^^^^^^^^^^^^^ + +- When using astropy in environments with sparse file systems (e.g., where the temporary directory and astropy data directory resides in different volumes), ``os.rename`` may fail with ``OSError: [Errno 18] Invalid cross-device link``. + This may affect some clean-up operations executed by the ``data`` module, causing them to fail. + This patch is to catch ``OSError`` with ``errno == EXDEV`` (i.e., Errno 18) when performing these operations and try to use ``shutil.move`` instead to relocate the data. [#13730] + +- Ensure masks are propagated correctly for ``outer`` methods of ufuncs also if + one of the inputs is not actually masked. [#14624] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- The location of a ``astropy.visualization.wcsaxes.frame.Spine`` in a plot is now + correctly calculated when the DPI of a figure changes between a WCSAxes being + created and the figure being drawn. [#13989] + +- ``CoordinateHelper.set_ticks()`` now accepts ``number=0``. Previously it errored. [#14160] + +- ``WCSAxes.plot_coord`` and ``plot_scatter`` now work correctly for APE 14 compliant WCSes where the units are not always converted to degrees. [#14251] + +- Fixed a bug where coordinate overlays did not automatically determine the + longitude wrap angle or the appropriate units. [#14326] + +astropy.wcs +^^^^^^^^^^^ + +- Fix bugs with high-level WCS API on ``wcs.WCS`` object when using ``-TAB`` + coordinates. [#13571] + +- Fixed a bug in how WCS handles ``PVi_ja`` header coefficients when ``CTYPE`` + has ``-SIP`` suffix and in how code detects TPV distortions. [#14295] + + +Other Changes and Additions +--------------------------- + +- The minimum supported version of Python is now 3.9, changing from 3.8. [#14286] + +- The minimum supported version of Numpy is now 1.21. [#14349] + +- The minimum supported version of matplotlib is now 3.3. [#14286, #14321] + +- ``astropy`` no longer publishes wheels for i686 architecture. [#14517] + +- Added a pre-commit configuration for codespell. [#13985] + +- Removed a large fraction of the bundled CFITSIO code and internally refactored + FITS compression-related code, which has resulted in a speedup when compiling + astropy from source (40% faster in some cases). [#14252] + +- The CFITSIO library is no longer bundled in full with astropy and + the option to build against an external installation of CFITSIO + has now been removed, so the ASTROPY_USE_SYSTEM_CFITSIO environment + variable will be ignored during building. [#14311] + +- Updated CDS URL for Sesame look-up as the old URL is deprecated. [#14681] + + +Version 5.2.2 (2023-03-28) +========================== + +Bug Fixes +--------- + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- CDS and MRT tables with units that contain with multiple divisions, such as + ``km/s/Mpc`` now parse correctly as being equal to ``km/(s.Mpc)``. [#14369] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix ``FITSDiff`` when table contains a VLA column with the Q type. [#14539] + +astropy.table +^^^^^^^^^^^^^ + +- Fix a bug when creating a ``QTable`` when a ``Quantity`` input column is present and the + ``units`` argument modifies the unit of that column. This now works as expected where + previously this caused an exception. [#14357] + +astropy.units +^^^^^^^^^^^^^ + +- CDS units with multiple divisions, such as ``km/s/Mpc`` now parse + correctly as being equal to ``km/(s.Mpc)``. [#14369] + +astropy.wcs +^^^^^^^^^^^ + +- Fixed a bug that caused subclasses of BaseHighLevelWCS and HighLevelWCSMixin to + not work correctly under certain conditions if they did not have ``world_n_dim`` + and ``pixel_n_dim`` defined on them. [#14495] + + +Version 5.2.1 (2023-01-06) +========================== + +Bug Fixes +--------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Fix to ITRS frame ``earth_location`` attribute to give the correct result for + a topocentric frame. [#14180] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- Bounds are no longer passed to the scipy minimizer for methods Brent and + Golden. The scipy minimizer never used the bounds but silently accepted them. + In scipy v1.11.0.dev0+ an error is raised, so we now pass None as the bounds + to the minimizer. Users should not be affected by this change. [#14232] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Tables with multidimensional variable length array can now be properly read + and written. [#13417] + +astropy.units +^^^^^^^^^^^^^ + +- Modified the behavior of ``numpy.histogram()``, + ``numpy.histogram_bin_edges()``, ``numpy.histogram2d()``, and + ``numpy.histogramdd()`` so that the ``range`` argument must a compatible + instance of ``astropy.units.Quantity`` if the other arguments are instances of + ``astropy.units.Quantity``. [#14213] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Improved the performance of drawing WCSAxes grids by skipping some unnecessary + computations. [#14164] + +- Fixed WCSAxes sometimes triggering a NumPy RuntimeWarning when determining the + coordinate range of the axes. [#14211] + +Other Changes and Additions +--------------------------- + +- Fix compatibility with Numpy 1.24. [#14193] + +Version 5.2 (2022-12-12) +======================== + +New Features +------------ + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Adds new topocentric ITRS frame and direct transforms to and from the observed + frames ``AltAz`` and ``HADec`` with the ability to add or remove refraction + corrections as required. Since these frames are all within the ITRS, there are + no corrections applied other than refraction in the transforms. This makes the + topocentric ITRS frame and these transforms convenient for observers of near + Earth objects where stellar aberration should be omitted. [#13398] + +- Allow comparing ``SkyCoord`` to frames with data. [#13477] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- Cosmology instance can be parsed from or converted to a HTML table using + the new HTML methods in Cosmology's ``to/from_format`` I/O. [#13075] + +- A new comparison function has been added -- ``cosmology_equal()`` -- that + mirrors its ``numpy`` counterpart but allows for the arguments to be converted + to a ``Cosmology`` and to compare flat cosmologies with their non-flat + equivalents. [#13104] + +- Cosmology equivalence for flat FLRW cosmologies has been generalized to apply + to all cosmologies using the FlatCosmology mixin. [#13261] + +- The cosmological redshift unit now has a physical type of ``"redshift"``. [#13561] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Add ability to read and write a fixed width ASCII table that includes additional + header rows specifying any or all of the column dtype, unit, format, and + description. This is available in the ``fixed_width`` and + ``fixed_width_two_line`` formats via the new ``header_rows`` keyword argument. [#13734] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Added support to the ``io.fits`` API for reading and writing file paths of the + form ``~/file.fits`` or ``~/file.fits``, referring to the home + directory of the current user or the specified user, respectively. [#13131] + +- Added support for opening remote and cloud-hosted FITS files using the + ``fsspec`` package, which has been added as an optional dependency. [#13238] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Added support in ``io.votable`` for reading and writing file paths of the form + ``~/file.xml`` or ``~/file.xml``, referring to the home directory of + the current user or the specified user, respectively. [#13149] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Add option to non-linear fitters which enables automatic + exclusion of non-finite values from the fit data. [#13259] + +astropy.nddata +^^^^^^^^^^^^^^ + +- Modified ``Cutout2D`` to allow objects of type ``astropy.io.fits.Section`` + to be passed to the ``data`` parameter. [#13238] + +- Add a PSF image representation to ``astropy.nddata.NDData`` and ``astropy.nddata.CCDData``. [#13743] + +astropy.table +^^^^^^^^^^^^^ + +- An Astropy table can now be converted to a scalar NumPy object array. For NumPy + >= 1.20, a list of Astropy tables can be converted to an NumPy object array of + tables. [#13469] + +astropy.time +^^^^^^^^^^^^ + +- Added the ``astropy.time.Time.mean()`` method which also enables the ``numpy.mean()`` function to be used on instances of ``astropy.time.Time``. [#13508] + +- Improve the performance of getting the string representation of a large ``Time`` + or ``TimeDelta`` object. This is done via a new ``to_string()`` method that does + the time string format conversion only for the outputted values. Previously the + entire array was formatted in advance. [#13555] + +astropy.units +^^^^^^^^^^^^^ + +- It is now possible to use unit format names as string format specifiers for a + ``Quantity``, e.g. ``f'{1e12*u.m/u.s:latex_inline}'`` now produces the string + ``'$1 \\times 10^{12} \\; \\mathrm{m\\,s^{-1}}$'``. [#13050] + +- Ensure that the ``argmin`` and ``argmax`` methods of ``Quantity`` support the + ``keepdims`` argument when numpy does (numpy version 1.22 and later). [#13329] + +- ``numpy.lib.recfunctions.merge_arrays()`` is registered with numpy overload for + ``Quantity``. [#13669] + +- Added SI prefixes for quecto ("q", :math:`10^{-30}`), ronto ("r", + :math:`10^{-27}`), ronna ("R", :math:`10^{27}`), and quetta ("Q", + :math:`10^{30}`). [#14046] + +astropy.utils +^^^^^^^^^^^^^ + +- Added the ``use_fsspec``, ``fsspec_kwargs``, and ``close_files`` arguments + to ``utils.data.get_readable_fileobj``. [#13238] + +- Ensure that the ``argmin`` and ``argmax`` methods of ``Masked`` instances + support the ``keepdims`` argument when numpy does (numpy version 1.22 and + later). [#13329] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Add helper functions for WCSAxes instances to draw the instrument beam and a physical scale. [#12102] + +- Add a ``scatter_coord`` method to the ``wcsaxes`` functionality based on the + existing ``plot_coord`` method but that calls ``matplotlib.pyplot.scatter``. [#13562] + +- Added a ``sinh`` stretch option to ``simple_norm``. [#13746] + +- It is now possible to define "tickable" gridlines for the purpose of placing ticks or tick labels in the interior of WCSAxes plots. [#13829] + + +API Changes +----------- + +astropy.convolution +^^^^^^^^^^^^^^^^^^^ + +- Removed deprecated ``MexicanHat1DKernel`` and ``MexicanHat2DKernel`` + classes. Please use ``RickerWavelet1DKernel`` and + ``RickerWavelet2DKernel`` instead. [#13300] + +astropy.units +^^^^^^^^^^^^^ + +- Multiplying a ``LogQuantity`` like ``Magnitude`` with dimensionless physical + units by an array will no longer downcast to ``Quantity``. [#12579] + +- Quantity normally upcasts integer dtypes to floats, unless the dtype is + specifically provided. + Before this happened when ``dtype=None``; now the default has been changed to + ``dtype=numpy.inexact`` and ``dtype=None`` has the same meaning as in `numpy`. [#12941] + +- In "in-place unit changes" of the form ``quantity <<= new_unit``, the result + will now share memory with the original only if the conversion could be done + through a simple multiplication with a scale factor. Hence, memory will not be + shared if the quantity has integer ```dtype``` or is structured, or when the + conversion is through an equivalency. [#13638] + +- When ``Quantity`` is constructed from a structured array and ``unit`` is + ``None``, the default unit is now structured like the input data. [#13676] + +astropy.utils +^^^^^^^^^^^^^ + +- ``astropy.utils.misc.suppress`` has been removed, use ``contextlib.suppress`` + instead. ``astropy.utils.namedtuple_asdict`` has been removed, instead use + method ``._asdict`` on a ``namedtuple``. ``override__dir__`` has been deprecated + and will be removed in a future version, see the docstring for the better + alternative. [#13636] + +- ``astropy.utils.misc.possible_filename`` has been removed. [#13661] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Rename number-of-samples keyword ``nsamples`` in ``ZScaleInterval`` to align + with the ``n_samples`` keyword used in all other ``Interval`` classes in + this module. [#13810] + + +Bug Fixes +--------- + +astropy.convolution +^^^^^^^^^^^^^^^^^^^ + +- Fixed convolution Kernels to ensure the that returned kernels + are normalized to sum to one (e.g., ``Gaussian1DKernel``, + ``Gaussian2DKernel``). Also fixed the Kernel ``truncation`` calculation. [#13299] + +- Fix import error with setuptools v65.6.0 by replacing + ``numpy.ctypeslib.load_library`` with Cython to load the C convolution + extension. [#14035] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- ``BaseCoordinateFrame.get_frame_attr_names()`` had a misleading name, + because it actually provided a ``dict`` of attribute names and + their default values. It is now deprecated and replaced by ``BaseCoordinateFrame.get_frame_attr_defaults()``. + The fastest way to obtain the attribute names is ``BaseFrame.frame_attributes.keys()``. [#13484] + +- Fixed bug that caused ``earth_orientation.nutation_matrix()`` to error instead of returning output. [#13572] + +- Ensure that ``angle.to_string()`` continues to work after pickling, + and that units passed on to ``to_string()`` or the ``Angle`` + initializer can be composite units (like ``u.hour**1``), which might + result from preceding calculations. [#13933] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- ``report_diff_values()`` have now two new parameters ``rtol`` and ``atol`` to make the + report consistent with ``numpy.allclose`` results. + This fixes ``FITSDiff`` with multi-dimensional columns. [#13465] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Fixed two bugs in validator.validator.make_validation_report: + - ProgressBar iterator was not called correctly. + - make_validation_report now handles input string urls correctly. [#14102] + +astropy.timeseries +^^^^^^^^^^^^^^^^^^ + +- Fixed a performance regression in ``timeseries.aggregate_downsample`` + introduced in Astropy 5.0 / #11266. [#13069] + +astropy.units +^^^^^^^^^^^^^ + +- Unit changes of the form ``quantity <<= new_unit`` will now work also if the + quantity is integer. The result will always be float. This means that the result + will not share memory with the original. [#13638] + +- Ensure dimensionless quantities can be added inplace to regular ndarray. [#13913] + +astropy.utils +^^^^^^^^^^^^^ + +- Fixed an incompatibility with latest Python 3.1x versions that kept + ``astropy.utils.data.download_file`` from switching to TLS+FTP mode. [#14092] + +- ``np.quantile`` and ``np.percentile`` can now be used on ``Masked`` + arrays and quantities also with ``keepdims=True``. [#14113] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Significantly improve performance of ``ManualInterval`` when both limits + are specified manually. [#13898] + + +Other Changes and Additions +--------------------------- + +- The deprecated private ``astropy._erfa`` module has been removed. Use + ``pyerfa``, which is a dependency of ``astropy`` and can be imported directly + using ``import erfa``. [#13317] + +- The minimum version required for numpy is now 1.20 and that for scipy 1.5. [#13885] + +- Updated the bundled CFITSIO library to 4.2.0. [#14020] + + +Version 5.1.1 (2022-10-23) +========================== + +API Changes +----------- + +astropy.wcs +^^^^^^^^^^^ + +- The ``pixel`` argument to ``astropy.visualization.wcsaxes.ticklabels.TickLabels.add`` + no longer does anything, is deprecated, and will be removed in a future + astropy version. It has been replaced by a new required ``data`` argument, which + should be used to specify the data coordinates of the tick label being added. + + This changes has been made because it is (in general) not possible to correctly + calculate pixel coordinates before Matplotlib is drawing a figure. [#12630] + + +Bug Fixes +--------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Fixed a bug that prevented ``SkyOffsetFrame`` instances to be pickled by adding + a custom ``__reduce__`` method to the class (see issue #9249). [#13305] + +- Fixed the check for invalid ``Latitude`` values for float32 values. + ``Latitude`` now accepts the float32 value of pi/2, which was rejected + before because a comparison was made using the slightly smaller float64 representation. + See issue #13708. [#13745] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Fixed confusing chained exception messages of ``read()`` function when it fails. [#13170] + +- When writing out a :class:`~astropy.table.Table` to HTML format, the + ``formats`` keyword argument to the :meth:`~astropy.table.Table.write` method + will now be applied. [#13453] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- ``heapsize`` is now checked for VLA tables. An error is thrown whether P format is used + but the heap size is bigger than what can be indexed with a 32 bit signed int. [#13429] + +- Fix parsing of ascii TFORM when precision is missing. [#13520] + +- A compressed image HDU created from the header of a PRIMARY HDU, now correctly updates + 'XTENSION' and 'SIMPLE' keywords. [#13557] + +- Empty variable-length arrays are now properly handled when pathological combinations of + heapoffset and heapsize are encountered. [#13621] + +- ``PCOUNT`` and ``GCOUNT`` keywords are now removed from an uncompressed Primary header, + for compliance with ``fitsverify`` behavior. [#13753] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Bugfix for using ``MagUnit`` units on model parameters. [#13158] + +- Fix bug in using non-linear fitters to fit 0-degree polynomials using weights. [#13628] + +astropy.table +^^^^^^^^^^^^^ + +- Fix a problem where accessing one field of a structured column returned a Column + with the same info as the original column. This resulted in unintuitive behavior + in general and an exception if the format for the column was set. [#13269] + +- Tables with columns with structured data can now be properly stacked and joined. [#13306] + +- Update jQuery to 3.6.0, to pick up security fixes. [#13438] + +- Fix a Python 3.11 compatibility issue. Ensure that when removing a table column + that the ``pprint_include_names`` or ``pprint_exclude_names`` attributes get + updated correctly. [#13639] + +- When using ``add_columns`` with same indexes in ``indexes`` option or without + specifying the option, the order of the new columns will now be kept. [#13783] + +- Fix a bug when printing or getting the representation of a multidimensional + table column that has a zero dimension. [#13838] + +- Ensure that mixin columns and their ``info`` are not shared between tables + even when their underlying data is shared with ``copy=False``. [#13842] + +astropy.time +^^^^^^^^^^^^ + +- Fix ``Time.insert()`` on times which have their ``out_subfmt`` set. [#12732] + +- Prevent ``Time()`` from being initialized with an invalid precision + leading to incorrect results when representing the time as a string. [#13068] + +- Fix a bug in Time where a date string like "2022-08-01.123" was being parsed + as an ISO-format time "2022-08-01 00:00:00.123". The fractional part at the + end of the string was being taken as seconds. Now this raises an exception + because the string is not in ISO format. [#13731] + +astropy.units +^^^^^^^^^^^^^ + +- Significantly improved the performance of parsing composite units with the FITS + format, by ensuring the ``detailed_exception`` argument is properly passed on + and thus used. [#12699] + +- Ensure that ``np.concatenate`` on quantities can take a ``dtype`` argument (added in numpy 1.20). [#13323] + +- Ensure that the units of any ``initial`` argument to reductions such as + ``np.add.reduce`` (which underlies ``np.sum``) are properly taken into account. [#13340] + +astropy.utils +^^^^^^^^^^^^^ + +- Ensure that ``np.concatenate`` on masked data can take a ``dtype`` argument (added in numpy 1.20). [#13323] + +- Fix error when suppressing download progress bar while using non-default + ``sys.stdout`` stream. [#13352] + +- Ensure ``str`` and ``repr`` work properly for ``Masked`` versions of + structured subarrays. [#13404] + +- If an attribute is created using ``deprecated_attribute()`` with the + ``alternative`` argument then getting or setting the value of the deprecated + attribute now accesses its replacement. [#13824] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Fixed calling ``.tight_layout()`` on a WCSAxes. [#12418] + +astropy.wcs +^^^^^^^^^^^ + +- ``WCS.pixel_to_world`` now creates an ``EarthLocation`` object using ``MJD-AVG`` + if present before falling back to the old behaviour of using ``MJD-OBS``. [#12598] + +- The locations of ``WCSAxes`` ticks and tick-labels are now correctly calculated + when the DPI of a figure changes between a WCSAxes being created and the figure + being drawn, or when a rasterized artist is added to the WCSAxes. [#12630] + +- Fix a bug where ``SlicedLowLevelWCS.world_to_pixel_values`` would break when + the result of the transform is dependent on the coordinate of a sliced out + pixel. [#13579] + +- Updated bundled WCSLIB version to 7.12. This update includes bug fixes to + ``wcssub()`` in how it handles temporal axes with -TAB and fixes handling + of status returns from ``linp2x()`` and ``linx2p()`` relating to distortion + functions, in particular affecting TPV distortions - see #13509. For a full + list of changes - see http://www.atnf.csiro.au/people/mcalabre/WCS/CHANGES or + `astropy/cextern/wcslib/CHANGES `_. [#13635] + +- Fixed WCS validation not working properly if HDUList is needed + for multi-extension FITS file. [#13668] + + +Other Changes and Additions +--------------------------- + +- Development wheels of astropy should now be installed from + https://pypi.anaconda.org/astropy/simple instead of from + https://pkgs.dev.azure.com/astropy-project/astropy/_packaging/nightly/pypi/simple. [#13431] + +- Compatibility with Python 3.11, 3.10.7, 3.9.14, 3.8.14 [#13614] + + +Version 5.1 (2022-05-24) +======================== + +New Features +------------ + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- The ephemeris used in ``astropy.coordinates`` can now be set to any version of + the JPL ephemeris available from https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/. [#12541] + +- ``Angle.to_string()`` now accepts the ``'latex_inline'`` unit format. [#13056] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- Cosmology instance can be parsed from or converted to a YAML string using + the new "yaml" format in Cosmology's ``to/from_format`` I/O. [#12279] + +- Register "astropy.row" into Cosmology's to/from format I/O, allowing a + Cosmology instance to be parse from or converted to an Astropy Table Row. [#12313] + +- A method ``clone`` has been added to ``Parameter`` to quickly deep copy the + object and change any constructor argument. + A supporting equality method is added, and ``repr`` is enhanced to be able to + roundtrip -- ``eval(repr(Parameter()))`` -- if the Parameter's arguments can + similarly roundtrip. + Parameter's arguments are made keyword-only. [#12479] + +- Add methods ``Otot`` and ``Otot0`` to FLRW cosmologies to calculate the total + energy density of the Universe. [#12590] + +- Add property ``is_flat`` to cosmologies to calculate the curvature of the Universe. + ``Cosmology`` is now an abstract class and subclasses must override the + abstract property ``is_flat``. [#12606] + +- For converting a cosmology to a mapping, two new boolean keyword arguments are + added: ``cosmology_as_str`` for turning the class reference to a string, + instead of the class object itself, and ``move_from_meta`` to merge the + metadata with the rest of the returned mapping instead of adding it as a + nested dictionary. [#12710] + +- Register format "astropy.cosmology" with Cosmology I/O. [#12736] + +- Cosmological equivalency (``Cosmology.is_equivalent``) can now be extended + to any Python object that can be converted to a Cosmology, using the new + keyword argument ``format``. + This allows e.g. a properly formatted Table to be equivalent to a Cosmology. [#12740] + +- The new module ``cosmology/tests/helper.py`` has been added to provide tools + for testing the cosmology module and related extensions. [#12966] + +- A new property ``nonflat`` has been added to flat cosmologies + (``FlatCosmologyMixin`` subclasses) to get an equivalent cosmology, but of the + corresponding non-flat class. [#13076] + +- ``clone`` has been enhanced to allow for flat cosmologies to clone on the + equivalent non-flat cosmology. [#13099] + +- ``cosmology`` file I/O uses the Unified Table I/O interface, which has added + support for reading and writing file paths of the form ``~/file.ecsv`` or + ``~/file.ecsv``, referring to the home directory of the current user + or the specified user, respectively. [#13129] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Simplify the way that the ``converters`` argument of ``io.ascii.read`` is + provided. Previously this required wrapping each data type as the tuple returned + by the ``io.ascii.convert_numpy()`` function and ensuring that the value is a + ``list``. With this update you can write ``converters={'col1': bool}`` to force + conversion as a ``bool`` instead of the previous syntax ``converters={'col1': + [io.ascii.convert_numpy(bool)]}``. Note that this update is back-compatible with + the old behavior. [#13073] + +- Added support in ``io.ascii`` for reading and writing file paths of the form + ``~/file.csv`` or ``~/file.csv``, referring to the home directory of + the current user or the specified user, respectively. [#13130] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Add option ``unit_parse_strict`` to ``astropy.io.fits.connect.read_table_fits`` + to enable warnings or errors about invalid FITS units when using ``astropy.table.Table.read``. + The default for this new option is ``"warn"``, which means warnings are now raised for + columns with invalid units. [#11843] + +- Changes default FITS behavior to use buffered I/O + rather than unbuffered I/O for performance reasons. [#12081] + +- ``astropy.io.fits.Header`` now has a method to calculate the size + (in bytes) of the data portion (with or without padding) following + that header. [#12110] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Allow serialization of model unit equivalencies. [#10198] + +- Built-in Cosmology subclasses can now be converted to/from YAML with the + functions ``dump`` and ``load`` in ``astropy.io.misc.yaml``. [#12279] + +- Add asdf support for ``Cosine1D``, ``Tangent1D``, ``ArcSine1D``, + ``ArcCosine1D``, and ``ArcTangent1D`` models. [#12895] + +- Add asdf support for ``Spline1D`` models. [#12897] + +astropy.io.registry +^^^^^^^^^^^^^^^^^^^ + +- Added support to the Unified Table I/O interface for reading and writing file + paths of the form ``~/file.csv`` or ``~/file.csv``, referring to the + home directory of the current user or the specified user, respectively. [#13129] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Add new fitters based on ``scipy.optimize.least_squares`` method of non-linear + least-squares optimization: [#12051] + + - ``TRFLSQFitter`` using the Trust Region Reflective algorithm. + - ``LMLSQFitter`` using the Levenberg-Marquardt algorithm (implemented by ``scipy.optimize.least_squares``). + - ``DogBoxLSQFitter`` using the dogleg algorithm. + +- Enable direct use of the ``ignored`` feature of ``ModelBoundingBox`` by users in + addition to its use as part of enabling ``CompoundBoundingBox``. [#12384] + +- Switch ``modeling.projections`` to use ``astropy.wcs.Prjprm`` wrapper internally and provide access to the ``astropy.wcs.Prjprm`` structure. [#12558] + +- Add error to non-finite inputs to the ``LevMarLSQFitter``, to protect against soft scipy failure. [#12811] + +- Allow the ``Ellipse2D`` and ``Sersic2D`` theta parameter to be input as + an angular quantity. [#13030] + +- Added ``Schechter1D`` model. [#13116] + +astropy.nddata +^^^^^^^^^^^^^^ + +- Add support for converting between uncertainty types. This uncertainty + conversion system uses a similar flow to the coordinate subsystem, where + Cartesian is used as the common system. In this case, variance is used as the + common system. [#12057] + +- The ``as_image_hdu`` option is now available for ``CCDData.to_hdu`` and + ``CCDData.write``. This option allows the user to get an ``ImageHDU`` as the + first item of the returned ``HDUList``, instead of the default ``PrimaryHDU``. [#12962] + +- File I/O through ``nddata.CCDData`` uses the Unified I/O interface, which has + added support for reading and writing file paths of the form ``~/file.csv`` or + ``~/file.csv``, referring to the home directory of the current user + or the specified user, respectively. [#13129] + +astropy.table +^^^^^^^^^^^^^ + +- A new keyword-only argument ``kind`` was added to the ``Table.sort`` method to + specify the sort algorithm. [#12637] + +- Columns which are ``numpy`` structured arrays are now fully supported, + effectively allowing tables within tables. This applies to ``Column``, + ``MaskedColumn``, and ``Quantity`` columns. These structured data columns + can be stored in ECSV, FITS, and HDF5 formats. [#12644] + +- Improve the performance of ``np.searchsorted`` by a factor of 1000 for a + bytes-type ``Column`` when the search value is ``str`` or an array of ``str``. + This happens commonly for string data stored in FITS or HDF5 format files. [#12680] + +- Add support for using mixin columns in group aggregation operations when the + mixin supports the specified operation (e.g. ``np.sum`` works for ``Quantity`` + but not ``Time``). In cases where the operation is not supported the code now + issues a warning and drops the column instead of raising an exception. [#12825] + +- Added support to the Unified Table I/O interface for reading and writing file + paths of the form ``~/file.csv`` or ``~/file.csv``, referring to the + home directory of the current user or the specified user, respectively. [#13129] + +astropy.time +^^^^^^^^^^^^ + +- Add support for calling ``numpy.linspace()`` with two ``Time`` instances to + generate a or multiple linearly spaced set(s) of times. [#13132] + +astropy.units +^^^^^^^^^^^^^ + +- ``structured_to_unstructured`` and ``unstructured_to_structured`` in + ``numpy.lib.recfunctions`` now work with Quantity. [#12486] + +- Implement multiplication and division of LogQuantities and numbers [#12566] + +- New ``doppler_redshift`` equivalency to convert between + Doppler redshift and radial velocity. [#12709] + +- Added the ``where`` keyword argument to the ``mean()``,``var()``, ``std()`` and ``nansum()`` methods of + ``astropy.units.Quantity``. Also added the ``initial`` keyword argument to ``astropy.units.Quantity.nansum()``. [#12891] + +- Added "Maxwell" as a unit for magnetic flux to the CGS module. [#12975] + +- ``Quantity.to_string()`` and ``FunctionUnitBase.to_string()`` now accept the + ``'latex_inline'`` unit format. The output of ``StructuredUnit.to_string()`` + when called with ``format='latex_inline'`` is now more consistent with the + output when called with ``format='latex'``. [#13056] + +astropy.utils +^^^^^^^^^^^^^ + +- Added the ``where`` keyword argument to the ``mean()``, ``var()``, ``std()``, ``any()``, and ``all()`` methods of + ``astropy.utils.masked.MaskedNDArray``. [#12891] + +- Improve handling of unavailable IERS-A (predictive future Earth rotation) data + in two ways. First, allow conversions with degraded accuracy if the IERS-A data + are missing or do not cover the required time span. This is done with a new + config item ``conf.iers_degraded_accuracy`` which specifies the behavior when + times are outside the range of IERS table. The options are 'error' (raise an + ``IERSRangeError``, default), 'warn' (issue a ``IERSDegradedAccuracyWarning``) + or 'ignore' (ignore the problem). Second, the logic for auto-downloads was + changed to guarantee that no matter what happens with the IERS download + operations, only warnings will be issued. [#13052] + +astropy.wcs +^^^^^^^^^^^ + +- ``astropy.wcs.Celprm`` and ``astropy.wcs.Prjprm`` have been added + to allow access to lower level WCSLIB functionality and to allow direct + access to the ``cel`` and ``prj`` members of ``Wcsprm``. [#12514] + +- Add ``temporal`` properties for convenient access of/selection of/testing for + the ``TIME`` axis introduced in WCSLIB version 7.8. [#13094] + +API Changes +----------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- The ``dms_to_degrees`` and ``hms_to_hours`` functions (and implicitly + tuple-based initialization of ``Angle``) is now deprecated, as it was + difficult to be sure about the intent of the user for signed values of + the degrees/hours, minutes, and seconds. [#13162] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- The already deprecated ``Planck18_arXiv_v2`` has been removed. + Use ``Planck18`` instead [#12354] + +- ``default_cosmology.get_cosmology_from_string`` is deprecated and will be + removed in two minor versions. + Use ``getattr(astropy.cosmology, )`` instead. [#12375] + +- In I/O, conversions of Parameters move more relevant information from the + Parameter to the Column. + The default Parameter ``format_spec`` is changed from ``".3g"`` to ``""``. [#12612] + +- Units of redshift are added to ``z_reion`` in built-in realizations' metadata. [#12624] + +- Cosmology realizations (e.g. ``Planck18``) and parameter dictionaries are now + lazily loaded from source files. [#12746] + +- The Cosmology Parameter argument "fmt" for specifying a format spec + has been deprecated in favor of using the built-in string representation from + the Parameter's value's dtype. [#13072] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- When reading an ECSV file, changed the type checking + to issue an ``InvalidEcsvDatatypeWarning`` instead of raising a ``ValueError`` + exception if the ``datatype`` of a column is not recognized in the ECSV standard. + This also applies to older versions of ECSV files which used to silently + proceed but now warn first. [#12841] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Removed deprecated ``clobber`` argument from functions in ``astropy.io.fits``. [#12258] + +- Add ``-s/--sort`` argument to ``fitsheader`` to sort the fitsort-mode output. [#13106] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Deprecate asdf in astropy core in favor of the asdf-astropy package. [#12903, #12930] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Made ``astropy.modeling.fitting._fitter_to_model_params`` and ``astropy.modeling.fitting._model_to_fit_params`` + public methods. [#12585] + +astropy.table +^^^^^^^^^^^^^ + +- Change the repr of the Table object to replace embedded newlines and tabs with + ``r'\n'`` and ``r'\t'`` respectively. This improves the display of such tables. [#12631] + +- A new keyword-only argument ``kind`` was added to the ``Table.sort`` method to + specify the sort algorithm. The signature of ``Table.sort`` was modified so that + the ``reverse`` argument is now keyword-only. Previously ``reverse`` could be + specified as the second positional argument. [#12637] + +- Changed behavior when a structured ``numpy.ndarray`` is added as a column to a + ``Table``. Previously this was converted to a ``NdarrayMixin`` subclass of + ``ndarray`` and added as a mixin column. This was because saving as a file (e.g. + HDF5, FITS, ECSV) was not supported for structured array columns. Now a + structured ``numpy.ndarray`` is added to the table as a native ``Column`` and + saving to file is supported. [#13236] + +astropy.tests +^^^^^^^^^^^^^ + +- Backward-compatible import of ``astropy.tests.disable_internet`` + has been removed; use ``pytest_remotedata.disable_internet`` + from ``pytest-remotedata`` instead. [#12633] + +- Backward-compatible import of ``astropy.tests.helper.remote_data`` + has been removed; use ``pytest.mark.remote_data`` from ``pytest-remotedata`` + instead. [#12633] + +- The following are deprecated and will be removed in a future release. + Use ``pytest`` warning and exception handling instead: [#12633] + + * ``astropy.io.ascii.tests.common.raises`` + * ``astropy.tests.helper.catch_warnings`` + * ``astropy.tests.helper.ignore_warnings`` + * ``astropy.tests.helper.raises`` + * ``astropy.tests.helper.enable_deprecations_as_exceptions`` + * ``astropy.tests.helper.treat_deprecations_as_exceptions`` + +- Backward-compatible plugin ``astropy.tests.plugins.display`` + has been removed; use ``pytest-astropy-header`` instead. [#12633] + +astropy.time +^^^^^^^^^^^^ + +- Creating an `~astropy.time.TimeDelta` object with numerical inputs + that do not have a unit and without specifying an explicit format, + for example ``TimeDelta(5)``, + now results in a `~astropy.time.TimeDeltaMissingUnitWarning`. + This also affects statements like ``Time("2020-01-01") + 5`` or + ``Time("2020-01-05") - Time("2020-01-03") < 5``, which implicitly + transform the right-hand side into an `~astropy.time.TimeDelta` instance. [#12888] + +Bug Fixes +--------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- The machinery that makes observatory locations available as ``EarthLocation`` + objects is now smarter about processing observatory names from its data files. + More names are available for use and the empty string is no longer considered + to be a valid name. [#12721] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Fixed ``io.ascii`` read and write functions for most formats to correctly handle + data fields with embedded newlines for both the fast and pure-Python readers and + writers. [#12631] + +- Fix an issue when writing ``Time`` table columns to a file when the time + ``format`` is one of ``datetime``, ``datetime64``, or ``ymdhms``. Previously, + writing a ``Time`` column with one of these formats could result in an exception + or else an incorrect output file that cannot be read back in. [#12842] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Add a ``mask_invalid`` option to ``Table.read`` to allow deactivating the + masking of NaNs in float columns and empty strings in string columns. This + option is necessary to allow effective use of memory-mapped reading with + ``memmap=True``. [#12544] + +- Fix ``CompImageHeader.clear()``. [#13102] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Bugfix for ``ignore`` functionality failing in ``ModelBoundingBox`` when using + ``ignore`` option alongside passing bounding box data as tuples. [#13032] + +astropy.table +^^^^^^^^^^^^^ + +- Fixed a bug in ``Table.show_in_browser`` using the ``jsviewer=True`` option + to display the table with sortable columns. Previously the sort direction arrows + were not being shown due to missing image files for the arrows. [#12716] + +- Fix an issue when writing ``Time`` table columns to a file when the time + ``format`` is one of ``datetime``, ``datetime64``, or ``ymdhms``. Previously, + writing a ``Time`` column with one of these formats could result in an exception + or else an incorrect output file that cannot be read back in. [#12842] + +- Fixed a bug where it is not possible to set the ``.info.format`` property of a + table structured column and get formatted output. [#13233] + +- Fixed a bug when adding a masked structured array to a table. Previously this + was auto-converted to a ``NdarrayMixin`` which loses the mask. With this fix + the data are added to the table as a ``MaskedColumn`` and the mask is preserved. [#13236] + +astropy.time +^^^^^^^^^^^^ + +- Fix an issue when writing ``Time`` table columns to a file when the time + ``format`` is one of ``datetime``, ``datetime64``, or ``ymdhms``. Previously, + writing a ``Time`` column with one of these formats could result in an exception + or else an incorrect output file that cannot be read back in. [#12842] + +astropy.utils +^^^^^^^^^^^^^ + +- Fixed a bug which caused ``numpy.interp`` to produce incorrect + results when ``Masked`` arrays were passed. [#12978] + +- Fixed HAS_YAML not working as intended. [#13066] + +astropy.wcs +^^^^^^^^^^^ + +- Convert ``NoConvergence`` errors to warnings in ``world_to_pixel_values`` so that callers can work at least with the non converged solution. [#11693] + +- Expose the ability to select TIME axis introduced in WCSLIB version 7.8. [#13062] + +- Do not call ``wcstab`` on ``wcscopy`` and copy ``wtb`` members from the original WCS. [#13063] + +- Updated bundled WCSLIB version to 7.11. This update together with 7.10 + includes bug fixes to ``tabini()`` and ``tabcpy()`` as well as several + print formatting enhancements. For a full list of + changes - see http://www.atnf.csiro.au/people/mcalabre/WCS/CHANGES [#13171] + +- Fixed error that occurred in ``WCS.world_to_pixel`` for ``WCS`` objects with a + spectral axis and observer location information when passing a ``SpectralCoord`` + that had missing observer or target information. [#13228] + +Version 5.0.4 (2022-03-31) +========================== + +Bug Fixes +--------- + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Fixed the ``Gaussian2D`` ``bounding_box`` when ``theta`` is an angular + ``Quantity``. [#13021] + +astropy.utils +^^^^^^^^^^^^^ + +- Reverted ``astropy.utils.iers.iers.IERS_A_URL`` to ``maia.usno.navy.mil`` domain instead + of NASA FTP to work around server issues. [#13004] + +Other Changes and Additions +--------------------------- + +- Updated bundled WCSLIB to version 7.9 with several bugfixes and added + support for time coordinate axes in ``wcsset()`` and ``wcssub()``. The + four-digit type code for the time axis will have the first digit set to 4, + i.e., four digit code will be 4xxx where x is a digit 0-9. For a full list of + bug fixes see https://www.atnf.csiro.au/people/mcalabre/WCS/CHANGES [#12994] + + +Version 5.0.3 (2022-03-25) +========================== + +Bug Fixes +--------- + +astropy.convolution +^^^^^^^^^^^^^^^^^^^ + +- Bugfix in ``astropy.convolution.utils.discretize_model`` which allows the function to handle a ``CompoundModel``. + Before this fix, ``discretize_model`` was confusing ``CompoundModel`` with a callable function. [#12959] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix write and read FITS tables with multidimensional items, using ``from_columns`` + without previously defined ``ColDefs`` structure. [#12863] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Fix VOTable linting to avoid use of shell option. [#12985] + +astropy.utils +^^^^^^^^^^^^^ + +- Fix XML linting to avoid use of shell option. [#12985] + +Other Changes and Additions +--------------------------- + +- Updated the bundled CFITSIO library to 4.1.0. [#12967] + +Version 5.0.2 (2022-03-10) +========================== + +Bug Fixes +--------- + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Bugfix to add backwards compatibility for reading ECSV version 0.9 files with + non-standard column datatypes (such as ``object``, ``str``, ``datetime64``, + etc.), which would raise a ValueError in ECSV version 1.0. [#12880] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Bugfix for ``units_mapping`` schema's property name conflicts. Changes: + * ``inputs`` to ``unit_inputs`` + * ``outputs`` to ``unit_outputs`` [#12800] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Fixed a bug where ``astropy.io.votable.validate`` was printing output to + ``sys.stdout`` when the ``output`` parameter was set to ``None``. ``validate`` + now returns a string when ``output`` is set to ``None``, as documented. + [#12604] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Fix handling of units on ``scale`` parameter in BlackBody model. [#12318] + +- Indexing on models can now be used with all types of integers + (like ``numpy.int64``) instead of just ``int``. [#12561] + +- Fix computation of the separability of a ``CompoundModel`` where another + ``CompoundModel`` is on the right hand side of the ``&`` operator. [#12907] + +- Provide a hook (``Model._calculate_separability_matrix``) to allow subclasses + of ``Model`` to define how to compute their separability matrix. [#12900] + +astropy.stats +^^^^^^^^^^^^^ + +- Fixed a bug in which running ``kuiper_false_positive_probability(D,N)`` on + distributions with many data points could produce NaN values for the false + positive probability of the Kuiper statistic. [#12896] + +astropy.wcs +^^^^^^^^^^^ + +- Fixed a bug due to which ``naxis``, ``pixel_shape``, and + ``pixel_bounds`` attributes of ``astropy.wcs.WCS`` were not restored when + an ``astropy.wcs.WCS`` object was unpickled. This fix also eliminates + ``FITSFixedWarning`` warning issued during unpiclikng of the WCS objects + related to the number of axes. This fix also eliminates errors when + unpickling WCS objects originally created using non-default values for + ``key``, ``colsel``, and ``keysel`` parameters. [#12844] + +Version 5.0.1 (2022-01-26) +========================== + +Bug Fixes +--------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Trying to create an instance of ``astropy.coordinates.Distance`` by providing + both ``z`` and ``parallax`` now raises the expected ``ValueError``. [#12531] + +- Fixed a bug where changing the wrap angle of the longitude component of a + representation could raise a warning or error in certain situations. [#12556] + +- ``astropy.coordinates.Distance`` constructor no longer ignores the ``unit`` + keyword when ``parallax`` is provided. [#12569] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- ``astropy.cosmology.utils.aszarr`` can now convert ``Column`` objects. [#12525] + +- Reading a cosmology from an ECSV will load redshift and Hubble parameter units + from the cosmology units module. [#12636] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix formatting issue in ``_dump_coldefs`` and add tests for ``tabledump`` and + ``tableload`` convenience functions. [#12526] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- YAML can now also represent quantities and arrays with structured dtype, + as well as structured scalars based on ``np.void``. [#12509] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Fixes error when fitting multiplication or division based compound models + where the sub-models have different output units. [#12475] + +- Bugfix for incorrectly initialized and filled ``parameters`` data for ``Spline1D`` model. [#12523] + +- Bugfix for ``keyerror`` thrown by ``Model.input_units_equivalencies`` when + used on ``fix_inputs`` models which have no set unit equivalencies. [#12597] + +astropy.table +^^^^^^^^^^^^^ + +- ``astropy.table.Table.keep_columns()`` and + ``astropy.table.Table.remove_columns()`` now work with generators of column + names. [#12529] + +- Avoid duplicate storage of info in serialized columns if the column + used to serialize already can hold that information. [#12607] + +astropy.timeseries +^^^^^^^^^^^^^^^^^^ + +- Fixed edge case bugs which emerged when using ``aggregate_downsample`` with custom bins. [#12527] + +astropy.units +^^^^^^^^^^^^^ + +- Structured units can be serialized to/from yaml. [#12492] + +- Fix bad typing problems by removing interaction with ``NDArray.__class_getitem__``. [#12511] + +- Ensure that ``Quantity.to_string(format='latex')`` properly typesets exponents + also when ``u.quantity.conf.latex_array_threshold = -1`` (i.e., when the threshold + is taken from numpy). [#12573] + +- Structured units can now be copied with ``copy.copy`` and ``copy.deepcopy`` + and also pickled and unpicked also for ``protocol`` >= 2. + This does not work for big-endian architecture with older ``numpy<1.21.1``. [#12583] + +astropy.utils +^^^^^^^^^^^^^ + +- Ensure that a ``Masked`` instance can be used to initialize (or viewed + as) a ``numpy.ma.Maskedarray``. [#12482] + +- Ensure ``Masked`` also works with numpy >=1.22, which has a keyword argument + name change for ``np.quantile``. [#12511] + +- ``astropy.utils.iers.LeapSeconds.auto_open()`` no longer emits unnecessary + warnings when ``astropy.utils.iers.conf.auto_max_age`` is set to ``None``. [#12713] + + +Version 5.0 (2021-11-15) +======================== + + +New Features +------------ + +astropy.convolution +^^^^^^^^^^^^^^^^^^^ + +- Added dealiasing support to ``convolve_fft``. [#11495] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Added missing coordinate transformations where the starting and ending frames + are the same (i.e., loopback transformations). [#10909] + +- Allow negation, multiplication and division also of representations that + include a differential (e.g., ``SphericalRepresentation`` with a + ``SphericalCosLatDifferential``). For all operations, the outcome is + equivalent to transforming the representation and differential to cartesian, + then operating on those, and transforming back to the original representation + (except for ``UnitSphericalRepresentation``, which will return a + ``SphericalRepresentation`` if there is a scale change). [#11470] + +- ``RadialRepresentation.transform`` can work with a multiplication matrix only. + All other matrices still raise an exception. [#11576] + +- ``transform`` methods are added to ``BaseDifferential`` and ``CartesianDifferential``. + All transform methods on Representations now delegate transforming differentials + to the differential objects. [#11654] + +- Adds new ``HADec`` built-in frame with transformations to/from ``ICRS`` and ``CIRS``. + This frame complements ``AltAz`` to give observed coordinates (hour angle and declination) + in the ``ITRS`` for an equatorially mounted telescope. [#11676] + +- ``SkyCoord`` objects now have a ``to_table()`` method, which allows them to be + converted to a ``QTable``. [#11743] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- Cosmologies now store metadata in a mutable parameter ``meta``. + The initialization arguments ``name`` and ``meta`` are keyword-only. [#11542] + +- A new unit, ``redshift``, is defined. It is a dimensionless unit to distinguish + redshift quantities from other non-redshift values. For compatibility with + dimensionless quantities the equivalency ``dimensionless_redshift`` is added. + This equivalency is enabled by default. [#11786] + +- Add equality operator for comparing Cosmology instances. Comparison is done on + all immutable fields (this excludes 'meta'). + + Now the following will work: + + .. code-block:: python + + >>> from astropy.cosmology import Planck13, Planck18 + >>> Planck13 == Planck18 + False + + >>> Planck18 == Planck18 + True [#11813] + +- Added ``read/write`` methods to Cosmology using the Unified I/O registry. + Now custom file format readers, writers, and format-identifier functions + can be registered to read, write, and identify, respectively, Cosmology + objects. Details are discussed in an addition to the docs. [#11948] + +- Added ``to_format/from_format`` methods to Cosmology using the Unified I/O + registry. Now custom format converters and format-identifier functions + can be registered to transform Cosmology objects. + The transformation between Cosmology and dictionaries is pre-registered. + Details are discussed in an addition to the docs. [#11998] + +- Added units module for defining and collecting cosmological units and + equivalencies. [#12092] + +- Flat cosmologies are now set by a mixin class, ``FlatCosmologyMixin`` and its + FLRW-specific subclass ``FlatFLRWMixin``. All ``FlatCosmologyMixin`` are flat, + but not all flat cosmologies are instances of ``FlatCosmologyMixin``. As + example, ``LambdaCDM`` **may** be flat (for the a specific set of parameter + values), but ``FlatLambdaCDM`` **will** be flat. + + Cosmology parameters are now descriptors. When accessed from a class they + transparently stores information, like the units and accepted equivalencies. + On a cosmology instance, the descriptor will return the parameter value. + Parameters can have custom ``getter`` methods. + + Cosmological equality is refactored to check Parameters (and the name) + A new method, ``is_equivalent``, is added to check Cosmology equivalence, so + a ``FlatLambdaCDM`` and flat ``LambdaCDM`` are equivalent. [#12136] + +- Replaced ``z = np.asarray(z)`` with ``z = u.Quantity(z, u.dimensionless_unscaled).value`` + in Cosmology methods. Input of values with incorrect units raises a UnitConversionError + or TypeError. [#12145] + +- Cosmology Parameters allow for custom value setters. + Values can be set once, but will error if set a second time. + If not specified, the default setter is used, which will assign units + using the Parameters ``units`` and ``equivalencies`` (if present). + Alternate setters may be registered with Parameter to be specified by a str, + not a decorator on the Cosmology. [#12190] + +- Cosmology instance conversion to dict now accepts keyword argument ``cls`` to + determine dict type, e.g. ``OrderedDict``. [#12209] + +- A new equivalency is added between redshift and the Hubble parameter and values + with units of little-h. + This equivalency is also available in the catch-all equivalency ``with_redshift``. [#12211] + +- A new equivalency is added between redshift and distance -- comoving, lookback, + and luminosity. This equivalency is also available in the catch-all equivalency + ``with_redshift``. [#12212] + +- Register Astropy Table into Cosmology's ``to/from_format`` I/O, allowing + a Cosmology instance to be parsed from or converted to a Table instance. + Also adds the ``__astropy_table__`` method allowing ``Table(cosmology)``. [#12213] + +- The WMAP1 and WMAP3 are accessible as builtin cosmologies. [#12248] + +- Register Astropy Model into Cosmology's ``to/from_format`` I/O, allowing + a Cosmology instance to be parsed from or converted to a Model instance. [#12269] + +- Register an ECSV reader and writer into Cosmology's I/O, allowing a Cosmology + instance to be read from from or written to an ECSV file. [#12321] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Added new way to specify the dtype for tables that are read: ``converters`` + can specify column names with wildcards. [#11892] + +- Added a new ``astropy.io.ascii.Mrt`` class to write tables in the American + Astronomical Society Machine-Readable Table format, + including documentation and tests for the same. [#11897, #12301, #12302] + +- When writing, the input data are no longer copied, improving performance. + Metadata that might be changed, such as format and serialization + information, is copied, hence users can continue to count on no + changes being made to the input data. [#11919] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Add Parquet serialization of Tables with pyarrow, including metadata support and + columnar access. [#12215] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Added fittable spline models to ``modeling``. [#11634] + +- Extensive refactor of ``BoundingBox`` for better usability and maintainability. [#11930] + +- Added ``CompoundBoundingBox`` feature to ``~astropy.modeling``, which allows more flexibility in + defining bounding boxes for models that are applied to images with many slices. [#11942] + +- Improved parameter support for ``astropy.modeling.core.custom_model`` created models. [#11984] + +- Added the following trigonometric models and linked them to their appropriate inverse models: + * ``Cosine1D`` [#12158] + * ``Tangent1D`` + * ``ArcSine1D`` + * ``ArcCosine1D`` + * ``ArcTangent1D`` [#12185] + +astropy.table +^^^^^^^^^^^^^ + +- Added a new method ``Table.update()`` which does a dictionary-style update of a + ``Table`` by adding or replacing columns. [#11904] + +- Masked quantities are now fully supported in tables. This includes ``QTable`` + automatically converting ``MaskedColumn`` instances to ``MaskedQuantity``, + and ``Table`` doing the reverse. [#11914] + +- Added new keyword arguments ``keys_left`` and ``keys_right`` to the table ``join`` + function to support joining tables on key columns with different names. In + addition the new keywords can accept a list of column-like objects which are + used as the match keys. This allows joining on arbitrary data which are not part + of the tables being joined. [#11954] + +- Formatting of any numerical values in the output of ``Table.info()`` and + ``Column.info()`` has been improved. [#12022] + +- It is now possible to add dask arrays as columns in tables + and have them remain as dask arrays rather than be converted + to Numpy arrays. [#12219] + +- Added a new registry for mixin handlers, which can be used + to automatically convert array-like Python objects into + mixin columns when assigned to a table column. [#12219] + +astropy.time +^^^^^^^^^^^^ + +- Adds a new method ``earth_rotation_angle`` to calculate the Local Earth Rotation Angle. + Also adjusts Local Sidereal Time for the Terrestrial Intermediate Origin (``TIO``) + and adds a rigorous correction for polar motion. The ``TIO`` adjustment is approximately + 3 microseconds per century from ``J2000`` and the polar motion correction is at most + about +/-50 nanoseconds. For models ``IAU1982`` and ``IAU1994``, no such adjustments are + made as they pre-date the TIO concept. [#11680] + +astropy.timeseries +^^^^^^^^^^^^^^^^^^ + +- A custom binning scheme is now available in ``aggregate_downsample``. + It allows ``time_bin_start`` and ``time_bin_size`` to be arrays, and adds + an optional ``time_bin_end``. + This scheme mirrors the API for ``BinnedTimeSeries``. [#11266] + +astropy.units +^^^^^^^^^^^^^ + +- ``Quantity`` gains a ``__class_getitem__`` to create unit-aware annotations + with the syntax ``Quantity[unit or physical_type, shape, numpy.dtype]``. + If the python version is 3.9+ or ``typing_extensions`` is installed, + these are valid static type annotations. [#10662] + +- Each physical type is added to ``astropy.units.physical`` + (e.g., ``physical.length`` or ``physical.electrical_charge_ESU``). + The attribute-accessible names (underscored, without parenthesis) also + work with ``astropy.units.physical.get_physical_type``. [#11691] + +- It is now possible to have quantities based on structured arrays in + which the unit has matching structure, giving each field its own unit, + using units constructed like ``Unit('AU,AU/day')``. [#11775] + +- The milli- prefix has been added to ``astropy.units.Angstrom``. [#11788] + +- Added attributes ``base``, ``coords``, and ``index`` and method ``copy()`` to + ``QuantityIterator`` to match ``numpy.ndarray.flatiter``. [#11796] + +- Added "angular frequency" and "angular velocity" as aliases for the "angular + speed" physical type. [#11865] + +- Add light-second to units of length [#12128] + +astropy.utils +^^^^^^^^^^^^^ + +- The ``astropy.utils.deprecated_renamed_argument()`` decorator now supports + custom warning messages. [#12305] + +- The NaN-aware numpy functions such as ``np.nansum`` now work on Masked + arrays, with masked values being treated as NaN, but without raising + warnings or exceptions. [#12454] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Added a feature so that SphericalCircle will accept center parameter as a SkyCoord object. [#11790] + +astropy.wcs +^^^^^^^^^^^ + +- ``astropy.wcs.utils.obsgeo_to_frame`` has been added to convert the obsgeo coordinate + array on ``astropy.wcs.WCS`` objects to an ``ITRS`` coordinate frame instance. [#11716] + +- Updated bundled ``WCSLIB`` to version 7.7 with several bugfixes. [#12034] + + +API Changes +----------- + +astropy.config +^^^^^^^^^^^^^^ + +- ``update_default_config`` and ``ConfigurationMissingWarning`` are deprecated. [#11502] + +astropy.constants +^^^^^^^^^^^^^^^^^ + +- Removed deprecated ``astropy.constants.set_enabled_constants`` context manager. [#12105] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Positions for the Moon using the 'builtin' ephemeris now use the new + ``erfa.moon98`` function instead of our own implementation of the Meeus + algorithm. As this also corrects a misunderstanding of the frame returned by + the Meeus, this improves the agreement with the JPL ephemeris from about 30 to + about 6 km rms. [#11753] + +- Removed deprecated ``representation`` attribute from + ``astropy.coordinates.BaseCoordinateFrame`` class. [#12257] + +- ``SpectralQuantity`` and ``SpectralCoord`` ``.to_value`` method can now be called without + ``unit`` argument in order to maintain a consistent interface with ``Quantity.to_value`` [#12440] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- ``z_at_value`` now works with arrays for all arguments (except ``func``, + ``verbose``, and ``method``). Consequently, ``coordinates.Distance.z`` can + be used when Distance is an array. [#11778] + +- Remove deprecation warning and error remapping in ``Cosmology.clone``. + Now unknown arguments will raise a ``TypeError``, not an ``AttributeError``. [#11785] + +- The ``read/write`` and ``to/from_format`` Unified I/O registries are separated + and apply only to ``Cosmology``. [#12015] + +- Cosmology parameters in ``cosmology.parameters.py`` now have units, + where applicable. [#12116] + +- The function ``astropy.cosmology.utils.inf_like()`` is deprecated. [#12175] + +- The function ``astropy.cosmology.utils.vectorize_if_needed()`` is deprecated. + A new function ``astropy.cosmology.utils.vectorize_redshift_method()`` is added + as replacement. [#12176] + +- Cosmology base class constructor now only accepts arguments ``name`` and ``meta``. + Subclasses should add relevant arguments and not pass them to the base class. [#12191] + +astropy.io +^^^^^^^^^^ + +- When ``astropy`` raises an ``OSError`` because a file it was told to write + already exists, the error message now always suggests the use of the + ``overwrite=True`` argument. The wording is now consistent for all I/O formats. [#12179] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Removed deprecated ``overwrite=None`` option for + ``astropy.io.ascii.ui.write()``. Overwriting existing files now only happens if + ``overwrite=True``. [#12171] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- The internal class _CardAccessor is no longer registered as a subclass of + the Sequence or Mapping ABCs. [#11923] + +- The deprecated ``clobber`` argument will be removed from the + ``astropy.io.fits`` functions in version 5.1, and the deprecation warnings now + announce that too. [#12311] + +astropy.io.registry +^^^^^^^^^^^^^^^^^^^ + +- The ``write`` function now is allowed to return possible content results, which + means that custom writers could, for example, create and return an instance of + some container class rather than a file on disk. [#11916] + +- The registry functions are refactored into a class-based system. + New Read-only, write-only, and read/write registries can be created. + All functions accept a new argument ``registry``, which if not specified, + defaults to the global default registry. [#12015] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Deprecated the ``pedantic`` keyword argument in the + ``astropy.io.votable.table.parse`` function and the corresponding configuration + setting. It has been replaced by the ``verify`` option. [#12129] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Refactored how ``astropy.modeling.Model`` handles model evaluation in order to better + organize the code. [#11931] + +- Removed the following deprecated modeling features: + ``astropy.modeling.utils.ExpressionTree`` class, + ``astropy.modeling.functional_models.MexicanHat1D`` model, + ``astropy.modeling.functional_models.MexicanHat2D`` model, + ``astropy.modeling.core.Model.inputs`` setting in model initialize, + ``astropy.modeling.core.CompoundModel.inverse`` setting in model initialize, and + ``astropy.modeling.core.CompoundModel.both_inverses_exist()`` method. [#11978] + +- Deprecated the ``AliasDict`` class in ``modeling.utils``. [#12411] + +astropy.nddata +^^^^^^^^^^^^^^ + +- Removed ``block_reduce`` and ``block_replicate`` functions from + ``nddata.utils``. These deprecated functions in ``nddata.utils`` were + moved to ``nddata.blocks``. [#12288] + +astropy.stats +^^^^^^^^^^^^^ + +- Removed the following deprecated features from ``astropy.stats``: + + * ``conf`` argument for ``funcs.binom_conf_interval()`` and + ``funcs.binned_binom_proportion()``, + * ``conflevel`` argument for ``funcs.poisson_conf_interval()``, and + * ``conf_lvl`` argument for ``jackknife.jackknife_stats()``. [#12200] + +astropy.table +^^^^^^^^^^^^^ + +- Printing a ``Table`` now shows the qualified class name of mixin columns in the + dtype header row instead of "object". This applies to all the ``Table`` formatted output + methods whenever ``show_dtype=True`` is selected. [#11660] + +- The 'overwrite' argument has been added to the jsviewer table writer. + Overwriting an existing file requires 'overwrite' to be True. [#11853] + +- The 'overwrite' argument has been added to the pandas table writers. + Overwriting an existing file requires 'overwrite' to be True. [#11854] + +- The table ``join`` function now accepts only the first four arguments ``left``, + ``right``, ``keys``, and ``join_type`` as positional arguments. All other + arguments must be supplied as keyword arguments. [#11954] + +- Adding a dask array to a Table will no longer convert + that dask to a Numpy array, so accessing t['dask_column'] + will now return a dask array instead of a Numpy array. [#12219] + +astropy.time +^^^^^^^^^^^^ + +- Along with the new method ``earth_rotation_angle``, ``sidereal_time`` now accepts + an ``EarthLocation`` as the ``longitude`` argument. [#11680] + +astropy.units +^^^^^^^^^^^^^ + +- Unit ``littleh`` and equivalency ``with_H0`` have been moved to the + ``cosmology`` module and are deprecated from ``astropy.units``. [#12092] + +astropy.utils +^^^^^^^^^^^^^ + +- ``astropy.utils.introspection.minversion()`` now uses + ``importlib.metadata.version()``. Therefore, its ``version_path`` keyword is no + longer used and deprecated. This keyword will be removed in a future release. [#11714] + +- Updated ``utils.console.Spinner`` to better resemble the API of + ``utils.console.ProgressBar``, including an ``update()`` method and + iterator support. [#11772] + +- Removed deprecated ``check_hashes`` in ``check_download_cache()``. The function also + no longer returns anything. [#12293] + +- Removed unused ``download_cache_lock_attempts`` configuration item in + ``astropy.utils.data``. Deprecation was not possible. [#12293] + +- Removed deprecated ``hexdigest`` keyword from ``import_file_to_cache()``. [#12293] + +- Setting ``remote_timeout`` configuration item in ``astropy.utils.data`` to 0 will + no longer disable download from the Internet; Set ``allow_internet`` configuration + item to ``False`` instead. [#12293] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Removed deprecated ``imshow_only_kwargs`` keyword from ``imshow_norm``. [#12290] + +astropy.wcs +^^^^^^^^^^^ + +- Move complex logic from ``HighLevelWCSMixin.pixel_to_world`` and + ``HighLevelWCSMixin.world_to_pixel`` into the helper functions + ``astropy.wcs.wcsapi.high_level_api.high_level_objects_to_values`` and + ``astropy.wcs.wcsapi.high_level_api.values_to_high_level_objects`` to allow + reuse in other places. [#11950] + + +Bug Fixes +--------- + +astropy.config +^^^^^^^^^^^^^^ + +- ``generate_config`` no longer outputs wrong syntax for list type. [#12037] + +astropy.constants +^^^^^^^^^^^^^^^^^ + +- Fixed a bug where an older constants version cannot be set directly after + astropy import. [#12084] + +astropy.convolution +^^^^^^^^^^^^^^^^^^^ + +- Passing an ``array`` argument for any Kernel1D or Kernel2D subclasses (with the + exception of CustomKernel) will now raise a ``TypeError``. [#11969] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- If a ``Table`` containing a ``SkyCoord`` object as a column is written to a + FITS, ECSV or HDF5 file then any velocity information that might be present + will be retained. [#11750] + +- The output of ``SkyCoord.apply_space_motion()`` now always has the same + differential type as the ``SkyCoord`` itself. [#11932] + +- Fixed bug where Angle, Latitude and Longitude with NaN values could not be printed. [#11943] + +- Fixed a bug with the transformation from ``PrecessedGeocentric`` to ``GCRS`` + where changes in ``obstime``, ``obsgeoloc``, or ``obsgeovel`` were ignored. + This bug would also affect loopback transformations from one ``PrecessedGeocentric`` + frame to another ``PrecessedGeocentric`` frame. [#12152] + +- Fixed a bug with the transformations between ``TEME`` and ``ITRS`` or between ``TEME`` + and itself where a change in ``obstime`` was ignored. [#12152] + +- Avoid unnecessary transforms through CIRS for AltAz and HADec and + use ICRS as intermediate frame for these transformations instead. [#12203] + +- Fixed a bug where instantiating a representation with a longitude component + could mutate input provided for that component even when copying is specified. [#12307] + +- Wrapping an ``Angle`` array will now ignore NaN values instead of attempting to wrap + them, which would produce unexpected warnings/errors when working with coordinates + and representations due to internal broadcasting. [#12317] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- Dictionaries for in-built cosmology realizations are not altered by creating + the realization and are also made immutable. [#12278] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Prevent zero-byte writes for FITS binary tables to + speed up writes on the Lustre filesystem. [#11955] + +- Enable ``json.dump`` for FITS_rec with variable length (VLF) arrays. [#11957] + +- Add support for reading and writing int8 images [#11996] + +- Ensure header passed to ``astropy.io.fits.CompImageHDU`` does not need to contain + standard cards that can be automatically generated, such as ``BITPIX`` and ``NAXIS``. [#12061] + +- Fixed a bug where ``astropy.io.fits.HDUDiff`` would ignore the ``ignore_blank_cards`` + keyword argument. [#12122] + +- Open uncompressed file even if extension says it's compressed [#12135] + +- Fix the computation of the DATASUM in a ``CompImageHDU`` when the data is >1D. [#12138] + +- Reading files where the SIMPLE card is present but with an invalid format now + issues a warning instead of raising an exception [#12234] + +- Convert UNDEFINED to None when iterating over card values. [#12310] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Update ASDF tag versions in ExtensionType subclasses to match ASDF Standard 1.5.0. [#11986] + +- Fix ASDF serialization of model inputs and outputs and add relevant assertion to + test helper. [#12381] + +- Fix bug preventing ASDF serialization of bounding box for models with only one input. [#12385] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Now accepting UCDs containing phot.color. [#11982] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Added ``Parameter`` descriptions to the implemented models which were + missing. [#11232] + +- The ``separable`` property is now correctly set on models constructed with + ``astropy.modeling.custom_model``. [#11744] + +- Minor bugfixes and improvements to modeling including the following: + * Fixed typos and clarified several errors and their messages throughout + modeling. + * Removed incorrect try/except blocks around scipy code in + ``convolution.py`` and ``functional_models.py``. + * Fixed ``Ring2D`` model's init to properly accept all combinations + of ``r_in``, ``r_out``, and ``width``. + * Fixed bug in ``tau`` validator for the ``Logarithmic1D`` and + ``Exponential1D`` models when using them as model sets. + * Fixed ``copy`` method for ``Parameter`` in order to prevent an + automatic ``KeyError``, and fixed ``bool`` for ``Parameter`` so + that it functions with vector values. + * Removed unreachable code from ``Parameter``, the ``_Tabular`` model, + and the ``Drude1D`` model. + * Fixed validators in ``Drude1D`` model so that it functions in a + model set. + * Removed duplicated code from ``polynomial.py`` for handing of + ``domain`` and ``window``. + * Fixed the ``Pix2Sky_HEALPixPolar`` and ``Sky2Pix_HEALPixPolar`` modes + so that their ``evaluate`` and ``inverse`` methods actually work + without raising an error. [#12232] + +astropy.nddata +^^^^^^^^^^^^^^ + +- Ensure that the ``wcs=`` argument to ``NDData`` is always parsed into a high + level WCS object. [#11985] + +astropy.stats +^^^^^^^^^^^^^ + +- Fixed a bug in sigma clipping where the bounds would not be returned for + completely empty or masked data. [#11994] + +- Fixed a bug in ``biweight_midvariance`` and ``biweight_scale`` where + output data units would be dropped for constant data and where the + result was a scalar NaN. [#12146] + +astropy.table +^^^^^^^^^^^^^ + +- Ensured that ``MaskedColumn.info`` is propagated in all cases, so that when + tables are sliced, writing will still be as requested on + ``info.serialize_method``. [#11917] + +- ``table.conf.replace_warnings`` and ``table.jsviewer.conf.css_urls`` configuration + items now have correct ``'string_list'`` type. [#12037] + +- Fixed an issue where initializing from a list of dict-like rows (Mappings) did + not work unless the row values were instances of ``dict``. Now any object that + is an instance of the more general ``collections.abc.Mapping`` will work. [#12417] + +astropy.uncertainty +^^^^^^^^^^^^^^^^^^^ + +- Ensure that scalar ``QuantityDistribution`` unit conversion in ufuncs + works properly again. [#12471] + +astropy.units +^^^^^^^^^^^^^ + +- Add quantity support for ``scipy.special`` dimensionless functions + erfinv, erfcinv, gammaln and loggamma. [#10934] + +- ``VOUnit.to_string`` output is now compliant with IVOA VOUnits 1.0 standards. [#11565] + +- Units initialization with unicode has been expanded to include strings such as + 'M☉' and 'eâģ'. [#11827] + +- Give a more informative ``NotImplementedError`` when trying to parse a unit + using an output-only format such as 'unicode' or 'latex'. [#11829] + +astropy.utils +^^^^^^^^^^^^^ + +- Fixed a bug in ``get_readable_fileobj`` that prevented the unified file read + interface from closing ASCII files. [#11809] + +- The function ``astropy.utils.decorators.deprecated_attribute()`` no longer + ignores its ``message``, ``alternative``, and ``pending`` arguments. [#12184] + +- Ensure that when taking the minimum or maximum of a ``Masked`` array, + any masked NaN values are ignored. [#12454] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- The tick labelling for radians has been fixed to remove a redundant ``.0`` in + the label for integer multiples of pi at 2pi and above. [#12221] + +- Fix a bug where non-``astropy.wcs.WCS`` WCS instances were not accepted in + ``WCSAxes.get_transform``. [#12286] + +- Fix compatibility with Matplotlib 3.5 when using the ``grid_type='contours'`` + mode for drawing grid lines. [#12447] + +astropy.wcs +^^^^^^^^^^^ + +- Enabled ``SlicedLowLevelWCS.pixel_to_world_values`` to handle slices including + non-``int`` integers, e.g. ``numpy.int64``. [#11980] + + +Other Changes and Additions +--------------------------- + +- In docstrings, Sphinx cross-reference targets now use intersphinx, even if the + target is an internal link (``link`` is now ``'astropy:link``). + When built in Astropy these links are interpreted as internal links. When built + in affiliate packages, the link target is set by the key 'astropy' in the + intersphinx mapping. [#11690] + +- Made PyYaml >= 3.13 a strict runtime dependency. [#11903] + +- Minimum version of required Python is now 3.8. [#11934] + +- Minimum version of required Scipy is now 1.3. [#11934] + +- Minimum version of required Matplotlib is now 3.1. [#11934] + +- Minimum version of required Numpy is now 1.18. [#11935] + +- Fix deprecation warnings with Python 3.10 [#11962] + +- Speed up ``minversion()`` in cases where a module with a ``__version__`` + attribute is passed. [#12174] + +- ``astropy`` now requires ``packaging``. [#12199] + +- Updated the bundled CFITSIO library to 4.0.0. When compiling with an external + library, version 3.35 or later is required. [#12272] + + +Version 4.3.1 (2021-08-11) +========================== + +Bug Fixes +--------- + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- In ``fits.io.getdata`` do not fall back to first non-primary extension when + user explicitly specifies an extension. [#11860] + +- Ensure multidimensional masked columns round-trip properly to FITS. [#11911] + +- Ensure masked times round-trip to FITS, even if multi-dimensional. [#11913] + +- Raise ``ValueError`` if an ``np.float32`` NaN/Inf value is assigned to a + header keyword. [#11922] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Fixed bug in ``fix_inputs`` handling of bounding boxes. [#11908] + +astropy.table +^^^^^^^^^^^^^ + +- Fix an error when converting to pandas any ``Table`` subclass that + automatically adds a table index when the table is created. An example is a + binned ``TimeSeries`` table. [#12018] + +astropy.units +^^^^^^^^^^^^^ + +- Ensure that unpickling quantities and units in new sessions does not change + hashes and thus cause problems with (de)composition such as getting different + answers from the ``.si`` attribute. [#11879] + +- Fixed cannot import name imperial from astropy.units namespace. [#11977] + +astropy.utils +^^^^^^^^^^^^^ + +- Ensure any ``.info`` on ``Masked`` instances is propagated correctly when + viewing or slicing. As a consequence, ``MaskedQuantity`` can now be correctly + written to, e.g., ECSV format with ``serialize_method='data_mask'``. [#11910] + + +Version 4.3 (2021-07-26) +======================== + +New Features +------------ + +astropy.convolution +^^^^^^^^^^^^^^^^^^^ + +- Change padding sizes for ``fft_pad`` in ``convolve_fft`` from powers of + 2 only to scipy-optimized numbers, applied separately to each dimension; + yielding some performance gains and avoiding potential large memory + impact for certain multi-dimensional inputs. [#11533] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Adds the ability to create topocentric ``CIRS`` frames. Using these, + ``AltAz`` calculations are now accurate down to the milli-arcsecond + level. [#10994] + +- Adds a direct transformation from ``ICRS`` to ``AltAz`` frames. This + provides a modest speedup of approximately 10 percent. [#11079] + +- Adds new ``WGS84GeodeticRepresentation``, ``WGS72GeodeticRepresentation``, + and ``GRS80GeodeticRepresentation``. These are mostly for use inside + ``EarthLocation`` but can also be used to convert between geocentric + (cartesian) and different geodetic representations directly. [#11086] + +- ``SkyCoord.guess_from_table`` now also searches for differentials in the table. + In addition, multiple regex matches can be resolved when they are exact + component names, e.g. having both columns “dec” and “pm_dec” no longer errors + and will be included in the SkyCoord. [#11417] + +- All representations now have a ``transform`` method, which allows them to be + transformed by a 3x3 matrix in a Cartesian basis. By default, transformations + are routed through ``CartesianRepresentation``. ``SphericalRepresentation`` and + ``PhysicssphericalRepresentation`` override this for speed and to prevent NaN + leakage from the distance to the angular components. + Also, the functions ``is_O3`` and ``is_rotation`` have been added to + ``matrix_utities`` for checking whether a matrix is in the O(3) group or is a + rotation (proper or improper), respectively. [#11444] + +- Moved angle formatting and parsing utilities to + ``astropy.coordinates.angle_formats``. + Added new functionality to ``astropy.coordinates.angle_utilities`` for + generating points on or in spherical surfaces, either randomly or on a grid. [#11628] + +- Added a new method to ``SkyCoord``, ``spherical_offsets_by()``, which is the + conceptual inverse of ``spherical_offsets_to()``: Given angular offsets in + longitude and latitude, this method returns a new coordinate with the offsets + applied. [#11635] + +- Refactor conversions between ``GCRS`` and ``CIRS,TETE`` for better accuracy + and substantially improved speed. [#11069] + +- Also refactor ``EarthLocation.get_gcrs`` for an increase in performance of + an order of magnitude, which enters as well in getting observed positions of + planets using ``get_body``. [#11073] + +- Refactored the usage of metaclasses in ``astropy.coordinates`` to instead use + ``__init_subclass__`` where possible. [#11090] + +- Removed duplicate calls to ```transform_to``` from ```match_to_catalog_sky``` + and ```match_to_catalog_3d```, improving their performance. [#11449] + +- The new DE440 and DE440s ephemerides are now available via shortcuts 'de440' + and 'de440s'. The DE 440s ephemeris will probably become the default + ephemeris when choosing 'jpl' in 5.0. [#11601] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- Cosmology parameter dictionaries now also specify the Cosmology class to which + the parameters correspond. For example, the dictionary for + ``astropy.cosmology.parameters.Planck18`` has the added key-value pair + ("cosmology", "FlatLambdaCDM"). [#11530] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Added support for reading and writing ASCII tables in QDP (Quick and Dandy + Plotter) format. [#11256] + +- Added support for reading and writing multidimensional column data (masked and + unmasked) to ECSV. Also added formal support for reading and writing object-type + column data which can contain items consisting of lists, dicts, and basic scalar + types. This can be used to store columns of variable-length arrays. Both of + these features use JSON to convert the object to a string that is stored in the + ECSV output. [#11569, #11662, #11720] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Added ``append`` keyword to append table objects to an existing FITS file [#2632, #11149] + +- Check that the SIMPLE card is present when opening a file, to ensure that the + file is a valid FITS file and raise a better error when opening a non FITS + one. ``ignore_missing_simple`` can be used to skip this verification. [#10895] + +- Expose ``Header.strip`` as a public method, to remove the most common + structural keywords. [#11174] + +- Enable the use of ``os.PathLike`` objects when dealing with (mainly FITS) files. [#11580] + +astropy.io.registry +^^^^^^^^^^^^^^^^^^^ + +- Readers and writers can now set a priority, to assist with resolving which + format to use. [#11214] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Version 1.4 VOTables now use the VOUnit format specification. [#11032] + +- When reading VOTables using the Unified File Read/Write Interface (i.e. using + the ``Table.read()`` or ``QTable.read()`` functions) it is now possible to + specify all keyword arguments that are valid for + ``astropy.io.votable.table.parse()``. [#11643] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Added a state attribute to models to allow preventing the syncing of + constraint values from the constituent models. This syncing can + greatly slow down fitting if there are large numbers of fit parameters. + model.sync_constraints = True means check constituent model constraints + for compound models every time the constraint is accessed, False, do not. + Fitters that support constraints will set this to False on the model copy + and then set back to True when the fit is complete before returning. [#11365] + +- The ``convolve_models_fft`` function implements model convolution so that one + insures that the convolution remains consistent across multiple different + inputs. [#11456] + +astropy.nddata +^^^^^^^^^^^^^^ + +- Prevent unnecessary copies of the data during ``NDData`` arithmetic when units + need to be added. [#11107] + +- NDData str representations now show units, if present. [#11553] + +astropy.stats +^^^^^^^^^^^^^ + +- Added the ability to specify stdfunc='mad_std' when doing sigma clipping, + which will use a built-in function and lead to significant performance + improvements if cenfunc is 'mean' or 'median'. [#11664] + + +- Significantly improved the performance of sigma clipping when cenfunc and + stdfunc are passed as strings and the ``grow`` option is not used. [#11219] + +- Improved performance of ``bayesian_blocks()`` by removing one ``np.log()`` + call [#11356] + +astropy.table +^^^^^^^^^^^^^ + +- Add table attributes to include or exclude columns from the output when + printing a table. This functionality includes a context manager to + include/exclude columns temporarily. [#11190] + +- Improved the string representation of objects related to ``Table.indices`` so + they now indicate the object type and relevant attributes. [#11333] + +astropy.timeseries +^^^^^^^^^^^^^^^^^^ + +- An exception is raised when ``n_bins`` is passed as an argument while + any of the parameters ``time_bin_start`` or ``time_bin_size`` is not + scalar. [#11463] + +astropy.units +^^^^^^^^^^^^^ + +- The ``physical_type`` attributes of each unit are now objects of the (new) + ``astropy.units.physical.PhysicalType`` class instead of strings and the + function ``astropy.units.physical.get_physical_type`` can now translate + strings to these objects. [#11204] + +- The function ``astropy.units.physical.def_physical_type`` was created to + either define entirely new physical types, or to add more physical type + names to an existing physical types. [#11204] + +- ``PhysicalType``'s can be operated on using operations multiplication, + division, and exponentiation are to facilitate dimensional analysis. [#11204] + +- It is now possible to define aliases for units using + ``astropy.units.set_enabled_aliases``. This can be used when reading files + that have misspelled units. [#11258] + +- Add a new "DN" unit, ``units.dn`` or ``units.DN``, representing data number + for a detector. [#11591] + +astropy.utils +^^^^^^^^^^^^^ + +- Added ``ssl_context`` and ``allow_insecure`` options to ``download_file``, + as well as the ability to optionally use the ``certifi`` package to provide + root CA certificates when downloading from sites secured with + TLS/SSL. [#10434] + +- ``astropy.utils.data.get_pkg_data_path`` is publicly scoped (previously the + private function ``_find_pkg_data_path``) for obtaining file paths without + checking if the file/directory exists, as long as the package and module + do. [#11006] + +- Deprecated ``astropy.utils.OrderedDescriptor`` and + ``astropy.utils.OrderedDescriptorContainer``, as new features in Python 3 + make their use less compelling. [#11094, #11099] + +- ``astropy.utils.masked`` provides a new ``Masked`` class/factory that can be + used to represent masked ``ndarray`` and all its subclasses, including + ``Quantity`` and its subclasses. These classes can be used inside + coordinates, but the mask is not yet exposed. Generally, the interface should + be considered experimental. [#11127, #11792] + +- Add new ``utils.parsing`` module to with helper wrappers around + ``ply``. [#11227] + +- Change the Time and IERS leap second handling so that the leap second table is + updated only when a Time transform involving UTC is performed. Previously this + update check was done the first time a ``Time`` object was created, which in + practice occurred when importing common astropy subpackages like + ``astropy.coordinates``. Now you can prevent querying internet resources (for + instance on a cluster) by setting ``iers.conf.auto_download = False``. This + can be done after importing astropy but prior to performing any ``Time`` + scale transformations related to UTC. [#11638] + + +- Added a new module at ``astropy.utils.compat.optional_deps`` to consolidate + the definition of ``HAS_x`` optional dependency flag variables, + like ``HAS_SCIPY``. [#11490] + +astropy.wcs +^^^^^^^^^^^ + +- Add IVOA UCD mappings for some FITS WCS keywords commonly used in solar + physics. [#10965] + +- Add ``STOKES`` FITS WCS keyword to the IVOA UCD mapping. [#11236] + +- Updated bundled version of WCSLIB to version 7.6. See + https://www.atnf.csiro.au/people/mcalabre/WCS/CHANGES for a list of + included changes. [#11549] + + +API Changes +----------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- For input to representations, subclasses of the class required for a + given attribute will now be allowed in. [#11113] + +- Except for ``UnitSphericalRepresentation``, shortcuts in representations now + allow for attached differentials. [#11467] + +- Allow coordinate name strings as input to + ``SkyCoord.is_transformable_to``. [#11552] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- Change ``z_at_value`` to use ``scipy.optimize.minimize_scalar`` with default + method ``Brent`` (other options ``Bounded`` and ``Golden``) and accept + ``bracket`` option to set initial search region. [#11080] + +- Clarified definition of inputs to ``angular_diameter_distance_z1z2``. + The function now emits ``AstropyUserWarning`` when ``z2`` is less than + ``z1``. [#11197] + +- Split cosmology realizations from core classes, moving the former to new file + ``realizations``. [#11345] + +- Since cosmologies are immutable, the initialization signature and values can + be stored, greatly simplifying cloning logic and extending it to user-defined + cosmology classes that do not have attributes with the same name as each + initialization argument. [#11515] + +- Cloning a cosmology with changed parameter(s) now appends "(modified)" to the + new instance's name, unless a name is explicitly passed to ``clone``. [#11536] + +- Allow ``m_nu`` to be input as any quantity-like or array-like -- Quantity, + array, float, str, etc. Input is passed to the Quantity constructor and + converted to eV, still with the prior mass-energy equivalence + enabled. [#11640] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- For conversion between FITS tables and astropy ``Table``, the standard mask + values of ``NaN`` for float and null string for string are now properly + recognized, leading to a ``MaskedColumn`` with appropriately set mask + instead of a ``Column`` with those values exposed. Conversely, when writing + an astropy ``Table`` to a FITS tables, masked values are now consistently + converted to the standard FITS mask values of ``NaN`` for float and null + string for string (i.e., not just for tables with ``masked=True``, which no + longer is guaranteed to signal the presence of ``MaskedColumn``). [#11222] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- The use of ``version='1.0'`` is now fully deprecated in constructing + a ``astropy.io.votable.tree.VOTableFile``. [#11659] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Removed deprecated ``astropy.modeling.blackbody`` module. [#10972] + +astropy.table +^^^^^^^^^^^^^ + +- Added ``Column.value`` as an alias for the existing ``Column.data`` attribute. + This makes accessing a column's underlying data array consistent with the + ``.value`` attribute available for ``Time`` and ``Quantity`` objects. [#10962] + +- In reading from a FITS tables, the standard mask values of ``NaN`` for float + and null string for string are properly recognized, leading to a + ``MaskedColumn`` with appropriately set mask. [#11222] + +- Changed the implementation of the ``table.index.Index`` class so instantiating + from this class now returns an ``Index`` object as expected instead of a + ``SlicedIndex`` object. [#11333] + +astropy.units +^^^^^^^^^^^^^ + +- The ``physical_type`` attribute of units now returns an instance of + ``astropy.units.physical.PhysicalType`` instead of a string. Because + ``PhysicalType`` instances can be compared to strings, no code changes + should be necessary when making comparisons. The string representations + of different physical types will differ from previous releases. [#11204] + +- Calling ``Unit()`` with no argument now returns a dimensionless unit, + as was documented but not implemented. [#11295] + +astropy.utils +^^^^^^^^^^^^^ + +- Removed deprecated ``utils.misc.InheritDocstrings`` and ``utils.timer``. [#10281] + +- Removed usage of deprecated ``ipython`` stream in ``utils.console``. [#10942] + +astropy.wcs +^^^^^^^^^^^ + +- Deprecate ``accuracy`` argument in ``all_world2pix`` which was mistakenly + *documented*, in the case ``accuracy`` was ever used. [#11055] + + +Bug Fixes +--------- + +astropy.convolution +^^^^^^^^^^^^^^^^^^^ + +- Fixes for ``convolve_fft`` documentation examples. [#11510] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Allow ``Distance`` instances with negative distance values as input for + ``SphericalRepresentation``. This was always noted as allowed in an + exception message when a negative ``Quantity`` with length units was + passed in, but was not actually possible to do. [#11113] + +- Makes the ``Angle.to_string`` method to follow the format described in the + docstring with up to 8 significant decimals instead of 4. [#11153] + +- Ensure that proper motions can be calculated when converting a ``SkyCoord`` + with cartesian representation to unit-spherical, by fixing the conversion of + ``CartesianDifferential`` to ``UnitSphericalDifferential``. [#11469] + +- When re-representing coordinates from spherical to unit-spherical and vice + versa, the type of differential will now be preserved. For instance, if only a + radial velocity was present, that will remain the case (previously, a zero + proper motion component was added). [#11482] + +- Ensure that wrapping of ``Angle`` does not raise a warning even if ``nan`` are + present. Also try to make sure that the result is within the wrapped range + even in the presence of rounding errors. [#11568] + +- Comparing a non-SkyCoord object to a ``SkyCoord`` using ``==`` no longer + raises an error. [#11666] + +- Different ``SkyOffsetFrame`` classes no longer interfere with each other, + causing difficult to debug problems with the ``origin`` attribute. The + ``origin`` attribute now no longer is propagated, so while it remains + available on a ``SkyCoord`` that is an offset, it no longer is available once + that coordinate is transformed to another frame. [#11730] [#11730] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- Cosmology instance names are now immutable. [#11535] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Fixed bug where writing a table that has comments defined (via + ``tbl.meta['comments']``) with the 'csv' format was failing. Since the + formally defined CSV format does not support comments, the comments are now + just ignored unless ``comment=`` is supplied to the + ``write()`` call. [#11475] + +- Fixed the issue where the CDS reader failed to treat columns + as nullable if the ReadMe file contains a limits specifier. [#11531] + +- Made sure that the CDS reader does not ignore an order specifier that + may be present after the null specifier '?'. Also made sure that it + checks null values only when an '=' symbol is present and reads + description text even if there is no whitespace after '?'. [#11593] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix ``ColDefs.add_col/del_col`` to allow in-place addition or removal of + a column. [#11338] + +- Fix indexing of ``fits.Header`` with Numpy integers. [#11387] + +- Do not delete ``EXTNAME`` for compressed image header if a default and + non-default ``EXTNAME`` are present. [#11396] + +- Prevent warnings about ``HIERARCH`` with ``CompImageHeader`` class. [#11404] + +- Fixed regression introduced in Astropy 4.0.5 and 4.2.1 with verification of + FITS headers with HISTORY or COMMENT cards with long (> 72 characters) + values. [#11487] + +- Fix reading variable-length arrays when there is a gap between the data and the + heap. [#11688] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- ``NumericArray`` converter now properly broadcasts scalar mask to array. [#11157] + +- VOTables are now written with the correct namespace and schema location + attributes. [#11659] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Fixes the improper propagation of ``bounding_box`` from + ``astropy.modeling.models`` to their inverses. For cases in which the inverses + ``bounding_box`` can be determined, the proper calculation has been + implemented. [#11414] + +- Bugfix to allow rotation models to accept arbitrarily-shaped inputs. [#11435] + +- Bugfixes for ``astropy.modeling`` to allow ``fix_inputs`` to accept empty + dictionaries and dictionaries with ``numpy`` integer keys. [#11443] + +- Bugfix for how ``SPECIAL_OPERATORS`` are handled. [#11512] + +- Fixes ``Model`` crashes when some inputs are scalars and during some types of + output reshaping. [#11548] + +- Fixed bug in ``LevMarLSQFitter`` when using weights and vector inputs. [#11603] + +astropy.stats +^^^^^^^^^^^^^ + +- Fixed a bug with the ``copy=False`` option when carrying out sigma + clipping - previously if ``masked=False`` this still copied the data, + but this will now change the array in-place. [#11219] + +astropy.table +^^^^^^^^^^^^^ + +- Ensure that adding a ``Quantity`` or other mixin column to a ``Table`` + does not have side effects, such as creating an associated ``info`` + instance (which would lead to slow-down of, e.g., slicing afterwards). [#11077] + +- When writing to a FITS tables, masked values are again always converted to + the standard FITS mask values of ``NaN`` for float and null string + for string, not just for table with ``masked=True``. [#11222] + +- Using ``Table.to_pandas()`` on an indexed ``Table`` with masked integer values + now correctly construct the ``pandas.DataFrame``. [#11432] + +- Fixed ``Table`` HTML representation in Jupyter notebooks so that it is + horizontally scrollable within Visual Studio Code. This was done by wrapping + the ```` in a ``
`` element. [#11476] + +- Fix a bug where a string-valued ``Column`` that happened to have a ``unit`` + attribute could not be added to a ``QTable``. Such columns are now simply + kept as ``Column`` instances (with a warning). [#11585] + +- Fix an issue in ``Table.to_pandas(index=)`` where the index column name + was not being set properly for the ``DataFrame`` index. This was introduced by + an API change in pandas version 1.3.0. Previously when creating a ``DataFrame`` + with the index set to an astropy ``Column``, the ``DataFrame`` index name was + automatically set to the column name. [#11921] + +astropy.time +^^^^^^^^^^^^ + +- Fix a thread-safety issue with initialization of the leap-second table + (which is only an issue when ERFA's built-in table is out of date). [#11234] + +- Fixed converting a zero-length time object from UTC to + UT1 when an empty array is passed. [#11516] + +astropy.uncertainty +^^^^^^^^^^^^^^^^^^^ + +- ``Distribution`` instances can now be used as input to ``Quantity`` to + initialize ``QuantityDistribution``. Hence, ``distribution * unit`` + and ``distribution << unit`` will work too. [#11210] + +astropy.units +^^^^^^^^^^^^^ + +- Move non-astronomy units from astrophys.py to a new misc.py file. [#11142] + +- The physical type of ``astropy.units.mol / astropy.units.m ** 3`` is now + defined as molar concentration. It was previously incorrectly defined + as molar volume. [#11204] + +- Make ufunc helper lookup thread-safe. [#11226] + +- Make ``Unit`` string parsing (as well as ``Angle`` parsing) thread-safe. [#11227] + +- Decorator ``astropy.units.decorators.quantity_input`` now only evaluates + return type annotations based on ``UnitBase`` or ``FunctionUnitBase`` types. + Other annotations are skipped over and are not attempted to convert to the + correct type. [#11506] + +astropy.utils +^^^^^^^^^^^^^ + +- Make ``lazyproperty`` and ``classdecorator`` thread-safe. This should fix a + number of thread safety issues. [#11224] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Fixed a bug that resulted in some parts of grid lines being visible when they + should have been hidden. [#11380] + +- Fixed a bug that resulted in ``time_support()`` failing for intervals of + a few months if one of the ticks was the month December. [#11615] + +astropy.wcs +^^^^^^^^^^^ + +- ``fit_wcs_from_points`` now produces a WCS with integer ``NAXIXn`` + values. [#10865] + +- Updated bundled version of ``WCSLIB`` to v7.4, fixing a bug that caused + the coefficients of the TPD distortion function to not be written to the + header. [#11260] + +- Fixed a bug in assigning type when converting ``colsel`` to + ``numpy.ndarray``. [#11431] + +- Added ``WCSCOMPARE_*`` constants to the list of WCSLIB constants + available/exposed through the ``astropy.wcs`` module. [#11647] + +- Fix a bug that caused APE 14 WCS transformations for FITS WCS with ZOPT, BETA, + VELO, VOPT, or VRAD CTYPE to not work correctly. [#11781] + + +Other Changes and Additions +--------------------------- + +- The configuration file is no longer created by default when importing astropy + and its existence is no longer required. Affiliated packages should update their + ``__init__.py`` module to remove the block using ``update_default_config`` and + ``ConfigurationDefaultMissingWarning``. [#10877] + +- Replace ``pkg_resources`` (from setuptools) with ``importlib.metadata`` which + comes from the stdlib, except for Python 3.7 where the backport package is added + as a new dependency. [#11091] + +- Turn on numpydoc's ``numpydoc_xref_param_type`` to create cross-references + for the parameter types in the Parameters, Other Parameters, Returns and Yields + sections of the docstrings. [#11118] + +- Docstrings across the package are standardized to enable references. + Also added is an Astropy glossary-of-terms to define standard inputs, + e.g. ``quantity-like`` indicates an input that can be interpreted by + ``astropy.units.Quantity``. [#11118] + +- Binary wheels are now built to the manylinux2010 specification. These wheels + should be supported on all versions of pip shipped with Python 3.7+. [#11377] + +- The name of the default branch for the astropy git repository has been renamed + to ``main``, and the documentation and tooling has been updated accordingly. + If you have made a local clone you may wish to update it following the + instructions in the repository's README. [#11379] + +- Sphinx cross-reference link targets are added for every ``PhysicalType``. + Now in the parameter types in the Parameters, Other Parameters, Returns and + Yields sections of the docstring, the physical type of a quantity can be + annotated in square brackets. + E.g. ``distance : `~astropy.units.Quantity` ['length']`` [#11595] + +- The minimum supported version of ``ipython`` is now 4.2. [#10942] + +- The minimum supported version of ``pyerfa`` is now 1.7.3. [#11637] + + +Version 4.2.1 (2021-04-01) +========================== + +Bug Fixes +--------- + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- Fixed an issue where specializations of the comoving distance calculation + for certain cosmologies could not handle redshift arrays. [#10980] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix bug where manual fixes to invalid header cards were not preserved when + saving a FITS file. [#11108] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- ``NumericArray`` converter now properly broadcasts scalar mask to array. + [#11157] + +astropy.table +^^^^^^^^^^^^^ + +- Fix bug when initializing a ``Table`` subclass that uses ``TableAttribute``'s. + If the data were an instance of the table then attributes provided in the + table initialization call could be ignored. [#11217] + +astropy.time +^^^^^^^^^^^^ + +- Change epoch of ``TimeUnixTAI`` (``"unix_tai"``) from ``1970-01-01T00:00:00 UTC`` + to ``1970-01-01T00:00:00 TAI`` to match the intended and documented behaviour. + This essentially changes the resulting times by 8.000082 seconds, the initial + offset between TAI and UTC. [#11249] + +astropy.units +^^^^^^^^^^^^^ + +- Fixed a bug with the ``quantity_input`` decorator where allowing + dimensionless inputs for an argument inadvertently disabled any checking of + compatible units for that argument. [#11283] + +astropy.utils +^^^^^^^^^^^^^ + +- Fix a bug so that ``np.shape``, ``np.ndim`` and ``np.size`` again work on + classes that use ``ShapedLikeNDArray``, like representations, frames, + sky coordinates, and times. [#11133] + +astropy.wcs +^^^^^^^^^^^ + +- Fix error when a user defined ``proj_point`` parameter is passed to ``fit_wcs_from_points``. [#11139] + + +Other Changes and Additions +--------------------------- + + +- Change epoch of ``TimeUnixTAI`` (``"unix_tai"``) from ``1970-01-01T00:00:00 UTC`` + to ``1970-01-01T00:00:00 TAI`` to match the intended and documented behaviour. + This essentially changes the resulting times by 8.000082 seconds, the initial + offset between TAI and UTC. [#11249] + + +Version 4.2 (2020-11-24) +======================== + +New Features +------------ + +astropy.convolution +^^^^^^^^^^^^^^^^^^^ + +- Methods ``convolve`` and ``convolve_fft`` both now return Quantity arrays + if user input is given in one. [#10822] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Numpy functions that broadcast, change shape, or index (like + ``np.broadcast_to``, ``np.rot90``, or ``np.roll``) now work on + coordinates, frames, and representations. [#10337] + +- Add a new science state ``astropy.coordinates.erfa_astrom.erfa_astrom`` and + two classes ``ErfaAstrom``, ``ErfaAstromInterpolator`` as wrappers to + the ``pyerfa`` astrometric functions used in the coordinate transforms. + Using ``ErfaAstromInterpolator``, which interpolates astrometric properties for + ``SkyCoord`` instances with arrays of obstime, can dramatically speed up + coordinate transformations while keeping microarcsecond resolution. + Depending on needed precision and the obstime array in question, speed ups + reach factors of 10x to >100x. [#10647] + +- ``galactocentric_frame_defaults`` can now also be used as a registry, with + user-defined parameter values and metadata. [#10624] + +- Method ``.realize_frame`` from coordinate frames now accepts ``**kwargs``, + including ``representation_type``. [#10727] + +- Avoid an unnecessary call to ``erfa.epv00`` in transformations between + ``CIRS`` and ``ICRS``, improving performance by 50 %. [#10814] + +- A new equatorial coordinate frame, with RA and Dec measured w.r.t to the True + Equator and Equinox (TETE). This frame is commonly known as "apparent place" + and is the correct frame for coordinates returned from JPL Horizons. [#10867] + +- Added a context manager ``impose_finite_difference_dt`` to the + ``TransformGraph`` class to override the finite-difference time step + attribute (``finite_difference_dt``) for all transformations in the graph + with that attribute. [#10341] + +- Improve performance of ``SpectralCoord`` by refactoring internal + implementation. [#10398] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- The final version of the Planck 2018 cosmological parameters are included + as the ``Planck18`` object, which is now the default cosmology. The + parameters are identical to those of the ``Planck18_arXiv_v2`` object, + which is now deprecated and will be removed in a future release. [#10915] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Added NFW profile and tests to modeling package [#10505] + +- Added missing logic for evaluate to compound models [#10002] + +- Stop iteration in ``FittingWithOutlierRemoval`` before reaching ``niter`` if + the masked points are no longer changing. [#10642] + +- Keep a (shallow) copy of ``fit_info`` from the last iteration of the wrapped + fitter in ``FittingWithOutlierRemoval`` and also record the actual number of + iterations performed in it. [#10642] + +- Added attributes for fitting uncertainties (covariance matrix, standard + deviations) to models. Parameter covariance matrix can be accessed via + ``model.cov_matrix``, standard deviations by ``model.stds`` or individually + for each parameter by ``parameter.std``. Currently implemented for + ``LinearLSQFitter`` and ``LevMarLSQFitter``. [#10552] + +- N-dimensional least-squares statistic and specific 1,2,3-D methods [#10670] + +astropy.stats +^^^^^^^^^^^^^ + +- Added ``circstd`` function to obtain a circular standard deviation. [#10690] + +astropy.table +^^^^^^^^^^^^^ + +- Allow initializing a ``Table`` using a list of ``names`` in conjunction with + a ``dtype`` from a numpy structured array. The list of ``names`` overrides the + names specified in the ``dtype``. [#10419] + +astropy.time +^^^^^^^^^^^^ + +- Add new ``isclose()`` method to ``Time`` and ``TimeDelta`` classes to allow + comparison of time objects to within a specified tolerance. [#10646] + +- Improve initialization time by a factor of four when creating a scalar ``Time`` + object in a format like ``unix`` or ``cxcsec`` (time delta from a reference + epoch time). [#10406] + +- Improve initialization time by a factor of ~25 or more for large arrays of + string times in ISO, ISOT or year day-of-year formats. This is done with a new + C-based time parser that can be adapted for other fixed-format custom time + formats. [#10360] + +- Numpy functions that broadcast, change shape, or index (like + ``np.broadcast_to``, ``np.rot90``, or ``np.roll``) now work on times. + [#10337, #10502] + +astropy.timeseries +^^^^^^^^^^^^^^^^^^ + +- Improve memory and speed performance when iterating over the entire time + column of a ``TimeSeries`` object. Previously this involved O(N^2) operations + and memory. [#10889] + +astropy.units +^^^^^^^^^^^^^ + +- ``Quantity.to`` has gained a ``copy`` option to allow copies to be avoided + when the units do not change. [#10517] + +- Added the ``spat`` unit of solid angle that represents the full sphere. + [#10726] + +astropy.utils +^^^^^^^^^^^^^ + +- ``ShapedLikeNDArray`` has gained the capability to use numpy functions + that broadcast, change shape, or index. [#10337] + +- ``get_free_space_in_dir`` now takes a new ``unit`` keyword and + ``check_free_space_in_dir`` takes ``size`` defined as ``Quantity``. [#10627] + +- New ``astropy.utils.data.conf.allow_internet`` configuration item to + control downloading data from the Internet. Setting ``allow_internet=False`` + is the same as ``remote_timeout=0``. Using ``remote_timeout=0`` to control + internet access will stop working in a future release. [#10632] + +- New ``is_url`` function so downstream packages do not have to secretly use + the hidden ``_is_url`` anymore. [#10684] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Added the ``Quadrangle`` patch for ``WCSAxes`` for a latitude-longitude + quadrangle. Unlike ``matplotlib.patches.Rectangle``, the edges of this + patch will be rendered as curved lines if appropriate for the WCS + transformation. [#10862] + +- The position of tick labels are now only calculated when needed. If any text + parameters are changed (color, font weight, size etc.) that don't effect the + tick label position, the positions are not recomputed, improving performance. + [#10806] + +astropy.wcs +^^^^^^^^^^^ + +- ``WCS.to_header()`` now appends comments to SIP coefficients. [#10480] + +- A new property ``dropped_world_dimensions`` has been added to + ``SlicedLowLevelWCS`` to record information about any world axes removed by + slicing a WCS. [#10195] + +- New ``WCS.proj_plane_pixel_scales()`` and ``WCS.proj_plane_pixel_area()`` + methods to return pixel scales and area, respectively, as Quantity. [#10872] + + +API Changes +----------- + +astropy.config +^^^^^^^^^^^^^^ + +- ``set_temp_config`` now preserves the existing cache rather than deleting + it and relying on reloading it from the previous config file. This ensures + that any programmatically made changes are preserved as well. [#10474] + +- Configuration path detection logic has changed: Now, it looks for ``~`` first + before falling back to older logic. In addition, ``HOMESHARE`` is no longer + used in Windows. [#10705] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- The passing of frame classes (as opposed to frame instances) to the + ``transform_to()`` methods of low-level coordinate-frame classes has been + deprecated. Frame classes can still be passed to the ``transform_to()`` + method of the high-level ``SkyCoord`` class, and using ``SkyCoord`` is + recommended for all typical use cases of transforming coordinates. [#10475] + +astropy.stats +^^^^^^^^^^^^^ + +- Added a ``grow`` parameter to ``SigmaClip``, ``sigma_clip`` and + ``sigma_clipped_stats``, to allow expanding the masking of each deviant + value to its neighbours within a specified radius. [#10613] + +- Passing float ``n`` to ``poisson_conf_interval`` when using + ``interval='kraft-burrows-nousek'`` now raises ``TypeError`` as its value + must be an integer. [#10838] + +astropy.table +^^^^^^^^^^^^^ + +- Change ``Table.columns.keys()`` and ``Table.columns.values()`` to both return + generators instead of a list. This matches the behavior for Python ``dict`` + objects. [#10543] + +- Removed the ``FastBST`` and ``FastRBT`` indexing engines because they depend + on the ``bintrees`` package, which is no longer maintained and is deprecated. + Instead, use the ``SCEngine`` indexing engine, which is similar in + performance and relies on the ``sortedcontainers`` package. [#10622] + +- When slicing a mixin column in a table that had indices, the indices are no + longer copied since they generally are not useful, having the wrong shape. + With this, the behaviour becomes the same as that for a regular ``Column``. + (Note that this does not affect slicing of a table; sliced columns in those + will continue to carry a sliced version of any indices). [#10890] + +- Change behavior so that when getting a single item out of a mixin column such + as ``Time``, ``TimeDelta``, ``SkyCoord`` or ``Quantity``, the ``info`` + attribute is no longer copied. This improves performance, especially when the + object is an indexed column in a ``Table``. [#10889] + +- Raise a TypeError when a scalar column is added to an unsized table. [#10476] + +- The order of columns when creating a table from a ``list`` of ``dict`` may be + changed. Previously, the order was alphabetical because the ``dict`` keys + were assumed to be in random order. Since Python 3.7, the keys are always in + order of insertion, so ``Table`` now uses the order of keys in the first row + to set the column order. To alphabetize the columns to match the previous + behavior, use ``t = t[sorted(t.colnames)]``. [#10900] + +astropy.time +^^^^^^^^^^^^ + +- Refactor ``Time`` and ``TimeDelta`` classes to inherit from a common + ``TimeBase`` class. The ``TimeDelta`` class no longer inherits from ``Time``. + A number of methods that only apply to ``Time`` (e.g. ``light_travel_time``) + are no longer available in the ``TimeDelta`` class. [#10656] + +astropy.units +^^^^^^^^^^^^^ + +- The ``bar`` unit is no longer wrongly considered an SI unit, meaning that + SI decompositions like ``(u.kg*u.s**-2* u.sr**-1 * u.nm**-1).si`` will + no longer include it. [#10586] + +astropy.utils +^^^^^^^^^^^^^ + +- Shape-related items from ``astropy.utils.misc`` -- ``ShapedLikeNDArray``, + ``check_broadcast``, ``unbroadcast``, and ``IncompatibleShapeError`` -- + have been moved to their own module, ``astropy.utils.shapes``. They remain + importable from ``astropy.utils``. [#10337] + +- ``check_hashes`` keyword in ``check_download_cache`` is deprecated and will + be removed in a future release. [#10628] + +- ``hexdigest`` keyword in ``import_file_to_cache`` is deprecated and will + be removed in a future release. [#10628] + + +Bug Fixes +--------- + +astropy.config +^^^^^^^^^^^^^^ + +- Fix a few issues with ``generate_config`` when used with other packages. + [#10893] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Fixed a bug in the coordinate-frame attribute ``CoordinateAttribute`` where + the internal transformation could behave differently depending on whether + the input was a low-level coordinate frame or a high-level ``SkyCoord``. + ``CoordinateAttribute`` now always performs a ``SkyCoord``-style internal + transformation, including the by-default merging of frame attributes. [#10475] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Fixed an issue of ``Model.render`` when the input ``out`` datatype is not + float64. [#10542] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Fix support for referencing WCSAxes coordinates by their world axes names. + [#10484] + +astropy.wcs +^^^^^^^^^^^ + +- Objective functions called by ``astropy.wcs.fit_wcs_from_points`` were + treating longitude and latitude distances equally. Now longitude scaled + properly. [#10759] + + +Other Changes and Additions +--------------------------- + +- Minimum version of required Python is now 3.7. [#10900] + +- Minimum version of required Numpy is now 1.17. [#10664] + +- Minimum version of required Scipy is now 1.1. [#10900] + +- Minimum version of required PyYAML is now 3.13. [#10900] + +- Minimum version of required Matplotlib is now 3.0. [#10900] + +- The private ``_erfa`` module has been converted to its own package, + ``pyerfa``, which is a required dependency for astropy, and can be imported + with ``import erfa``. Importing ``_erfa`` from ``astropy`` will give a + deprecation warning. [#10329] + +- Added ``optimize=True`` flag to calls of ``yacc.yacc`` (as already done for + ``lex.lex``) to allow running in ``python -OO`` session without raising an + exception in ``astropy.units.format``. [#10379] + +- Shortened FITS comment strings for some D2IM and CPDIS FITS keywords to + reduce the number of FITS ``VerifyWarning`` warnings when working with WCSes + containing lookup table distortions. [#10513] + +- When importing astropy without first building the extension modules first, + raise an error directly instead of trying to auto-build. [#10883] + + + +Version 4.1 (2020-10-21) +======================== + +New Features +------------ + +astropy.config +^^^^^^^^^^^^^^ + +- Add new function ``generate_config`` to generate the configuration file and + include it in the documentation. [#10148] + +- ``ConfigNamespace.__iter__`` and ``ConfigNamespace.keys`` now yield ``ConfigItem`` + names defined within it. Similarly, ``items`` and ``values`` would yield like a + Python dictionary would. [#10139] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Added a new ``SpectralCoord`` class that can be used to define spectral + coordinates and transform them between different velocity frames. [#10185] + +- Angle parsing now supports ``cardinal direction`` in the cases + where angles are initialized as ``string`` instances. eg ``"17°53'27"W"``.[#9859] + +- Allow in-place modification of array-valued ``Frame`` and ``SkyCoord`` objects. + This provides limited support for updating coordinate data values from another + coordinate object of the same class and equivalent frame attributes. [#9857] + +- Added a robust equality operator for comparing ``SkyCoord``, frame, and + representation objects. A comparison like ``sc1 == sc2`` will now return a + boolean or boolean array where the objects are strictly equal in all relevant + frame attributes and coordinate representation values. [#10154] + +- Added the True Equator Mean Equinox (TEME) frame. [#10149] + +- The ``Galactocentric`` frame will now use the "latest" parameter definitions + by default. This currently corresponds to the values defined in v4.0, but will + change with future releases. [#10238] + +- The ``SkyCoord.from_name()`` and Sesame name resolving functionality now is + able to cache results locally and will do so by default. [#9162] + +- Allow in-place modification of array-valued ``Representation`` and ``Differential`` + objects, including of representations with attached differentials. [#10210] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Functional Units can now be processed in CDS-tables. [#9971] + +- Allow reading in ASCII tables which have duplicate column names. [#9939] + +- Fixed failure of ASCII ``fast_reader`` to handle ``names``, ``include_names``, + ``exclude_names`` arguments for ``RDB`` formatted tables. Homogenised checks + and exceptions for invalid ``names`` arguments. Improved performance when + parsing "wide" tables with many columns. [#10306] + +- Added type validation of key arguments in calls to ``io.ascii.read()`` and + ``io.ascii.write()`` functions. [#10005] + +astropy.io.misc +^^^^^^^^^^^^^^^ +- Added serialization of parameter constraints fixed and bounds. [#10082] + +- Added 'functional_models.py' and 'physical_models.py' to asdf/tags/transform, + with to allow serialization of all functional and physical models. [#10028, #10293] + +- Fix ASDF serialization of circular model inverses, and remove explicit calls + to ``asdf.yamlutil`` functions that became unnecessary in asdf 2.6.0. [#10189, #10384] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Added support for writing Dask arrays to disk efficiently for ``ImageHDU`` and + ``PrimaryHDU``. [#9742] + +- Add HDU name and ver to FITSDiff report where appropriate [#10197] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- New ``exceptions.conf.max_warnings`` configuration item to control the number of times a + type of warning appears before being suppressed. [#10152] + +- No longer ignore attributes whose values were specified as empty + strings. [#10583] + +astropy.modeling +^^^^^^^^^^^^^^^^ +- Added Plummer1D model to ``functional_models``. [#9896] + +- Added ``UnitsMapping`` model and ``Model.coerce_units`` to support units on otherwise + unitless models. [#9936] + +- Added ``domain`` and ``window`` attributes to ``repr`` and ``str``. Fixed bug with + ``_format_repr`` in core.py. [#9941] + +- Polynomial attributes ``domain`` and ``window`` are now tuples of size 2 and are + validated. `repr` and `print` show only their non-default values. [#10145] + +- Added ``replace_submodel()`` method to ``CompoundModel`` to modify an + existing instance. [#10176] + +- Delay construction of ``CompoundModel`` inverse until property is accessed, + to support ASDF deserialization of circular inverses in component models. [#10384] + +astropy.nddata +^^^^^^^^^^^^^^ + +- Added support in the ``bitmask`` module for using mnemonic bit flag names + when specifying the bit flags to be used or ignored when converting a bit + field to a boolean. [#10095, #10208] + +- Added ``reshape_as_blocks`` function to reshape a data array into + blocks, which is useful to efficiently apply functions on block + subsets of the data instead of using loops. The reshaped array is a + view of the input data array. [#10214] + +- Added a ``cache`` keyword option to allow caching for ``CCDData.read`` if + filename is a URL. [#10265] + +astropy.table +^^^^^^^^^^^^^ + +- Added ability to specify a custom matching function for table joins. In + particular this makes it possible to do cross-match table joins on ``SkyCoord``, + ``Quantity``, or standard columns, where column entries within a specified + distance are considered to be matched. [#10169] + +- Added ``units`` and ``descriptions`` keyword arguments to the Table object + initialization and ``Table.read()`` methods. This allows directly setting + the ``unit`` and ``description`` for the table columns at the time of + creating or reading the table. [#9671] + +- Make table ``Row`` work as mappings, by adding ``.keys()`` and ``.values()`` + methods. With this ``**row`` becomes possible, as does, more simply, turning + a ``Row`` into a dictionary with ``dict(row)``. [#9712] + +- Added two new ``Table`` methods ``.items()`` and ``.values()``, which return + respectively ``tbl.columns.items()`` (iterator over name, column tuples) and + ``tbl.columns.values()`` (list of columns) for a ``Table`` object ``tbl``. [#9780] + +- Added new ``Table`` method ``.round()``, which rounds numeric columns to the + specified number of decimals. [#9862] + +- Updated ``to_pandas()`` and ``from_pandas()`` to use and support Pandas + nullable integer data type for masked integer data. [#9541] + +- The HDF5 writer, ``write_table_hdf5()``, now allows passing through + additional keyword arguments to the ``h5py.Group.create_dataset()``. [#9602] + +- Added capability to add custom table attributes to a ``Table`` subclass. + These attributes are persistent and can be set during table creation. [#10097] + +- Added support for ``SkyCoord`` mixin columns in ``dstack``, ``vstack`` and + ``insert_row`` functions. [#9857] + +- Added support for coordinate ``Representation`` and ``Differential`` mixin + columns. [#10210] + +astropy.time +^^^^^^^^^^^^ + +- Added a new time format ``unix_tai`` which is essentially Unix time but with + leap seconds included. More precisely, this is the number of seconds since + ``1970-01-01 00:00:08 TAI`` and corresponds to the ``CLOCK_TAI`` clock + available on some linux platforms. [#10081] + +astropy.units +^^^^^^^^^^^^^ + +- Added ``torr`` pressure unit. [#9787] + +- Added the ``equal_nan`` keyword argument to ``isclose`` and ``allclose``, and + updated the docstrings. [#9849] + +- Added ``Rankine`` temperature unit. [#9916] + +- Added integrated flux unit conversion to ``spectral_density`` equivalency. + [#10015] + +- Changed ``pixel_scale`` equivalency to allow scales defined in any unit. + [#10123] + +- The ``quantity_input`` decorator now optionally allows passing through + numeric values or numpy arrays with numeric dtypes to arguments where + ``dimensionless_unscaled`` is an allowed unit. [#10232] + +astropy.utils +^^^^^^^^^^^^^ + +- Added a new ``MetaAttribute`` class to support easily adding custom attributes + to a subclass of classes like ``Table`` or ``NDData`` that have a ``meta`` + attribute. [#10097] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Added ``invalid`` keyword to ``SqrtStretch``, ``LogStretch``, + ``PowerStretch``, and ``ImageNormalize`` classes and the + ``simple_norm`` function. This keyword is used to replace generated + NaN values. [#10182] + +- Fixed an issue where ticks were sometimes not drawn at the edges of a spherical + projection on a WCSAxes. [#10442] + +astropy.wcs +^^^^^^^^^^^ + +- WCS objects with a spectral axis will now return ``SpectralCoord`` + objects when calling ``pixel_to_world`` instead of ``Quantity``, + and can now take either ``Quantity`` or ``SpectralCoord`` as input + to ``pixel_to_world``. [#10185] + +- Implemented support for the ``-TAB`` algorithm (WCS Paper III). [#9641] + +- Added an ``_as_mpl_axes`` method to the ``HightLevelWCSWrapper`` class. [#10138] + +- Add .upper() to ctype or ctype names to wcsapi/fitwcs.py to mitigate bugs from + unintended lower/upper case issues [#10557] + +API Changes +----------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- The equality operator for comparing ``SkyCoord``, frame, and representation + objects was changed. A comparison like ``sc1 == sc2`` was previously + equivalent to ``sc1 is sc2``. It will now return a boolean or boolean array + where the objects are strictly equal in all relevant frame attributes and + coordinate representation values. If the objects have different frame + attributes or representation types then an exception will be raised. [#10154] + +- ```SkyCoord.radial_velocity_correction``` now allows you to pass an ```obstime``` directly + when the ```SkyCoord``` also has an ```obstime``` set. In this situation, the position of the + ```SkyCoord``` has space motion applied to correct to the passed ```obstime```. This allows + mm/s radial velocity precision for objects with large space motion. [#10094] + +- For consistency with other astropy classes, coordinate ``Representations`` + and ``Differentials`` can now be initialized with an instance of their own class + if that instance is passed in as the first argument. [#10210] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Changed the behavior when reading a table where both the ``names`` argument + is provided (to specify the output column names) and the ``converters`` + argument is provided (to specify column conversion functions). Previously the + ``converters`` dict names referred to the *input* table column names, but now + they refer to the *output* table column names. [#9739] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- For FIELDs with datatype="char", store the values as strings instead + of bytes. [#9505] + +astropy.table +^^^^^^^^^^^^^ + +- ``Table.from_pandas`` now supports a ``units`` dictionary as argument to pass units + for columns in the ``DataFrame``. [#9472] + +astropy.time +^^^^^^^^^^^^ + +- Require that ``in_subfmt`` and ``out_subfmt`` properties of a ``Time`` object + have allowed values at the time of being set, either when creating the object + or when setting those properties on an existing ``Time`` instance. Previously + the validation of those properties was not strictly enforced. [#9868] + +astropy.utils +^^^^^^^^^^^^^ + +- Changed the exception raised by ``get_readable_fileobj`` on missing + compression modules (for ``bz2`` or ``lzma``/``xz`` support) to + ``ModuleNotFoundError``, consistent with ``io.fits`` file handlers. [#9761] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Deprecated the ``imshow_only_kwargs`` keyword in ``imshow_norm``. + [#9915] + +- Non-finite input values are now automatically excluded in + ``HistEqStretch`` and ``InvertedHistEqStretch``. [#10177] + +- The ``PowerDistStretch`` and ``InvertedPowerDistStretch`` ``a`` + value is restricted to be ``a >= 0`` in addition to ``a != 1``. + [#10177] + +- The ``PowerStretch``, ``LogStretch``, and ``InvertedLogStretch`` + ``a`` value is restricted to be ``a > 0``. [#10177] + +- The ``AsinhStretch`` and ``SinhStretch`` ``a`` value is restricted + to be ``0 < a <= 1``. [#10177] + +Bug Fixes +--------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Fix a bug where for light deflection by the Sun it was always assumed that the + source was at infinite distance, which in the (rare and) absolute worst-case + scenario could lead to errors up to 3 arcsec. [#10666] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- For FIELDs with datatype="char", store the values as strings instead + of bytes. [#9505] + +astropy.table +^^^^^^^^^^^^^ + +- Fix a bug that prevented ``Time`` columns from being used to sort a table. + [#10824] + +astropy.wcs +^^^^^^^^^^^ + +- WCS objects with a spectral axis will now return ``SpectralCoord`` + objects when calling ``pixel_to_world`` instead of ``Quantity`` + (note that ``SpectralCoord`` is a sub-class of ``Quantity``). [#10185] + +- Add .upper() to ctype or ctype names to wcsapi/fitwcs.py to mitigate bugs from + unintended lower/upper case issues [#10557] + +- Added bounds to ``fit_wcs_from_points`` to ensure CRPIX is on + input image. [#10346] + + +Other Changes and Additions +--------------------------- + +- The way in which users can specify whether to build astropy against + existing installations of C libraries rather than the bundled one + has changed, and should now be done via environment variables rather + than setup.py flags (e.g. --use-system-erfa). The available variables + are ``ASTROPY_USE_SYSTEM_CFITSIO``, ``ASTROPY_USE_SYSTEM_ERFA``, + ``ASTROPY_USE_SYSTEM_EXPAT``, ``ASTROPY_USE_SYSTEM_WCSLIB``, and + ``ASTROPY_USE_SYSTEM_ALL``. These should be set to ``1`` to build + against the system libraries. [#9730] + +- The infrastructure of the package has been updated in line with the + APE 17 roadmap (https://github.com/astropy/astropy-APEs/blob/main/APE17.rst). + The main changes are that the ``python setup.py test`` and + ``python setup.py build_docs`` commands will no longer work. The easiest + way to replicate these commands is to install the tox + (https://tox.readthedocs.io) package and run ``tox -e test`` and + ``tox -e build_docs``. It is also possible to run pytest and sphinx + directly. Other significant changes include switching to setuptools_scm to + manage the version number, and adding a ``pyproject.toml`` to opt in to + isolated builds as described in PEP 517/518. [#9726] + +- Bundled ``expat`` is updated to version 2.2.9. [#10038] + +- Increase minimum asdf version to 2.6.0. [#10189] + +- The bundled version of PLY was updated to 3.11. [#10258] + +- Removed dependency on scikit-image. [#10214] + +Version 4.0.5 (2021-03-26) +========================== + +Bug Fixes +--------- + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix bug where manual fixes to invalid header cards were not preserved when + saving a FITS file. [#11108] + +- Fix parsing of RVKC header card patterns that were not recognised + where multiple spaces were separating field-specifier and value like + "DP1.AXIS.1: 1". [#11301] + +- Fix misleading missing END card error when extra data are found at the end + of the file. [#11285] + +- Fix incorrect wrapping of long card values as CONTINUE cards when some + words in the value are longer than a single card. [#11304] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Fixed problem when writing serialized metadata to HDF5 using h5py >= 3.0. + With the newer h5py this was writing the metadata table as a variable-length + string array instead of the previous fixed-length bytes array. Fixed astropy + to force using a fixed-length bytes array. [#11359] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Change ``Voigt1D`` function to use Humlicek's approximation to avoid serious + inaccuracies + option to use (compiled) ``scipy.special.wofz`` error function + for yet more accurate results. [#11177] + +astropy.table +^^^^^^^^^^^^^ + +- Fixed bug when initializing a ``Table`` with a column as list of ``Quantity``, + for example ``Table({'x': [1*u.m, 2*u.m]})``. Previously this resulted in an + ``object`` dtype with no column ``unit`` set, but now gives a float array with + the correct unit. [#11329] + +- Fixed byteorder conversion in ``to_pandas()``, which had incorrectly + triggered swapping when native endianness was stored with explicit + ``dtype`` code ``'<'`` (or ``'>'``) instead of ``'='``. [#11288, #11294] + +- Fixed a compatibility issue with numpy 1.21. Initializing a Table with a + column like ``['str', np.ma.masked]`` was failing in tests due to a change in + numpy. [#11364] + +- Fixed bug when validating the inputs to ``table.hstack``, ``table.vstack``, + and ``table.dstack``. Previously, mistakenly calling ``table.hstack(t1, t2)`` + (instead of ``table.hstack([t1, t2]))`` would return ``t1`` instead of raising + an exception. [#11336] + +- Fixed byteorder conversion in ``to_pandas()``, which had incorrectly + triggered swapping when native endianness was stored with explicit + ``dtype`` code ``'<'`` (or ``'>'``) instead of ``'='``. [#11288] + +astropy.time +^^^^^^^^^^^^ + +- Fix leap second update when using a non english locale. [#11062] + +- Fix default assumed location to be the geocenter when transforming times + to and from solar-system barycenter scales. [#11134] + +- Fix inability to write masked times with ``formatted_value``. [#11195] + +astropy.units +^^^^^^^^^^^^^ + +- Ensure ``keepdims`` works for taking ``mean``, ``std``, and ``var`` of + ``Quantity``. [#11198] + +- For ``Quantity.to_string()``, ensure that the precision argument is also + used when the format is not latex. [#11145] + +astropy.wcs +^^^^^^^^^^^ + +- Allow "un-setting" of auxiliary WCS parameters in the ``aux`` attribute of + ``Wcsprm``. [#11166] + + + + + +Version 4.0.4 (2020-11-24) +========================== + +Bug Fixes +--------- + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- The ``norm()`` method for ``RadialDifferential`` no longer requires ``base`` + to be specified. The ``norm()`` method for other non-Cartesian differential + classes now gives a clearer error message if ``base`` is not specified. [#10969] + +- The transformations between ``ICRS`` and any of the heliocentric ecliptic + frames (``HeliocentricMeanEcliptic``, ``HeliocentricTrueEcliptic``, and + ``HeliocentricEclipticIAU76``) now correctly account for the small motion of + the Sun when transforming a coordinate with velocity information. [#10970] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Partially fixed a performance issue when reading in parallel mode. Parallel + reading currently has substantially worse performance than the default serial + reading, so we now ignore the parallel option and fall back to serial reading. + [#10880] + +- Fixed a bug where "" (blank string) as input data for a boolean type column + was causing an exception instead of indicating a masked value. As a + consequence of the fix, the values "0" and "1" are now also allowed as valid + inputs for boolean type columns. These new allowed values apply for both ECSV + and for basic character-delimited data files ('basic' format with appropriate + ``converters`` specified). [#10995] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Fixed use of weights with ``LinearLSQFitter``. [#10687] + +astropy.stats +^^^^^^^^^^^^^ + +- Fixed an issue in biweight stats when MAD=0 to give the same output + with and without an input ``axis``. [#10912] + +astropy.time +^^^^^^^^^^^^ + +- Fix a problem with the ``plot_date`` format for matplotlib >= 3.3 caused by + a change in the matplotlib plot date default reference epoch in that release. + [#10876] + +- Improve initialization time by a factor of four when creating a scalar ``Time`` + object in a format like ``unix`` or ``cxcsec`` (time delta from a reference + epoch time). [#10406] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Fixed the calculation of the tight bounding box of a ``WCSAxes``. This should + also significantly improve the application of ``tight_layout()`` to figures + containing ``WCSAxes``. [#10797] + + +Version 4.0.3 (2020-10-14) +========================== + +Bug Fixes +--------- + +astropy.table +^^^^^^^^^^^^^ + +- Fixed a small bug where initializing an empty ``Column`` with a structured dtype + with a length and a shape failed to give the requested dtype. [#10819] + +Other Changes and Additions +--------------------------- + +- Fixed installation of the source distribution with pip<19. [#10837, #10852] + + +Version 4.0.2 (2020-10-10) +========================== + +New Features +------------ + +astropy.utils +^^^^^^^^^^^^^ + +- ``astropy.utils.data.download_file`` now supports FTPS/FTP over TLS. [#9964] + +- ``astropy.utils.data`` now uses a lock-free mechanism for caching. This new + mechanism uses a new cache layout and so ignores caches created using earlier + mechanisms (which were causing lockups on clusters). The two cache formats can + coexist but do not share any files. [#10437, #10683] + +- ``astropy.utils.data`` now ignores the config item + ``astropy.utils.data.conf.download_cache_lock_attempts`` since no locking is + done. [#10437, #10683] + +- ``astropy.utils.data.download_file`` and related functions now interpret the + parameter or config file setting ``timeout=0`` to mean they should make no + attempt to download files. [#10437, #10683] + +- ``astropy.utils.import_file_to_cache`` now accepts a keyword-only argument + ``replace``, defaulting to True, to determine whether it should replace existing + files in the cache, in a way as close to atomic as possible. [#10437, #10683] + +- ``astropy.utils.data.download_file`` and related functions now treat + ``http://example.com`` and ``http://example.com/`` as equivalent. [#10631] + +astropy.wcs +^^^^^^^^^^^ + +- The new auxiliary WCS parameters added in WCSLIB 7.1 are now exposed as + the ``aux`` attribute of ``Wcsprm``. [#10333] + +- Updated bundled version of ``WCSLIB`` to v7.3. [#10433] + + +Bug fixes +--------- + +astropy.config +^^^^^^^^^^^^^^ + +- Added an extra fallback to ``os.expanduser('~')`` when trying to find the + user home directory. [#10570] + +astropy.constants +^^^^^^^^^^^^^^^^^ + +- Corrected definition of parsec to 648 000 / pi AU following IAU 2015 B2 [#10569] + +astropy.convolution +^^^^^^^^^^^^^^^^^^^ + +- Fixed a bug where a float-typed integers in the argument ``x_range`` of + ``astropy.convolution.utils.discretize_oversample_1D`` (and the 2D version as + well) fails because it uses ``numpy.linspace``, which requires an ``int``. + [#10696] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Ensure that for size-1 array ``SkyCoord`` and coordinate frames + the attributes also properly become scalars when indexed with 0. + [#10113] + +- Fixed a bug where ``SkyCoord.separation()`` and ``SkyCoord.separation_3d`` + were not accepting a frame object. [#10332] + +- Ensure that the ``lon`` values in ``SkyOffsetFrame`` are wrapped correctly at + 180 degree regardless of how the underlying data is represented. [#10163] + +- Fixed an error in the obliquity of the ecliptic when transforming to/from the + ``*TrueEcliptic`` coordinate frames. The error would primarily result in an + inaccuracy in the ecliptic latitude on the order of arcseconds. [#10129] + +- Fixed an error in the computation of the location of solar system bodies where the + Earth location of the observer was ignored during the correction for light travel + time. [#10292] + +- Ensure that coordinates with proper motion that are transformed to other + coordinate frames still can be represented properly. [#10276] + +- Improve the error message given when trying to get a cartesian representation + for coordinates that have both proper motion and radial velocity, but no + distance. [#10276] + +- Fixed an error where ``SkyCoord.apply_space_motion`` would return incorrect + results when no distance is set and proper motion is high. [#10296] + +- Make the parsing of angles thread-safe so that ``Angle`` can be used in + Python multithreading. [#10556] + +- Fixed reporting of ``EarthLocation.info`` which previously raised an exception. + [#10592] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Fixed a bug with the C ``fast_reader`` not correctly parsing newlines when + ``delimiter`` was also set to ``\n`` or ``\r``; ensured consistent handling + of input strings without newline characters. [#9929] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix integer formats of ``TFORMn=Iw`` columns in ASCII tables to correctly read + values exceeding int32 - setting int16, int32 or int64 according to ``w``. [#9901] + +- Fix unclosed memory-mapped FITS files in ``FITSDiff`` when difference found. + [#10159] + +- Fix crash when reading an invalid table file. [#10171] + +- Fix duplication issue when setting a keyword ending with space. [#10482] + +- Fix ResourceWarning with ``fits.writeto`` and ``pathlib.Path`` object. + [#10599] + +- Fix repr for commentary cards and strip spaces for commentary keywords. + [#10640] + +- Fix compilation of cfitsio with Xcode 12. [#10772] + +- Fix handling of 1-dimensional arrays with a single element in ``BinTableHDU`` [#10768] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Fix id URL in ``baseframe-1.0.0`` ASDF schema. [#10223] + +- Write keys to ASDF only if the value is present, to account + for a change in behavior in asdf 2.8. [#10674] + +astropy.io.registry +^^^^^^^^^^^^^^^^^^^ + +- Fix ``Table.(read|write).help`` when reader or writer has no docstring. [#10460] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Fixed parsing failure of VOTable with no fields. When detecting a non-empty + table with no fields, the following warning/exception is issued: + E25 "No FIELDs are defined; DATA section will be ignored." [#10192] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Fixed a problem with mapping ``input_units`` and ``return_units`` + of a ``CompoundModel`` to the units of the constituent models. [#10158] + +- Removed hard-coded names of inputs and outputs. [#10174] + +- Fixed a problem where slicing a ``CompoundModel`` by name will crash if + there ``fix_inputs`` operators are present. [#10224] + +- Removed a limitation of fitting of data with units with compound models + without units when the expression involves operators other than addition + and subtraction. [#10415] + +- Fixed a problem with fitting ``Linear1D`` and ``Planar2D`` in model sets. [#10623] + +- Fixed reported module name of ``math_functions`` model classes. [#10694] + +- Fixed reported module name of ``tabular`` model classes. [#10709] + +- Do not create new ``math_functions`` models for ufuncs that are + only aliases (divide and mod). [#10697] + +- Fix calculation of the ``Moffat2D`` derivative with respect to gamma. [#10784] + +astropy.stats +^^^^^^^^^^^^^ + +- Fixed an API regression where ``SigmaClip.__call__`` would convert masked + elements to ``nan`` and upcast the dtype to ``float64`` in its output + ``MaskedArray`` when using the ``axis`` parameter along with the defaults + ``masked=True`` and ``copy=True``. [#10610] + +- Fixed an issue where fully masked ``MaskedArray`` input to + ``sigma_clipped_stats`` gave incorrect results. [#10099] + +- Fixed an issue where ``sigma_clip`` and ``SigmaClip.__call__`` + would return a masked array instead of a ``ndarray`` when + ``masked=False`` and the input was a full-masked ``MaskedArray``. + [#10099] + +- Fixed bug with ``funcs.poisson_conf_interval`` where an integer for N + with ``interval='kraft-burrows-nousek'`` would throw an error with + mpmath backend. [#10427] + +- Fixed bug in ``funcs.poisson_conf_interval`` with + ``interval='kraft-burrows-nousek'`` where certain combinations of source + and background count numbers led to ``ValueError`` due to the choice of + starting value for numerical optimization. [#10618] + +astropy.table +^^^^^^^^^^^^^ + +- Fixed a bug when writing a table with mixin columns to FITS, ECSV or HDF5. + If one of the data attributes of the mixin (e.g. ``skycoord.ra``) had the + same name as one of the table column names (``ra``), the column (``ra``) + would be dropped when reading the table back. [#10222] + +- Fixed a bug when sorting an indexed table on the indexed column after first + sorting on another column. [#10103] + +- Fixed a bug in table argsort when called with ``reverse=True`` for an + indexed table. [#10103] + +- Fixed a performance regression introduced in #9048 when initializing a table + from Python lists. Also fixed incorrect behavior (for data types other than + float) when those lists contain ``np.ma.masked`` elements to indicate masked + data. [#10636] + +- Avoid modifying ``.meta`` when serializing columns to FITS. [#10485] + +- Avoid crash when reading a FITS table that contains mixin info and PyYAML + is missing. [#10485] + +astropy.time +^^^^^^^^^^^^ + +- Ensure that for size-1 array ``Time``, the location also properly becomes + a scalar when indexed with 0. [#10113] + +astropy.units +^^^^^^^^^^^^^ + +- Refined test_parallax to resolve difference between 2012 and 2015 definitions. [#10569] + +astropy.utils +^^^^^^^^^^^^^ + +- The default IERS server has been updated to use the FTPS server hosted by + CDDIS. [#9964] + +- Fixed memory allocation on 64-bit systems within ``xml.iterparse`` [#10076] + +- Fix case where ``None`` could be used in a numerical computation. [#10126] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Fixed a bug where the ``ImageNormalize`` ``clip`` keyword was + ignored when used with calling the object on data. [#10098] + +- Fixed a bug where ``axes.xlabel``/``axes.ylabel`` where not correctly set + nor returned on an ``EllipticalFrame`` class ``WCSAxes`` plot. [#10446] + +astropy.wcs +^^^^^^^^^^^ + +- Handled WCS 360 -> 0 deg crossover in ``fit_wcs_from_points`` [#10155] + +- Do not issue ``DATREF`` warning when ``MJDREF`` has default value. [#10440] + +- Fixed a bug due to which ``naxis`` argument was ignored if ``header`` + was supplied during the initialization of a WCS object. [#10532] + +Other Changes and Additions +--------------------------- + +- Improved the speed of sorting a large ``Table`` on a single column by a factor + of around 5. [#10103] + +- Ensure that astropy can be used inside Application bundles built with + pyinstaller. [#8795] + +- Updated the bundled CFITSIO library to 3.49. See + ``cextern/cfitsio/docs/changes.txt`` for additional information. + [#10256, #10665] + +- ``extract_array`` raises a ``ValueError`` if the data type of the + input array is inconsistent with the ``fill_value``. [#10602] + + +Version 4.0.1 (2020-03-27) +========================== + +Bug fixes +--------- + +astropy.config +^^^^^^^^^^^^^^ + +- Fixed a bug where importing a development version of a package that uses + ``astropy`` configuration system can result in a + ``~/.astropy/config/package..cfg`` file. [#9975] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Fixed a bug where a vestigal trace of a frame class could persist in the + transformation graph even after the removal of all transformations involving + that frame class. [#9815] + +- Fixed a bug with ``TransformGraph.remove_transform()`` when the "from" and + "to" frame classes are not explicitly specified. [#9815] + +- Read-only longitudes can now be passed in to ``EarthLocation`` even if + they include angles outside of the range of -180 to 180 degrees. [#9900] + +- ```SkyCoord.radial_velocity_correction``` no longer raises an Exception + when space motion information is present on the SkyCoord. [#9980] + +astropy.io +^^^^^^^^^^ + +- Fixed a bug that prevented the unified I/O infrastructure from working with + datasets that are represented by directories rather than files. [#9866] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Fixed a bug in the ``fast_reader`` C parsers incorrectly returning entries + of isolated positive/negative signs as ``float`` instead of ``str``. [#9918] + +- Fixed a segmentation fault in the ``fast_reader`` C parsers when parsing an + invalid file with ``guess=True`` and the file contains inconsistent column + numbers in combination with a quoted field; e.g., ``"1 2\n 3 4 '5'"``. + [#9923] + +- Magnitude, decibel, and dex can now be stored in ``ecsv`` files. [#9933] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Magnitude, decibel, and dex can now be stored in ``hdf5`` files. [#9933] + +- Fixed serialization of polynomial models to include non default values of + domain and window values. [#9956, #9961] + +- Fixed a bug which affected overwriting tables within ``hdf5`` files. + Overwriting an existing path with associated column meta data now also + overwrites the meta data associated with the table. [#9950] + +- Fixed serialization of Time objects with location under time-1.0.0 + ASDF schema. [#9983] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Fix regression with ``GroupsHDU`` which needs to modify the header to handle + invalid headers, and fix accessing ``.data`` for empty HDU. [#9711, #9934] + +- Fix ``fitsdiff`` when its arguments are directories that contain other + directories. [#9711] + +- Fix writing noncontiguous data to a compressed HDU. [#9958] + +- Added verification of ``disp`` (``TDISP``) keyword to ``fits.Column`` and + extended tests for ``TFORM`` and ``TDISP`` validation. [#9978] + +- Fix checksum verification to process all HDUs instead of only the first one + because of the lazy loading feature. [#10012] + +- Allow passing ``output_verify`` to ``.close`` when using the context manager. + [#10030] + +- Prevent instantiation of ``PrimaryHDU`` and ``ImageHDU`` with a scalar. + [#10041] + +- Fix column access by attribute with FITS_rec: columns with scaling or columns + from ASCII tables where not properly converted when accessed by attribute + name. [#10069] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Magnitude, decibel, and dex can now be stored in ``hdf5`` files. [#9933] + +- Fixed serialization of polynomial models to include non default values of + domain and window values. [#9956, #9961] + +- Fixed a bug which affected overwriting tables within ``hdf5`` files. + Overwriting an existing path with associated column meta data now also + overwrites the meta data associated with the table. [#9950] + +- Fixed serialization of Time objects with location under time-1.0.0 + ASDF schema. [#9983] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Fixed a bug in setting default values of parameters of orthonormal + polynomials when constructing a model set. [#9987] + +astropy.table +^^^^^^^^^^^^^ + +- Fixed bug in ``Table.reverse`` for tables that contain non-mutable mixin columns + (like ``SkyCoord``) for which in-place item update is not allowed. [#9839] + +- Tables containing Magnitude, decibel, and dex columns can now be saved to + ``ecsv`` files. [#9933] + +- Fixed bug where adding or inserting a row fails on a table with an index + defined on a column that is not the first one. [#10027] + +- Ensured that ``table.show_in_browser`` also worked for mixin columns like + ``Time`` and ``SkyCoord``. [#10068] + +astropy.time +^^^^^^^^^^^^ + +- Fix inaccuracy when converting between TimeDelta and datetime.timedelta. [#9679] + +- Fixed exception when changing ``format`` in the case when ``out_subfmt`` is + defined and is incompatible with the new format. [#9812] + +- Fixed exceptions in ``Time.to_value()``: when supplying any ``subfmt`` argument + for string-based formats like 'iso', and for ``subfmt='long'`` for the formats + 'byear', 'jyear', and 'decimalyear'. [#9812] + +- Fixed bug where the location attribute was lost when creating a new ``Time`` + object from an existing ``Time`` or list of ``Time`` objects. [#9969] + +- Fixed a bug where an exception occurred when creating a ``Time`` object + if the ``val1`` argument was a regular double and the ``val2`` argument + was a ``longdouble``. [#10034] + +astropy.timeseries +^^^^^^^^^^^^^^^^^^ + +- Fixed issue with reference time for the ``transit_time`` parameter returned by + the ``BoxLeastSquares`` periodogram. Now, the ``transit_time`` will be within + the range of the input data and arbitrary time offsets/zero points no longer + affect results. [#10013] + +astropy.units +^^^^^^^^^^^^^ + +- Fix for ``quantity_input`` annotation raising an exception on iterable + types that don't define a general ``__contains__`` for checking if ``None`` + is contained (e.g. Enum as of python3.8), by instead checking for instance of + Sequence. [#9948] + +- Fix for ``u.Quantity`` not taking into account ``ndmin`` if constructed from + another ``u.Quantity`` instance with different but convertible unit [#10066] + +astropy.utils +^^^^^^^^^^^^^ + +- Fixed ``deprecated_renamed_argument`` not passing in user value to + deprecated keyword when the keyword has no new name. [#9981] + +- Fixed ``deprecated_renamed_argument`` not issuing a deprecation warning when + deprecated keyword without new name is passed in as positional argument. + [#9985] + +- Fixed detection of read-only filesystems in the caching code. [#10007] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Fixed bug from matplotlib >=3.1 where an empty Quantity array is + sent for unit conversion as an empty list. [#9848] + +- Fix bug in ``ZScaleInterval`` to return the array minimum and + maximum when there are less then ``min_npixels`` in the input array. [#9913] + +- Fix a bug in simplifying axis labels that affected non-rectangular frames. + [#8004, #9991] + + +Other Changes and Additions +--------------------------- + +- Increase minimum asdf version to 2.5.2. [#9996, #9819] + +- Updated bundled version of ``WCSLIB`` to v7.2. [#10021] + + + +Version 4.0 (2019-12-16) +======================== + +New Features +------------ + +astropy.config +^^^^^^^^^^^^^^ + +- The config and cache directories and the name of the config file are now + customizable. This allows affiliated packages to put their configuration + files in locations other than ``CONFIG_DIR/.astropy/``. [#8237] + +astropy.constants +^^^^^^^^^^^^^^^^^ + +- The version of constants can be specified via ScienceState in a way + that ``constants`` and ``units`` will be consistent. [#8517] + +- Default constants now use CODATA 2018 and IAU 2015 definitions. [#8761] + +- Constants can be pickled and unpickled. [#9377] + +astropy.convolution +^^^^^^^^^^^^^^^^^^^ + +- Fixed a bug [#9168] where having a kernel defined using unitless astropy + quantity objects would result in a crash [#9300] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Changed ``coordinates.solar_system_ephemeris`` to also accept local files + as input. The ephemeris can now be selected by either keyword (e.g. 'jpl', + 'de430'), URL or file path. [#8767] + +- Added a ``cylindrical`` property to ``SkyCoord`` for shorthand access to a + ``CylindricalRepresentation`` of the coordinate, as is already available + for other common representations. [#8857] + +- The default parameters for the ``Galactocentric`` frame are now controlled by + a ``ScienceState`` subclass, ``galactocentric_frame_defaults``. New + parameter sets will be added to this object periodically to keep up with + ever-improved measurements of the solar position and motion. [#9346] + +- Coordinate frame classes can now have multiple aliases by assigning a list + of aliases to the class variable ``name``. Any of the aliases can be used + for attribute-style access or as the target of ``tranform_to()`` calls. + [#8834] + +- Passing a NaN to ``Distance`` no longer raises a warning. [#9598] + +astropy.cosmology +^^^^^^^^^^^^^^^^^ + +- The pre-publication Planck 2018 cosmological parameters are included as the + ``Planck2018_arXiv_v2`` object. Please note that the values are preliminary, + and when the paper is accepted a final version will be included as + ``Planck18``. [#8111] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Removed incorrect warnings on ``Overflow`` when reading in + ``FloatType`` 0.0 with ``use_fast_converter``; synchronised + ``IntType`` ``Overflow`` warning messages. [#9082] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Eliminate deprecated compatibility mode when writing ``Table`` metadata to + HDF5 format. [#8899] + +- Add support for orthogonal polynomial models to ASDF. [#9107] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Changed the ``fitscheck`` and ``fitsdiff`` script to use the ``argparse`` + module instead of ``optparse``. [#9148] + +- Allow writing of ``Table`` objects with ``Time`` columns that are also table + indices to FITS files. [#8077] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Support VOTable version 1.4. The main addition is the new element, TIMESYS, + which allows defining of metadata for temporal coordinates much like COOSYS + defines metadata for celestial coordinates. [#9475] + +astropy.logger +^^^^^^^^^^^^^^ + +- Added a configuration option to specify the text encoding of the log file, + with the default behavior being the platform-preferred encoding. [#9203] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Major rework of modeling internals. `See modeling documentation for details. + `_ . [#8769] + +- Add ``Tabular1D.inverse``. [#9083] + +- ``Model.rename`` was changed to add the ability to rename ``Model.inputs`` + and ``Model.outputs``. [#9220] + +- New function ``fix_inputs`` to generate new models from others by fixing + specific inputs variable values to constants. [#9135] + +- ``inputs`` and ``outputs`` are now model instance attributes, and ``n_inputs`` + and ``n_outputs`` are class attributes. Backwards compatible default + values of ``inputs`` and ``outputs`` are generated. ``Model.inputs`` and + ``Model.outputs`` are now settable which allows renaming them on per user + case. [#9298] + +- Add a new model representing a sequence of rotations in 3D around an + arbitrary number of axes. [#9369] + +- Add many of the numpy ufunc functions as models. [#9401] + +- Add ``BlackBody`` model. [#9282] + +- Add ``Drude1D`` model. [#9452] + +- Added analytical King model (KingProjectedAnalytic1D). [#9084] + +- Added Exponential1D and Logarithmic1D models. [#9351] + +astropy.nddata +^^^^^^^^^^^^^^ + +- Add a way for technically invalid but unambiguous units in a fits header + to be parsed by ``CCDData``. [#9397] + +- ``NDData`` now only accepts WCS objects which implement either the high, or + low level APE 14 WCS API. All WCS objects are converted to a high level WCS + object, so ``NDData.wcs`` now always returns a high level APE 14 object. Not + all array slices are valid for wcs objects, so some slicing operations which + used to work may now fail. [#9067] + +astropy.stats +^^^^^^^^^^^^^ + +- The ``biweight_location``, ``biweight_scale``, and + ``biweight_midvariance`` functions now allow for the ``axis`` + keyword to be a tuple of integers. [#9309] + +- Added an ``ignore_nan`` option to the ``biweight_location``, + ``biweight_scale``, and ``biweight_midvariance`` functions. [#9457] + +- A numpy ``MaskedArray`` can now be input to the ``biweight_location``, + ``biweight_scale``, and ``biweight_midvariance`` functions. [#9466] + +- Removed the warning related to p0 in the Bayesian blocks algorithm. The + caveat related to p0 is described in the docstring for ``Events``. [#9567] + +astropy.table +^^^^^^^^^^^^^ + +- Improved the implementation of ``Table.replace_column()`` to provide + a speed-up of 5 to 10 times for wide tables. The method can now accept + any input which convertible to a column of the correct length, not just + ``Column`` subclasses. [#8902] + +- Improved the implementation of ``Table.add_column()`` to provide a speed-up + of 2 to 10 (or more) when adding a column to tables, with increasing benefit + as the number of columns increases. The method can now accept any input + which is convertible to a column of the correct length, not just ``Column`` + subclasses. [#8933] + +- Changed the implementation of ``Table.add_columns()`` to use the new + ``Table.add_column()`` method. In most cases the performance is similar + or slightly faster to the previous implementation. [#8933] + +- ``MaskedColumn.data`` will now return a plain ``MaskedArray`` rather than + the previous (unintended) ``masked_BaseColumn``. [#8855] + +- Added depth-wise stacking ``dstack()`` in higher level table operation. + It help will in stacking table column depth-wise. [#8939] + +- Added a new table equality method ``values_equal()`` which allows comparison + table values to another table, list, or value, and returns an + element-by-element equality table. [#9068] + +- Added new ``join_type='cartesian'`` option to the ``join`` operation. [#9288] + +- Allow adding a table column as a list of mixin-type objects, for instance + ``t['q'] = [1 * u.m, 2 * u.m]``. [#9165] + +- Allow table ``join()`` using any sortable key column (e.g. Time), not + just ndarray subclasses. A column is considered sortable if there is a + ``.info.get_sortable_arrays()`` method that is implemented. [#9340] + +- Added ``Table.iterrows()`` for making row-wise iteration faster. [#8969] + +- Allow table to be initialized with a list of dict where the dict keys + are not the same in every row. The table column names are the set of all keys + found in the input data, and any missing key/value pairs are turned into + missing data in the table. [#9425] + +- Prevent unnecessary ERFA warnings when indexing by ``Time`` columns. [#9545] + +- Added support for sorting tables which contain non-mutable mixin columns + (like ``SkyCoord``) for which in-place item update is not allowed. [#9549] + +- Ensured that inserting ``np.ma.masked`` (or any other value with a mask) into + a ``MaskedColumn`` causes a masked entry to be inserted. [#9623] + +- Fixed a bug that caused an exception when initializing a ``MaskedColumn`` from + another ``MaskedColumn`` that has a structured dtype. [#9651] + +astropy.tests +^^^^^^^^^^^^^ + +- The plugin that handles the custom header in the test output has been + moved to the ``pytest-astropy-header plugin`` package. `See the README at + `__ for information about + using this new plugin. [#9214] + +astropy.time +^^^^^^^^^^^^ + +- Added a new time format ``ymdhms`` for representing times via year, month, + day, hour, minute, and second attributes. [#7644] + +- ``TimeDelta`` gained a ``to_value`` method, so that it becomes easier to + use it wherever a ``Quantity`` with units of time could be used. [#8762] + +- Made scalar ``Time`` and ``TimeDelta`` objects hashable based on JD, time + scale, and location attributes. [#8912] + +- Improved error message when bad input is used to initialize a ``Time`` or + ``TimeDelta`` object and the format is specified. [#9296] + +- Allow numeric time formats to be initialized with numpy ``longdouble``, + ``Decimal`` instances, and strings. One can select just one of these + using ``in_subfmt``. The output can be similarly set using ``out_subfmt``. + [#9361] + +- Introduce a new ``.to_value()`` method for ``Time`` (and adjusted the + existing method for ``TimeDelta``) so that one can get values in a given + ``format`` and possible ``subfmt`` (e.g., ``to_value('mjd', 'str')``. [#9361] + +- Prevent unnecessary ERFA warnings when sorting ``Time`` objects. [#9545] + +astropy.timeseries +^^^^^^^^^^^^^^^^^^ + +- Adding ``epoch_phase``, ``wrap_phase`` and ``normalize_phase`` keywords to + ``TimeSeries.fold()`` to control the phase of the epoch and to return + normalized phase rather than time for the folded TimeSeries. [#9455] + +astropy.uncertainty +^^^^^^^^^^^^^^^^^^^ + +- ``Distribution`` was rewritten such that it deals better with subclasses. + As a result, Quantity distributions now behave correctly with ``to`` methods + yielding new distributions of the kind expected for the starting + distribution, and ``to_value`` yielding ``NdarrayDistribution`` instances. + [#9429, #9442] + +- The ``pdf_*`` properties that were used to calculate statistical properties + of ``Distribution`` instances were changed into methods. This allows one + to pass parameters such as ``ddof`` to ``pdf_std`` and ``pdf_var`` (which + generally should equal 1 instead of the default 0), and reflects that these + are fairly involved calculations, not just "properties". [#9613] + +astropy.units +^^^^^^^^^^^^^ + +- Support for unicode parsing. Currently supported are superscripts, Ohm, + ÅngstrÃļm, and the micro-sign. [#9348] + +- Accept non-unit type annotations in @quantity_input. [#8984] + +- For numpy 1.17 and later, the new ``__array_function__`` protocol is used to + ensure that all top-level numpy functions interact properly with + ``Quantity``, preserving units also in operations like ``np.concatenate``. + [#8808] + +- Add equivalencies for surface brightness units to spectral_density. [#9282] + +astropy.utils +^^^^^^^^^^^^^ + +- ``astropy.utils.data.download_file`` and + ``astropy.utils.data.get_readable_fileobj`` now provides an ``http_headers`` + keyword to pass in specific request headers for the download. It also now + defaults to providing ``User-Agent: Astropy`` and ``Accept: */*`` + headers. The default ``User-Agent`` value can be set with a new + ``astropy.data.conf.default_http_user_agent`` configuration item. + [#9508, #9564] + +- Added a new ``astropy.utils.misc.unbroadcast`` function which can be used + to return the smallest array that can be broadcasted back to the initial + array. [#9209] + +- The specific IERS Earth rotation parameter table used for time and + coordinate transformations can now be set, either in a context or per + session, using ``astropy.utils.iers.earth_rotation_table``. [#9244] + +- Added ``export_cache`` and ``import_cache`` to permit transporting + downloaded data to machines with no Internet connection. Several new + functions are available to investigate the cache contents; e.g., + ``check_download_cache`` can be used to confirm that the persistent + cache has not become damaged. [#9182] + +- A new ``astropy.utils.iers.LeapSeconds`` class has been added to track + leap seconds. [#9365] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- Added a new ``time_support`` context manager/function for making it easy to + plot and format ``Time`` objects in Matplotlib. [#8782] + +- Added support for plotting any WCS compliant with the generalized (APE 14) + WCS API with WCSAxes. [#8885, #9098] + +- Improved display of information when inspecting ``WCSAxes.coords``. [#9098] + +- Improved error checking for the ``slices=`` argument to ``WCSAxes``. [#9098] + +- Added support for more solar frames in WCSAxes. [#9275] + +- Add support for one dimensional plots to ``WCSAxes``. [#9266] + +- Add a ``get_format_unit`` to ``wcsaxes.CoordinateHelper``. [#9392] + +- ``WCSAxes`` now, by default, sets a default label for plot axes which is the + WCS physical type (and unit) for that axis. This can be disabled using the + ``coords[i].set_auto_axislabel(False)`` or by explicitly setting an axis + label. [#9392] + +- Fixed the display of tick labels when plotting all sky images that have a + coord_wrap less than 360. [#9542] + +astropy.wcs +^^^^^^^^^^^ + +- Added a ``astropy.wcs.wcsapi.pixel_to_pixel`` function that can be used to + transform pixel coordinates in one dataset with a WCS to pixel coordinates + in another dataset with a different WCS. This function is designed to be + efficient when the input arrays are broadcasted views of smaller + arrays. [#9209] + +- Added a ``local_partial_pixel_derivatives`` function that can be used to + determine a matrix of partial derivatives of each world coordinate with + respect to each pixel coordinate. [#9392] + +- Updated wcslib to v6.4. [#9125] + +- Improved the ``SlicedLowLevelWCS`` class in ``astropy.wcs.wcsapi`` to avoid + storing chains of nested ``SlicedLowLevelWCS`` objects when applying multiple + slicing operations in turn. [#9210] + +- Added a ``wcs_info_str`` function to ``astropy.wcs.wcsapi`` to show a summary + of an APE-14-compliant WCS as a string. [#8546, #9207] + +- Added two new optional attributes to the APE 14 low-level WCS: + ``pixel_axis_names`` and ``world_axis_names``. [#9156] + +- Updated the WCS class to now correctly take and return ``Time`` objects in + the high-level APE 14 API (e.g. ``pixel_to_world``. [#9376] + +- ``SlicedLowLevelWCS`` now raises ``IndexError`` rather than ``ValueError`` on + an invalid slice. [#9067] + +- Added ``fit_wcs_from_points`` function to ``astropy.wcs.utils``. Fits a WCS + object to set of matched detector/sky coordinates. [#9469] + +- Fix various bugs in ``SlicedLowLevelWCS`` when the WCS being sliced was one + dimensional. [#9693] + + +API Changes +----------- + +astropy.constants +^^^^^^^^^^^^^^^^^ + +- Deprecated ``set_enabled_constants`` context manager. Use + ``astropy.physical_constants`` and ``astropy.astronomical_constants``. + [#9025] + +astropy.convolution +^^^^^^^^^^^^^^^^^^^ + +- Removed the deprecated keyword argument ``interpolate_nan`` from + ``convolve_fft``. [#9356] + +- Removed the deprecated keyword argument ``stddev`` from + ``Gaussian2DKernel``. [#9356] + +- Deprecated and renamed ``MexicanHat1DKernel`` and ``MexicanHat2DKernel`` + to ``RickerWavelet1DKernel`` and ``RickerWavelet2DKernel``. [#9445] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- Removed the ``recommended_units`` attribute from Representations; it was + deprecated since 3.0. [#8892] + +- Removed the deprecated frame attribute classes, ``FrameAttribute``, + ``TimeFrameAttribute``, ``QuantityFrameAttribute``, + ``CartesianRepresentationFrameAttribute``; deprecated since 3.0. [#9326] + +- Removed ``longitude`` and ``latitude`` attributes from ``EarthLocation``; + deprecated since 2.0. [#9326] + +- The ``DifferentialAttribute`` for frame classes now passes through any input + to the ``allowed_classes`` if only one allowed class is specified, i.e. this + now allows passing a quantity in for frame attributes that use + ``DifferentialAttribute``. [#9325] + +- Removed the deprecated ``galcen_ra`` and ``galcen_dec`` attributes from the + ``Galactocentric`` frame. [#9346] + +astropy.extern +^^^^^^^^^^^^^^ + +- Remove the bundled ``six`` module. [#8315] + +astropy.io.ascii +^^^^^^^^^^^^^^^^ + +- Masked column handling has changed, see ``astropy.table`` entry below. + [#8789] + +astropy.io.misc +^^^^^^^^^^^^^^^ + +- Masked column handling has changed, see ``astropy.table`` entry below. + [#8789] + +- Removed deprecated ``usecPickle`` kwarg from ``fnunpickle`` and + ``fnpickle``. [#8890] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Masked column handling has changed, see ``astropy.table`` entry below. + [#8789] + +- ``io.fits.Header`` has been made safe for subclasses for copying and slicing. + As a result of this change, the private subclass ``CompImageHeader`` + now always should be passed an explicit ``image_header``. [#9229] + +- Removed the deprecated ``tolerance`` option in ``fitsdiff`` and + ``io.fits.diff`` classes. [#9520] + +- Removed deprecated keyword arguments for ``CompImageHDU``: + ``compressionType``, ``tileSize``, ``hcompScale``, ``hcompSmooth``, + ``quantizeLevel``. [#9520] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Changed ``pedantic`` argument to ``verify`` and change it to have three + string-based options (``ignore``, ``warn``, and ``exception``) instead of + just being a boolean. In addition, changed default to ``ignore``, which means + that warnings will not be shown by default when loading VO tables. [#8715] + +astropy.modeling +^^^^^^^^^^^^^^^^ + +- Eliminates support for compound classes (but not compound instances!) [#8769] + +- Slicing compound models more restrictive. [#8769] + +- Shape of parameters now includes n_models as dimension. [#8769] + +- Parameter instances now hold values instead of models. [#8769] + +- Compound model parameters now share instance and value with + constituent models. [#8769] + +- No longer possible to assign slices of parameter values to model parameters + attribute (it is possible to replace it with a complete array). [#8769] + +- Many private attributes and methods have changed (see documentation). [#8769] + +- Deprecated ``BlackBody1D`` model and ``blackbody_nu`` and + ``blackbody_lambda`` functions. [#9282] + +- The deprecated ``rotations.rotation_matrix_from_angle`` was removed. [#9363] + +- Deprecated and renamed ``MexicanHat1D`` and ``MexicanHat2D`` + to ``RickerWavelet1D`` and ``RickerWavelet2D``. [#9445] + +- Deprecated ``modeling.utils.ExpressionTree``. [#9576] + +astropy.stats +^^^^^^^^^^^^^ + +- Removed the ``iters`` keyword from sigma clipping stats functions. [#8948] + +- Renamed the ``a`` parameter to ``data`` in biweight stat functions. [#8948] + +- Renamed the ``a`` parameter to ``data`` in ``median_absolute_deviation``. + [#9011] + +- Renamed the ``conflevel`` keyword to ``confidence_level`` in + ``poisson_conf_interval``. Usage of ``conflevel`` now issues + ``AstropyDeprecationWarning``. [#9408] + +- Renamed the ``conf`` keyword to ``confidence_level`` in + ``binom_conf_interval`` and ``binned_binom_proportion``. Usage of ``conf`` + now issues ``AstropyDeprecationWarning``. [#9408] + +- Renamed the ``conf_lvl`` keyword to ``confidence_level`` in + ``jackknife_stats``. Usage of ``conf_lvl`` now issues + ``AstropyDeprecationWarning``. [#9408] + +astropy.table +^^^^^^^^^^^^^ + +- The handling of masked columns in the ``Table`` class has changed in a way + that may impact program behavior. Now a ``Table`` with ``masked=False`` + may contain both ``Column`` and ``MaskedColumn`` objects, and adding a + masked column or row to a table no longer "upgrades" the table and columns + to masked. This means that tables with masked data which are read via + ``Table.read()`` will now always have ``masked=False``, though specific + columns will be masked as needed. Two new table properties + ``has_masked_columns`` and ``has_masked_values`` were added. See the + `Masking change in astropy 4.0 section within + `_ for + details. [#8789] + +- Table operation functions such as ``join``, ``vstack``, ``hstack``, etc now + always return a table with ``masked=False``, though the individual columns + may be masked as necessary. [#8957] + +- Changed implementation of ``Table.add_column()`` and ``Table.add_columns()`` + methods. Now it is possible add any object(s) which can be converted or + broadcasted to a valid column for the table. ``Table.__setitem__`` now + just calls ``add_column``. [#8933] + +- Changed default table configuration setting ``replace_warnings`` from + ``['slice']`` to ``[]``. This removes the default warning when replacing + a table column that is a slice of another column. [#9144] + +- Removed the non-public method + ``astropy.table.np_utils.recarray_fromrecords``. [#9165] + +astropy.tests +^^^^^^^^^^^^^ + +- In addition to ``DeprecationWarning``, now ``FutureWarning`` and + ``ImportWarning`` would also be turned into exceptions. [#8506] + +- ``warnings_to_ignore_by_pyver`` option in + ``enable_deprecations_as_exceptions()`` has changed. Please refer to API + documentation. [#8506] + +- Default settings for ``warnings_to_ignore_by_pyver`` are updated to remove + very old warnings that are no longer relevant and to add a new warning + caused by ``pytest-doctestplus``. [#8506] + +astropy.time +^^^^^^^^^^^^ + +- ``Time.get_ut1_utc`` now uses the auto-updated ``IERS_Auto`` by default, + instead of the bundled ``IERS_B`` file. [#9226] + +- Time formats that do not use ``val2`` now raise ValueError instead of + silently ignoring a provided value. [#9373] + +- Custom time formats can now accept floating-point types with extended + precision. Existing time formats raise exceptions rather than discarding + extended precision through conversion to ordinary floating-point. [#9368] + +- Time formats (implemented in subclasses of ``TimeFormat``) now have + their input and output routines more thoroughly validated, making it more + difficult to create damaged ``Time`` objects. [#9375] + +- The ``TimeDelta.to_value()`` method now can also take the ``format`` name + as its argument, in which case the value will be calculated using the + ``TimeFormat`` machinery. For this case, one can also pass a ``subfmt`` + argument to retrieve the value in another form than ``float``. [#9361] + +astropy.timeseries +^^^^^^^^^^^^^^^^^^ + +- Keyword ``midpoint_epoch`` is renamed to ``epoch_time``. [#9455] + +astropy.uncertainty +^^^^^^^^^^^^^^^^^^^ + +- ``Distribution`` was rewritten such that it deals better with subclasses. + As a result, Quantity distributions now behave correctly with ``to`` methods + yielding new distributions of the kind expected for the starting distribution, + and ``to_value`` yielding ``NdarrayDistribution`` instances. [#9442] + +astropy.units +^^^^^^^^^^^^^ + +- For consistency with ``ndarray``, scalar ``Quantity.value`` will now return + a numpy scalar rather than a python one. This should help keep track of + precision better, but may lead to unexpected results for the rare cases + where numpy scalars behave differently than python ones (e.g., taking the + square root of a negative number). [#8876] + +- Removed the ``magnitude_zero_points`` module, which was deprecated in + favour of ``astropy.units.photometric`` since 3.1. [#9353] + +- ``EquivalentUnitsList`` now has a ``_repr_html_`` method to output a HTML + table on a call to ``find_equivalent_units`` in Jupyter notebooks. [#9495] + +astropy.utils +^^^^^^^^^^^^^ + +- ``download_file`` and related functions now accept a list of fallback + sources, and they are able to update the cache at the user's request. [#9182] + +- Allow ``astropy.utils.console.ProgressBarOrSpinner.map`` and + ``.map_unordered`` to take an argument ``multiprocessing_start_method`` to + control how subprocesses are started; the different methods (``fork``, + ``spawn``, and ``forkserver``) have different implications in terms of + security, efficiency, and behavioural anomalies. The option is useful in + particular for cross-platform testing because Windows supports only ``spawn`` + while Linux defaults to ``fork``. [#9182] + +- All operations that act on the astropy download cache now take an argument + ``pkgname`` that allows one to specify which package's cache to use. + [#8237, #9182] + +- Removed deprecated ``funcsigs`` and ``futures`` from + ``astropy.utils.compat``. [#8909] + +- Removed the deprecated ``astropy.utils.compat.numpy`` module. [#8910] + +- Deprecated ``InheritDocstrings`` as it is natively supported by + Sphinx 1.7 or higher. [#8881] + +- Deprecated ``astropy.utils.timer`` module, which has been moved to + ``astroquery.utils.timer`` and will be part of ``astroquery`` 0.4.0. [#9038] + +- Deprecated ``astropy.utils.misc.set_locale`` function, as it is meant for + internal use only. [#9471] + +- The implementation of ``data_info.DataInfo`` has changed (for a considerable + performance boost). Generally, this should not affect simple subclasses, but + because the class now uses ``__slots__`` any attributes on the class have to + be explicitly given a slot. [#8998] + +- ``IERS`` tables now use ``nan`` to mark missing values + (rather than ``1e20``). [#9226] + +astropy.visualization +^^^^^^^^^^^^^^^^^^^^^ + +- The default ``clip`` value is now ``False`` in ``ImageNormalize``. [#9478] + +- The default ``clip`` value is now ``False`` in ``simple_norm``. + [#9698] + +- Infinite values are now excluded when calculating limits in + ``ManualInterval`` and ``MinMaxInterval``. They were already excluded in + all other interval classes. [#9480] + + +Bug Fixes +--------- + +astropy.convolution +^^^^^^^^^^^^^^^^^^^ + +- Fixed ``nan_treatment='interpolate'`` option to ``convolve_fft`` to properly + take into account ``fill_value``. [#8122] + +astropy.coordinates +^^^^^^^^^^^^^^^^^^^ + +- The ``QuantityAttribute`` class now supports a None default value if a unit + is specified. [#9345] + +- When ``Representation`` classes with the same name are defined, this no + longer leads to a ``ValueError``, but instead to a warning and the removal + of both from the name registry (i.e., one either has to use the class itself + to set, e.g., ``representation_type``, or refer to the class by its fully + qualified name). [#8561] + +astropy.io.fits +^^^^^^^^^^^^^^^ + +- Implemented skip (after warning) of header cards with reserved + keywords in ``table_to_hdu``. [#9390] + +- Add ``AstropyDeprecationWarning`` to ``read_table_fits`` when ``hdu=`` is + selected, but does not match single present table HDU. [#9512] + +astropy.io.votable +^^^^^^^^^^^^^^^^^^ + +- Address issue #8995 by ignoring BINARY2 null mask bits for string values + on parsing a VOTable. In this way, the reader should never create masked + values for string types. [#9057] + +- Corrected a spurious warning issued for the ``value`` attribute of the + ``
+ + + + + + + + + + + +
cosmology name H0 Om0 Tcmb0Neff m_nu Ob0
FlatLambdaCDM Planck18 67.66 0.309662.7255 3.046 0.0 0.0 0.060.04897
+ + + + + +The cosmology's metadata is not included in the file. + +To save the cosmology in an existing file, use ``overwrite=True``; otherwise, an +error will be raised. + + >>> Planck18.write(file, overwrite=True) + +To use a different table class as the underlying writer, use the ``cls`` kwarg. For +more information on the available table classes, see the documentation on Astropy's +table classes and on ``Cosmology.to_format("astropy.table")``. + +By default the parameter names are not converted to LaTeX / MathJax format. To +enable this, set ``latex_names=True``. + + >>> file = Path(temp_dir.name) / "file2.html" + >>> Planck18.write(file, latex_names=True) + >>> with open(file) as f: print(f.read()) + + ... + + + cosmology + name + $$H_{0}$$ + $$\Omega_{m,0}$$ + $$T_{0}$$ + $$N_{eff}$$ + $$m_{nu}$$ + $$\Omega_{b,0}$$ + + ... + +.. note:: + + A HTML file containing a Cosmology HTML table should have scripts enabling MathJax. + + .. code-block:: html + + + +.. testcleanup:: + + >>> temp_dir.cleanup() +""" + +from typing import Any, TypeVar + +import astropy.units as u +from astropy.table import QTable, Table + +# isort: split +import astropy.cosmology.units as cu +from astropy.cosmology._src.core import Cosmology +from astropy.cosmology._src.io.connect import readwrite_registry +from astropy.cosmology._src.parameter import Parameter +from astropy.cosmology._src.typing import _CosmoT +from astropy.io.typing import PathLike, ReadableFileLike, WriteableFileLike + +from .table import from_table, to_table + +_TableT = TypeVar("_TableT", bound=Table) + +# Format look-up for conversion, {original_name: new_name} +# TODO! move this information into the Parameters themselves +_FORMAT_TABLE = { + "H0": "$$H_{0}$$", + "Om0": "$$\\Omega_{m,0}$$", + "Ode0": "$$\\Omega_{\\Lambda,0}$$", + "Tcmb0": "$$T_{0}$$", + "Neff": "$$N_{eff}$$", + "m_nu": "$$m_{nu}$$", + "Ob0": "$$\\Omega_{b,0}$$", + "w0": "$$w_{0}$$", + "wa": "$$w_{a}$$", + "wz": "$$w_{z}$$", + "wp": "$$w_{p}$$", + "zp": "$$z_{p}$$", +} + + +def read_html_table( + filename: PathLike | ReadableFileLike[Table], + index: int | str | None = None, + *, + move_to_meta: bool = False, + cosmology: str | type[_CosmoT] | None = None, + latex_names: bool = True, + **kwargs: Any, +) -> _CosmoT: + r"""Read a |Cosmology| from an HTML file. + + Parameters + ---------- + filename : path-like or file-like + From where to read the Cosmology. + index : int or str or None, optional + Needed to select the row in tables with multiple rows. ``index`` can be an + integer for the row number or, if the table is indexed by a column, the value of + that column. If the table is not indexed and ``index`` is a string, the "name" + column is used as the indexing column. + + move_to_meta : bool, optional keyword-only + Whether to move keyword arguments that are not in the Cosmology class' signature + to the Cosmology's metadata. This will only be applied if the Cosmology does NOT + have a keyword-only argument (e.g. ``**kwargs``). Arguments moved to the + metadata will be merged with existing metadata, preferring specified metadata in + the case of a merge conflict (e.g. for ``Cosmology(meta={'key':10}, key=42)``, + the ``Cosmology.meta`` will be ``{'key': 10}``). + cosmology : str or |Cosmology| class or None, optional keyword-only + The cosmology class (or string name thereof) to use when constructing the + cosmology instance. The class also provides default parameter values, filling in + any non-mandatory arguments missing in 'table'. + latex_names : bool, optional keyword-only + Whether the |Table| (might) have latex column names for the parameters that need + to be mapped to the correct parameter name -- e.g. $$H_{0}$$ to 'H0'. This is + `True` by default, but can be turned off (set to `False`) if there is a known + name conflict (e.g. both an 'H0' and '$$H_{0}$$' column) as this will raise an + error. In this case, the correct name ('H0') is preferred. + **kwargs : Any + Passed to ``QTable.read``. ``format`` is set to 'ascii.html', regardless of + input. + + Returns + ------- + |Cosmology| subclass instance + + Raises + ------ + ValueError + If the keyword argument 'format' is given and is not "ascii.html". + """ + # Check that the format is 'ascii.html' (or not specified) + format = kwargs.pop("format", "ascii.html") + if format != "ascii.html": + raise ValueError(f"format must be 'ascii.html', not {format}") + + # Reading is handled by `QTable`. + with u.add_enabled_units(cu): # (cosmology units not turned on by default) + table = QTable.read(filename, format="ascii.html", **kwargs) + + # Need to map the table's column names to Cosmology inputs (parameter + # names). + # TODO! move the `latex_names` into `from_table` + if latex_names: + table_columns = set(table.colnames) + for name, latex in _FORMAT_TABLE.items(): + if latex in table_columns: + table.rename_column(latex, name) + + # Build the cosmology from table, using the private backend. + return from_table( + table, index=index, move_to_meta=move_to_meta, cosmology=cosmology, rename=None + ) + + +def write_html_table( + cosmology: Cosmology, + file: PathLike | WriteableFileLike[_TableT], + *, + overwrite: bool = False, + cls: type[_TableT] = QTable, + latex_names: bool = False, + **kwargs: Any, +) -> None: + r"""Serialize the |Cosmology| into a HTML table. + + Parameters + ---------- + cosmology : |Cosmology| subclass instance + The cosmology to serialize. + file : path-like or file-like + Where to write the html table. + + overwrite : bool, optional keyword-only + Whether to overwrite the file, if it exists. + cls : |Table| class, optional keyword-only + Astropy |Table| (sub)class to use when writing. Default is |QTable| class. + latex_names : bool, optional keyword-only + Whether to format the parameters (column) names to latex -- e.g. 'H0' to + $$H_{0}$$. + **kwargs : Any + Passed to ``cls.write``. + + Raises + ------ + TypeError + If the optional keyword-argument 'cls' is not a subclass of |Table|. + ValueError + If the keyword argument 'format' is given and is not "ascii.html". + + Examples + -------- + We assume the following setup: + + >>> from pathlib import Path + >>> from tempfile import TemporaryDirectory + >>> temp_dir = TemporaryDirectory() + + Writing a cosmology to a html file will produce a table with the cosmology's type, + name, and parameters as columns. + + >>> from astropy.cosmology import Planck18 + >>> file = Path(temp_dir.name) / "file.html" + >>> Planck18.write(file, overwrite=True) + >>> with open(file) as f: print(f.read()) + + + + + + + + + + + + + + + + + + +
cosmology name H0 Om0 Tcmb0Neff m_nu Ob0
FlatLambdaCDM Planck18 67.66 0.309662.7255 3.046 0.0 0.0 0.060.04897
+ + + + + + The cosmology's metadata is not included in the file. + + To save the cosmology in an existing file, use ``overwrite=True``; otherwise, an + error will be raised. + + >>> Planck18.write(file, overwrite=True) + + To use a different table class as the underlying writer, use the ``cls`` kwarg. For + more information on the available table classes, see the documentation on Astropy's + table classes and on ``Cosmology.to_format("astropy.table")``. + + By default the parameter names are not converted to LaTeX / MathJax format. To + enable this, set ``latex_names=True``. + + >>> file = Path(temp_dir.name) / "file2.html" + >>> Planck18.write(file, latex_names=True) + >>> with open(file) as f: print(f.read()) + + ... + + + cosmology + name + $$H_{0}$$ + $$\Omega_{m,0}$$ + $$T_{0}$$ + $$N_{eff}$$ + $$m_{nu}$$ + $$\Omega_{b,0}$$ + + ... + + .. testcleanup:: + + >>> temp_dir.cleanup() + + Notes + ----- + A HTML file containing a Cosmology HTML table should have scripts enabling MathJax. + + .. code-block:: html + + + """ + # Check that the format is 'ascii.html' (or not specified) + format = kwargs.pop("format", "ascii.html") + if format != "ascii.html": + raise ValueError(f"format must be 'ascii.html', not {format}") + + # Set cosmology_in_meta as false for now since there is no metadata being kept + table = to_table(cosmology, cls=cls, cosmology_in_meta=False) + + cosmo_cls = type(cosmology) + for name, col in table.columns.items(): + param = cosmo_cls.parameters.get(name) + if not isinstance(param, Parameter) or param.unit in (None, u.one): + continue + # Replace column with unitless version + table.replace_column(name, (col << param.unit).value, copy=False) + + if latex_names: + new_names = [_FORMAT_TABLE.get(k, k) for k in cosmology.parameters] + table.rename_columns(tuple(cosmology.parameters), new_names) + + # Write HTML, using table I/O + table.write(file, overwrite=overwrite, format="ascii.html", **kwargs) + + +def html_identify( + origin: object, filepath: object, *args: object, **kwargs: object +) -> bool: + """Identify if an object uses the HTML Table format. + + Parameters + ---------- + origin : object + Not used. + filepath : object + From where to read the Cosmology. + *args : object + Not used. + **kwargs : object + Not used. + + Returns + ------- + bool + If the filepath is a string ending with '.html'. + """ + return isinstance(filepath, str) and filepath.endswith(".html") + + +# =================================================================== +# Register + +readwrite_registry.register_reader("ascii.html", Cosmology, read_html_table) +readwrite_registry.register_writer("ascii.html", Cosmology, write_html_table) +readwrite_registry.register_identifier("ascii.html", Cosmology, html_identify) diff --git a/astropy/cosmology/_src/io/builtin/latex.py b/astropy/cosmology/_src/io/builtin/latex.py new file mode 100644 index 000000000000..af71a42e31de --- /dev/null +++ b/astropy/cosmology/_src/io/builtin/latex.py @@ -0,0 +1,214 @@ +r"""|Cosmology| <-> LaTeX I/O, using |Cosmology.read| and |Cosmology.write|. + +We assume the following setup: + + >>> from pathlib import Path + >>> from tempfile import TemporaryDirectory + >>> temp_dir = TemporaryDirectory() + +Writing a cosmology to a LaTeX file will produce a table with the cosmology's type, +name, and parameters as columns. + + >>> from astropy.cosmology import Cosmology, Planck18 + >>> file = Path(temp_dir.name) / "file.tex" + + >>> Planck18.write(file, format="ascii.latex") + >>> with open(file) as f: print(f.read()) + \begin{table} + \begin{tabular}{cccccccc} + cosmology & name & $H_0$ & $\Omega_{m,0}$ & $T_{0}$ & $N_{eff}$ & $m_{nu}$ & $\Omega_{b,0}$ \\ + & & $\mathrm{km\,Mpc^{-1}\,s^{-1}}$ & & $\mathrm{K}$ & & $\mathrm{eV}$ & \\ + FlatLambdaCDM & Planck18 & 67.66 & 0.30966 & 2.7255 & 3.046 & 0.0 .. 0.06 & 0.04897 \\ + \end{tabular} + \end{table} + + +The cosmology's metadata is not included in the table. + +To save the cosmology in an existing file, use ``overwrite=True``; otherwise, an +error will be raised. + + >>> Planck18.write(file, format="ascii.latex", overwrite=True) + +To use a different table class as the underlying writer, use the ``cls`` kwarg. For +more information on the available table classes, see the documentation on Astropy's +table classes and on ``Cosmology.to_format("astropy.table")``. + +By default the parameter names are converted to LaTeX format. To disable this, set +``latex_names=False``. + + >>> file = Path(temp_dir.name) / "file2.tex" + >>> Planck18.write(file, format="ascii.latex", latex_names=False) + >>> with open(file) as f: print(f.read()) + \begin{table} + \begin{tabular}{cccccccc} + cosmology & name & H0 & Om0 & Tcmb0 & Neff & m_nu & Ob0 \\ + & & $\mathrm{km\,Mpc^{-1}\,s^{-1}}$ & & $\mathrm{K}$ & & $\mathrm{eV}$ & \\ + FlatLambdaCDM & Planck18 & 67.66 & 0.30966 & 2.7255 & 3.046 & 0.0 .. 0.06 & 0.04897 \\ + \end{tabular} + \end{table} + + +.. testcleanup:: + + >>> temp_dir.cleanup() +""" + +from typing import Any, TypeVar + +import astropy.units as u +from astropy.cosmology._src.core import Cosmology +from astropy.cosmology._src.io.connect import readwrite_registry +from astropy.cosmology._src.parameter import Parameter +from astropy.io.typing import PathLike, WriteableFileLike +from astropy.table import QTable, Table + +from .table import to_table + +_TableT = TypeVar("_TableT", bound=Table) + +_FORMAT_TABLE = { + "H0": "$H_0$", + "Om0": r"$\Omega_{m,0}$", + "Ode0": r"$\Omega_{\Lambda,0}$", + "Tcmb0": "$T_{0}$", + "Neff": "$N_{eff}$", + "m_nu": "$m_{nu}$", + "Ob0": r"$\Omega_{b,0}$", + "w0": "$w_{0}$", + "wa": "$w_{a}$", + "wz": "$w_{z}$", + "wp": "$w_{p}$", + "zp": "$z_{p}$", +} + + +def write_latex( + cosmology: Cosmology, + file: PathLike | WriteableFileLike[_TableT], + *, + overwrite: bool = False, + cls: type[_TableT] = QTable, + latex_names: bool = True, + **kwargs: Any, +) -> None: + r"""Serialize the |Cosmology| into a LaTeX. + + Parameters + ---------- + cosmology : `~astropy.cosmology.Cosmology` subclass instance + The cosmology to serialize. + file : path-like or file-like + Location to save the serialized cosmology. + + overwrite : bool + Whether to overwrite the file, if it exists. + cls : type, optional keyword-only + Astropy :class:`~astropy.table.Table` (sub)class to use when writing. Default is + :class:`~astropy.table.QTable`. + latex_names : bool, optional keyword-only + Whether to use LaTeX names for the parameters. Default is `True`. + **kwargs + Passed to ``cls.write`` + + Raises + ------ + TypeError + If kwarg (optional) 'cls' is not a subclass of `astropy.table.Table` + + Examples + -------- + We assume the following setup: + + >>> from pathlib import Path + >>> from tempfile import TemporaryDirectory + >>> temp_dir = TemporaryDirectory() + + Writing a cosmology to a LaTeX file will produce a table with the cosmology's type, + name, and parameters as columns. + + >>> from astropy.cosmology import Planck18 + >>> file = Path(temp_dir.name) / "file.tex" + + >>> Planck18.write(file, format="ascii.latex") + >>> with open(file) as f: print(f.read()) + \begin{table} + \begin{tabular}{cccccccc} + cosmology & name & $H_0$ & $\Omega_{m,0}$ & $T_{0}$ & $N_{eff}$ & $m_{nu}$ & $\Omega_{b,0}$ \\ + & & $\mathrm{km\,Mpc^{-1}\,s^{-1}}$ & & $\mathrm{K}$ & & $\mathrm{eV}$ & \\ + FlatLambdaCDM & Planck18 & 67.66 & 0.30966 & 2.7255 & 3.046 & 0.0 .. 0.06 & 0.04897 \\ + \end{tabular} + \end{table} + + + The cosmology's metadata is not included in the table. + + To save the cosmology in an existing file, use ``overwrite=True``; otherwise, an + error will be raised. + + >>> Planck18.write(file, format="ascii.latex", overwrite=True) + + To use a different table class as the underlying writer, use the ``cls`` kwarg. For + more information on the available table classes, see the documentation on Astropy's + table classes and on ``Cosmology.to_format("astropy.table")``. + + By default the parameter names are converted to LaTeX format. To disable this, set + ``latex_names=False``. + + >>> file = Path(temp_dir.name) / "file2.tex" + >>> Planck18.write(file, format="ascii.latex", latex_names=False) + >>> with open(file) as f: print(f.read()) + \begin{table} + \begin{tabular}{cccccccc} + cosmology & name & H0 & Om0 & Tcmb0 & Neff & m_nu & Ob0 \\ + & & $\mathrm{km\,Mpc^{-1}\,s^{-1}}$ & & $\mathrm{K}$ & & $\mathrm{eV}$ & \\ + FlatLambdaCDM & Planck18 & 67.66 & 0.30966 & 2.7255 & 3.046 & 0.0 .. 0.06 & 0.04897 \\ + \end{tabular} + \end{table} + + + .. testcleanup:: + + >>> temp_dir.cleanup() + """ + # Check that the format is 'latex', 'ascii.latex' (or not specified) + fmt = kwargs.pop("format", "ascii.latex") + if fmt != "ascii.latex": + raise ValueError(f"format must be 'ascii.latex', not {fmt}") + + # Set cosmology_in_meta as false for now since there is no metadata being kept + table = to_table(cosmology, cls=cls, cosmology_in_meta=False) + + cosmo_cls = type(cosmology) + for name in table.columns.keys(): + param = cosmo_cls.parameters.get(name) + if not isinstance(param, Parameter) or param.unit in (None, u.one): + continue + # Get column to correct unit + table[name] <<= param.unit + + # Convert parameter names to LaTeX format + if latex_names: + new_names = [_FORMAT_TABLE.get(k, k) for k in cosmology.parameters] + table.rename_columns(tuple(cosmology.parameters), new_names) + + table.write(file, overwrite=overwrite, format="ascii.latex", **kwargs) + + +def latex_identify( + origin: object, filepath: str | None, *args: object, **kwargs: object +) -> bool: + """Identify if object uses the Table format. + + Returns + ------- + bool + """ + return filepath is not None and filepath.endswith(".tex") + + +# =================================================================== +# Register + +readwrite_registry.register_writer("ascii.latex", Cosmology, write_latex) +readwrite_registry.register_identifier("ascii.latex", Cosmology, latex_identify) diff --git a/astropy/cosmology/_src/io/builtin/mapping.py b/astropy/cosmology/_src/io/builtin/mapping.py new file mode 100644 index 000000000000..8e46ee9bf82a --- /dev/null +++ b/astropy/cosmology/_src/io/builtin/mapping.py @@ -0,0 +1,432 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""|Cosmology| <-> Mapping I/O, using |Cosmology.to_format| and |Cosmology.from_format|. + +This module provides functions to transform a |Cosmology| instance to a mapping +(`dict`-like) object and vice versa, from a mapping object back to a |Cosmology| +instance. The functions are registered with ``convert_registry`` under the format name +"mapping". The mapping object is a `dict`-like object, with the cosmology's parameters +and metadata as items. `dict` is a fundamental data structure in Python, and this +representation of a |Cosmology| is useful for translating between many serialization and +storage formats, or even passing arguments to functions. + +We start with the simple case of outputting a |Cosmology| as a mapping. + + >>> from astropy.cosmology import Cosmology, Planck18 + >>> cm = Planck18.to_format('mapping') + >>> cm + {'cosmology': , + 'name': 'Planck18', 'H0': , 'Om0': 0.30966, + 'Tcmb0': , 'Neff': 3.046, + 'm_nu': , 'Ob0': 0.04897, + 'meta': ... + +``cm`` is a `dict`, with the cosmology's parameters and metadata as items. + +How might we use this `dict`? One use is to unpack the `dict` into a function: + + >>> def function(H0, Tcmb0, **kwargs): ... + >>> function(**cm) + +Another use is to merge the `dict` with another `dict`: + + >>> cm2 = {'H0': 70, 'Tcmb0': 2.7} + >>> cm | cm2 + {'cosmology': , ..., 'H0': 70, ...} + +Most saliently, the `dict` can also be used to construct a new cosmological instance +identical to the |Planck18| cosmology from which it was generated. + + >>> cosmo = Cosmology.from_format(cm, format="mapping") + >>> cosmo + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=, Ob0=0.04897) + +How did |Cosmology.from_format| know to return an instance of the |FlatLambdaCDM| class? +The mapping object has a field ``cosmology`` which can be either the string name of the +cosmology class (e.g. "FlatLambdaCDM") or the class itself. + +This field can be omitted under two conditions. + +1. If the cosmology class is passed as the ``cosmology`` keyword argument to + |Cosmology.from_format|, +2. If a specific cosmology class, e.g. |FlatLambdaCDM|, is used to parse the data. + +To the first point, we can pass the cosmology class as the ``cosmology`` keyword +argument to |Cosmology.from_format|. + + >>> del cm["cosmology"] # remove cosmology class + + >>> Cosmology.from_format(cm, cosmology="FlatLambdaCDM") + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=, Ob0=0.04897) + +To the second point, we can use specific cosmology class to parse the data. + + >>> from astropy.cosmology import FlatLambdaCDM + >>> FlatLambdaCDM.from_format(cm) + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=, Ob0=0.04897) + +Also, the class' default parameter values are used to fill in any information missing in +the data. For example, if ``Tcmb0`` is missing, the default value of 0.0 K is used. + + >>> del cm["Tcmb0"] # show FlatLambdaCDM provides default + >>> FlatLambdaCDM.from_format(cm) + FlatLambdaCDM(name='Planck18', H0=..., Tcmb0=, ...) + +If instead of *missing* information, there is *extra* information, there are a few +options. The first is to use the ``move_to_meta`` keyword argument to move fields that +are not in the Cosmology constructor to the Cosmology's metadata. + + >>> cm2 = cm | {"extra": 42, "cosmology": "FlatLambdaCDM"} + >>> cosmo = Cosmology.from_format(cm2, move_to_meta=True) + >>> cosmo.meta + {'extra': 42, ...} + +Alternatively, the ``rename`` keyword argument can be used to rename keys in the mapping +to fields of the |Cosmology|. This is crucial when the mapping has keys that are not +valid arguments to the |Cosmology| constructor. + + >>> cm3 = dict(cm) # copy + >>> cm3["cosmo_cls"] = "FlatLambdaCDM" + >>> cm3["cosmo_name"] = cm3.pop("name") + + >>> rename = {'cosmo_cls': 'cosmology', 'cosmo_name': 'name'} + >>> Cosmology.from_format(cm3, rename=rename) + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=None, Ob0=0.04897) + +Let's take a closer look at |Cosmology.to_format|, because there a lot of options, to +tailor the output to specific needs. + +The dictionary type may be changed with the ``cls`` keyword argument: + + >>> from collections import OrderedDict + >>> Planck18.to_format('mapping', cls=OrderedDict) + OrderedDict({'cosmology': , 'name': 'Planck18', 'H0': , 'Om0': 0.30966, 'Tcmb0': , 'Neff': 3.046, 'm_nu': , 'Ob0': 0.04897, 'meta': {...}}) + +Sometimes it is more useful to have the name of the cosmology class, not the type +itself. The keyword argument ``cosmology_as_str`` may be used: + + >>> Planck18.to_format('mapping', cosmology_as_str=True) + {'cosmology': 'FlatLambdaCDM', ... + +The metadata is normally included as a nested mapping. To move the metadata into the +main mapping, use the keyword argument ``move_from_meta``. This kwarg inverts +``move_to_meta`` in ``Cosmology.to_format("mapping", move_to_meta=...)`` where extra +items are moved to the metadata (if the cosmology constructor does not have a variable +keyword-only argument -- ``**kwargs``). + + >>> from astropy.cosmology import Planck18 + >>> Planck18.to_format('mapping', move_from_meta=True) + {'cosmology': , + 'name': 'Planck18', 'Oc0': 0.2607, 'n': 0.9665, 'sigma8': 0.8102, ... + +Lastly, the keys in the mapping may be renamed with the ``rename`` keyword. + + >>> rename = {'cosmology': 'cosmo_cls', 'name': 'cosmo_name'} + >>> Planck18.to_format('mapping', rename=rename) + {'cosmo_cls': , + 'cosmo_name': 'Planck18', ... +""" + +__all__: list[str] = [] # nothing is publicly scoped + +import copy +import inspect +from collections.abc import Mapping, MutableMapping +from typing import Any, TypeVar + +from astropy.cosmology._src.core import _COSMOLOGY_CLASSES, Cosmology +from astropy.cosmology._src.io.connect import convert_registry +from astropy.cosmology._src.typing import _CosmoT + +_MapT = TypeVar("_MapT", bound=MutableMapping[str, Any]) + + +def _rename_map( + map: Mapping[str, Any], /, renames: Mapping[str, str] +) -> dict[str, Any]: + """Apply rename to map.""" + if common_names := set(renames.values()).intersection(map): + raise ValueError( + "'renames' values must be disjoint from 'map' keys, " + f"the common keys are: {common_names}" + ) + return {renames.get(k, k): v for k, v in map.items()} # dict separate from input + + +def _get_cosmology_class( + cosmology: type[_CosmoT] | str | None, params: dict[str, Any], / +) -> type[_CosmoT]: + # get cosmology + # 1st from argument. Allows for override of the cosmology, if on file. + # 2nd from params. This MUST have the cosmology if 'kwargs' did not. + if cosmology is None: + cosmology = params.pop("cosmology") + else: + params.pop("cosmology", None) # pop, but don't use + # if string, parse to class + return _COSMOLOGY_CLASSES[cosmology] if isinstance(cosmology, str) else cosmology + + +def from_mapping( + mapping: Mapping[str, Any], + /, + *, + move_to_meta: bool = False, + cosmology: str | type[_CosmoT] | None = None, + rename: Mapping[str, str] | None = None, +) -> _CosmoT: + """Load `~astropy.cosmology.Cosmology` from mapping object. + + Parameters + ---------- + mapping : Mapping + Arguments into the class -- like "name" or "meta". If 'cosmology' is None, must + have field "cosmology" which can be either the string name of the cosmology + class (e.g. "FlatLambdaCDM") or the class itself. + + move_to_meta : bool (optional, keyword-only) + Whether to move keyword arguments that are not in the Cosmology class' signature + to the Cosmology's metadata. This will only be applied if the Cosmology does NOT + have a keyword-only argument (e.g. ``**kwargs``). Arguments moved to the + metadata will be merged with existing metadata, preferring specified metadata in + the case of a merge conflict (e.g. for ``Cosmology(meta={'key':10}, key=42)``, + the ``Cosmology.meta`` will be ``{'key': 10}``). + + cosmology : str, |Cosmology| class, or None (optional, keyword-only) + The cosmology class (or string name thereof) to use when constructing the + cosmology instance. The class also provides default parameter values, filling in + any non-mandatory arguments missing in 'map'. + + rename : Mapping[str, str] or None (optional, keyword-only) + A mapping of keys in ``map`` to fields of the `~astropy.cosmology.Cosmology`. + + Returns + ------- + `~astropy.cosmology.Cosmology` subclass instance + + Examples + -------- + To see loading a `~astropy.cosmology.Cosmology` from a dictionary with + ``from_mapping``, we will first make a mapping using + :meth:`~astropy.cosmology.Cosmology.to_format`. + + >>> from astropy.cosmology import Cosmology, Planck18 + >>> cm = Planck18.to_format('mapping') + >>> cm + {'cosmology': , + 'name': 'Planck18', 'H0': , 'Om0': 0.30966, + 'Tcmb0': , 'Neff': 3.046, + 'm_nu': , 'Ob0': 0.04897, + 'meta': ... + + Now this dict can be used to load a new cosmological instance identical to the + |Planck18| cosmology from which it was generated. + + >>> cosmo = Cosmology.from_format(cm, format="mapping") + >>> cosmo + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=, Ob0=0.04897) + + The ``cosmology`` field can be omitted if the cosmology class (or its string name) + is passed as the ``cosmology`` keyword argument to |Cosmology.from_format|. + + >>> del cm["cosmology"] # remove cosmology class + >>> Cosmology.from_format(cm, cosmology="FlatLambdaCDM") + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=, Ob0=0.04897) + + Alternatively, specific cosmology classes can be used to parse the data. + + >>> from astropy.cosmology import FlatLambdaCDM + >>> FlatLambdaCDM.from_format(cm) + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=, Ob0=0.04897) + + When using a specific cosmology class, the class' default parameter values are used + to fill in any missing information. + + >>> del cm["Tcmb0"] # show FlatLambdaCDM provides default + >>> FlatLambdaCDM.from_format(cm) + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=None, Ob0=0.04897) + + The ``move_to_meta`` keyword argument can be used to move fields that are not in the + Cosmology constructor to the Cosmology's metadata. This is useful when the + dictionary contains extra information that is not part of the Cosmology. + + >>> cm2 = cm | {"extra": 42, "cosmology": "FlatLambdaCDM"} + >>> cosmo = Cosmology.from_format(cm2, move_to_meta=True) + >>> cosmo.meta + {'extra': 42, ...} + + The ``rename`` keyword argument can be used to rename keys in the mapping to fields + of the |Cosmology|. This is crucial when the mapping has keys that are not valid + arguments to the |Cosmology| constructor. + + >>> cm3 = dict(cm) # copy + >>> cm3["cosmo_cls"] = "FlatLambdaCDM" + >>> cm3["cosmo_name"] = cm3.pop("name") + + >>> rename = {'cosmo_cls': 'cosmology', 'cosmo_name': 'name'} + >>> Cosmology.from_format(cm3, rename=rename) + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=None, Ob0=0.04897) + """ + # Rename keys, if given a ``renames`` dict. + # Also, make a copy of the mapping, so we can pop from it. + params = _rename_map(dict(mapping), renames=rename or {}) + + # Get cosmology class + cosmology = _get_cosmology_class(cosmology, params) + + # select arguments from mapping that are in the cosmo's signature. + sig = inspect.signature(cosmology) + ba = sig.bind_partial() # blank set of args + ba.apply_defaults() # fill in the defaults + for k in sig.parameters.keys(): + if k in params: # transfer argument, if in params + ba.arguments[k] = params.pop(k) + + # deal with remaining params. If there is a **kwargs use that, else + # allow to transfer to metadata. Raise TypeError if can't. + lastp = next(reversed(sig.parameters.values())) + if lastp.kind == 4: # variable keyword-only + ba.arguments[lastp.name] = params + elif move_to_meta: # prefers current meta, which was explicitly set + meta = ba.arguments["meta"] or {} # (None -> dict) + ba.arguments["meta"] = {**params, **meta} + elif params: + raise TypeError(f"there are unused parameters {params}.") + # else: pass # no kwargs, no move-to-meta, and all the params are used + + return cosmology(*ba.args, **ba.kwargs) + + +def to_mapping( + cosmology: Cosmology, + *args: object, + cls: type[_MapT] = dict, + cosmology_as_str: bool = False, + move_from_meta: bool = False, + rename: Mapping[str, str] | None = None, +) -> _MapT: + """Return the cosmology class, parameters, and metadata as a `dict`. + + Parameters + ---------- + cosmology : :class:`~astropy.cosmology.Cosmology` + The cosmology instance to convert to a mapping. + *args : object + Not used. Needed for compatibility with + `~astropy.io.registry.UnifiedReadWriteMethod` + cls : type (optional, keyword-only) + `dict` or `collections.Mapping` subclass. + The mapping type to return. Default is `dict`. + cosmology_as_str : bool (optional, keyword-only) + Whether the cosmology value is the class (if `False`, default) or + the semi-qualified name (if `True`). + move_from_meta : bool (optional, keyword-only) + Whether to add the Cosmology's metadata as an item to the mapping (if + `False`, default) or to merge with the rest of the mapping, preferring + the original values (if `True`) + rename : Mapping[str, str] or None (optional, keyword-only) + A mapping of field names of the :class:`~astropy.cosmology.Cosmology` to keys in + the map. + + Returns + ------- + MutableMapping[str, Any] + A mapping of type ``cls``, by default a `dict`. + Has key-values for the cosmology parameters and also: + - 'cosmology' : the class + - 'meta' : the contents of the cosmology's metadata attribute. + If ``move_from_meta`` is `True`, this key is missing and the + contained metadata are added to the main `dict`. + + Examples + -------- + A Cosmology as a mapping will have the cosmology's name and + parameters as items, and the metadata as a nested dictionary. + + >>> from astropy.cosmology import Planck18 + >>> Planck18.to_format('mapping') + {'cosmology': , + 'name': 'Planck18', 'H0': , 'Om0': 0.30966, + 'Tcmb0': , 'Neff': 3.046, + 'm_nu': , 'Ob0': 0.04897, + 'meta': ... + + The dictionary type may be changed with the ``cls`` keyword argument: + + >>> from collections import OrderedDict + >>> Planck18.to_format('mapping', cls=OrderedDict) + OrderedDict({'cosmology': , 'name': 'Planck18', 'H0': , 'Om0': 0.30966, 'Tcmb0': , 'Neff': 3.046, 'm_nu': , 'Ob0': 0.04897, 'meta': {...}}) + + Sometimes it is more useful to have the name of the cosmology class, not + the type itself. The keyword argument ``cosmology_as_str`` may be used: + + >>> Planck18.to_format('mapping', cosmology_as_str=True) + {'cosmology': 'FlatLambdaCDM', ... + + The metadata is normally included as a nested mapping. To move the metadata + into the main mapping, use the keyword argument ``move_from_meta``. This + kwarg inverts ``move_to_meta`` in + ``Cosmology.to_format("mapping", move_to_meta=...)`` where extra items + are moved to the metadata (if the cosmology constructor does not have a + variable keyword-only argument -- ``**kwargs``). + + >>> from astropy.cosmology import Planck18 + >>> Planck18.to_format('mapping', move_from_meta=True) + {'cosmology': , + 'name': 'Planck18', 'Oc0': 0.2607, 'n': 0.9665, 'sigma8': 0.8102, ... + + Lastly, the keys in the mapping may be renamed with the ``rename`` keyword. + + >>> rename = {'cosmology': 'cosmo_cls', 'name': 'cosmo_name'} + >>> Planck18.to_format('mapping', rename=rename) + {'cosmo_cls': , + 'cosmo_name': 'Planck18', ... + """ + if not issubclass(cls, (dict, Mapping)): + raise TypeError(f"'cls' must be a (sub)class of dict or Mapping, not {cls}") + + m = cls() + # start with the cosmology class & name + m["cosmology"] = ( + cosmology.__class__.__qualname__ if cosmology_as_str else cosmology.__class__ + ) + m["name"] = cosmology.name # here only for dict ordering + + meta = copy.deepcopy(cosmology.meta) # metadata (mutable) + if move_from_meta: + # Merge the mutable metadata. Since params are added later they will + # be preferred in cases of overlapping keys. Likewise, need to pop + # cosmology and name from meta. + meta.pop("cosmology", None) + meta.pop("name", None) + m.update(meta) + + # Add all the immutable inputs + m.update(cosmology.parameters) + # Lastly, add the metadata, if haven't already (above) + if not move_from_meta: + m["meta"] = meta # TODO? should meta be type(cls) + # Rename keys + return m if rename is None else _rename_map(m, rename) + + +def mapping_identify( + origin: str, format: str | None, *args: object, **kwargs: object +) -> bool: + """Identify if object uses the mapping format. + + Returns + ------- + bool + """ + itis = False + if origin == "read": + itis = isinstance(args[1], Mapping) and (format in (None, "mapping")) + return itis + + +# =================================================================== +# Register + +convert_registry.register_reader("mapping", Cosmology, from_mapping) +convert_registry.register_writer("mapping", Cosmology, to_mapping) +convert_registry.register_identifier("mapping", Cosmology, mapping_identify) diff --git a/astropy/cosmology/_src/io/builtin/model.py b/astropy/cosmology/_src/io/builtin/model.py new file mode 100644 index 000000000000..6bbfeb9124bd --- /dev/null +++ b/astropy/cosmology/_src/io/builtin/model.py @@ -0,0 +1,299 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""|Cosmology| <-> Model I/O, using |Cosmology.to_format| and |Cosmology.from_format|. + +This module provides functions to transform a |Cosmology| object to and from a +:class:`~astropy.modeling.Model`. The functions are registered with ``convert_registry`` +under the format name "astropy.model". + +Using ``format="astropy.model"`` any redshift(s) method of a cosmology may be turned +into a :class:`astropy.modeling.Model`. Each |Cosmology| +:class:`~astropy.cosmology.Parameter` is converted to a :class:`astropy.modeling.Model` +:class:`~astropy.modeling.Parameter` and the redshift-method to the model's ``__call__ / +evaluate``. Now you can fit cosmologies with data! + +.. code-block:: + + >>> from astropy.cosmology import Cosmology, Planck18 + >>> model = Planck18.to_format("astropy.model", method="lookback_time") + >>> model + + +The |Planck18| cosmology can be recovered with |Cosmology.from_format|. + + >>> print(Cosmology.from_format(model)) + FlatLambdaCDM(name="Planck18", H0=67.66 km / (Mpc s), Om0=0.30966, + Tcmb0=2.7255 K, Neff=3.046, m_nu=[0. 0. 0.06] eV, Ob0=0.04897) +""" + +import abc +import copy +import inspect +from dataclasses import replace +from typing import Generic + +import numpy as np + +from astropy.modeling import FittableModel, Model +from astropy.utils.decorators import classproperty + +# isort: split +from astropy.cosmology._src.core import Cosmology +from astropy.cosmology._src.io.connect import convert_registry +from astropy.cosmology._src.typing import _CosmoT + +from .utils import convert_parameter_to_model_parameter + +__all__: list[str] = [] # nothing is publicly scoped + + +class _CosmologyModel(FittableModel, Generic[_CosmoT]): + """Base class for Cosmology redshift-method Models. + + .. note:: + + This class is not publicly scoped so should not be used directly. + Instead, from a Cosmology instance use ``.to_format("astropy.model")`` + to create an instance of a subclass of this class. + + `_CosmologyModel` (subclasses) wrap a redshift-method of a + :class:`~astropy.cosmology.Cosmology` class, converting each non-`None` + |Cosmology| :class:`~astropy.cosmology.Parameter` to a + :class:`astropy.modeling.Model` :class:`~astropy.modeling.Parameter` + and the redshift-method to the model's ``__call__ / evaluate``. + + See Also + -------- + astropy.cosmology.Cosmology.to_format + """ + + @abc.abstractmethod + def _cosmology_class(self) -> type[_CosmoT]: + """Cosmology class as a private attribute. + + Set in subclasses. + """ + + @abc.abstractmethod + def _method_name(self) -> str: + """Cosmology method name as a private attribute. + + Set in subclasses. + """ + + @classproperty + def cosmology_class(cls) -> type[_CosmoT]: + """|Cosmology| class.""" + return cls._cosmology_class + + @classproperty(lazy=True) + def _cosmology_class_sig(cls): + """Signature of |Cosmology| class.""" + return inspect.signature(cls._cosmology_class) + + @property + def cosmology(self) -> _CosmoT: + """Return |Cosmology| using `~astropy.modeling.Parameter` values.""" + return self._cosmology_class( + name=self.name, + **{ + k: (v.value if not (v := getattr(self, k)).unit else v.quantity) + for k in self.param_names + }, + ) + + @classproperty + def method_name(self) -> str: + """Redshift-method name on |Cosmology| instance.""" + return self._method_name + + # --------------------------------------------------------------- + + # NOTE: cannot add type annotations b/c of how Model introspects + def evaluate(self, *args, **kwargs): + """Evaluate method {method!r} of {cosmo_cls!r} Cosmology. + + The Model wraps the :class:`~astropy.cosmology.Cosmology` method, + converting each |Cosmology| :class:`~astropy.cosmology.Parameter` to a + :class:`astropy.modeling.Model` :class:`~astropy.modeling.Parameter` + (unless the Parameter is None, in which case it is skipped). + Here an instance of the cosmology is created using the current + Parameter values and the method is evaluated given the input. + + Parameters + ---------- + *args, **kwargs + The first ``n_inputs`` of ``*args`` are for evaluating the method + of the cosmology. The remaining args and kwargs are passed to the + cosmology class constructor. + Any unspecified Cosmology Parameter use the current value of the + corresponding Model Parameter. + + Returns + ------- + Any + Results of evaluating the Cosmology method. + """ + # TODO: speed up using ``replace`` + + # create BoundArgument with all available inputs beyond the Parameters, + # which will be filled in next + ba = self._cosmology_class_sig.bind_partial(*args[self.n_inputs :], **kwargs) + + # fill in missing Parameters + for k in self.param_names: + if k not in ba.arguments: + v = getattr(self, k) + ba.arguments[k] = v.value if not v.unit else v.quantity + + # unvectorize, since Cosmology is not vectorized + # TODO! remove when vectorized + if np.shape(ba.arguments[k]): # only in __call__ + # m_nu is a special case # TODO! fix by making it 'structured' + if k == "m_nu" and len(ba.arguments[k].shape) == 1: + continue + ba.arguments[k] = ba.arguments[k][0] + + # make instance of cosmology + cosmo = self._cosmology_class(**ba.arguments) + # evaluate method + return getattr(cosmo, self._method_name)(*args[: self.n_inputs]) + + +############################################################################## + + +def from_model(model: _CosmologyModel[_CosmoT]) -> _CosmoT: + """Load |Cosmology| from `~astropy.modeling.Model` object. + + Parameters + ---------- + model : `_CosmologyModel` subclass instance + See ``Cosmology.to_format.help("astropy.model") for details. + + Returns + ------- + `~astropy.cosmology.Cosmology` subclass instance + + Examples + -------- + >>> from astropy.cosmology import Cosmology, Planck18 + >>> model = Planck18.to_format("astropy.model", method="lookback_time") + >>> print(Cosmology.from_format(model)) + FlatLambdaCDM(name="Planck18", H0=67.66 km / (Mpc s), Om0=0.30966, + Tcmb0=2.7255 K, Neff=3.046, m_nu=[0. 0. 0.06] eV, Ob0=0.04897) + """ + cosmo = model.cosmology + + # assemble the metadata + meta = copy.deepcopy(model.meta) + for n in model.param_names: + p = getattr(model, n) + meta[p.name] = { + n: getattr(p, n) + for n in dir(p) + if not (n.startswith("_") or callable(getattr(p, n))) + } + return replace(cosmo, meta=meta) + + +def to_model(cosmology: _CosmoT, *_: object, method: str) -> _CosmologyModel[_CosmoT]: + """Convert a `~astropy.cosmology.Cosmology` to a `~astropy.modeling.Model`. + + Parameters + ---------- + cosmology : `~astropy.cosmology.Cosmology` subclass instance + method : str, keyword-only + The name of the method on the ``cosmology``. + + Returns + ------- + `_CosmologyModel` subclass instance + The Model wraps the |Cosmology| method, converting each non-`None` + :class:`~astropy.cosmology.Parameter` to a + :class:`astropy.modeling.Model` :class:`~astropy.modeling.Parameter` + and the method to the model's ``__call__ / evaluate``. + + Examples + -------- + >>> from astropy.cosmology import Planck18 + >>> model = Planck18.to_format("astropy.model", method="lookback_time") + >>> model + + """ + cosmo_cls = cosmology.__class__ + + # get bound method & sig from cosmology (unbound if class). + if not hasattr(cosmology, method): + raise AttributeError(f"{method} is not a method on {cosmology.__class__}.") + func = getattr(cosmology, method) + if not callable(func): + raise ValueError(f"{cosmology.__class__}.{method} is not callable.") + msig = inspect.signature(func) + + # introspect for number of positional inputs, ignoring "self" + n_inputs = len([p for p in tuple(msig.parameters.values()) if (p.kind in (0, 1))]) + + attrs = {} # class attributes + attrs["_cosmology_class"] = cosmo_cls + attrs["_method_name"] = method + attrs["n_inputs"] = n_inputs + attrs["n_outputs"] = 1 + + params = { + k: convert_parameter_to_model_parameter( + cosmo_cls.parameters[k], v, meta=cosmology.meta.get(k) + ) + for k, v in cosmology.parameters.items() + if v is not None + } + + # class name is cosmology name + Cosmology + method name + Model + clsname = ( + cosmo_cls.__qualname__.replace(".", "_") + + "Cosmology" + + method.replace("_", " ").title().replace(" ", "") + + "Model" + ) + + # make Model class + CosmoModel = type(clsname, (_CosmologyModel,), {**attrs, **params}) + # override __signature__ and format the doc. + CosmoModel.evaluate.__signature__ = msig + if CosmoModel.evaluate.__doc__ is not None: + # guard against PYTHONOPTIMIZE mode + CosmoModel.evaluate.__doc__ = CosmoModel.evaluate.__doc__.format( + cosmo_cls=cosmo_cls.__qualname__, method=method + ) + + # instantiate class using default values + return CosmoModel( + **cosmology.parameters, name=cosmology.name, meta=copy.deepcopy(cosmology.meta) + ) + + +def model_identify( + origin: str, format: str | None, *args: object, **kwargs: object +) -> bool: + """Identify if object uses the :class:`~astropy.modeling.Model` format. + + Returns + ------- + bool + """ + itis = False + if origin == "read": + itis = isinstance(args[1], Model) and (format in (None, "astropy.model")) + + return itis + + +# =================================================================== +# Register + +convert_registry.register_reader("astropy.model", Cosmology, from_model) +convert_registry.register_writer("astropy.model", Cosmology, to_model) +convert_registry.register_identifier("astropy.model", Cosmology, model_identify) diff --git a/astropy/cosmology/_src/io/builtin/mrt.py b/astropy/cosmology/_src/io/builtin/mrt.py new file mode 100644 index 000000000000..1217fcd11748 --- /dev/null +++ b/astropy/cosmology/_src/io/builtin/mrt.py @@ -0,0 +1,324 @@ +r"""|Cosmology| <-> MRT I/O, using |Cosmology.read| and |Cosmology.write|. + +We assume the following setup: + + >>> from pathlib import Path + >>> from tempfile import TemporaryDirectory + >>> temp_dir = TemporaryDirectory() + +Writing a cosmology to a mrt file will produce a table with the cosmology's type, +name, and parameters as columns. Note that the cosmology class is also included as +a column since MRT format does not preserve table metadata. + + >>> from astropy.cosmology import Planck18 + >>> file = Path(temp_dir.name) / "file.mrt" + >>> Planck18.write(file) + >>> with open(file) as f: print(f.read()) + Title: + Authors: + Table: + ================================================================================ + Byte-by-byte Description of file: table.dat + -------------------------------------------------------------------------------- + Bytes Format Units Label Explanations + -------------------------------------------------------------------------------- + 1-13 A13 --- cosmology Description of cosmology + 15-22 A8 --- name Description of name + 24-28 F5.2 km.Mpc-1.s-1 H0 [67.66/67.66] Hubble constant at z=0. + 30-36 F7.5 --- Om0 [0.3/0.31] Omega matter; matter + density/critical density at z=0. + 38-43 F6.4 K Tcmb0 [2.72/2.73] Temperature of the CMB at z=0. + 45-49 F5.3 --- Neff [3.04/3.05] Number of effective neutrino + species. + 51-64 A14 --- m_nu [[0. 0. 0.06]] Mass of neutrino + species. + 66-72 F7.5 --- Ob0 [0.04/0.05] Omega baryon; baryonic matter + density/critical density at z=0. + -------------------------------------------------------------------------------- + ... + + +.. testcleanup:: + + >>> temp_dir.cleanup() +""" + +__all__ = ("mrt_identify", "read_mrt", "write_mrt") + +import contextlib +import json +from typing import Any, TypeVar + +import astropy.cosmology.units as cu +import astropy.units as u +from astropy.cosmology._src.core import Cosmology +from astropy.cosmology._src.io.connect import readwrite_registry +from astropy.cosmology._src.typing import _CosmoT +from astropy.io.typing import PathLike, ReadableFileLike, WriteableFileLike +from astropy.table import Column, QTable, Table + +from .table import from_table, to_table + +_TableT = TypeVar("_TableT", bound=Table) + + +def read_mrt( + filename: PathLike | ReadableFileLike[Table], + /, + index: int | str | None = None, + *, + move_to_meta: bool = False, + cosmology: str | type[_CosmoT] | None = None, + **kwargs: Any, +) -> _CosmoT: + r"""Read a `~astropy.cosmology.Cosmology` from an MRT file. + + Parameters + ---------- + filename : path-like or file-like + From where to read the Cosmology. + + index : int, str, or None, optional + Needed to select the row in tables with multiple rows. ``index`` can be an + integer for the row number or, if the table is indexed by a column, the value of + that column. If the table is not indexed and ``index`` is a string, the "name" + column is used as the indexing column. + + move_to_meta : bool (optional, keyword-only) + Whether to move keyword arguments that are not in the Cosmology class' signature + to the Cosmology's metadata. This will only be applied if the Cosmology does NOT + have a keyword-only argument (e.g. ``**kwargs``). Arguments moved to the + metadata will be merged with existing metadata, preferring specified metadata in + the case of a merge conflict (e.g. for ``Cosmology(meta={'key':10}, key=42)``, + the ``Cosmology.meta`` will be ``{'key': 10}``). + + cosmology : str or type or None (optional, keyword-only) + The cosmology class (or string name thereof) to use when constructing the + cosmology instance. The class also provides default parameter values, filling in + any non-mandatory arguments missing in 'table'. + + **kwargs + Passed to ``QTable.read`` + + Returns + ------- + `~astropy.cosmology.Cosmology` subclass instance + + + Examples + -------- + We assume the following setup: + + >>> from pathlib import Path + >>> from tempfile import TemporaryDirectory + >>> temp_dir = TemporaryDirectory() + + Writing a cosmology to a Mrt file will produce a table with the cosmology's type, + name, and parameters as columns. + + >>> from astropy.cosmology import Planck18 + >>> file = Path(temp_dir.name) / "file.mrt" + + >>> Planck18.write(file, format="ascii.mrt") + >>> with open(file) as f: print(f.read()) + Title: + Authors: + Table: + ================================================================================ + Byte-by-byte Description of file: table.dat + -------------------------------------------------------------------------------- + Bytes Format Units Label Explanations + -------------------------------------------------------------------------------- + 1-13 A13 --- cosmology Description of cosmology + 15-22 A8 --- name Description of name + 24-28 F5.2 km.Mpc-1.s-1 H0 [67.66/67.66] Hubble constant at z=0. + 30-36 F7.5 --- Om0 [0.3/0.31] Omega matter; matter + density/critical density at z=0. + 38-43 F6.4 K Tcmb0 [2.72/2.73] Temperature of the CMB at z=0. + 45-49 F5.3 --- Neff [3.04/3.05] Number of effective neutrino + species. + 51-64 A14 --- m_nu [[0. 0. 0.06]] Mass of neutrino + species. + 66-72 F7.5 --- Ob0 [0.04/0.05] Omega baryon; baryonic matter + density/critical density at z=0. + -------------------------------------------------------------------------------- + ... + + .. testcleanup:: + + >>> temp_dir.cleanup() + """ + if (fmt := kwargs.pop("format", "ascii.mrt")) != "ascii.mrt": + raise ValueError(f"format must be 'ascii.mrt',not {fmt}") + + with u.add_enabled_units(cu): + table = QTable.read(filename, format="ascii.mrt", **kwargs) + + # Decode JSON-encoded columns (for arrays) + for col in table.itercols(): + # Check if this might be a JSON-encoded column (string type with array-like content) + if col.dtype.kind in ("U", "S", "O"): # Unicode, byte string, or object + with contextlib.suppress(json.JSONDecodeError, ValueError, TypeError): + # Try to decode the first value to see if it's JSON + first_val = col[0] + if isinstance(first_val, str | bytes): + decoded = json.loads(first_val) + # If successful and it's a list, decode all values + if isinstance(decoded, list): + decoded_data = [json.loads(val) for val in col] + # Replace the column with decoded values + table[col.name] = decoded_data + + # Build the cosmology from table, using the private backend. + return from_table( + table, index=index, move_to_meta=move_to_meta, cosmology=cosmology + ) + + +def write_mrt( + cosmo: Cosmology, + /, + file: PathLike | WriteableFileLike[_TableT], + *, + overwrite: bool = False, + cls: type[_TableT] = QTable, + **kwargs: Any, +): + r"""Serialize the |Cosmology| into a MRT table. + + Parameters + ---------- + cosmology : |Cosmology| subclass instance + The cosmology to serialize. + file : path-like or file-like + Where to write the MRT table. + overwrite : bool, optional keyword-only + Whether to overwrite the file, if it exists. + cls : |Table| class, optional keyword-only + Astropy |Table| (sub)class to use when writing. Default is |QTable| class. + **kwargs : Any + Passed to ``cls.write``. + + Raises + ------ + TypeError + If the optional keyword-argument 'cls' is not a subclass of |Table|. + ValueError + If the keyword argument 'format' is given and is not "ascii.mrt". + + Examples + -------- + We assume the following setup: + + >>> from pathlib import Path + >>> from tempfile import TemporaryDirectory + >>> temp_dir = TemporaryDirectory() + + Writing a cosmology to a MRT file will produce a table with the cosmology's type, + name, and parameters as columns. The cosmology class is included as a column + since MRT format does not preserve table metadata. + + >>> from astropy.cosmology import Planck18 + >>> file = Path(temp_dir.name) / "file.mrt" + >>> Planck18.write(file, overwrite=True) + >>> with open(file) as f: print(f.read()) + Title: + Authors: + Table: + ================================================================================ + Byte-by-byte Description of file: table.dat + -------------------------------------------------------------------------------- + Bytes Format Units Label Explanations + -------------------------------------------------------------------------------- + 1-13 A13 --- cosmology Description of cosmology + 15-22 A8 --- name Description of name + 24-28 F5.2 km.Mpc-1.s-1 H0 [67.66/67.66] Hubble constant at z=0. + 30-36 F7.5 --- Om0 [0.3/0.31] Omega matter; matter + density/critical density at z=0. + 38-43 F6.4 K Tcmb0 [2.72/2.73] Temperature of the CMB at z=0. + 45-49 F5.3 --- Neff [3.04/3.05] Number of effective neutrino + species. + 51-64 A14 --- m_nu [[0. 0. 0.06]] Mass of neutrino + species. + 66-72 F7.5 --- Ob0 [0.04/0.05] Omega baryon; baryonic matter + density/critical density at z=0. + -------------------------------------------------------------------------------- + ... + + + .. testcleanup:: + + >>> temp_dir.cleanup() + + Notes + ----- + + """ + if (fmt := kwargs.pop("format", "ascii.mrt")) != "ascii.mrt": + raise ValueError(f"format must be 'ascii.mrt', not {fmt}") + + table = to_table(cosmo, cls=cls, cosmology_in_meta=False) + + Parameters = cosmo.__class__.parameters # dict of Parameter objects + for name, col in table.columns.items(): + # MRT can't serialize redshift units, so remove them + if col.unit is cu.redshift: + table[name] <<= u.dimensionless_unscaled + + # check if col is multi dimensional + if len(col.shape) > 1 or col.info.dtype.kind == "0": + + def format_col_item(idx): + obj = col[idx] + # Get the value in the default units + if hasattr(obj, "to_value"): + obj = obj.to_value(Parameters[name].unit) + with contextlib.suppress(AttributeError): + obj = obj.tolist() + + return json.dumps(obj, separators=(",", ":")) + + try: + table[name] = Column( + data=[format_col_item(idx) for idx in range(len(col))], + name=name, + description=str(col.value) + " " + col.info.description, + ) + except TypeError as exc: + msg = f"could not convert column {col.info.name!r} to string: {exc}" + raise TypeError(msg) from exc + + # Write MRT + table.write(file, overwrite=overwrite, format="ascii.mrt", **kwargs) + + +def mrt_identify( + _: object, filepath: str | None, /, *args: object, **kwargs: object +) -> bool: + """Identify if an object uses the HTML Table format. + + Parameters + ---------- + origin : object + Not used. + filepath : object + From where to read the Cosmology. + *args : object + Not used. + **kwargs : object + Not used. + + Returns + ------- + bool + If the filepath is a string ending with '.mrt'. + """ + return filepath is not None and filepath.endswith(".mrt") + + +# =================================================================== +# Register + +readwrite_registry.register_reader("ascii.mrt", Cosmology, read_mrt) +readwrite_registry.register_writer("ascii.mrt", Cosmology, write_mrt) +readwrite_registry.register_identifier("ascii.mrt", Cosmology, mrt_identify) diff --git a/astropy/cosmology/_src/io/builtin/row.py b/astropy/cosmology/_src/io/builtin/row.py new file mode 100644 index 000000000000..eacc89827137 --- /dev/null +++ b/astropy/cosmology/_src/io/builtin/row.py @@ -0,0 +1,283 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""|Cosmology| <-> |Row| I/O, using |Cosmology.to_format| and |Cosmology.from_format|. + +A `~astropy.cosmology.Cosmology` as a `~astropy.table.Row` will have +the cosmology's name and parameters as columns. + + >>> from astropy.cosmology import Planck18 + >>> cr = Planck18.to_format("astropy.row") + >>> cr + + cosmology name H0 Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str13 str8 float64 float64 float64 float64 float64[3] float64 + ------------- -------- ------------ ------- ------- ------- ----------- ------- + FlatLambdaCDM Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + +The cosmological class and other metadata, e.g. a paper reference, are in +the Table's metadata. + + >>> cr.meta + {'Oc0': 0.2607, 'n': 0.9665, ...} + +Now this row can be used to load a new cosmological instance identical +to the ``Planck18`` cosmology from which it was generated. + + >>> cosmo = Cosmology.from_format(cr, format="astropy.row") + >>> cosmo + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=, Ob0=0.04897) + +For more information on the argument options, see :ref:`cosmology_io_builtin-table`. +""" + +import copy +from collections import defaultdict +from collections.abc import Mapping + +from astropy.table import QTable, Row, Table + +# isort: split +from astropy.cosmology._src.core import Cosmology +from astropy.cosmology._src.io.connect import convert_registry +from astropy.cosmology._src.typing import _CosmoT + +from .mapping import from_mapping + + +def from_row( + row: Row, + *, + move_to_meta: bool = False, + cosmology: str | type[_CosmoT] | None = None, + rename: Mapping[str, str] | None = None, +) -> _CosmoT: + """Instantiate a `~astropy.cosmology.Cosmology` from a `~astropy.table.Row`. + + Parameters + ---------- + row : `~astropy.table.Row` + The object containing the Cosmology information. + move_to_meta : bool (optional, keyword-only) + Whether to move keyword arguments that are not in the Cosmology class' + signature to the Cosmology's metadata. This will only be applied if the + Cosmology does NOT have a keyword-only argument (e.g. ``**kwargs``). + Arguments moved to the metadata will be merged with existing metadata, + preferring specified metadata in the case of a merge conflict + (e.g. for ``Cosmology(meta={'key':10}, key=42)``, the ``Cosmology.meta`` + will be ``{'key': 10}``). + + cosmology : str, type, or None (optional, keyword-only) + The cosmology class (or string name thereof) to use when constructing + the cosmology instance. The class also provides default parameter values, + filling in any non-mandatory arguments missing in 'table'. + + rename : Mapping[str, str] or None (optional, keyword-only) + A mapping of column names in the row to field names of the |Cosmology|. + + Returns + ------- + `~astropy.cosmology.Cosmology` + + Examples + -------- + To see loading a `~astropy.cosmology.Cosmology` from a Row with + ``from_row``, we will first make a `~astropy.table.Row` using + :func:`~astropy.cosmology.Cosmology.to_format`. + + >>> from astropy.cosmology import Cosmology, Planck18 + >>> cr = Planck18.to_format("astropy.row") + >>> cr + + cosmology name H0 Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str13 str8 float64 float64 float64 float64 float64[3] float64 + ------------- -------- ------------ ------- ------- ------- ----------- ------- + FlatLambdaCDM Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + + Now this row can be used to load a new cosmological instance identical + to the ``Planck18`` cosmology from which it was generated. + + >>> cosmo = Cosmology.from_format(cr, format="astropy.row") + >>> cosmo + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=, Ob0=0.04897) + + The ``cosmology`` information (column or metadata) may be omitted if the cosmology + class (or its string name) is passed as the ``cosmology`` keyword argument to + |Cosmology.from_format|. + + >>> del cr.columns["cosmology"] # remove cosmology from metadata + >>> Cosmology.from_format(cr, cosmology="FlatLambdaCDM") + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=, Ob0=0.04897) + + Alternatively, specific cosmology classes can be used to parse the data. + + >>> from astropy.cosmology import FlatLambdaCDM + >>> FlatLambdaCDM.from_format(cr) + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=, Ob0=0.04897) + + When using a specific cosmology class, the class' default parameter values are used + to fill in any missing information. + + >>> del cr.columns["Tcmb0"] # show FlatLambdaCDM provides default + >>> FlatLambdaCDM.from_format(cr) + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=None, Ob0=0.04897) + + If a `~astropy.table.Row` object has columns that do not match the fields of the + `~astropy.cosmology.Cosmology` class, they can be mapped using the ``rename`` + keyword argument. + + >>> renamed = Planck18.to_format("astropy.row", rename={"H0": "Hubble"}) + >>> renamed + + cosmology name Hubble Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str13 str8 float64 float64 float64 float64 float64[3] float64 + ------------- -------- ------------ ------- ------- ------- ----------- ------- + FlatLambdaCDM Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + + >>> cosmo = Cosmology.from_format(renamed, format="astropy.row", + ... rename={"Hubble": "H0"}) + >>> cosmo == Planck18 + True + """ + inv_rename = {v: k for k, v in rename.items()} if rename is not None else {} + kname = inv_rename.get("name", "name") + kmeta = inv_rename.get("meta", "meta") + kcosmo = inv_rename.get("cosmology", "cosmology") + + # special values + name = row.get(kname) + + meta = defaultdict(dict, copy.deepcopy(row.meta)) + # Now need to add the Columnar metadata. This is only available on the + # parent table. If Row is ever separated from Table, this should be moved + # to ``to_table``. + for col in row._table.itercols(): + if col.info.meta: # Only add metadata if not empty + meta[col.name].update(col.info.meta) + + # turn row into mapping, filling cosmo if not in a column + mapping = dict(row) + mapping[kname] = name + mapping.setdefault(kcosmo, meta.pop(kcosmo, None)) + mapping[kmeta] = dict(meta) + + # build cosmology from map + return from_mapping( + mapping, move_to_meta=move_to_meta, cosmology=cosmology, rename=rename + ) + + +def to_row( + cosmology: Cosmology, + *args: object, + cosmology_in_meta: bool = False, + table_cls: type[Table] = QTable, + rename: Mapping[str, str] | None = None, +) -> Row: + """Serialize the cosmology into a `~astropy.table.Row`. + + Parameters + ---------- + cosmology : `~astropy.cosmology.Cosmology` + The cosmology instance to convert to a mapping. + *args + Not used. Needed for compatibility with + `~astropy.io.registry.UnifiedReadWriteMethod` + table_cls : type (optional, keyword-only) + Astropy :class:`~astropy.table.Table` class or subclass type to use. Default is + :class:`~astropy.table.QTable`. + cosmology_in_meta : bool + Whether to put the cosmology class in the Table metadata (if `True`) or as the + first column (if `False`, default). + rename : Mapping[str, str] or None (optional, keyword-only) + A mapping of field names of the |Cosmology| to column names in the row. + + Returns + ------- + `~astropy.table.Row` + With columns for the cosmology parameters, and metadata in the Table's ``meta`` + attribute. The cosmology class name will either be a column or in ``meta``, + depending on 'cosmology_in_meta'. + + Examples + -------- + A `~astropy.cosmology.Cosmology` as a `~astropy.table.Row` will have the cosmology's + name and parameters as columns. + + >>> from astropy.cosmology import Planck18 + >>> cr = Planck18.to_format("astropy.row") + >>> cr + + cosmology name H0 Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str13 str8 float64 float64 float64 float64 float64[3] float64 + ------------- -------- ------------ ------- ------- ------- ----------- ------- + FlatLambdaCDM Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + + The cosmological class and other metadata, e.g. a paper reference, are in the + Table's metadata. + + >>> cr.meta + {'Oc0': 0.2607, 'n': 0.9665, ...} + + To move the cosmology class from a column to the Table's metadata, set the + ``cosmology_in_meta`` argument to `True`: + + >>> Planck18.to_format("astropy.table", cosmology_in_meta=True) + + name H0 Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str8 float64 float64 float64 float64 float64[3] float64 + -------- ------------ ------- ------- ------- ----------- ------- + Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + + In Astropy, Row objects are always part of a Table. :class:`~astropy.table.QTable` + is recommended for tables with |Quantity| columns. However the returned type may be + overridden using the ``cls`` argument: + + >>> from astropy.table import Table + >>> Planck18.to_format("astropy.table", cls=Table) + + ... + + The columns can be renamed using the ``rename`` keyword argument. + + >>> renamed = Planck18.to_format("astropy.row", rename={"H0": "Hubble"}) + >>> renamed + + cosmology name Hubble Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str13 str8 float64 float64 float64 float64 float64[3] float64 + ------------- -------- ------------ ------- ------- ------- ----------- ------- + FlatLambdaCDM Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + """ + from .table import to_table + + table = to_table( + cosmology, cls=table_cls, cosmology_in_meta=cosmology_in_meta, rename=rename + ) + return table[0] # extract row from table + + +def row_identify( + origin: str, format: str | None, *args: object, **kwargs: object +) -> bool: + """Identify if object uses the `~astropy.table.Row` format. + + Returns + ------- + bool + """ + itis = False + if origin == "read": + itis = isinstance(args[1], Row) and (format in (None, "astropy.row")) + return itis + + +# =================================================================== +# Register + +convert_registry.register_reader("astropy.row", Cosmology, from_row) +convert_registry.register_writer("astropy.row", Cosmology, to_row) +convert_registry.register_identifier("astropy.row", Cosmology, row_identify) diff --git a/astropy/cosmology/_src/io/builtin/table.py b/astropy/cosmology/_src/io/builtin/table.py new file mode 100644 index 000000000000..76c5bc061de0 --- /dev/null +++ b/astropy/cosmology/_src/io/builtin/table.py @@ -0,0 +1,484 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""|Cosmology| <-> |Table| I/O, using |Cosmology.to_format| and |Cosmology.from_format|. + +This module provides functions to transform a |Cosmology| object to and from a |Table| +object. The functions are registered with ``convert_registry`` under the format name +"astropy.table". |Table| itself has an abundance of I/O methods, making this conversion +useful for further interoperability with other formats. + +A Cosmology as a `~astropy.table.QTable` will have the cosmology's name and parameters +as columns. + + >>> from astropy.cosmology import Planck18 + >>> ct = Planck18.to_format("astropy.table") + >>> ct + + name H0 Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str8 float64 float64 float64 float64 float64[3] float64 + -------- ------------ ------- ------- ------- ----------- ------- + Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + +The cosmological class and other metadata, e.g. a paper reference, are in the Table's +metadata. + + >>> ct.meta + {..., 'cosmology': 'FlatLambdaCDM'} + + +Cosmology supports the astropy Table-like protocol (see :ref:`Table-like Objects`) to +the same effect: + +.. code-block:: + + >>> from astropy.table import QTable + >>> QTable(Planck18) + + name H0 Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str8 float64 float64 float64 float64 float64[3] float64 + -------- ------------ ------- ------- ------- ----------- ------- + Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + + +To move the cosmology class from the metadata to a Table row, set the +``cosmology_in_meta`` argument to `False`: + + >>> Planck18.to_format("astropy.table", cosmology_in_meta=False) + + cosmology name H0 Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str13 str8 float64 float64 float64 float64 float64[3] float64 + ------------- -------- ------------ ------- ------- ------- ----------- ------- + FlatLambdaCDM Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + +Astropy recommends `~astropy.table.QTable` for tables with |Quantity| columns. However +the returned type may be overridden using the ``cls`` argument: + + >>> from astropy.table import Table + >>> Planck18.to_format("astropy.table", cls=Table) +
+ ... + +Fields of the cosmology may be renamed using the ``rename`` argument. + + >>> Planck18.to_format("astropy.table", rename={"H0": "Hubble"}) + + name Hubble Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str8 float64 float64 float64 float64 float64[3] float64 + -------- ------------ ------- ------- ------- ----------- ------- + Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + +Appropriately formatted tables can be converted to |Cosmology| instances. Since the +|Table| can hold arbitrary metadata, we can faithfully round-trip a |Cosmology| through +|Table|, e.g. to construct a ``Planck18`` cosmology identical to the instance from which +it was generated. + + >>> ct = Planck18.to_format("astropy.table") + + >>> cosmo = Cosmology.from_format(ct, format="astropy.table") + >>> cosmo + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=, Ob0=0.04897) + >>> cosmo == Planck18 + True + +The ``cosmology`` information (row or metadata) may be omitted if the cosmology class +(or its string name) is passed as the ``cosmology`` keyword argument to +|Cosmology.from_format|. + + >>> del ct.meta["cosmology"] # remove cosmology from metadata + >>> Cosmology.from_format(ct, cosmology="FlatLambdaCDM") + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=, Ob0=0.04897) + +Alternatively, specific cosmology classes can be used to parse the data. + + >>> from astropy.cosmology import FlatLambdaCDM + >>> FlatLambdaCDM.from_format(ct) + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=, Ob0=0.04897) + +When using a specific cosmology class, the class' default parameter values are used to +fill in any missing information. + + >>> del ct["Tcmb0"] # show FlatLambdaCDM provides default + >>> FlatLambdaCDM.from_format(ct) + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=None, Ob0=0.04897) + +For tables with multiple rows of cosmological parameters, the ``index`` argument is +needed to select the correct row. The index can be an integer for the row number or, if +the table is indexed by a column, the value of that column. If the table is not indexed +and ``index`` is a string, the "name" column is used as the indexing column. + +Here is an example where ``index`` is needed and can be either an integer (for the row +number) or the name of one of the cosmologies, e.g. 'Planck15'. + + >>> from astropy.cosmology import Planck13, Planck15, Planck18 + >>> from astropy.table import vstack + >>> cts = vstack([c.to_format("astropy.table") + ... for c in (Planck13, Planck15, Planck18)], + ... metadata_conflicts='silent') + >>> cts + + name H0 Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str8 float64 float64 float64 float64 float64[3] float64 + -------- ------------ ------- ------- ------- ----------- -------- + Planck13 67.77 0.30712 2.7255 3.046 0.0 .. 0.06 0.048252 + Planck15 67.74 0.3075 2.7255 3.046 0.0 .. 0.06 0.0486 + Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + + >>> cosmo = Cosmology.from_format(cts, index=1, format="astropy.table") + >>> cosmo == Planck15 + True + +Fields in the table can be renamed to match the `~astropy.cosmology.Cosmology` class' +signature using the ``rename`` argument. This is useful when the table's column names do +not match the class' parameter names. + + >>> renamed_table = Planck18.to_format("astropy.table", rename={"H0": "Hubble"}) + >>> renamed_table + + name Hubble Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str8 float64 float64 float64 float64 float64[3] float64 + -------- ------------ ------- ------- ------- ----------- ------- + Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + + >>> cosmo = Cosmology.from_format(renamed_table, format="astropy.table", + ... rename={"Hubble": "H0"}) + >>> cosmo == Planck18 + True +""" + +from collections.abc import Mapping +from typing import TypeVar + +import numpy as np + +from astropy.cosmology._src.core import Cosmology +from astropy.cosmology._src.io.connect import convert_registry +from astropy.cosmology._src.typing import _CosmoT +from astropy.table import Column, QTable, Table + +from .mapping import to_mapping +from .row import from_row +from .utils import convert_parameter_to_column + +_TableT = TypeVar("_TableT", bound=Table) + + +def from_table( + table: Table, + index: int | str | None = None, + *, + move_to_meta: bool = False, + cosmology: str | type[_CosmoT] | None = None, + rename: Mapping[str, str] | None = None, +) -> _CosmoT: + """Instantiate a `~astropy.cosmology.Cosmology` from a |QTable|. + + Parameters + ---------- + table : `~astropy.table.Table` + The object to parse into a |Cosmology|. + index : int, str, or None, optional + Needed to select the row in tables with multiple rows. ``index`` can be an + integer for the row number or, if the table is indexed by a column, the value of + that column. If the table is not indexed and ``index`` is a string, the "name" + column is used as the indexing column. + + move_to_meta : bool (optional, keyword-only) + Whether to move keyword arguments that are not in the Cosmology class' signature + to the Cosmology's metadata. This will only be applied if the Cosmology does NOT + have a keyword-only argument (e.g. ``**kwargs``). Arguments moved to the + metadata will be merged with existing metadata, preferring specified metadata in + the case of a merge conflict (e.g. for ``Cosmology(meta={'key':10}, key=42)``, + the ``Cosmology.meta`` will be ``{'key': 10}``). + + cosmology : str or type or None (optional, keyword-only) + The cosmology class (or string name thereof) to use when constructing the + cosmology instance. The class also provides default parameter values, filling in + any non-mandatory arguments missing in 'table'. + + rename : Mapping[str, str] or None (optional, keyword-only) + A mapping of column names in 'table' to field names of the |Cosmology| class. + + Returns + ------- + `~astropy.cosmology.Cosmology` + + Examples + -------- + To see loading a `~astropy.cosmology.Cosmology` from a Table with ``from_table``, we + will first make a |QTable| using :func:`~astropy.cosmology.Cosmology.to_format`. + + >>> from astropy.cosmology import Cosmology, Planck18 + >>> ct = Planck18.to_format("astropy.table") + >>> ct + + name H0 Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str8 float64 float64 float64 float64 float64[3] float64 + -------- ------------ ------- ------- ------- ----------- ------- + Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + + Now this table can be used to load a new cosmological instance identical to the + ``Planck18`` cosmology from which it was generated. + + >>> cosmo = Cosmology.from_format(ct, format="astropy.table") + >>> cosmo + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=, Ob0=0.04897) + + The ``cosmology`` information (column or metadata) may be omitted if the cosmology + class (or its string name) is passed as the ``cosmology`` keyword argument to + |Cosmology.from_format|. + + >>> del ct.meta["cosmology"] # remove cosmology from metadata + >>> Cosmology.from_format(ct, cosmology="FlatLambdaCDM") + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=, Ob0=0.04897) + + Alternatively, specific cosmology classes can be used to parse the data. + + >>> from astropy.cosmology import FlatLambdaCDM + >>> FlatLambdaCDM.from_format(ct) + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=, Ob0=0.04897) + + When using a specific cosmology class, the class' default parameter values are used + to fill in any missing information. + + >>> del ct["Tcmb0"] # show FlatLambdaCDM provides default + >>> FlatLambdaCDM.from_format(ct) + FlatLambdaCDM(name='Planck18', H0=, Om0=0.30966, Tcmb0=, Neff=3.046, m_nu=None, Ob0=0.04897) + + For tables with multiple rows of cosmological parameters, the ``index`` argument is + needed to select the correct row. The index can be an integer for the row number or, + if the table is indexed by a column, the value of that column. If the table is not + indexed and ``index`` is a string, the "name" column is used as the indexing column. + + Here is an example where ``index`` is needed and can be either an integer (for the + row number) or the name of one of the cosmologies, e.g. 'Planck15'. + + >>> from astropy.cosmology import Planck13, Planck15, Planck18 + >>> from astropy.table import vstack + >>> cts = vstack([c.to_format("astropy.table") + ... for c in (Planck13, Planck15, Planck18)], + ... metadata_conflicts='silent') + >>> cts + + name H0 Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str8 float64 float64 float64 float64 float64[3] float64 + -------- ------------ ------- ------- ------- ----------- -------- + Planck13 67.77 0.30712 2.7255 3.046 0.0 .. 0.06 0.048252 + Planck15 67.74 0.3075 2.7255 3.046 0.0 .. 0.06 0.0486 + Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + + >>> cosmo = Cosmology.from_format(cts, index="Planck15", format="astropy.table") + >>> cosmo == Planck15 + True + + Fields in the table can be renamed to match the `~astropy.cosmology.Cosmology` + class' signature using the ``rename`` argument. This is useful when the table's + column names do not match the class' parameter names. + + >>> renamed_table = Planck18.to_format("astropy.table", rename={"H0": "Hubble"}) + >>> renamed_table + + name Hubble Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str8 float64 float64 float64 float64 float64[3] float64 + -------- ------------ ------- ------- ------- ----------- ------- + Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + + >>> cosmo = Cosmology.from_format(renamed_table, format="astropy.table", + ... rename={"Hubble": "H0"}) + >>> cosmo == Planck18 + True + + For further examples, see :doc:`astropy:cosmology/io`. + """ + # Get row from table + # string index uses the indexed column on the table to find the row index. + if isinstance(index, str): + if not table.indices: # no indexing column, find by string match + nc = "name" # default name column + if rename is not None: # from inverted `rename` + for key, value in rename.items(): + if value == "name": + nc = key + break + + indices = np.where(table[nc] == index)[0] + else: # has indexing column + indices = table.loc_indices[index] # need to convert to row index (int) + + if isinstance(indices, (int, np.integer)): # loc_indices + index = indices + elif len(indices) == 1: # only happens w/ np.where + index = indices[0] + elif len(indices) == 0: # matches from loc_indices + raise KeyError(f"No matches found for key {indices}") + else: # like the Highlander, there can be only 1 Cosmology + raise ValueError(f"more than one cosmology found for key {indices}") + + # no index is needed for a 1-row table. For a multi-row table... + if index is None: + if len(table) != 1: # multi-row table and no index + raise ValueError( + "need to select a specific row (e.g. index=1) when " + "constructing a Cosmology from a multi-row table." + ) + else: # single-row table + index = 0 + row = table[index] # index is now the row index (int) + + # parse row to cosmo + return from_row(row, move_to_meta=move_to_meta, cosmology=cosmology, rename=rename) + + +def to_table( + cosmology: Cosmology, + *args: object, + cls: type[_TableT] = QTable, + cosmology_in_meta: bool = True, + rename: Mapping[str, str] | None = None, +) -> _TableT: + """Serialize the cosmology into a `~astropy.table.QTable`. + + Parameters + ---------- + cosmology : `~astropy.cosmology.Cosmology` + The cosmology instance to convert to a table. + *args : object + Not used. Needed for compatibility with + `~astropy.io.registry.UnifiedReadWriteMethod` + cls : type (optional, keyword-only) + Astropy :class:`~astropy.table.Table` class or subclass type to return. + Default is :class:`~astropy.table.QTable`. + cosmology_in_meta : bool (optional, keyword-only) + Whether to put the cosmology class in the Table metadata (if `True`, + default) or as the first column (if `False`). + rename : Mapping[str, str] or None (optional, keyword-only) + A mapping of field names of the |Cosmology| class to column names in the + |Table|. + + Returns + ------- + `~astropy.table.QTable` + With columns for the cosmology parameters, and metadata and + cosmology class name in the Table's ``meta`` attribute + + Raises + ------ + TypeError + If kwarg (optional) 'cls' is not a subclass of `astropy.table.Table` + + Examples + -------- + A Cosmology as a `~astropy.table.QTable` will have the cosmology's name and + parameters as columns. + + >>> from astropy.cosmology import Planck18 + >>> ct = Planck18.to_format("astropy.table") + >>> ct + + name H0 Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str8 float64 float64 float64 float64 float64[3] float64 + -------- ------------ ------- ------- ------- ----------- ------- + Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + + The cosmological class and other metadata, e.g. a paper reference, are in + the Table's metadata. + + >>> ct.meta + {..., 'cosmology': 'FlatLambdaCDM'} + + To move the cosmology class from the metadata to a Table column, set the + ``cosmology_in_meta`` argument to `False`: + + >>> Planck18.to_format("astropy.table", cosmology_in_meta=False) + + cosmology name H0 Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str13 str8 float64 float64 float64 float64 float64[3] float64 + ------------- -------- ------------ ------- ------- ------- ----------- ------- + FlatLambdaCDM Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + + Astropy recommends `~astropy.table.QTable` for tables with |Quantity| columns. + However the returned type may be overridden using the ``cls`` argument: + + >>> from astropy.table import Table + >>> Planck18.to_format("astropy.table", cls=Table) +
+ ... + + Fields of the cosmology may be renamed using the ``rename`` argument. + + >>> Planck18.to_format("astropy.table", rename={"H0": "Hubble"}) + + name Hubble Om0 Tcmb0 Neff m_nu Ob0 + km / (Mpc s) K eV + str8 float64 float64 float64 float64 float64[3] float64 + -------- ------------ ------- ------- ------- ----------- ------- + Planck18 67.66 0.30966 2.7255 3.046 0.0 .. 0.06 0.04897 + """ + if not issubclass(cls, Table): + raise TypeError(f"'cls' must be a (sub)class of Table, not {type(cls)}") + + # Start by getting a map representation. + data = to_mapping(cosmology) + data["cosmology"] = data["cosmology"].__qualname__ # change to str + + # Metadata + meta = data.pop("meta") # remove the meta + if cosmology_in_meta: + meta["cosmology"] = data.pop("cosmology") + + # Need to turn everything into something Table can process: + # - Column for Parameter + # - list for anything else + cosmo_cls = cosmology.__class__ + for k, v in data.items(): + if k in cosmology.parameters: + col = convert_parameter_to_column( + cosmo_cls.parameters[k], v, cosmology.meta.get(k) + ) + else: + col = Column([v]) + data[k] = col + + tbl = cls(data, meta=meta) + + # Renames + renames = rename or {} + for name in tbl.colnames: + tbl.rename_column(name, renames.get(name, name)) + + # Add index + tbl.add_index(renames.get("name", "name"), unique=True) + + return tbl + + +def table_identify( + origin: str, format: str | None, *args: object, **kwargs: object +) -> bool: + """Identify if object uses the Table format. + + Returns + ------- + bool + """ + itis = False + if origin == "read": + itis = isinstance(args[1], Table) and (format in (None, "astropy.table")) + return itis + + +# =================================================================== +# Register + +convert_registry.register_reader("astropy.table", Cosmology, from_table) +convert_registry.register_writer("astropy.table", Cosmology, to_table) +convert_registry.register_identifier("astropy.table", Cosmology, table_identify) diff --git a/astropy/cosmology/_src/io/builtin/utils.py b/astropy/cosmology/_src/io/builtin/utils.py new file mode 100644 index 000000000000..58e8d2e49da6 --- /dev/null +++ b/astropy/cosmology/_src/io/builtin/utils.py @@ -0,0 +1,70 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import numpy as np + +from astropy.modeling import Parameter as ModelParameter +from astropy.table import Column + + +def convert_parameter_to_column(parameter, value, meta=None): + """Convert a |Cosmology| Parameter to a Table |Column|. + + Parameters + ---------- + parameter : `astropy.cosmology._src.parameter.Parameter` + value : Any + meta : dict or None, optional + Information from the Cosmology's metadata. + + Returns + ------- + `astropy.table.Column` + """ + shape = (1,) + np.shape(value) # minimum of 1d + + return Column( + data=np.reshape(value, shape), + name=parameter.name, + dtype=None, # inferred from the data + description=parameter.__doc__, + format=None, + meta=meta, + ) + + +def convert_parameter_to_model_parameter(parameter, value, meta=None): + """Convert a Cosmology Parameter to a Model Parameter. + + Parameters + ---------- + parameter : `astropy.cosmology._src.parameter.Parameter` + value : Any + meta : dict or None, optional + Information from the Cosmology's metadata. + This function will use any of: 'getter', 'setter', 'fixed', 'tied', + 'min', 'max', 'bounds', 'prior', 'posterior'. + + Returns + ------- + `astropy.modeling.Parameter` + """ + # Get from meta information relevant to Model + attrs = ( + "getter", + "setter", + "fixed", + "tied", + "min", + "max", + "bounds", + "prior", + "posterior", + ) + extra = {k: v for k, v in (meta or {}).items() if k in attrs} + + return ModelParameter( + description=parameter.__doc__, + default=value, + unit=getattr(value, "unit", None), + **extra, + ) diff --git a/astropy/cosmology/_src/io/builtin/yaml.py b/astropy/cosmology/_src/io/builtin/yaml.py new file mode 100644 index 000000000000..8a0f0ac77c73 --- /dev/null +++ b/astropy/cosmology/_src/io/builtin/yaml.py @@ -0,0 +1,265 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +r"""|Cosmology| <-> YAML I/O, using |Cosmology.to_format| and |Cosmology.from_format|. + +This module provides functions to transform a |Cosmology| object to and from a `yaml +`_ representation. The functions are registered with +``convert_registry`` under the format name "yaml". This format is primarily intended for +use by other I/O functions, e.g. |Table|'s metadata serialization, which themselves +require YAML serialization. + + >>> from astropy.cosmology import Planck18 + >>> yml = Planck18.to_format("yaml") + >>> yml # doctest: +NORMALIZE_WHITESPACE + "!astropy.cosmology...FlatLambdaCDM\nH0: !astropy.units.Quantity... + + >>> print(Cosmology.from_format(yml, format="yaml")) + FlatLambdaCDM(name="Planck18", H0=67.66 km / (Mpc s), Om0=0.30966, + Tcmb0=2.7255 K, Neff=3.046, m_nu=[0. 0. 0.06] eV, Ob0=0.04897) +""" # this is shown in the docs. + +__all__: list[str] = [] # nothing is publicly scoped + +from collections.abc import Callable + +from yaml import MappingNode + +import astropy.units as u +from astropy.io.misc.yaml import AstropyDumper, AstropyLoader, dump, load + +# isort: split +import astropy.cosmology.units as cu +from astropy.cosmology._src.core import _COSMOLOGY_CLASSES, Cosmology +from astropy.cosmology._src.io.connect import convert_registry +from astropy.cosmology._src.typing import _CosmoT + +from .mapping import from_mapping + +FULLQUALNAME_SUBSTITUTIONS = { + "astropy.cosmology._src.flrw.base.FLRW": "astropy.cosmology.FLRW", + "astropy.cosmology._src.flrw.lambdacdm.LambdaCDM": "astropy.cosmology.LambdaCDM", + "astropy.cosmology._src.flrw.lambdacdm.FlatLambdaCDM": ( + "astropy.cosmology.FlatLambdaCDM" + ), + "astropy.cosmology._src.flrw.w0wacdm.w0waCDM": "astropy.cosmology.w0waCDM", + "astropy.cosmology._src.flrw.w0wacdm.Flatw0waCDM": "astropy.cosmology.Flatw0waCDM", + "astropy.cosmology._src.flrw.w0wzcdm.w0wzCDM": "astropy.cosmology.w0wzCDM", + "astropy.cosmology._src.flrw.w0cdm.wCDM": "astropy.cosmology.wCDM", + "astropy.cosmology._src.flrw.w0cdm.FlatwCDM": "astropy.cosmology.FlatwCDM", + "astropy.cosmology._src.flrw.wpwazpcdm.wpwaCDM": "astropy.cosmology.wpwaCDM", + # ==== Paths removed in v8.0 ==== + "astropy.cosmology.flrw.base.FLRW": "astropy.cosmology.FLRW", + "astropy.cosmology.flrw.lambdacdm.LambdaCDM": "astropy.cosmology.LambdaCDM", + "astropy.cosmology.flrw.lambdacdm.FlatLambdaCDM": "astropy.cosmology.FlatLambdaCDM", + "astropy.cosmology.flrw.w0wacdm.w0waCDM": "astropy.cosmology.w0waCDM", + "astropy.cosmology.flrw.w0wacdm.Flatw0waCDM": "astropy.cosmology.Flatw0waCDM", + "astropy.cosmology.flrw.w0wzcdm.w0wzCDM": "astropy.cosmology.w0wzCDM", + "astropy.cosmology.flrw.w0cdm.wCDM": "astropy.cosmology.wCDM", + "astropy.cosmology.flrw.w0cdm.FlatwCDM": "astropy.cosmology.FlatwCDM", + "astropy.cosmology.flrw.wpwazpcdm.wpwaCDM": "astropy.cosmology.wpwaCDM", +} +"""Substitutions mapping the actual qualified name to its preferred value.""" + + +############################################################################## +# Serializer Functions +# these do Cosmology <-> YAML through a modified dictionary representation of +# the Cosmology object. The Unified-I/O functions are just wrappers to the YAML +# that calls these functions. + + +_representer_doc = """Cosmology yaml representer function for {}. + +Parameters +---------- +dumper : :class:`~astropy.io.misc.yaml.AstropyDumper` + The dumper object with which to serialize the |Cosmology| object. +obj : :class:`~astropy.cosmology.Cosmology` + The |Cosmology| object to serialize. + +Returns +------- +str + :mod:`yaml` representation of |Cosmology| object. +""" + + +def yaml_representer(tag: str) -> Callable[[AstropyDumper, Cosmology], str]: + """`yaml `_ representation of |Cosmology| object. + + Parameters + ---------- + tag : str + The class tag, e.g. '!astropy.cosmology.LambdaCDM' + + Returns + ------- + representer : callable[[`~astropy.io.misc.yaml.AstropyDumper`, |Cosmology|], str] + Function to construct :mod:`yaml` representation of |Cosmology| object. + """ + + def representer(dumper: AstropyDumper, obj: Cosmology) -> str: + # convert to mapping + map = obj.to_format("mapping") + # remove the cosmology class info. It's already recorded in `tag` + map.pop("cosmology") + # make the metadata serializable in an order-preserving way. + map["meta"] = tuple(map["meta"].items()) + + return dumper.represent_mapping(tag, map) + + representer.__doc__ = _representer_doc.format(tag) + + return representer + + +def yaml_constructor( + cls: type[_CosmoT], +) -> Callable[[AstropyLoader, MappingNode], _CosmoT]: + """Cosmology| object from :mod:`yaml` representation. + + Parameters + ---------- + cls : type + The class type, e.g. `~astropy.cosmology.LambdaCDM`. + + Returns + ------- + constructor : callable + Function to construct |Cosmology| object from :mod:`yaml` representation. + """ + + def constructor(loader: AstropyLoader, node: MappingNode) -> _CosmoT: + """Cosmology yaml constructor function. + + Parameters + ---------- + loader : `~astropy.io.misc.yaml.AstropyLoader` + node : `yaml.nodes.MappingNode` + yaml representation of |Cosmology| object. + + Returns + ------- + `~astropy.cosmology.Cosmology` subclass instance + """ + # create mapping from YAML node + map = loader.construct_mapping(node) + # restore metadata to dict + map["meta"] = dict(map["meta"]) + # get cosmology class qualified name from node + cosmology = str(node.tag).split(".")[-1] + # create Cosmology from mapping + return from_mapping(map, move_to_meta=False, cosmology=cosmology) + + return constructor + + +def register_cosmology_yaml(cosmo_cls: type[Cosmology]) -> None: + """Register :mod:`yaml` for Cosmology class. + + Parameters + ---------- + cosmo_cls : `~astropy.cosmology.Cosmology` class + """ + fqn = f"{cosmo_cls.__module__}.{cosmo_cls.__qualname__}" + fqn = FULLQUALNAME_SUBSTITUTIONS.get(fqn, fqn) # Possibly sub for a preferred path + tag = "!" + fqn + + AstropyDumper.add_representer(cosmo_cls, yaml_representer(tag)) + AstropyLoader.add_constructor(tag, yaml_constructor(cosmo_cls)) + + +############################################################################## +# Unified-I/O Functions + + +def from_yaml(yml: str, *, cosmology: type[_CosmoT] | None = None) -> _CosmoT: + """Load `~astropy.cosmology.Cosmology` from :mod:`yaml` object. + + Parameters + ---------- + yml : str + :mod:`yaml` representation of |Cosmology| object + cosmology : str, |Cosmology| class, or None (optional, keyword-only) + The expected cosmology class (or string name thereof). This argument is + is only checked for correctness if not `None`. + + Returns + ------- + `~astropy.cosmology.Cosmology` subclass instance + + Raises + ------ + TypeError + If the |Cosmology| object loaded from ``yml`` is not an instance of + the ``cosmology`` (and ``cosmology`` is not `None`). + + Examples + -------- + >>> from astropy.cosmology import Cosmology, Planck18 + >>> yml = Planck18.to_format("yaml") + >>> print(Cosmology.from_format(yml, format="yaml")) + FlatLambdaCDM(name="Planck18", H0=67.66 km / (Mpc s), Om0=0.30966, + Tcmb0=2.7255 K, Neff=3.046, m_nu=[0. 0. 0.06] eV, Ob0=0.04897) + """ + with u.add_enabled_units(cu): + cosmo = load(yml) + + # Check argument `cosmology`, if not None + # This kwarg is required for compatibility with |Cosmology.from_format| + if isinstance(cosmology, str): + cosmology = _COSMOLOGY_CLASSES[cosmology] + if cosmology is not None and not isinstance(cosmo, cosmology): + raise TypeError(f"cosmology {cosmo} is not an {cosmology} instance.") + + return cosmo + + +def to_yaml(cosmology: Cosmology, *args: object) -> str: + r"""Return the cosmology class, parameters, and metadata as a :mod:`yaml` object. + + Parameters + ---------- + cosmology : `~astropy.cosmology.Cosmology` subclass instance + The cosmology to serialize. + *args : Any + Not used. Needed for compatibility with + `~astropy.io.registry.UnifiedReadWriteMethod` + + Returns + ------- + str + :mod:`yaml` representation of |Cosmology| object + + Examples + -------- + >>> from astropy.cosmology import Planck18 + >>> Planck18.to_format("yaml") + "!astropy.cosmology...FlatLambdaCDM\nH0: !astropy.units.Quantity... + """ + return dump(cosmology) + + +# ``read`` cannot handle non-path strings. +# TODO! this says there should be different types of I/O registries. +# not just hacking object conversion on top of file I/O. +# def yaml_identify(origin, format, *args, **kwargs): +# """Identify if object uses the yaml format. +# +# Returns +# ------- +# bool +# """ +# itis = False +# if origin == "read": +# itis = isinstance(args[1], str) and args[1][0].startswith("!") +# itis &= format in (None, "yaml") +# +# return itis + + +# =================================================================== +# Register + +convert_registry.register_reader("yaml", Cosmology, from_yaml) +convert_registry.register_writer("yaml", Cosmology, to_yaml) +# convert_registry.register_identifier("yaml", Cosmology, yaml_identify) diff --git a/astropy/cosmology/_src/io/connect.py b/astropy/cosmology/_src/io/connect.py new file mode 100644 index 000000000000..650988912e2a --- /dev/null +++ b/astropy/cosmology/_src/io/connect.py @@ -0,0 +1,369 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +__all__ = ( + # classes + "CosmologyFromFormat", + "CosmologyRead", + "CosmologyToFormat", + "CosmologyWrite", +) + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Literal, TypeVar, overload + +from astropy.cosmology._src.typing import _CosmoT +from astropy.io import registry as io_registry +from astropy.table import Row, Table +from astropy.units import add_enabled_units + +# isort: split +import astropy.cosmology._src.units as cu + +if TYPE_CHECKING: + import astropy.cosmology + + +__doctest_skip__ = __all__ + + +# NOTE: private b/c RTD error +_MT = TypeVar("_MT", bound=Mapping) # type: ignore[type-arg] + + +# ============================================================================== +# Read / Write + +readwrite_registry = io_registry.UnifiedIORegistry() + + +class CosmologyRead(io_registry.UnifiedReadWrite): + """Read and parse data to a `~astropy.cosmology.Cosmology`. + + This function provides the Cosmology interface to the Astropy unified I/O + layer. This allows easily reading a file in supported data formats using + syntax such as:: + + >>> from astropy.cosmology import Cosmology + >>> cosmo1 = Cosmology.read('') + + When the ``read`` method is called from a subclass the subclass will + provide a keyword argument ``cosmology=`` to the registered read + method. The method uses this cosmology class, regardless of the class + indicated in the file, and sets parameters' default values from the class' + signature. + + Get help on the available readers using the ``help()`` method:: + + >>> Cosmology.read.help() # Get help reading and list supported formats + >>> Cosmology.read.help(format='') # Get detailed help on a format + >>> Cosmology.read.list_formats() # Print list of available formats + + See also: https://docs.astropy.org/en/stable/io/unified.html + + Parameters + ---------- + *args + Positional arguments passed through to data reader. If supplied the + first argument is typically the input filename. + format : str (optional, keyword-only) + File format specifier. + **kwargs + Keyword arguments passed through to data reader. + + Returns + ------- + out : `~astropy.cosmology.Cosmology` subclass instance + `~astropy.cosmology.Cosmology` corresponding to file contents. + + Notes + ----- + """ + + def __init__( + self, + instance: "astropy.cosmology.Cosmology", + cosmo_cls: type["astropy.cosmology.Cosmology"], + ) -> None: + super().__init__(instance, cosmo_cls, "read", registry=readwrite_registry) + + def __call__(self, *args: Any, **kwargs: Any) -> "astropy.cosmology.Cosmology": + from astropy.cosmology._src.core import Cosmology + + # so subclasses can override, also pass the class as a kwarg. + # allows for `FlatLambdaCDM.read` and + # `Cosmology.read(..., cosmology=FlatLambdaCDM)` + if self._cls is not Cosmology: + kwargs.setdefault("cosmology", self._cls) # set, if not present + # check that it is the correct cosmology, can be wrong if user + # passes in e.g. `w0wzCDM.read(..., cosmology=FlatLambdaCDM)` + valid = (self._cls, self._cls.__qualname__) + if kwargs["cosmology"] not in valid: + raise ValueError( + "keyword argument `cosmology` must be either the class " + f"{valid[0]} or its qualified name '{valid[1]}'" + ) + + with add_enabled_units(cu): + return self.registry.read(self._cls, *args, **kwargs) + + +class CosmologyWrite(io_registry.UnifiedReadWrite): + """Write this Cosmology object out in the specified format. + + This function provides the Cosmology interface to the astropy unified I/O + layer. This allows easily writing a file in supported data formats + using syntax such as:: + + >>> from astropy.cosmology import Planck18 + >>> Planck18.write('') + + Get help on the available writers for ``Cosmology`` using the ``help()`` + method:: + + >>> Cosmology.write.help() # Get help writing and list supported formats + >>> Cosmology.write.help(format='') # Get detailed help on format + >>> Cosmology.write.list_formats() # Print list of available formats + + Parameters + ---------- + *args + Positional arguments passed through to data writer. If supplied the + first argument is the output filename. + format : str (optional, keyword-only) + File format specifier. + **kwargs + Keyword arguments passed through to data writer. + + Notes + ----- + """ + + def __init__( + self, + instance: "astropy.cosmology.Cosmology", + cls: type["astropy.cosmology.Cosmology"], + ) -> None: + super().__init__(instance, cls, "write", registry=readwrite_registry) + + def __call__(self, *args: Any, **kwargs: Any) -> None: + self.registry.write(self._instance, *args, **kwargs) + + +# ============================================================================== +# Format Interchange +# for transforming instances, e.g. Cosmology <-> dict + +convert_registry = io_registry.UnifiedIORegistry() + + +class CosmologyFromFormat(io_registry.UnifiedReadWrite): + """Transform object to a `~astropy.cosmology.Cosmology`. + + This function provides the Cosmology interface to the Astropy unified I/O + layer. This allows easily parsing supported data formats using + syntax such as:: + + >>> from astropy.cosmology import Cosmology + >>> cosmo1 = Cosmology.from_format(cosmo_mapping, format='mapping') + + When the ``from_format`` method is called from a subclass the subclass will + provide a keyword argument ``cosmology=`` to the registered parser. + The method uses this cosmology class, regardless of the class indicated in + the data, and sets parameters' default values from the class' signature. + + Get help on the available readers using the ``help()`` method:: + + >>> Cosmology.from_format.help() # Get help and list supported formats + >>> Cosmology.from_format.help('') # Get detailed help on a format + >>> Cosmology.from_format.list_formats() # Print list of available formats + + See also: https://docs.astropy.org/en/stable/io/unified.html + + Parameters + ---------- + obj : object + The object to parse according to 'format' + *args + Positional arguments passed through to data parser. + format : str or None, optional keyword-only + Object format specifier. For `None` (default) CosmologyFromFormat tries + to identify the correct format. + **kwargs + Keyword arguments passed through to data parser. + Parsers should accept the following keyword arguments: + + - cosmology : the class (or string name thereof) to use / check when + constructing the cosmology instance. + + Returns + ------- + out : `~astropy.cosmology.Cosmology` subclass instance + `~astropy.cosmology.Cosmology` corresponding to ``obj`` contents. + """ + + def __init__( + self, + instance: "astropy.cosmology.Cosmology", + cosmo_cls: type["astropy.cosmology.Cosmology"], + ) -> None: + super().__init__(instance, cosmo_cls, "read", registry=convert_registry) + + # =============================================================== + # __call__ overloads + # note: format: ... | None means the format can be auto-detected from the input. + + @overload + def __call__( + self, + obj: _CosmoT, + *args: Any, + format: Literal["astropy.cosmology"] | None, + **kwargs: Any, + ) -> _CosmoT: ... + + @overload + def __call__( + self, + obj: "astropy.cosmology._src.io.builtin.model._CosmologyModel", + *args: Any, + format: Literal["astropy.model"] | None, + **kwargs: Any, + ) -> "astropy.cosmology.Cosmology": ... + + @overload + def __call__( + self, obj: Row, *args: Any, format: Literal["astropy.row"] | None, **kwargs: Any + ) -> "astropy.cosmology.Cosmology": ... + + @overload + def __call__( + self, + obj: Table, + *args: Any, + format: Literal["astropy.table"] | None, + **kwargs: Any, + ) -> "astropy.cosmology.Cosmology": ... + + @overload + def __call__( + self, + obj: Mapping[str, Any], + *args: Any, + format: Literal["mapping"] | None, + **kwargs: Any, + ) -> "astropy.cosmology.Cosmology": ... + + @overload + def __call__( + self, obj: str, *args: Any, format: Literal["yaml"], **kwargs: Any + ) -> "astropy.cosmology.Cosmology": ... + + @overload + def __call__( + self, obj: Any, *args: Any, format: str | None = None, **kwargs: Any + ) -> "astropy.cosmology.Cosmology": ... + + def __call__( + self, obj: Any, *args: Any, format: str | None = None, **kwargs: Any + ) -> "astropy.cosmology.Cosmology": + from astropy.cosmology._src.core import Cosmology + + # so subclasses can override, also pass the class as a kwarg. + # allows for `FlatLambdaCDM.read` and + # `Cosmology.read(..., cosmology=FlatLambdaCDM)` + if self._cls is not Cosmology: + kwargs.setdefault("cosmology", self._cls) # set, if not present + # check that it is the correct cosmology, can be wrong if user + # passes in e.g. `w0wzCDM.read(..., cosmology=FlatLambdaCDM)` + valid = (self._cls, self._cls.__qualname__) + if kwargs["cosmology"] not in valid: + raise ValueError( + "keyword argument `cosmology` must be either the class " + f"{valid[0]} or its qualified name '{valid[1]}'" + ) + + with add_enabled_units(cu): + return self.registry.read(self._cls, obj, *args, format=format, **kwargs) + + +class CosmologyToFormat(io_registry.UnifiedReadWrite): + """Transform this Cosmology to another format. + + This function provides the Cosmology interface to the astropy unified I/O + layer. This allows easily transforming to supported data formats + using syntax such as:: + + >>> from astropy.cosmology import Planck18 + >>> Planck18.to_format("mapping") + {'cosmology': astropy.cosmology.core.FlatLambdaCDM, + 'name': 'Planck18', + 'H0': , + 'Om0': 0.30966, + ... + + Get help on the available representations for ``Cosmology`` using the + ``help()`` method:: + + >>> Cosmology.to_format.help() # Get help and list supported formats + >>> Cosmology.to_format.help('') # Get detailed help on format + >>> Cosmology.to_format.list_formats() # Print list of available formats + + Parameters + ---------- + format : str + Format specifier. + *args + Positional arguments passed through to data writer. If supplied the + first argument is the output filename. + **kwargs + Keyword arguments passed through to data writer. + """ + + def __init__( + self, + instance: "astropy.cosmology.Cosmology", + cls: type["astropy.cosmology.Cosmology"], + ) -> None: + super().__init__(instance, cls, "write", registry=convert_registry) + + # =============================================================== + # __call__ overloads + + @overload + def __call__( + self, format: Literal["astropy.cosmology"], *args: Any, **kwargs: Any + ) -> "astropy.cosmology.Cosmology": ... + + @overload + def __call__( + self, format: Literal["astropy.model"], *args: Any, **kwargs: Any + ) -> "astropy.cosmology._src.io.builtin.model._CosmologyModel": ... + + @overload + def __call__( + self, format: Literal["astropy.row"], *args: Any, **kwargs: Any + ) -> Row: ... + + @overload + def __call__( + self, format: Literal["astropy.table"], *args: Any, **kwargs: Any + ) -> Table: ... + + @overload # specific mapping option, where the mapping class is specified. + def __call__( + self, format: Literal["mapping"], *args: Any, cls: _MT, **kwargs: Any + ) -> _MT: ... + + @overload + def __call__( + self, format: Literal["mapping"], *args: Any, **kwargs: Any + ) -> dict[str, Any]: ... + + @overload + def __call__(self, format: Literal["yaml"], *args: Any, **kwargs: Any) -> str: ... + + @overload + def __call__(self, format: str, *args: Any, **kwargs: Any) -> Any: ... + + def __call__(self, format: str, *args: Any, **kwargs: Any) -> Any: + return self.registry.write(self._instance, None, *args, format=format, **kwargs) diff --git a/astropy/cosmology/_src/parameter/__init__.py b/astropy/cosmology/_src/parameter/__init__.py new file mode 100644 index 000000000000..b75f8a2273f6 --- /dev/null +++ b/astropy/cosmology/_src/parameter/__init__.py @@ -0,0 +1,7 @@ +"""Cosmological Parameters. Private API.""" +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +from .converter import * +from .core import * +from .descriptors import * +from .utils import * diff --git a/astropy/cosmology/_src/parameter/converter.py b/astropy/cosmology/_src/parameter/converter.py new file mode 100644 index 000000000000..d2a35c8c03aa --- /dev/null +++ b/astropy/cosmology/_src/parameter/converter.py @@ -0,0 +1,122 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +__all__ = ( + "validate_non_negative", + "validate_to_float", + "validate_to_scalar", + "validate_with_unit", +) + +from collections.abc import Callable +from typing import TYPE_CHECKING, Any + +from numpy.typing import NDArray + +import astropy.units as u + +if TYPE_CHECKING: + import astropy.cosmology + +FValidateCallable = Callable[[object, object, Any], Any] +_REGISTRY_FVALIDATORS: dict[str, FValidateCallable] = {} + + +def _register_validator( + key: str, fvalidate: FValidateCallable | None = None +) -> FValidateCallable | Callable[[FValidateCallable], FValidateCallable]: + """Decorator to register a new kind of validator function. + + Parameters + ---------- + key : str + fvalidate : callable[[object, object, Any], Any] or None, optional + Value validation function. + + Returns + ------- + ``validator`` or callable[``validator``] + if validator is None returns a function that takes and registers a + validator. This allows ``register_validator`` to be used as a + decorator. + """ + if key in _REGISTRY_FVALIDATORS: + raise KeyError(f"validator {key!r} already registered with Parameter.") + + # fvalidate directly passed + if fvalidate is not None: + _REGISTRY_FVALIDATORS[key] = fvalidate + return fvalidate + + # for use as a decorator + def register(fvalidate): + """Register validator function. + + Parameters + ---------- + fvalidate : callable[[object, object, Any], Any] + Validation function. + + Returns + ------- + ``validator`` + """ + _REGISTRY_FVALIDATORS[key] = fvalidate + return fvalidate + + return register + + +# ====================================================================== + + +@_register_validator("default") +def validate_with_unit( + cosmology: "astropy.cosmology.Cosmology", + param: "astropy.cosmology.Parameter", + value: Any, +) -> Any: + """Default Parameter value validator. + + Adds/converts units if Parameter has a unit. + """ + if param.unit is not None: + with u.add_enabled_equivalencies(param.equivalencies): + value = u.Quantity(value, param.unit) + return value + + +@_register_validator("float") +def validate_to_float( + cosmology: "astropy.cosmology.Cosmology", + param: "astropy.cosmology.Parameter", + value: Any, +) -> float: + """Parameter value validator with units, and converted to float.""" + value = validate_with_unit(cosmology, param, value) + return float(value) + + +@_register_validator("scalar") +def validate_to_scalar( + cosmology: "astropy.cosmology.Cosmology", + param: "astropy.cosmology.Parameter", + value: Any, +) -> NDArray: + """Parameter value validator where value is a scalar.""" + value = validate_with_unit(cosmology, param, value) + if not value.isscalar: + raise ValueError(f"{param.name} is a non-scalar quantity") + return value + + +@_register_validator("non-negative") +def validate_non_negative( + cosmology: "astropy.cosmology.Cosmology", + param: "astropy.cosmology.Parameter", + value: Any, +) -> float: + """Parameter value validator where value is a positive float.""" + value = validate_to_float(cosmology, param, value) + if value < 0.0: + raise ValueError(f"{param.name} cannot be negative.") + return value diff --git a/astropy/cosmology/_src/parameter/core.py b/astropy/cosmology/_src/parameter/core.py new file mode 100644 index 000000000000..7c0d3fa72741 --- /dev/null +++ b/astropy/cosmology/_src/parameter/core.py @@ -0,0 +1,304 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +__all__ = ("MISSING", "Parameter") + +import copy +from collections.abc import Sequence +from dataclasses import KW_ONLY, dataclass, field, fields, is_dataclass, replace +from enum import Enum, auto +from typing import TYPE_CHECKING, Any, Union + +import astropy.units as u + +from .converter import _REGISTRY_FVALIDATORS, FValidateCallable, _register_validator + +if TYPE_CHECKING: + import astropy.cosmology + + +class Sentinel(Enum): + """Sentinel values for Parameter fields.""" + + MISSING = auto() + """A sentinel value signifying a missing default.""" + + def __repr__(self) -> str: + return f"<{self.name}>" + + +MISSING = Sentinel.MISSING + + +@dataclass(frozen=True) +class _UnitField: + # TODO: rm this class when py3.13+ allows for `field(converter=...)` + + def __get__( + self, obj: Union["Parameter", None], objcls: type["Parameter"] | None + ) -> u.Unit | None: + if obj is None: # calling `Parameter.unit` from the class + return None + return getattr(obj, "_unit", None) + + def __set__(self, obj: "Parameter", value: Any) -> None: + object.__setattr__(obj, "_unit", u.Unit(value) if value is not None else None) + + +@dataclass(frozen=True) +class _FValidateField: + default: FValidateCallable | str = "default" + + def __get__( + self, obj: Union["Parameter", None], objcls: type["Parameter"] | None + ) -> FValidateCallable | str: + if obj is None: # calling `Parameter.fvalidate` from the class + return self.default + return obj._fvalidate # calling `Parameter.fvalidate` from an instance + + def __set__(self, obj: "Parameter", value: Any) -> None: + # Always store input fvalidate. + object.__setattr__(obj, "_fvalidate_in", value) + + # Process to the callable. + if value in _REGISTRY_FVALIDATORS: + value = _REGISTRY_FVALIDATORS[value] + elif isinstance(value, str): + msg = f"`fvalidate`, if str, must be in {_REGISTRY_FVALIDATORS.keys()}" + raise ValueError(msg) + elif not callable(value): + msg = f"`fvalidate` must be a function or {_REGISTRY_FVALIDATORS.keys()}" + raise TypeError(msg) + object.__setattr__(obj, "_fvalidate", value) + + +@dataclass(frozen=True) +class Parameter: + r"""Cosmological parameter (descriptor). + + Should only be used with a :class:`~astropy.cosmology.Cosmology` subclass. + + Parameters + ---------- + default : Any (optional, keyword-only) + Default value of the Parameter. If not given the + Parameter must be set when initializing the cosmology. + derived : bool (optional, keyword-only) + Whether the Parameter is 'derived', default `False`. + Derived parameters behave similarly to normal parameters, but are not + sorted by the |Cosmology| signature (probably not there) and are not + included in all methods. For reference, see ``Ode0`` in + ``FlatFLRWMixin``, which removes :math:`\Omega_{de,0}`` as an + independent parameter (:math:`\Omega_{de,0} \equiv 1 - \Omega_{tot}`). + unit : unit-like or None (optional, keyword-only) + The `~astropy.units.Unit` for the Parameter. If None (default) no + unit as assumed. + equivalencies : `~astropy.units.Equivalency` or sequence thereof + Unit equivalencies for this Parameter. + fvalidate : callable[[object, object, Any], Any] or str (optional, keyword-only) + Function to validate the Parameter value from instances of the + cosmology class. If "default", uses default validator to assign units + (with equivalencies), if Parameter has units. + For other valid string options, see ``Parameter._registry_validators``. + 'fvalidate' can also be set through a decorator with + :meth:`~astropy.cosmology.Parameter.validator`. + doc : str or None (optional, keyword-only) + Parameter description. + + Examples + -------- + For worked examples see :class:`~astropy.cosmology.FLRW`. + """ + + _: KW_ONLY + + default: Any = MISSING + """Default value of the Parameter. + + By default set to ``MISSING``, which indicates the parameter must be set + when initializing the cosmology. + """ + + derived: bool = False + """Whether the Parameter can be set, or is derived, on the cosmology.""" + + # Units + unit: _UnitField = _UnitField() + """The unit of the Parameter (can be `None` for unitless).""" + + equivalencies: u.Equivalency | Sequence[u.Equivalency] = field(default_factory=list) + """Unit equivalencies available when setting the parameter.""" + + # Setting + fvalidate: _FValidateField = _FValidateField(default="default") + """Function to validate/convert values when setting the Parameter.""" + + # Info + doc: str | None = None + """Parameter description.""" + + name: str = field(init=False, compare=True, default=None, repr=False) + """The name of the Parameter on the Cosmology. + + Cannot be set directly. + """ + + def __post_init__(self) -> None: + self._fvalidate_in: FValidateCallable | str + self._fvalidate: FValidateCallable + object.__setattr__(self, "__doc__", self.doc) + # Now setting a dummy attribute name. The cosmology class will call + # `__set_name__`, passing the real attribute name. However, if Parameter is not + # init'ed as a descriptor then this ensures that all declared fields exist. + self.__set_name__(None, "name not initialized") + + def __set_name__(self, cosmo_cls: type, name: str) -> None: + # attribute name on container cosmology class + object.__setattr__(self, "name", name) + + # ------------------------------------------- + # descriptor and property-like methods + + def __get__( + self, + cosmology: Union["astropy.cosmology.Cosmology", None], + cosmo_cls: Union["type[astropy.cosmology.Cosmology]", None] = None, + ) -> Any: + # Get from class + if cosmology is None: + # If the Parameter is being set as part of a dataclass constructor, then we + # raise an AttributeError if the default is MISSING. This is to prevent the + # Parameter from being set as the default value of the dataclass field and + # erroneously included in the class' __init__ signature. + if self.default is MISSING and ( + not is_dataclass(cosmo_cls) + or self.name not in cosmo_cls.__dataclass_fields__ + ): + raise AttributeError + return self + # Get from instance + return cosmology.__dict__[self.name] + + def __set__(self, cosmology: "astropy.cosmology.Cosmology", value: Any) -> None: + """Allows attribute setting once. + + Raises AttributeError subsequently. + """ + # Raise error if setting 2nd time. The built-in Cosmology objects are frozen + # dataclasses and this is redundant, however user defined cosmology classes do + # not have to be frozen. + if self.name in cosmology.__dict__: + raise AttributeError(f"cannot assign to field {self.name!r}") + + # Change `self` to the default value if default is MISSING. + # This is done for backwards compatibility only - so that Parameter can be used + # in a dataclass and still return `self` when accessed from a class. + # Accessing the Parameter object via `cosmo_cls.param_name` will be removed + # in favor of `cosmo_cls.parameters["param_name"]`. + if value is self: + value = self.default + + # Validate value, generally setting units if present + value = self.validate(cosmology, copy.deepcopy(value)) + + # Make the value read-only, if ndarray-like + if hasattr(value, "setflags"): + value.setflags(write=False) + + # Set the value on the cosmology + cosmology.__dict__[self.name] = value + + # ------------------------------------------- + # validate value + + def validator(self, fvalidate: FValidateCallable) -> "Parameter": + """Make new Parameter with custom ``fvalidate``. + + Note: ``Parameter.fvalidator`` must be the top-most descriptor decorator. + + Parameters + ---------- + fvalidate : callable[[type, type, Any], Any] + + Returns + ------- + `~astropy.cosmology.Parameter` + Copy of this Parameter but with custom ``fvalidate``. + """ + return self.clone(fvalidate=fvalidate) + + def validate(self, cosmology: "astropy.cosmology.Cosmology", value: Any) -> Any: + """Run the validator on this Parameter. + + Parameters + ---------- + cosmology : `~astropy.cosmology.Cosmology` instance + value : Any + The object to validate. + + Returns + ------- + Any + The output of calling ``fvalidate(cosmology, self, value)`` + (yes, that parameter order). + """ + return self._fvalidate(cosmology, self, value) + + @staticmethod + def register_validator(key, fvalidate: FValidateCallable | None = None) -> Any: + """Decorator to register a new kind of validator function. + + Parameters + ---------- + key : str + fvalidate : callable[[object, object, Any], Any] or None, optional + Value validation function. + + Returns + ------- + ``validator`` or callable[``validator``] + if validator is None returns a function that takes and registers a + validator. This allows ``register_validator`` to be used as a + decorator. + """ + return _register_validator(key, fvalidate=fvalidate) + + # ------------------------------------------- + + def clone(self, **kw: Any) -> "Parameter": + """Clone this `Parameter`, changing any constructor argument. + + Parameters + ---------- + **kw + Passed to constructor. The current values, eg. ``fvalidate`` are + used as the default values, so an empty ``**kw`` is an exact copy. + + Examples + -------- + >>> p = Parameter() + >>> p + Parameter(derived=False, unit=None, equivalencies=[], + fvalidate='default', doc=None) + + >>> p.clone(unit="km") + Parameter(derived=False, unit=Unit("km"), equivalencies=[], + fvalidate='default', doc=None) + """ + kw.setdefault("fvalidate", self._fvalidate_in) # prefer the input fvalidate + cloned = replace(self, **kw) + # Transfer over the __set_name__ stuff. If `clone` is used to make a + # new descriptor, __set_name__ will be called again, overwriting this. + cloned.__set_name__(None, self.name) + + return cloned + + def __repr__(self) -> str: + """Return repr(self).""" + fields_repr = ( + # Get the repr, using the input fvalidate over the processed value + f"{f.name}={(getattr(self, f.name if f.name != 'fvalidate' else '_fvalidate_in'))!r}" + for f in fields(self) + # Only show fields that should be displayed and are not sentinel values + if f.repr and (f.name != "default" or self.default is not MISSING) + ) + return f"{self.__class__.__name__}({', '.join(fields_repr)})" diff --git a/astropy/cosmology/_src/parameter/descriptors.py b/astropy/cosmology/_src/parameter/descriptors.py new file mode 100644 index 000000000000..b7d570424475 --- /dev/null +++ b/astropy/cosmology/_src/parameter/descriptors.py @@ -0,0 +1,69 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +__all__: list[str] = ["ParametersAttribute"] + +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, NoReturn, Union + +if TYPE_CHECKING: + import astropy.cosmology + + +@dataclass(frozen=True, slots=True) +class ParametersAttribute: + """Immutable mapping of the :class:`~astropy.cosmology.Parameter` objects or values. + + If accessed from the :class:`~astropy.cosmology.Cosmology` class, this returns a + mapping of the :class:`~astropy.cosmology.Parameter` objects themselves. If + accessed from an instance, this returns a mapping of the values of the Parameters. + + This class is used to implement :obj:`astropy.cosmology.Cosmology.parameters`. + + Parameters + ---------- + attr_name : str + The name of the class attribute that is a `~types.MappingProxyType[str, + astropy.cosmology.Parameter]` of all the cosmology's parameters. When accessed + from the class, this attribute is returned. When accessed from an instance, a + mapping of the cosmology instance's values for each key is returned. + + Examples + -------- + The normal usage of this class is the ``parameters`` attribute of + :class:`~astropy.cosmology.Cosmology`. + + >>> from astropy.cosmology import FlatLambdaCDM, Planck18 + + >>> FlatLambdaCDM.parameters + mappingproxy({'H0': Parameter(...), ...}) + + >>> Planck18.parameters + mappingproxy({'H0': , ...}) + """ + + attr_name: str + """Class attribute name on Cosmology for the mapping of Parameter objects.""" + + _name: str = field(init=False) + """The name of the descriptor on the containing class.""" + + def __set_name__(self, owner: Any, name: str) -> None: + object.__setattr__(self, "_name", name) + + def __get__( + self, + instance: Union["astropy.cosmology.Cosmology", None], + owner: type["astropy.cosmology.Cosmology"] | None, + ) -> MappingProxyType[str, Any]: + # Called from the class + if instance is None: + return getattr(owner, self.attr_name) + # Called from the instance + return MappingProxyType( + {n: getattr(instance, n) for n in getattr(instance, self.attr_name)} + ) + + def __set__(self, instance: Any, value: Any) -> NoReturn: + msg = f"cannot set {self._name!r} of {instance!r}." + raise AttributeError(msg) diff --git a/astropy/cosmology/_src/parameter/utils.py b/astropy/cosmology/_src/parameter/utils.py new file mode 100644 index 000000000000..00e733190c85 --- /dev/null +++ b/astropy/cosmology/_src/parameter/utils.py @@ -0,0 +1,33 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +__all__ = ("all_parameters",) + +import functools +import operator +from dataclasses import Field +from typing import TypeGuard + +from .core import Parameter + + +def is_parameter_or_field(obj: object, /) -> TypeGuard[Parameter | Field[Parameter]]: + """Return if object is a Parameter or dataclass field thereof.""" + return isinstance(obj, Parameter) or ( + isinstance(obj, Field) and isinstance(obj.default, Parameter) + ) + + +def all_parameters(obj: object, /) -> dict[str, Parameter]: + """Get all `Parameter` fields of an object. + + Returns all fields of the object, including those not yet finalized in the class, if + it's still under construction, e.g. in ``__init_subclass__``. + """ + cls = obj if isinstance(obj, type) else obj.__class__ + all_cls_vars = functools.reduce(operator.__or__, map(vars, cls.mro()[::-1])) + + return { + k: (v if isinstance(v, Parameter) else v.default) + for k, v in all_cls_vars.items() + if is_parameter_or_field(v) + } diff --git a/astropy/cosmology/_src/scipy_compat.py b/astropy/cosmology/_src/scipy_compat.py new file mode 100644 index 000000000000..414c2d819574 --- /dev/null +++ b/astropy/cosmology/_src/scipy_compat.py @@ -0,0 +1,22 @@ +"""Scipy compatibility.""" + +__all__ = ("ellipkinc", "hyp2f1", "quad") + +from typing import Any, Never + +from astropy.utils.compat.optional_deps import HAS_SCIPY + +if HAS_SCIPY: + from scipy.integrate import quad + from scipy.special import ellipkinc, hyp2f1 + +else: + + def quad(*args: Any, **kwargs: Any) -> Never: + raise ModuleNotFoundError("No module named 'scipy.integrate'") + + def ellipkinc(*args: Any, **kwargs: Any) -> Never: + raise ModuleNotFoundError("No module named 'scipy.special'") + + def hyp2f1(*args: Any, **kwargs: Any) -> Never: + raise ModuleNotFoundError("No module named 'scipy.special'") diff --git a/astropy/cosmology/_src/setup_package.py b/astropy/cosmology/_src/setup_package.py new file mode 100644 index 000000000000..7d3fbce6feb5 --- /dev/null +++ b/astropy/cosmology/_src/setup_package.py @@ -0,0 +1,29 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import sys +from os.path import relpath +from pathlib import Path + +from setuptools import Extension + +ASTROPY_COSMOLOGY_SRC_ROOT = Path(__file__).parent + + +if sys.platform.startswith("win"): + # on windows, -Werror (and possibly -Wall too) isn't recognized + extra_compile_args = [] +else: + # be extra careful with this extension as it calls PyErr_WarnEx + # with a formatted message whose size cannot be determined at compile time, + # which is never done within the standard library + extra_compile_args = ["-Werror", "-Wall"] + + +def get_extensions(): + return [ + Extension( + "astropy.cosmology._src.signature_deprecations", + [relpath(Path(ASTROPY_COSMOLOGY_SRC_ROOT, "signature_deprecations.c"))], + extra_compile_args=extra_compile_args, + ), + ] diff --git a/astropy/cosmology/_src/signature_deprecations.c b/astropy/cosmology/_src/signature_deprecations.c new file mode 100644 index 000000000000..7668a69fc886 --- /dev/null +++ b/astropy/cosmology/_src/signature_deprecations.c @@ -0,0 +1,292 @@ +// This extension is adapted from the positional_defaults PyPI package +// https://pypi.org/project/positional-defaults/ version 2023.4.19 +// MIT. see licenses/POSITIONAL_DEFAULTS.rst + + +#include +#include // snprintf +#include + +#define SINCE_CHAR_SIZE 32 +#define NAMES_CHAR_SIZE 128 +#define MSG_SIZE 512 + +typedef struct { + PyObject_HEAD + PyObject *dict; + PyObject *wrapped; + PyObject *names; + PyObject *since; +} DeprKwsObject; + + +static void depr_kws_wrap_dealloc(DeprKwsObject *self) +{ + Py_XDECREF(self->wrapped); + Py_XDECREF(self->names); + Py_XDECREF(self->since); + PyTypeObject *type = Py_TYPE((PyObject *)self); + freefunc free_func = PyType_GetSlot(type, Py_tp_free); + free_func((PyObject *)self); + Py_DECREF((PyObject *)type); +} + + +static PyObject *depr_kws_wrap_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + allocfunc alloc_func = PyType_GetSlot(type, Py_tp_alloc); + DeprKwsObject *self = (DeprKwsObject *)alloc_func(type, 0); + + if (self != NULL) { + self->names = PyTuple_New(0); + if (self->names == NULL) { + Py_DECREF(self); + return NULL; + } + + Py_INCREF(Py_None); + self->wrapped = Py_None; + + Py_INCREF(Py_None); + self->since = Py_None; + } + + return (PyObject *)self; +} + + +static int depr_kws_wrap_init(DeprKwsObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"wrapped", "names", "since", NULL}; + Py_ssize_t i, n_names; + PyObject *wrapped, *names, *since, *tmp; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOO:wrap", kwlist, &wrapped, &names, &since)) { + return -1; + } + + if (!PyTuple_Check(names)) { + PyErr_SetString(PyExc_TypeError, "names must be a tuple"); + return -1; + } + + n_names = PyTuple_Size(names); + + for (i = 0; i < n_names; ++i) { + PyObject *name = PyTuple_GetItem(names, i); + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, "names[%zd] must be a string", i); + return -1; + } + } + + if (!PyUnicode_Check(since)) { + PyErr_Format(PyExc_TypeError, "since must be a string", i); + return -1; + } + + tmp = self->wrapped; + Py_INCREF(wrapped); + self->wrapped = wrapped; + Py_XDECREF(tmp); + + tmp = self->names; + Py_INCREF(names); + self->names = names; + Py_XDECREF(tmp); + + tmp = self->since; + Py_INCREF(since); + self->since = since; + Py_XDECREF(tmp); + return 0; +} + + +static PyMemberDef depr_kws_wrap_members[] = { + {"__dict__", T_OBJECT, offsetof(DeprKwsObject, dict), READONLY}, + {"__dictoffset__", T_PYSSIZET, offsetof(DeprKwsObject, dict), READONLY}, + {"wrapped", T_OBJECT, offsetof(DeprKwsObject, wrapped), READONLY}, + {"names", T_OBJECT, offsetof(DeprKwsObject, names), READONLY}, + {"since", T_OBJECT, offsetof(DeprKwsObject, since), READONLY}, + {NULL}, +}; + + +static PyObject *depr_kws_wrap_call(DeprKwsObject *self, PyObject *args, PyObject *kwds) +{ + // step 0: return early whenever possible + if (self->wrapped == NULL) { + Py_RETURN_NONE; + } + + if (kwds == NULL) { + return PyObject_Call(self->wrapped, args, kwds); + } + + // step 1: detect any deprecated keyword arguments, return if none. + Py_ssize_t n_names = PyTuple_Size(self->names); + PyObject *deprecated_kwargs = PyList_New(n_names); + Py_INCREF(deprecated_kwargs); + PyObject *name = NULL; + Py_ssize_t i = 0; + int has_kw = -2; + + Py_ssize_t n_depr = 0; + for (i = 0; i < n_names; ++i) { + name = PyTuple_GetItem(self->names, i); + has_kw = PyDict_Contains(kwds, name); + if (has_kw) { + PyList_SetItem(deprecated_kwargs, n_depr, name); + ++n_depr; + } + } + + if (n_depr == 0) { + return PyObject_Call(self->wrapped, args, kwds); + } + + // step 2: create and emit warning message + char names_char[NAMES_CHAR_SIZE]; + char *s, *arguments, *respectively, *pronoun; + + PyObject *names_unicode; + if (n_depr > 1) { + names_unicode = PyObject_Str(PyList_GetSlice(deprecated_kwargs, 0, n_depr)); + s = "s"; + arguments = " arguments"; + respectively = ", respectively"; + pronoun = "them"; + } + else { + names_unicode = PyObject_Repr(PyList_GetItem(deprecated_kwargs, 0)); + s = arguments = respectively = ""; + pronoun = "it"; + } + const char *names_utf8 = PyUnicode_AsUTF8AndSize(names_unicode, NULL); + snprintf(names_char, NAMES_CHAR_SIZE, "%s", names_utf8); + + PyObject *since_unicode = PyObject_Str(self->since); + const char *since_utf8 = PyUnicode_AsUTF8AndSize(since_unicode, NULL); + char since_char[SINCE_CHAR_SIZE]; + snprintf(since_char, SINCE_CHAR_SIZE, "%s", since_utf8); + + char msg[MSG_SIZE]; + snprintf( + msg, + MSG_SIZE, + "Passing %s%s as keyword%s " + "is deprecated since version %s " + "and will stop working in a future release. " + "Pass %s positionally to suppress this warning.", + names_char, + arguments, + s, + since_char, + pronoun + ); + const char *msg_ptr = msg; + + int status = PyErr_WarnEx(PyExc_FutureWarning, msg_ptr, 2); + if (status == -1) { + // avoid leaking memory if Warning is promoted to Exception + Py_DECREF(deprecated_kwargs); + } + + return PyObject_Call(self->wrapped, args, kwds); +} + +// replace PyMethod_New (not part of the limited API) +// this is adapted from Cython's ObjectHandling.c +// https://github.com/cython/cython/blob/dba606bc50e90dbaf37850779d1f84f1c0a22c7a/Cython/Utility/ObjectHandling.c#L2977 +#ifdef Py_LIMITED_API +static PyObject *CachedMethodType = NULL; + + +static PyObject *PyMethod_New(PyObject *func, PyObject *self) +{ + PyObject *result; +#if Py_LIMITED_API >= 0x030C0000 + { + PyObject *args[] = {func, self}; + result = PyObject_Vectorcall(CachedMethodType, args, 2, NULL); + } +#else + result = PyObject_CallFunctionObjArgs(CachedMethodType, func, self, NULL); +#endif + return result; +} +#endif + +static PyObject *depr_kws_wrap_get(PyObject *self, PyObject *obj, PyObject *type) +{ + if (obj == Py_None || obj == NULL) { + Py_INCREF(self); + return self; + } + return PyMethod_New(self, obj); +} + + +static PyType_Spec DeprKwsWrapType_spec = { + .name = "signature_deprecations.wrap", + .basicsize = sizeof(DeprKwsObject), + .itemsize = 0, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_METHOD_DESCRIPTOR, + .slots = (PyType_Slot[]){ + {Py_tp_doc, PyDoc_STR("wrap a function with deprecated keyword arguments")}, + {Py_tp_new, depr_kws_wrap_new}, + {Py_tp_init, (initproc)depr_kws_wrap_init}, + {Py_tp_dealloc, (destructor)depr_kws_wrap_dealloc}, + {Py_tp_members, depr_kws_wrap_members}, + {Py_tp_call, (ternaryfunc)depr_kws_wrap_call}, + {Py_tp_descr_get, depr_kws_wrap_get}, + {0, NULL}, + }, +}; + +static PyObject *DeprKwsWrapType = NULL; + +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + .m_name = "signature_deprecations", + .m_doc = PyDoc_STR("fast decorators to mark signature details as deprecated"), + .m_size = -1, +}; + + +PyMODINIT_FUNC PyInit_signature_deprecations(void) +{ + PyObject *m; + + m = PyModule_Create(&moduledef); + if (m == NULL) { + return NULL; + } +#if Py_LIMITED_API + { + PyObject *typesModule = PyImport_ImportModule("types"); + if (!typesModule) { + return NULL; + } + CachedMethodType = PyObject_GetAttrString(typesModule, "MethodType"); + Py_DECREF(typesModule); + if (!CachedMethodType) { + return NULL; + } + } +#endif + DeprKwsWrapType = PyType_FromModuleAndSpec(m, &DeprKwsWrapType_spec, NULL); + if (DeprKwsWrapType == NULL) { + return NULL; + } + + if (PyModule_AddObject(m, "_depr_kws_wrap", DeprKwsWrapType) < 0) { + Py_DECREF(DeprKwsWrapType); + Py_DECREF(m); + return NULL; + } + + return m; +} diff --git a/astropy/analytic_functions/tests/__init__.py b/astropy/cosmology/_src/tests/__init__.py similarity index 100% rename from astropy/analytic_functions/tests/__init__.py rename to astropy/cosmology/_src/tests/__init__.py diff --git a/astropy/cosmology/_src/tests/conftest.py b/astropy/cosmology/_src/tests/conftest.py new file mode 100644 index 000000000000..cd1f3649bd63 --- /dev/null +++ b/astropy/cosmology/_src/tests/conftest.py @@ -0,0 +1,6 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +"""Configure the tests for :mod:`astropy.cosmology`.""" + +from astropy.cosmology._src.tests.helper import clean_registry # noqa: F401 +from astropy.tests.helper import pickle_protocol # noqa: F401 diff --git a/astropy/extern/bundled/__init__.py b/astropy/cosmology/_src/tests/flrw/__init__.py similarity index 100% rename from astropy/extern/bundled/__init__.py rename to astropy/cosmology/_src/tests/flrw/__init__.py diff --git a/astropy/cosmology/_src/tests/flrw/conftest.py b/astropy/cosmology/_src/tests/flrw/conftest.py new file mode 100644 index 000000000000..20adfe0015aa --- /dev/null +++ b/astropy/cosmology/_src/tests/flrw/conftest.py @@ -0,0 +1,32 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +"""Configure the tests for :mod:`astropy.cosmology`.""" + +from collections.abc import Iterable, Mapping, Sequence +from typing import TypeVar + +from astropy.cosmology._src.tests.helper import clean_registry # noqa: F401 +from astropy.tests.helper import pickle_protocol # noqa: F401 + +K = TypeVar("K") +V = TypeVar("V") + + +def filter_keys_from_items( + m: Mapping[K, V], /, filter_out: Sequence[K] +) -> Iterable[K, V]: + """Filter ``m``, returning key-value pairs not including keys in ``filter``. + + Parameters + ---------- + m : mapping[K, V] + A mapping from which to remove keys in ``filter_out``. + filter_out : sequence[K] + Sequence of keys to filter out from ``m``. + + Returns + ------- + iterable[K, V] + Iterable of ``(key, value)`` pairs with the ``filter_out`` keys removed. + """ + return ((k, v) for k, v in m.items() if k not in filter_out) diff --git a/astropy/cosmology/_src/tests/flrw/data/cosmo_closed.ecsv b/astropy/cosmology/_src/tests/flrw/data/cosmo_closed.ecsv new file mode 100644 index 000000000000..adaec308ff07 --- /dev/null +++ b/astropy/cosmology/_src/tests/flrw/data/cosmo_closed.ecsv @@ -0,0 +1,61 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: redshift, unit: redshift, datatype: float64} +# - {name: dm, unit: Mpc, datatype: float64} +# - {name: da, unit: Mpc, datatype: float64} +# - {name: dl, unit: Mpc, datatype: float64} +# meta: !!omap +# - {source: icosmo (icosmo.org)} +# - {Om: 2} +# - {w: -1} +# - {h: 0.7} +# - {Ol: 0.1} +# - __serialized_columns__: +# da: +# __class__: astropy.units.quantity.Quantity +# unit: &id001 !astropy.units.Unit {unit: Mpc} +# value: !astropy.table.SerializedColumn {name: da} +# dl: +# __class__: astropy.units.quantity.Quantity +# unit: *id001 +# value: !astropy.table.SerializedColumn {name: dl} +# dm: +# __class__: astropy.units.quantity.Quantity +# unit: *id001 +# value: !astropy.table.SerializedColumn {name: dm} +# redshift: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: redshift} +# value: !astropy.table.SerializedColumn {name: redshift} +# schema: astropy-2.0 +redshift dm da dl +0.0 0.0 0.0 0.0 +0.1625 601.8016 517.67879 699.59436 +0.325 1057.9502 798.45297 1401.784 +0.5 1438.2161 958.81076 2157.3242 +0.6625 1718.6778 1033.7912 2857.3019 +0.825 1948.24 1067.5288 3555.5381 +1.0 2152.7954 1076.3977 4305.5908 +1.1625 2312.3427 1069.2914 5000.441 +1.325 2448.9755 1053.3228 5693.8681 +1.5 2575.6795 1030.2718 6439.1988 +1.6625 2677.9671 1005.8092 7130.0873 +1.825 2768.1157 979.86398 7819.927 +2.0 2853.9222 951.30739 8561.7665 +2.1625 2924.8116 924.84161 9249.7167 +2.325 2988.5333 898.80701 9936.8732 +2.5 3050.3065 871.51614 10676.073 +2.6625 3102.1909 847.01459 11361.774 +2.825 3149.5043 823.39982 12046.854 +3.0 3195.9966 798.99915 12783.986 +3.1625 3235.5334 777.30533 13467.908 +3.325 3271.9832 756.5279 14151.327 +3.5 3308.1758 735.15017 14886.791 +3.6625 3339.2521 716.19347 15569.263 +3.825 3368.1489 698.06195 16251.319 +4.0 3397.0803 679.41605 16985.401 +4.1625 3422.1142 662.87926 17666.664 +4.325 3445.5542 647.05243 18347.576 +4.5 3469.1805 630.76008 19080.493 +4.6625 3489.7534 616.29199 19760.729 diff --git a/astropy/cosmology/_src/tests/flrw/data/cosmo_flat.ecsv b/astropy/cosmology/_src/tests/flrw/data/cosmo_flat.ecsv new file mode 100644 index 000000000000..ba6dd55e911c --- /dev/null +++ b/astropy/cosmology/_src/tests/flrw/data/cosmo_flat.ecsv @@ -0,0 +1,61 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: redshift, unit: redshift, datatype: float64} +# - {name: dm, unit: Mpc, datatype: float64} +# - {name: da, unit: Mpc, datatype: float64} +# - {name: dl, unit: Mpc, datatype: float64} +# meta: !!omap +# - {source: icosmo (icosmo.org)} +# - {Om: 0.3} +# - {w: -1} +# - {h: 0.7} +# - {Ol: 0.7} +# - __serialized_columns__: +# da: +# __class__: astropy.units.quantity.Quantity +# unit: &id001 !astropy.units.Unit {unit: Mpc} +# value: !astropy.table.SerializedColumn {name: da} +# dl: +# __class__: astropy.units.quantity.Quantity +# unit: *id001 +# value: !astropy.table.SerializedColumn {name: dl} +# dm: +# __class__: astropy.units.quantity.Quantity +# unit: *id001 +# value: !astropy.table.SerializedColumn {name: dm} +# redshift: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: redshift} +# value: !astropy.table.SerializedColumn {name: redshift} +# schema: astropy-2.0 +redshift dm da dl +0.0 0.0 0.0 0.0 +0.1625 669.77536 576.15085 778.61386 +0.325 1285.5964 970.26143 1703.4152 +0.5 1888.6254 1259.0836 2832.9381 +0.6625 2395.5489 1440.9317 3982.6 +0.825 2855.5732 1564.6976 5211.421 +1.0 3303.8288 1651.9144 6607.6577 +1.1625 3681.1867 1702.2829 7960.5663 +1.325 4025.5229 1731.4077 9359.3408 +1.5 4363.8558 1745.5423 10909.64 +1.6625 4651.483 1747.0359 12384.573 +1.825 4916.597 1740.3883 13889.387 +2.0 5179.8621 1726.6207 15539.586 +2.1625 5406.0204 1709.4136 17096.54 +2.325 5616.5075 1689.1752 18674.888 +2.5 5827.5418 1665.012 20396.396 +2.6625 6010.4886 1641.089 22013.414 +2.825 6182.1688 1616.2533 23646.796 +3.0 6355.6855 1588.9214 25422.742 +3.1625 6507.2491 1563.3031 27086.425 +3.325 6650.452 1537.6768 28763.205 +3.5 6796.1499 1510.2555 30582.674 +3.6625 6924.2096 1485.0852 32284.127 +3.825 7045.8876 1460.2876 33996.408 +4.0 7170.3664 1434.0733 35851.832 +4.1625 7280.3423 1410.2358 37584.767 +4.325 7385.3277 1386.916 39326.87 +4.5 7493.2222 1362.404 41212.722 +4.6625 7588.9589 1340.2135 42972.48 diff --git a/astropy/cosmology/_src/tests/flrw/data/cosmo_open.ecsv b/astropy/cosmology/_src/tests/flrw/data/cosmo_open.ecsv new file mode 100644 index 000000000000..d39dd7a8d587 --- /dev/null +++ b/astropy/cosmology/_src/tests/flrw/data/cosmo_open.ecsv @@ -0,0 +1,61 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: redshift, unit: redshift, datatype: float64} +# - {name: dm, unit: Mpc, datatype: float64} +# - {name: da, unit: Mpc, datatype: float64} +# - {name: dl, unit: Mpc, datatype: float64} +# meta: !!omap +# - {source: icosmo (icosmo.org)} +# - {Om: 0.3} +# - {w: -1} +# - {h: 0.7} +# - {Ol: 0.1} +# - __serialized_columns__: +# da: +# __class__: astropy.units.quantity.Quantity +# unit: &id001 !astropy.units.Unit {unit: Mpc} +# value: !astropy.table.SerializedColumn {name: da} +# dl: +# __class__: astropy.units.quantity.Quantity +# unit: *id001 +# value: !astropy.table.SerializedColumn {name: dl} +# dm: +# __class__: astropy.units.quantity.Quantity +# unit: *id001 +# value: !astropy.table.SerializedColumn {name: dm} +# redshift: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: redshift} +# value: !astropy.table.SerializedColumn {name: redshift} +# schema: astropy-2.0 +redshift dm da dl +0.0 0.0 0.0 0.0 +0.1625 643.08185 553.18868 747.58265 +0.325 1200.9858 906.40441 1591.3062 +0.5 1731.6262 1154.4175 2597.4393 +0.6625 2174.3252 1307.8648 3614.8157 +0.825 2578.7616 1413.0201 4706.2399 +1.0 2979.346 1489.673 5958.692 +1.1625 3324.2002 1537.2024 7188.5829 +1.325 3646.8432 1568.5347 8478.9104 +1.5 3972.8407 1589.1363 9932.1017 +1.6625 4258.1131 1599.2913 11337.226 +1.825 4528.5346 1603.0211 12793.11 +2.0 4804.9314 1601.6438 14414.794 +2.1625 5049.2007 1596.5852 15968.097 +2.325 5282.6693 1588.7727 17564.875 +2.5 5523.0914 1578.0261 19330.82 +2.6625 5736.9813 1566.4113 21011.694 +2.825 5942.5803 1553.6158 22730.37 +3.0 6155.4289 1538.8572 24621.716 +3.1625 6345.6997 1524.4924 26413.975 +3.325 6529.3655 1509.6799 28239.506 +3.5 6720.2676 1493.3928 30241.204 +3.6625 6891.5474 1478.0799 32131.84 +3.825 7057.4213 1462.678 34052.058 +4.0 7230.3723 1446.0745 36151.862 +4.1625 7385.9998 1430.7021 38130.224 +4.325 7537.1112 1415.4199 40135.117 +4.5 7695.0718 1399.104 42322.895 +4.6625 7837.551 1384.115 44380.133 diff --git a/astropy/cosmology/_src/tests/flrw/test_base.py b/astropy/cosmology/_src/tests/flrw/test_base.py new file mode 100644 index 000000000000..48cc07ebbe4c --- /dev/null +++ b/astropy/cosmology/_src/tests/flrw/test_base.py @@ -0,0 +1,568 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +"""Testing :mod:`astropy.cosmology.flrw.base`. + +This module sets up the tests for subclasses of :class:`astropy.cosmology.FLRW`. The +tests for the specific abstract class :class:`astropy.cosmology.FLRW` are in +``test_flrw``. + +""" + +import abc +from functools import cached_property + +import numpy as np +import pytest + +import astropy.constants as const +import astropy.units as u +from astropy.cosmology import FLRW, FlatLambdaCDM, LambdaCDM, Planck18 +from astropy.cosmology._src.core import _COSMOLOGY_CLASSES, dataclass_decorator +from astropy.cosmology._src.flrw.base import a_B_c2 +from astropy.cosmology._src.tests.helper import get_redshift_methods +from astropy.cosmology._src.tests.test_core import ( + CosmologyTest, + FlatCosmologyMixinTest, + invalid_zs, + valid_zs, +) +from astropy.tests.helper import assert_quantity_allclose +from astropy.utils.compat.optional_deps import HAS_PANDAS, HAS_SCIPY + +from .conftest import filter_keys_from_items +from .test_parameters import ( + ParameterFlatOde0TestMixin, + ParameterH0TestMixin, + Parameterm_nuTestMixin, + ParameterNeffTestMixin, + ParameterOb0TestMixin, + ParameterOde0TestMixin, + ParameterOm0TestMixin, + ParameterTcmb0TestMixin, +) + +############################################################################## +# TESTS +############################################################################## + + +class FLRWTest( + CosmologyTest, + ParameterH0TestMixin, + ParameterOm0TestMixin, + ParameterOde0TestMixin, + ParameterTcmb0TestMixin, + ParameterNeffTestMixin, + Parameterm_nuTestMixin, + ParameterOb0TestMixin, +): + abstract_w = False + + @abc.abstractmethod + def setup_class(self): + """Setup for testing.""" + super().setup_class(self) + + # Default cosmology args and kwargs + self._cls_args = dict( + H0=70 * u.km / u.s / u.Mpc, Om0=0.27 * u.one, Ode0=0.73 * u.one + ) + self.cls_kwargs = dict( + Tcmb0=3.0 * u.K, + Ob0=0.03 * u.one, + name=self.__class__.__name__, + meta={"a": "b"}, + ) + + @pytest.fixture(scope="class") + @classmethod + def nonflatcosmo(cls): + """A non-flat cosmology used in equivalence tests.""" + return LambdaCDM(70, 0.4, 0.8) + + # =============================================================== + # Method & Attribute Tests + + def test_init(self, cosmo_cls): + """Test initialization.""" + super().test_init(cosmo_cls) + + # TODO! tests for initializing calculated values, e.g. `h` + # TODO! transfer tests for initializing neutrinos + + def test_init_Tcmb0_zeroing(self, cosmo_cls, ba): + """Test if setting Tcmb0 parameter to 0 influences other parameters. + + TODO: consider moving this test to ``FLRWTest`` + """ + ba.arguments["Tcmb0"] = 0.0 + cosmo = cosmo_cls(*ba.args, **ba.kwargs) + + assert cosmo.Ogamma0 == 0.0 + assert cosmo.Onu0 == 0.0 + + if not self.abstract_w: + assert u.allclose(cosmo.Ogamma(1.5), [0, 0, 0, 0]) + assert u.allclose(cosmo.Ogamma([0, 1, 2, 3]), [0, 0, 0, 0]) + assert u.allclose(cosmo.Onu(1.5), [0, 0, 0, 0]) + assert u.allclose(cosmo.Onu([0, 1, 2, 3]), [0, 0, 0, 0]) + + # --------------------------------------------------------------- + # Properties + + def test_Odm0(self, cosmo_cls, cosmo): + """Test ``cached_property`` ``Odm0``.""" + # on the class + assert isinstance(cosmo_cls.Odm0, cached_property) + + # on the instance + assert np.allclose(cosmo.Odm0, cosmo.Om0 - cosmo.Ob0) + + def test_Ok0(self, cosmo_cls, cosmo): + """Test ``cached_property`` ``Ok0``.""" + # on the class + assert isinstance(cosmo_cls.Ok0, cached_property) + + # on the instance + assert np.allclose( + cosmo.Ok0, 1.0 - (cosmo.Om0 + cosmo.Ode0 + cosmo.Ogamma0 + cosmo.Onu0) + ) + + def test_is_flat(self, cosmo_cls, cosmo): + """Test property ``is_flat``.""" + # on the class + assert isinstance(cosmo_cls.is_flat, property) + assert cosmo_cls.is_flat.fset is None # immutable + + # on the instance + assert isinstance(cosmo.is_flat, bool) + assert cosmo.is_flat is bool((cosmo.Ok0 == 0.0) and (cosmo.Otot0 == 1.0)) + + def test_Tnu0(self, cosmo_cls, cosmo): + """Test ``cached_property`` ``Tnu0``.""" + # on the class + assert isinstance(cosmo_cls.Tnu0, cached_property) + + # on the instance + assert cosmo.Tnu0.unit == u.K + assert u.allclose(cosmo.Tnu0, 0.7137658555036082 * cosmo.Tcmb0, rtol=1e-5) + + def test_has_massive_nu(self, cosmo_cls, cosmo): + """Test property ``has_massive_nu``.""" + # on the class + assert isinstance(cosmo_cls.has_massive_nu, property) + assert cosmo_cls.has_massive_nu.fset is None # immutable + + # on the instance + if cosmo.Tnu0 == 0: + assert cosmo.has_massive_nu is False + else: + assert cosmo.has_massive_nu is cosmo._nu_info.has_massive_nu + + def test_h(self, cosmo_cls, cosmo): + """Test ``cached_property`` ``h``.""" + # on the class + assert isinstance(cosmo_cls.h, cached_property) + + # on the instance + assert np.allclose(cosmo.h, cosmo.H0.value / 100.0) + + def test_hubble_time(self, cosmo_cls, cosmo): + """Test ``cached_property`` ``hubble_time``.""" + # on the class + assert isinstance(cosmo_cls.hubble_time, cached_property) + + # on the instance + assert u.allclose(cosmo.hubble_time, (1 / cosmo.H0) << u.Gyr) + + def test_hubble_distance(self, cosmo_cls, cosmo): + """Test ``cached_property`` ``hubble_distance``.""" + # on the class + assert isinstance(cosmo_cls.hubble_distance, cached_property) + + # on the instance + assert cosmo.hubble_distance == (const.c / cosmo.H0).to(u.Mpc) + + def test_critical_density0(self, cosmo_cls, cosmo): + """Test ``cached_property`` ``critical_density0``.""" + # on the class + assert isinstance(cosmo_cls.critical_density0, cached_property) + + # on the instance + assert cosmo.critical_density0.unit == u.g / u.cm**3 + assert u.allclose( # sanity check + cosmo.critical_density0, 3 * cosmo.H0**2 / (8 * np.pi * const.G) + ) + + def test_Ogamma0(self, cosmo_cls, cosmo): + """Test ``cached_property`` ``Ogamma0``.""" + # on the class + assert isinstance(cosmo_cls.Ogamma0, cached_property) + + # on the instance + # Ogamma cor \propto T^4/rhocrit + expect = a_B_c2 * cosmo.Tcmb0.value**4 / cosmo.critical_density0.value + assert np.allclose(cosmo.Ogamma0, expect) + # check absolute equality to 0 if Tcmb0 is 0 + if cosmo.Tcmb0 == 0: + assert cosmo.Ogamma0 == 0 + + def test_Onu0(self, cosmo_cls, cosmo): + """Test ``cached_property`` ``Onu0``.""" + # on the class + assert isinstance(cosmo_cls.Onu0, cached_property) + + # on the instance + # neutrino temperature <= photon temperature since the neutrinos + # decouple first. + if cosmo.has_massive_nu: # Tcmb0 > 0 & has massive + # check the expected formula + assert cosmo.Onu0 == cosmo.Ogamma0 * cosmo.nu_relative_density(0) + # a sanity check on on the ratio of neutrinos to photons + # technically it could be 1, but not for any of the tested cases. + assert cosmo.nu_relative_density(0) <= 1 + elif cosmo.Tcmb0 == 0: + assert cosmo.Onu0 == 0 + else: + # check the expected formula + assert cosmo.Onu0 == 0.22710731766 * cosmo.__dict__["Neff"] * cosmo.Ogamma0 + # and check compatibility with nu_relative_density + assert np.allclose( + cosmo.nu_relative_density(0), 0.22710731766 * cosmo.__dict__["Neff"] + ) + + def test_Otot0(self, cosmo): + """Test :attr:`astropy.cosmology.FLRW.Otot0`.""" + assert ( + cosmo.Otot0 + == cosmo.Om0 + cosmo.Ogamma0 + cosmo.Onu0 + cosmo.Ode0 + cosmo.Ok0 + ) + + # --------------------------------------------------------------- + # Methods + + _FLRW_redshift_methods = get_redshift_methods( + FLRW, include_private=True, include_z2=False + ) + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy is not installed") + @pytest.mark.parametrize("z, exc", invalid_zs) + @pytest.mark.parametrize("method", sorted(_FLRW_redshift_methods)) + def test_redshift_method_bad_input(self, cosmo, method, z, exc): + """Test all the redshift methods for bad input.""" + with pytest.raises(exc): + getattr(cosmo, method)(z) + + @pytest.mark.parametrize("z", valid_zs) + @abc.abstractmethod + def test_w(self, cosmo, z): + """Test :meth:`astropy.cosmology.FLRW.w`. + + Since ``w`` is abstract, each test class needs to define further tests. + """ + # super().test_w(cosmo, z) # NOT b/c abstract `w(z)` + w = cosmo.w(z) + assert np.shape(w) == np.shape(z) # test same shape + assert u.Quantity(w).unit == u.one # test no units or dimensionless + + # ------------------------------------------- + + @pytest.mark.parametrize("z", valid_zs) + def test_Otot(self, cosmo, z): + """Test :meth:`astropy.cosmology.FLRW.Otot`.""" + # super().test_Otot(cosmo) # NOT b/c abstract `w(z)` + assert np.allclose( + cosmo.Otot(z), + cosmo.Om(z) + cosmo.Ogamma(z) + cosmo.Onu(z) + cosmo.Ode(z) + cosmo.Ok(z), + ) + + def test_scale_factor0(self, cosmo): + """Test :meth:`astropy.cosmology.FLRW.scale_factor`.""" + assert isinstance(cosmo.scale_factor0, u.Quantity) + assert cosmo.scale_factor0.unit == u.one + assert cosmo.scale_factor0 == 1 + assert np.allclose(cosmo.scale_factor0, cosmo.scale_factor(0)) + + @pytest.mark.parametrize("z", valid_zs) + def test_scale_factor(self, cosmo, z): + """Test :meth:`astropy.cosmology.FLRW.scale_factor`.""" + assert np.allclose(cosmo.scale_factor(z), 1 / (1 + np.array(z))) + + # ------------------------------------------- + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy required for this test.") + def test_comoving_distance_1arg_equal_to_2arg(self, cosmo): + """Test :meth:`astropy.cosmology.FLRW.comoving_distance`.""" + # Special case of z1 = 0 + z = np.linspace(0, 1, 10) + assert u.allclose(cosmo.comoving_distance(z), cosmo.comoving_distance(0, z)) + + # General case of z1, z2 + z1 = z + z2 = z + 1 + assert u.allclose( + cosmo.comoving_distance(z2) - cosmo.comoving_distance(z1), + cosmo.comoving_distance(z1, z2), + ) + + @pytest.mark.skipif( + not (HAS_PANDAS and HAS_SCIPY), reason="requires pandas and scipy" + ) + def test_luminosity_distance_pandas(self, cosmo): + """Test :meth:`astropy.cosmology.FLRW.luminosity_distance`. + + Regression test for https://github.com/astropy/astropy/issues/15576. + """ + import pandas as pd + + z = pd.Series([0.1, 0.2, 0.3]) + d = cosmo.luminosity_distance(z) + + assert isinstance(d, u.Quantity) + assert d.unit == u.Mpc + np.testing.assert_array_equal(d, cosmo.luminosity_distance(np.array(z))) + + # --------------------------------------------------------------- + + def test_efunc_vs_invefunc(self, cosmo): + """Test that ``efunc`` and ``inv_efunc`` give inverse values. + + Note that the test doesn't need scipy because it doesn't need to call + ``de_density_scale``. + """ + # super().test_efunc_vs_invefunc(cosmo) # NOT b/c abstract `w(z)` + z0 = 0.5 + z = np.array([0.5, 1.0, 2.0, 5.0]) + + assert np.allclose(cosmo.efunc(z0), 1.0 / cosmo.inv_efunc(z0)) + assert np.allclose(cosmo.efunc(z), 1.0 / cosmo.inv_efunc(z)) + + # --------------------------------------------------------------- + # from Cosmology + + def test_clone_change_param(self, cosmo): + """Test method ``.clone()`` changing a(many) Parameter(s).""" + super().test_clone_change_param(cosmo) + + # don't change any values + kwargs = dict(cosmo.parameters) + c = cosmo.clone(**kwargs) + assert c.__class__ == cosmo.__class__ + assert c == cosmo + + # change ``H0`` + # Note that H0 affects Ode0 because it changes Ogamma0 + c = cosmo.clone(H0=100) + assert c.__class__ == cosmo.__class__ + assert c.name == cosmo.name + " (modified)" + assert c.H0.value == 100 + for n, v in filter_keys_from_items(c.parameters, ("H0",)): + v_expect = getattr(cosmo, n) + assert_quantity_allclose(v, v_expect, atol=1e-4 * getattr(v, "unit", 1)) + assert not u.allclose(c.Ogamma0, cosmo.Ogamma0) + assert not u.allclose(c.Onu0, cosmo.Onu0) + + # change multiple things + c = cosmo.clone(name="new name", H0=100, Tcmb0=2.8, meta=dict(zz="tops")) + assert c.__class__ == cosmo.__class__ + assert c.name == "new name" + assert c.H0.value == 100 + assert c.Tcmb0.value == 2.8 + assert c.meta == {**cosmo.meta, **dict(zz="tops")} + for n, v in filter_keys_from_items(c.parameters, ("H0", "Tcmb0")): + v_expect = getattr(cosmo, n) + assert_quantity_allclose(v, v_expect, atol=1e-4 * getattr(v, "unit", 1)) + assert not u.allclose(c.Ogamma0, cosmo.Ogamma0) + assert not u.allclose(c.Onu0, cosmo.Onu0) + assert not u.allclose(c.Tcmb0.value, cosmo.Tcmb0.value) + + def test_is_equivalent(self, cosmo): + """Test :meth:`astropy.cosmology.FLRW.is_equivalent`.""" + super().test_is_equivalent(cosmo) # pass to CosmologyTest + + # test against a FlatFLRWMixin + # case (3) in FLRW.is_equivalent + if isinstance(cosmo, FlatLambdaCDM): + assert cosmo.is_equivalent(Planck18) + assert Planck18.is_equivalent(cosmo) + else: + assert not cosmo.is_equivalent(Planck18) + assert not Planck18.is_equivalent(cosmo) + + # =============================================================== + # Usage Tests + + # TODO: this test should be subsumed by other tests + @pytest.mark.parametrize("method", ("Om", "Ode", "w", "de_density_scale")) + def test_distance_broadcast(self, cosmo, method): + """Test distance methods broadcast z correctly.""" + g = getattr(cosmo, method) + z = np.linspace(0.1, 1, 6) + z2d = z.reshape(2, 3) + z3d = z.reshape(3, 2, 1) + + value_flat = g(z) + assert value_flat.shape == z.shape + + value_2d = g(z2d) + assert value_2d.shape == z2d.shape + + value_3d = g(z3d) + assert value_3d.shape == z3d.shape + assert u.allclose(value_flat, value_2d.flatten()) + assert u.allclose(value_flat, value_3d.flatten()) + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy required for this test.") + def test_comoving_distance_example(self, cosmo_cls, args, kwargs, expected): + """Test :meth:`astropy.cosmology.LambdaCDM.comoving_distance`. + + These do not come from external codes -- they are just internal checks to make + sure nothing changes if we muck with the distance calculators. + """ + z = np.array([1.0, 2.0, 3.0, 4.0]) + + cosmo = cosmo_cls(*args, **kwargs) + assert u.allclose(cosmo.comoving_distance(z), expected, rtol=1e-4) + + +# ============================================================================== + + +class FlatFLRWMixinTest(FlatCosmologyMixinTest, ParameterFlatOde0TestMixin): + """Tests for :class:`astropy.cosmology.FlatFLRWMixin` subclasses. + + E.g to use this class:: + + class TestFlatSomeFLRW(FlatFLRWMixinTest, TestSomeFLRW): + ... + """ + + def setup_class(self): + """Setup for testing. + + Set up as for regular FLRW test class, but remove dark energy component + since flat cosmologies are forbidden Ode0 as an argument, + see ``test_init_subclass``. + """ + super().setup_class(self) + self._cls_args.pop("Ode0") + + # =============================================================== + # Method & Attribute Tests + + # --------------------------------------------------------------- + # class-level + + def test_init_subclass(self, cosmo_cls): + """Test initializing subclass, mostly that can't have Ode0 in init.""" + super().test_init_subclass(cosmo_cls) + + with pytest.raises(TypeError, match="subclasses of"): + + @dataclass_decorator + class HASOde0SubClass(cosmo_cls): + def __init__(self, Ode0): + pass + + _COSMOLOGY_CLASSES.pop(HASOde0SubClass.__qualname__, None) + + # --------------------------------------------------------------- + # instance-level + + def test_init(self, cosmo_cls): + super().test_init(cosmo_cls) + + cosmo = cosmo_cls(*self.cls_args, **self.cls_kwargs) + assert cosmo.Ok0 == 0.0 + assert cosmo.Ode0 == 1.0 - (cosmo.Om0 + cosmo.Ogamma0 + cosmo.Onu0 + cosmo.Ok0) + + def test_Ok0(self, cosmo_cls, cosmo): + """Test property ``Ok0``.""" + super().test_Ok0(cosmo_cls, cosmo) + + # for flat cosmologies, Ok0 is not *close* to 0, it *is* 0 + assert cosmo.Ok0 == 0.0 + + def test_Otot0(self, cosmo): + """Test :attr:`astropy.cosmology.FLRW.Otot0`. Should always be 1.""" + super().test_Otot0(cosmo) + + # for flat cosmologies, Otot0 is not *close* to 1, it *is* 1 + assert cosmo.Otot0 == 1.0 + + @pytest.mark.parametrize("z", valid_zs) + def test_Otot(self, cosmo, z): + """Test :meth:`astropy.cosmology.FLRW.Otot`. Should always be 1.""" + super().test_Otot(cosmo, z) + + # for flat cosmologies, Otot is 1, within precision. + assert u.allclose(cosmo.Otot(z), 1.0) + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy is not installed") + @pytest.mark.parametrize("z, exc", invalid_zs) + @pytest.mark.parametrize( + "method", sorted(FLRWTest._FLRW_redshift_methods - {"Otot"}) + ) + def test_redshift_method_bad_input(self, cosmo, method, z, exc): + """Test all the redshift methods for bad input.""" + super().test_redshift_method_bad_input(cosmo, method, z, exc) + + # --------------------------------------------------------------- + + def test_clone_to_nonflat_change_param(self, cosmo): + """Test method ``.clone()`` changing a(many) Parameter(s).""" + super().test_clone_to_nonflat_change_param(cosmo) + + # change Ode0, without non-flat + msg = "Cannot set 'Ode0' in clone unless 'to_nonflat=True'. " + with pytest.raises(ValueError, match=msg): + cosmo.clone(Ode0=1) + + # change to non-flat + nc = cosmo.clone(to_nonflat=True, Ode0=cosmo.Ode0) + assert isinstance(nc, cosmo.__nonflatclass__) + assert nc == cosmo.nonflat + + nc = cosmo.clone(to_nonflat=True, Ode0=1) + assert nc.Ode0 == 1.0 + assert nc.name == cosmo.name + " (modified)" + + # --------------------------------------------------------------- + + def test_is_equivalent(self, cosmo, nonflatcosmo): + """Test :meth:`astropy.cosmology.FLRW.is_equivalent`.""" + super().test_is_equivalent(cosmo) # pass to TestFLRW + + # against non-flat Cosmology + assert not cosmo.is_equivalent(nonflatcosmo) + assert not nonflatcosmo.is_equivalent(cosmo) + + # non-flat version of class + nonflat_cosmo_cls = cosmo.__nonflatclass__ + # keys check in `test_is_equivalent_nonflat_class_different_params` + + # non-flat + nonflat = nonflat_cosmo_cls(*self.cls_args, Ode0=0.9, **self.cls_kwargs) + assert not nonflat.is_equivalent(cosmo) + assert not cosmo.is_equivalent(nonflat) + + # Flat, but not FlatFLRWMixin + # This will require forcing flatness by overriding attribute values. + # Since Cosmology is frozen, the easiest way is via __dict__. + flat = nonflat_cosmo_cls( + *self.cls_args, + Ode0=1.0 - cosmo.Om0 - cosmo.Ogamma0 - cosmo.Onu0, + **self.cls_kwargs, + ) + flat.__dict__["Ok0"] = 0.0 # manually forcing flatness by setting `Ok0`. + assert flat.is_equivalent(cosmo) + assert cosmo.is_equivalent(flat) + + def test_repr(self, cosmo_cls, cosmo): + """ + Test method ``.__repr__()``. Skip non-flat superclass test. + e.g. `TestFlatLambdaCDDM` -> `FlatFLRWMixinTest` + vs `TestFlatLambdaCDDM` -> `TestLambdaCDDM` -> `FlatFLRWMixinTest` + """ + # test eliminated Ode0 from parameters + assert "Ode0" not in repr(cosmo) diff --git a/astropy/cosmology/_src/tests/flrw/test_flrw.py b/astropy/cosmology/_src/tests/flrw/test_flrw.py new file mode 100644 index 000000000000..2a9afcc86b7e --- /dev/null +++ b/astropy/cosmology/_src/tests/flrw/test_flrw.py @@ -0,0 +1,112 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +"""Testing :mod:`astropy.cosmology.FLRW`.""" + +from typing import final + +import pytest + +from astropy.cosmology import FLRW +from astropy.cosmology._src.core import _COSMOLOGY_CLASSES, dataclass_decorator +from astropy.cosmology._src.tests.helper import get_redshift_methods +from astropy.cosmology._src.tests.test_core import invalid_zs +from astropy.utils.compat.optional_deps import HAS_SCIPY + +from .test_base import FLRWTest + + +@dataclass_decorator +class SubFLRW(FLRW): + def w(self, z): + return super().w(z) + + +@final +class TestFLRW(FLRWTest): + """Test :class:`astropy.cosmology.FLRW`.""" + + abstract_w = True + + def setup_class(self): + """ + Setup for testing. + FLRW is abstract, so tests are done on a subclass. + """ + super().setup_class(self) + + # make sure SubCosmology is known + _COSMOLOGY_CLASSES["SubFLRW"] = SubFLRW + + self.cls = SubFLRW + + def teardown_class(self): + super().teardown_class(self) + _COSMOLOGY_CLASSES.pop("SubFLRW", None) + + # =============================================================== + # Method & Attribute Tests + + # --------------------------------------------------------------- + # Methods + + def test_w(self, cosmo): + """Test abstract :meth:`astropy.cosmology.FLRW.w`.""" + with pytest.raises(NotImplementedError, match="not implemented"): + cosmo.w(1) + + def test_Otot(self, cosmo): + """Test :meth:`astropy.cosmology.FLRW.Otot`.""" + exception = NotImplementedError if HAS_SCIPY else ModuleNotFoundError + with pytest.raises(exception): + assert cosmo.Otot(1) + + def test_efunc_vs_invefunc(self, cosmo): + """ + Test that efunc and inv_efunc give inverse values. + Here they just fail b/c no ``w(z)`` or no scipy. + """ + exception = NotImplementedError if HAS_SCIPY else ModuleNotFoundError + + with pytest.raises(exception): + cosmo.efunc(0.5) + + with pytest.raises(exception): + cosmo.inv_efunc(0.5) + + @pytest.mark.skip(reason="w(z) is abstract") + def test_luminosity_distance_pandas(self, cosmo): + """Test :meth:`astropy.cosmology.FLRW.luminosity_distance`.""" + + _FLRW_redshift_methods = get_redshift_methods( + FLRW, include_private=True, include_z2=False + ) - {"w"} + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy is not installed") + @pytest.mark.parametrize("z, exc", invalid_zs) + @pytest.mark.parametrize("method", sorted(_FLRW_redshift_methods)) + def test_redshift_method_bad_input(self, cosmo, method, z, exc): + """Test all the redshift methods for bad input.""" + with pytest.raises(exc): + getattr(cosmo, method)(z) + + # =============================================================== + # Usage Tests + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy required for this test.") + @pytest.mark.parametrize("method", ("Om", "Ode", "w", "de_density_scale")) + def test_distance_broadcast(self, cosmo, method): + with pytest.raises(NotImplementedError): + super().test_distance_broadcast(cosmo, method) + + @pytest.mark.skip(reason="w(z) is abstract") + def test_comoving_distance_1arg_equal_to_2arg(self, cosmo): + """Test :meth:`astropy.cosmology.FLRW.luminosity_distance`.""" + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy required for this test.") + @pytest.mark.parametrize( + ("args", "kwargs", "expected"), + [((70, 0.27, 0.73), {"Tcmb0": 3.0, "Ob0": 0.03}, None)], + ) + def test_comoving_distance_example(self, cosmo_cls, args, kwargs, expected): + with pytest.raises(NotImplementedError): + super().test_comoving_distance_example(cosmo_cls, args, kwargs, expected) diff --git a/astropy/cosmology/_src/tests/flrw/test_lambdacdm.py b/astropy/cosmology/_src/tests/flrw/test_lambdacdm.py new file mode 100644 index 000000000000..b25954339331 --- /dev/null +++ b/astropy/cosmology/_src/tests/flrw/test_lambdacdm.py @@ -0,0 +1,1059 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +"""Testing :mod:`astropy.cosmology.flrw.lambdacdm`.""" + +import pathlib +import re + +import numpy as np +import pytest + +import astropy.constants as const +import astropy.cosmology.units as cu +import astropy.units as u +from astropy.cosmology import FlatLambdaCDM, LambdaCDM +from astropy.cosmology._src.flrw.lambdacdm import ellipkinc, hyp2f1 +from astropy.cosmology._src.tests.helper import get_redshift_methods +from astropy.cosmology._src.tests.test_core import invalid_zs, valid_zs +from astropy.table import QTable +from astropy.utils.compat.optional_deps import HAS_SCIPY +from astropy.utils.exceptions import AstropyDeprecationWarning, AstropyUserWarning + +from .test_base import FlatFLRWMixinTest, FLRWTest + +############################################################################## +# TESTS +############################################################################## + + +@pytest.mark.skipif(HAS_SCIPY, reason="scipy is installed") +def test_optional_deps_functions(): + """Test stand-in functions when optional dependencies not installed.""" + with pytest.raises(ModuleNotFoundError, match="No module named 'scipy.special'"): + ellipkinc() + + with pytest.raises(ModuleNotFoundError, match="No module named 'scipy.special'"): + hyp2f1() + + +############################################################################## + + +class TestLambdaCDM(FLRWTest): + """Test :class:`astropy.cosmology.LambdaCDM`.""" + + def setup_class(self): + """Setup for testing.""" + super().setup_class(self) + self.cls = LambdaCDM + + # =============================================================== + # Method & Attribute Tests + + _FLRW_redshift_methods = get_redshift_methods( + LambdaCDM, include_private=True, include_z2=False + ) - {"_dS_age"} + # `_dS_age` is removed because it doesn't strictly rely on the value of `z`, + # so any input that doesn't trip up ``np.shape`` is "valid" + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy is not installed") + @pytest.mark.parametrize("z, exc", invalid_zs) + @pytest.mark.parametrize("method", sorted(_FLRW_redshift_methods)) + def test_redshift_method_bad_input(self, cosmo, method, z, exc): + """Test all the redshift methods for bad input.""" + super().test_redshift_method_bad_input(cosmo, method, z, exc) + + @pytest.mark.parametrize("z", valid_zs) + def test_w(self, cosmo, z): + """Test :meth:`astropy.cosmology.LambdaCDM.w`.""" + super().test_w(cosmo, z) + + w = cosmo.w(z) + assert u.allclose(w, -1.0) + + def test_repr(self, cosmo): + """Test method ``.__repr__()``.""" + assert repr(cosmo) == ( + "LambdaCDM(name='ABCMeta', H0=, Om0=0.27," + " Ode0=0.73, Tcmb0=, Neff=3.04," + " m_nu=, Ob0=0.03)" + ) + + def test_str(self, cosmo): + """Test method ``.__str__()``.""" + assert str(cosmo) == ( + 'LambdaCDM(name="ABCMeta", H0=70.0 km / (Mpc s), Om0=0.27, Ode0=0.73,' + " Tcmb0=3.0 K, Neff=3.04, m_nu=[0. 0. 0.] eV, Ob0=0.03)" + ) + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy is not installed") + @pytest.mark.parametrize( + ("args", "kwargs", "expected"), + [ + ( # no relativistic species + (75.0, 0.25, 0.5), + {"Tcmb0": 0.0}, + [2953.93001902, 4616.7134253, 5685.07765971, 6440.80611897] * u.Mpc, + ), + ( # massless neutrinos + (75.0, 0.25, 0.6), + {"Tcmb0": 3.0, "Neff": 3, "m_nu": u.Quantity(0.0, u.eV)}, + [3037.12620424, 4776.86236327, 5889.55164479, 6671.85418235] * u.Mpc, + ), + ( # massive neutrinos + (75.0, 0.3, 0.4), + {"Tcmb0": 3.0, "Neff": 3, "m_nu": u.Quantity(10.0, u.eV)}, + [2471.80626824, 3567.1902565, 4207.15995626, 4638.20476018] * u.Mpc, + ), + ], + ) + def test_comoving_distance_example(self, cosmo_cls, args, kwargs, expected): + """Test :meth:`astropy.cosmology.LambdaCDM.comoving_distance`. + + These do not come from external codes -- they are just internal checks to make + sure nothing changes if we muck with the distance calculators. + """ + super().test_comoving_distance_example(cosmo_cls, args, kwargs, expected) + + +# ----------------------------------------------------------------------------- + + +class TestFlatLambdaCDM(FlatFLRWMixinTest, TestLambdaCDM): + """Test :class:`astropy.cosmology.FlatLambdaCDM`.""" + + def setup_class(self): + """Setup for testing.""" + super().setup_class(self) + self.cls = FlatLambdaCDM + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy is not installed") + @pytest.mark.parametrize("z, exc", invalid_zs) + @pytest.mark.parametrize( + "method", sorted(TestLambdaCDM._FLRW_redshift_methods - {"Otot"}) + ) + def test_redshift_method_bad_input(self, cosmo, method, z, exc): + """Test all the redshift methods for bad input.""" + super().test_redshift_method_bad_input(cosmo, method, z, exc) + + # =============================================================== + # Method & Attribute Tests + + def test_repr(self, cosmo): + """Test method ``.__repr__()``.""" + assert repr(cosmo) == ( + "FlatLambdaCDM(name='ABCMeta', H0=, Om0=0.27," + " Tcmb0=, Neff=3.04, m_nu=," + " Ob0=0.03)" + ) + + def test_str(self, cosmo): + """Test method ``.__str__()``.""" + assert str(cosmo) == ( + 'FlatLambdaCDM(name="ABCMeta", H0=70.0 km / (Mpc s), Om0=0.27, ' + "Tcmb0=3.0 K, Neff=3.04, m_nu=[0. 0. 0.] eV, Ob0=0.03)" + ) + + # =============================================================== + # Usage Tests + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy is not installed") + @pytest.mark.parametrize( + ("args", "kwargs", "expected"), + [ + ( # no relativistic species + (75.0, 0.25), + {"Tcmb0": 0.0}, + [3180.83488552, 5060.82054204, 6253.6721173, 7083.5374303] * u.Mpc, + ), + ( # massless neutrinos + (75.0, 0.25), + {"Tcmb0": 3.0, "Neff": 3, "m_nu": u.Quantity(0.0, u.eV)}, + [3180.42662867, 5059.60529655, 6251.62766102, 7080.71698117] * u.Mpc, + ), + ( # massive neutrinos + (75.0, 0.25), + {"Tcmb0": 3.0, "Neff": 3, "m_nu": u.Quantity(10.0, u.eV)}, + [2337.54183142, 3371.91131264, 3988.40711188, 4409.09346922] * u.Mpc, + ), + ( # work the scalar nu density functions + (75.0, 0.25), + {"Tcmb0": 3.0, "m_nu": u.Quantity([10.0, 0, 0], u.eV)}, + [2777.71589173, 4186.91111666, 5046.0300719, 5636.10397302] * u.Mpc, + ), + ( # work the scalar nu density functions + (75.0, 0.25), + {"Tcmb0": 3.0, "m_nu": u.Quantity([10.0, 5, 0], u.eV)}, + [2636.48149391, 3913.14102091, 4684.59108974, 5213.07557084] * u.Mpc, + ), + ( # work the scalar nu density functions + (75.0, 0.25), + {"Tcmb0": 3.0, "m_nu": u.Quantity([4.0, 5, 9], u.eV)}, + [2563.5093049, 3776.63362071, 4506.83448243, 5006.50158829] * u.Mpc, + ), + ( # work the scalar nu density functions + (75.0, 0.25), + {"Tcmb0": 3.0, "Neff": 4.2, "m_nu": u.Quantity([1.0, 4.0, 5, 9], u.eV)}, + [2525.58017482, 3706.87633298, 4416.58398847, 4901.96669755] * u.Mpc, + ), + ], + ) + def test_comoving_distance_example(self, cosmo_cls, args, kwargs, expected): + """Test :meth:`astropy.cosmology.LambdaCDM.comoving_distance`. + + These do not come from external codes -- they are just internal checks to make + sure nothing changes if we muck with the distance calculators. + """ + super().test_comoving_distance_example(cosmo_cls, args, kwargs, expected) + + +############################################################################## +# Comparison to Other Codes + + +@pytest.mark.skipif(not HAS_SCIPY, reason="requires scipy.") +def test_flat_z1(): + """Test a flat cosmology at z=1 against several other on-line calculators. + + Test values were taken from the following web cosmology calculators on + 2012-02-11: + + Wright: http://www.astro.ucla.edu/~wright/CosmoCalc.html + (https://ui.adsabs.harvard.edu/abs/2006PASP..118.1711W) + Kempner: http://www.kempner.net/cosmic.php + iCosmos: http://www.icosmos.co.uk/index.html + """ + cosmo = FlatLambdaCDM(H0=70, Om0=0.27, Tcmb0=0.0) + + # The order of values below is Wright, Kempner, iCosmos' + assert u.allclose( + cosmo.comoving_distance(1), [3364.5, 3364.8, 3364.7988] * u.Mpc, rtol=1e-4 + ) + assert u.allclose( + cosmo.angular_diameter_distance(1), + [1682.3, 1682.4, 1682.3994] * u.Mpc, + rtol=1e-4, + ) + assert u.allclose( + cosmo.luminosity_distance(1), [6729.2, 6729.6, 6729.5976] * u.Mpc, rtol=1e-4 + ) + assert u.allclose( + cosmo.lookback_time(1), [7.841, 7.84178, 7.843] * u.Gyr, rtol=1e-3 + ) + assert u.allclose( + cosmo.lookback_distance(1), [2404.0, 2404.24, 2404.4] * u.Mpc, rtol=1e-3 + ) + + +############################################################################## +# Regression Tests + + +SPECIALIZED_COMOVING_DISTANCE_COSMOLOGIES = [ + FlatLambdaCDM(H0=70, Om0=0.0, Tcmb0=0.0), # de Sitter + FlatLambdaCDM(H0=70, Om0=1.0, Tcmb0=0.0), # Einstein - de Sitter + FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=0.0), # Hypergeometric + LambdaCDM(H0=70, Om0=0.3, Ode0=0.6, Tcmb0=0.0), # Elliptic +] + +ITERABLE_REDSHIFTS = [ + (0, 1, 2, 3, 4), # tuple + [0, 1, 2, 3, 4], # list + np.array([0, 1, 2, 3, 4]), # array +] + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +@pytest.mark.parametrize("cosmo", SPECIALIZED_COMOVING_DISTANCE_COSMOLOGIES) +@pytest.mark.parametrize("z", ITERABLE_REDSHIFTS) +def test_comoving_distance_iterable_argument(cosmo, z): + """ + Regression test for #10980 + Test that specialized comoving distance methods handle iterable arguments. + """ + + assert u.allclose( + cosmo.comoving_distance(z), cosmo._integral_comoving_distance_z1z2(0.0, z) + ) + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +@pytest.mark.parametrize("cosmo", SPECIALIZED_COMOVING_DISTANCE_COSMOLOGIES) +def test_comoving_distance_broadcast(cosmo): + """ + Regression test for #10980 + Test that specialized comoving distance methods broadcast array arguments. + """ + + z1 = np.zeros((2, 5)) + z2 = np.ones((3, 1, 5)) + z3 = np.ones((7, 5)) + output_shape = np.broadcast(z1, z2).shape + + # Check compatible array arguments return an array with the correct shape + assert cosmo.comoving_distance(z1, z2).shape == output_shape + + # Check incompatible array arguments raise an error + with pytest.raises(ValueError, match="z1 and z2 have different shapes"): + cosmo.comoving_distance(z1, z3) + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +def test_elliptic_comoving_distance_z1z2(): + """Regression test for #8388.""" + cosmo = LambdaCDM(70.0, 2.3, 0.05, Tcmb0=0) + z = 0.2 + assert u.allclose( + cosmo.comoving_distance(z), cosmo._integral_comoving_distance_z1z2(0.0, z) + ) + assert u.allclose( + cosmo._elliptic_comoving_distance_z1z2(0.0, z), + cosmo._integral_comoving_distance_z1z2(0.0, z), + ) + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +def test_ogamma(): + """Tests the effects of changing the temperature of the CMB""" + + # Tested against Ned Wright's advanced cosmology calculator, + # Sep 7 2012. The accuracy of our comparison is limited by + # how many digits it outputs, which limits our test to about + # 0.2% accuracy. The NWACC does not allow one + # to change the number of nuetrino species, fixing that at 3. + # Also, inspection of the NWACC code shows it uses inaccurate + # constants at the 0.2% level (specifically, a_B), + # so we shouldn't expect to match it that well. The integral is + # also done rather crudely. Therefore, we should not expect + # the NWACC to be accurate to better than about 0.5%, which is + # unfortunate, but reflects a problem with it rather than this code. + # More accurate tests below using Mathematica + z = np.array([1.0, 10.0, 500.0, 1000.0]) + + cosmo = FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=0, Neff=3) + assert u.allclose( + cosmo.angular_diameter_distance(z), + [1651.9, 858.2, 26.855, 13.642] * u.Mpc, + rtol=5e-4, + ) + + cosmo = FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=2.725, Neff=3) + assert u.allclose( + cosmo.angular_diameter_distance(z), + [1651.8, 857.9, 26.767, 13.582] * u.Mpc, + rtol=5e-4, + ) + + cosmo = FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=4.0, Neff=3) + assert u.allclose( + cosmo.angular_diameter_distance(z), + [1651.4, 856.6, 26.489, 13.405] * u.Mpc, + rtol=5e-4, + ) + + # Next compare with doing the integral numerically in Mathematica, + # which allows more precision in the test. It is at least as + # good as 0.01%, possibly better + cosmo = FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=0, Neff=3.04) + assert u.allclose( + cosmo.angular_diameter_distance(z), + [1651.91, 858.205, 26.8586, 13.6469] * u.Mpc, + rtol=1e-5, + ) + + cosmo = FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=2.725, Neff=3.04) + assert u.allclose( + cosmo.angular_diameter_distance(z), + [1651.76, 857.817, 26.7688, 13.5841] * u.Mpc, + rtol=1e-5, + ) + + cosmo = FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=4.0, Neff=3.04) + assert u.allclose( + cosmo.angular_diameter_distance(z), + [1651.21, 856.411, 26.4845, 13.4028] * u.Mpc, + rtol=1e-5, + ) + + # Just to be really sure, we also do a version where the integral + # is analytic, which is a Ode = 0 flat universe. In this case + # Integrate(1/E(x),{x,0,z}) = 2 ( sqrt((1+Or z)/(1+z)) - 1 )/(Or - 1) + # Recall that c/H0 * Integrate(1/E) is FLRW.comoving_distance. + Ogamma0h2 = 4 * 5.670373e-8 / 299792458.0**3 * 2.725**4 / 1.87837e-26 + Onu0h2 = Ogamma0h2 * 7.0 / 8.0 * (4.0 / 11.0) ** (4.0 / 3.0) * 3.04 + Or0 = (Ogamma0h2 + Onu0h2) / 0.7**2 + Om0 = 1.0 - Or0 + hubdis = (299792.458 / 70.0) * u.Mpc + cosmo = FlatLambdaCDM(H0=70, Om0=Om0, Tcmb0=2.725, Neff=3.04) + targvals = 2.0 * hubdis * (np.sqrt((1.0 + Or0 * z) / (1.0 + z)) - 1.0) / (Or0 - 1.0) + assert u.allclose(cosmo.comoving_distance(z), targvals, rtol=1e-5) + + # And integers for z + assert u.allclose(cosmo.comoving_distance(z.astype(int)), targvals, rtol=1e-5) + + # Try Tcmb0 = 4 + Or0 *= (4.0 / 2.725) ** 4 + Om0 = 1.0 - Or0 + cosmo = FlatLambdaCDM(H0=70, Om0=Om0, Tcmb0=4.0, Neff=3.04) + targvals = 2.0 * hubdis * (np.sqrt((1.0 + Or0 * z) / (1.0 + z)) - 1.0) / (Or0 - 1.0) + assert u.allclose(cosmo.comoving_distance(z), targvals, rtol=1e-5) + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +@pytest.mark.parametrize( + "file_name", ["cosmo_flat.ecsv", "cosmo_open.ecsv", "cosmo_closed.ecsv"] +) +def test_flat_open_closed_icosmo(file_name): + """Test against the tabulated values generated from icosmo.org + with three example cosmologies (flat, open and closed). + """ + with u.add_enabled_units(cu): + tbl = QTable.read(pathlib.Path(__file__).parent / "data" / file_name) + cosmo = LambdaCDM( + H0=100 * tbl.meta["h"], Om0=tbl.meta["Om"], Ode0=tbl.meta["Ol"], Tcmb0=0.0 + ) + assert u.allclose(cosmo.comoving_transverse_distance(tbl["redshift"]), tbl["dm"]) + assert u.allclose(cosmo.angular_diameter_distance(tbl["redshift"]), tbl["da"]) + assert u.allclose(cosmo.luminosity_distance(tbl["redshift"]), tbl["dl"]) + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +def test_comoving_transverse_distance_z1z2(): + tcos = FlatLambdaCDM(100, 0.3, Tcmb0=0.0) + + with pytest.raises(ValueError): # test diff size z1, z2 fail + tcos._comoving_transverse_distance_z1z2((1, 2), (3, 4, 5)) + + # Tests that should actually work, target values computed with + # http://www.astro.multivax.de:8000/phillip/angsiz_prog/README.HTML + # Kayser, Helbig, and Schramm (Astron.Astrophys. 318 (1997) 680-686) + assert u.allclose( + tcos._comoving_transverse_distance_z1z2(1, 2), 1313.2232194828466 * u.Mpc + ) + + # In a flat universe comoving distance and comoving transverse + # distance are identical + z1 = 0, 0, 2, 0.5, 1 + z2 = 2, 1, 1, 2.5, 1.1 + + assert u.allclose( + tcos.comoving_distance(z1, z2), + tcos._comoving_transverse_distance_z1z2(z1, z2), + ) + + # Test Flat Universe with Omega_M > 1. Rarely used, but perfectly valid. + tcos = FlatLambdaCDM(100, 1.5, Tcmb0=0.0) + results = ( + 2202.72682564, + 1559.51679971, + -643.21002593, + 1408.36365679, + 85.09286258, + ) * u.Mpc + + assert u.allclose(tcos._comoving_transverse_distance_z1z2(z1, z2), results) + + # In a flat universe comoving distance and comoving transverse + # distance are identical + z1 = 0, 0, 2, 0.5, 1 + z2 = 2, 1, 1, 2.5, 1.1 + + assert u.allclose( + tcos.comoving_distance(z1, z2), + tcos._comoving_transverse_distance_z1z2(z1, z2), + ) + # Test non-flat cases to avoid simply testing + # comoving_distance. Test array, array case. + tcos = LambdaCDM(100, 0.3, 0.5, Tcmb0=0.0) + results = ( + 3535.931375645655, + 2226.430046551708, + -1208.6817970036532, + 2595.567367601969, + 151.36592003406884, + ) * u.Mpc + + assert u.allclose(tcos._comoving_transverse_distance_z1z2(z1, z2), results) + + # Test positive curvature with scalar, array combination. + tcos = LambdaCDM(100, 1.0, 0.2, Tcmb0=0.0) + z1 = 0.1 + z2 = 0, 0.1, 0.2, 0.5, 1.1, 2 + results = ( + -281.31602666724865, + 0.0, + 248.58093707820436, + 843.9331377460543, + 1618.6104987686672, + 2287.5626543279927, + ) * u.Mpc + + assert u.allclose(tcos._comoving_transverse_distance_z1z2(z1, z2), results) + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +def test_angular_diameter_distance_z1z2(): + tcos = FlatLambdaCDM(70.4, 0.272, Tcmb0=0.0) + + with pytest.raises(ValueError): # test diff size z1, z2 fail + tcos.angular_diameter_distance([1, 2], [3, 4, 5]) + + # Tests that should actually work, target values computed with + # http://www.astro.multivax.de:8000/phillip/angsiz_prog/README.HTML + # Kayser, Helbig, and Schramm (Astron.Astrophys. 318 (1997) 680-686) + assert u.allclose(tcos.angular_diameter_distance(1, 2), 646.22968662822018 * u.Mpc) + + z1 = 2 # Separate test for z2 Om0 errors + tba.arguments["Ob0"] = tba.arguments["Om0"] + 0.1 + with pytest.raises(ValueError, match="baryonic density can not be larger"): + cosmo_cls(*tba.args, **tba.kwargs) + + # In FLRW `Ob(z)` requires `w(z)`. + if not self.abstract_w: + assert cosmo.Ob(1) == 0 + + # The default value is None + assert cosmo_cls.parameters["Ob0"].default == 0.0 + + def test_Ob0_cannot_be_None(self, cosmo_cls: type[Cosmology], ba: BoundArguments): + """Test that Ob0 cannot be None.""" + ba.arguments["Ob0"] = None + with pytest.raises(TypeError): + cosmo_cls(*ba.args, **ba.kwargs) + + +# ============================================================================= + + +class ParameterFlatOde0TestMixin(ParameterOde0TestMixin): + """Tests for `astropy.cosmology.Parameter` Ode0 on a flat Cosmology. + + This will augment or override some tests in ``ParameterOde0TestMixin``. + + Ode0 is a descriptor, which are tested by mixin, here with ``TestFLRW``. + These tests expect dicts ``_cls_args`` and ``cls_kwargs`` which give the + args and kwargs for the cosmology class, respectively. See ``TestFLRW``. + """ + + def test_Parameter_Ode0(self, cosmo_cls: type[Cosmology]): + """Test Parameter ``Ode0`` on the class.""" + super().test_Parameter_Ode0(cosmo_cls) + Ode0 = cosmo_cls.parameters.get("Ode0", cosmo_cls._derived_parameters["Ode0"]) + assert Ode0.derived in (True, np.True_) + + def test_Ode0(self, cosmo: Cosmology): + """Test no-longer-Parameter ``Ode0``.""" + assert cosmo.Ode0 is cosmo.__dict__["Ode0"] + assert cosmo.Ode0 == 1.0 - (cosmo.Om0 + cosmo.Ogamma0 + cosmo.Onu0) + + def test_init_Ode0(self, cosmo_cls: type[Cosmology], ba: BoundArguments): + """Test initialization for values of ``Ode0``.""" + cosmo = cosmo_cls(*ba.args, **ba.kwargs) + assert cosmo.Ode0 == 1.0 - (cosmo.Om0 + cosmo.Ogamma0 + cosmo.Onu0 + cosmo.Ok0) + + # Ode0 is not in the signature + with pytest.raises(TypeError, match="Ode0"): + cosmo_cls(*ba.args, **ba.kwargs, Ode0=1) diff --git a/astropy/cosmology/_src/tests/flrw/test_w.py b/astropy/cosmology/_src/tests/flrw/test_w.py new file mode 100644 index 000000000000..a19aa6aa6dba --- /dev/null +++ b/astropy/cosmology/_src/tests/flrw/test_w.py @@ -0,0 +1,89 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +"""Testing :mod:`astropy.cosmology.FLRW` neutrinos.""" + +import numpy as np +import pytest + +import astropy.units as u +from astropy.cosmology import FLRW, wCDM +from astropy.utils.compat.optional_deps import HAS_SCIPY + +############################################################################## +# TYPES + + +class W1(FLRW): + """ + This class is to test whether the routines work correctly if one only overloads w(z). + """ + + def __init__(self): + super().__init__(70.0, 0.27, 0.73, Tcmb0=0.0, name="test_cos") + self.__dict__["w0"] = -0.9 + + def w(self, z): + return self.w0 * np.ones_like(z) + + +class W1nu(FLRW): + """Similar, but with neutrinos.""" + + def __init__(self): + super().__init__( + 70.0, 0.27, 0.73, Tcmb0=3.0, m_nu=0.1 * u.eV, name="test_cos_nu" + ) + self.__dict__["w0"] = -0.8 + + def w(self, z): + return self.w0 * np.ones_like(z) + + +############################################################################## +# TESTS +############################################################################## + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +def test_de_subclass(): + z = [0.2, 0.4, 0.6, 0.9] + + # This is the comparison object + cosmo = wCDM(H0=70, Om0=0.27, Ode0=0.73, w0=-0.9, Tcmb0=0.0) + # Values taken from Ned Wrights advanced cosmo calculator, Aug 17 2012 + assert u.allclose( + cosmo.luminosity_distance(z), [975.5, 2158.2, 3507.3, 5773.1] * u.Mpc, rtol=1e-3 + ) + + # Now try the subclass that only gives w(z) + cosmo = W1() + assert u.allclose( + cosmo.luminosity_distance(z), [975.5, 2158.2, 3507.3, 5773.1] * u.Mpc, rtol=1e-3 + ) + # Test efunc + assert u.allclose(cosmo.efunc(1.0), 1.7489240754, rtol=1e-5) + assert u.allclose(cosmo.efunc([0.5, 1.0]), [1.31744953, 1.7489240754], rtol=1e-5) + assert u.allclose(cosmo.inv_efunc([0.5, 1.0]), [0.75904236, 0.57178011], rtol=1e-5) + # Test de_density_scale + assert u.allclose(cosmo.de_density_scale(1.0), 1.23114444, rtol=1e-4) + assert u.allclose( + cosmo.de_density_scale([0.5, 1.0]), [1.12934694, 1.23114444], rtol=1e-4 + ) + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +def test_efunc_vs_invefunc_flrw(): + """Test that efunc and inv_efunc give inverse values""" + z0 = 0.5 + z = np.array([0.5, 1.0, 2.0, 5.0]) + + # FLRW is abstract, so requires W1 defined earlier + # This requires scipy, unlike the built-ins, because it + # calls de_density_scale, which has an integral in it + cosmo = W1() + assert u.allclose(cosmo.efunc(z0), 1.0 / cosmo.inv_efunc(z0)) + assert u.allclose(cosmo.efunc(z), 1.0 / cosmo.inv_efunc(z)) + # Add neutrinos + cosmo = W1nu() + assert u.allclose(cosmo.efunc(z0), 1.0 / cosmo.inv_efunc(z0)) + assert u.allclose(cosmo.efunc(z), 1.0 / cosmo.inv_efunc(z)) diff --git a/astropy/cosmology/_src/tests/flrw/test_w0cdm.py b/astropy/cosmology/_src/tests/flrw/test_w0cdm.py new file mode 100644 index 000000000000..2cdf0e2e42b6 --- /dev/null +++ b/astropy/cosmology/_src/tests/flrw/test_w0cdm.py @@ -0,0 +1,211 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +"""Testing :mod:`astropy.cosmology.w0cdm`.""" + +import numpy as np +import pytest + +import astropy.units as u +from astropy.cosmology import FlatwCDM, wCDM +from astropy.cosmology._src.parameter import Parameter +from astropy.cosmology._src.tests.test_core import ParameterTestMixin, valid_zs +from astropy.tests.helper import assert_quantity_allclose +from astropy.utils.compat.optional_deps import HAS_SCIPY + +from .conftest import filter_keys_from_items +from .test_base import FlatFLRWMixinTest, FLRWTest + +############################################################################## +# TESTS +############################################################################## + + +class Parameterw0TestMixin(ParameterTestMixin): + """Tests for `astropy.cosmology.Parameter` w0 on a Cosmology. + + w0 is a descriptor, which are tested by mixin, here with ``TestFLRW``. + These tests expect dicts ``_cls_args`` and ``cls_kwargs`` which give the + args and kwargs for the cosmology class, respectively. See ``TestFLRW``. + """ + + def test_w0(self, cosmo_cls, cosmo): + """Test Parameter ``w0``.""" + # on the class + w0 = cosmo_cls.parameters["w0"] + assert isinstance(w0, Parameter) + assert "Dark energy equation of state" in w0.__doc__ + assert w0.unit is None + assert w0.default == -1.0 + + # on the instance + assert cosmo.w0 is cosmo.__dict__["w0"] + assert cosmo.w0 == self.cls_kwargs["w0"] + + def test_init_w0(self, cosmo_cls, ba): + """Test initialization for values of ``w0``.""" + # test that it works with units + ba.arguments["w0"] = ba.arguments["w0"] << u.one # ensure units + cosmo = cosmo_cls(*ba.args, **ba.kwargs) + assert cosmo.w0 == ba.arguments["w0"] + + # also without units + ba.arguments["w0"] = ba.arguments["w0"].value # strip units + cosmo = cosmo_cls(*ba.args, **ba.kwargs) + assert cosmo.w0 == ba.arguments["w0"] + + # must be dimensionless + ba.arguments["w0"] = 10 * u.km + with pytest.raises(TypeError): + cosmo_cls(*ba.args, **ba.kwargs) + + +class TestwCDM(FLRWTest, Parameterw0TestMixin): + """Test :class:`astropy.cosmology.wCDM`.""" + + def setup_class(self): + """Setup for testing.""" + super().setup_class(self) + + self.cls = wCDM + self.cls_kwargs.update(w0=-0.5) + + # =============================================================== + # Method & Attribute Tests + + def test_clone_change_param(self, cosmo): + """Test method ``.clone()`` changing a(many) Parameter(s).""" + super().test_clone_change_param(cosmo) + + # `w` params + c = cosmo.clone(w0=0.1) + assert c.w0 == 0.1 + for n, v in filter_keys_from_items(c.parameters, ("w0",)): + v_expect = getattr(cosmo, n) + assert_quantity_allclose(v, v_expect, atol=1e-4 * getattr(v, "unit", 1)) + + @pytest.mark.parametrize("z", valid_zs) + def test_w(self, cosmo, z): + """Test :meth:`astropy.cosmology.wCDM.w`.""" + super().test_w(cosmo, z) + + w = cosmo.w(z) + assert u.allclose(w, self.cls_kwargs["w0"]) + + def test_repr(self, cosmo_cls, cosmo): + """Test method ``.__repr__()``.""" + assert repr(cosmo) == ( + "wCDM(name='ABCMeta', H0=, Om0=0.27, " + "Ode0=0.73, Tcmb0=, Neff=3.04, " + "m_nu=, Ob0=0.03, w0=-0.5)" + ) + + # =============================================================== + # Usage Tests + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy is not installed") + @pytest.mark.parametrize( + ("args", "kwargs", "expected"), + [ + ( # no relativistic species + (75.0, 0.25, 0.4), + {"w0": -0.9, "Tcmb0": 0.0}, + [2849.6163356, 4428.71661565, 5450.97862778, 6179.37072324] * u.Mpc, + ), + ( # massless neutrinos + (75.0, 0.25, 0.4), + {"w0": -1.1, "Tcmb0": 3.0, "Neff": 3, "m_nu": u.Quantity(0.0, u.eV)}, + [2904.35580229, 4511.11471267, 5543.43643353, 6275.9206788] * u.Mpc, + ), + ( # massive neutrinos + (75.0, 0.25, 0.4), + {"w0": -0.9, "Tcmb0": 3.0, "Neff": 3, "m_nu": u.Quantity(10.0, u.eV)}, + [2473.32522734, 3581.54519631, 4232.41674426, 4671.83818117] * u.Mpc, + ), + ], + ) + def test_comoving_distance_example(self, cosmo_cls, args, kwargs, expected): + """Test :meth:`astropy.cosmology.LambdaCDM.comoving_distance`. + + These do not come from external codes -- they are just internal checks to make + sure nothing changes if we muck with the distance calculators. + """ + super().test_comoving_distance_example(cosmo_cls, args, kwargs, expected) + + +# ----------------------------------------------------------------------------- + + +class TestFlatwCDM(FlatFLRWMixinTest, TestwCDM): + """Test :class:`astropy.cosmology.FlatwCDM`.""" + + def setup_class(self): + """Setup for testing.""" + super().setup_class(self) + self.cls = FlatwCDM + self.cls_kwargs.update(w0=-0.5) + + def test_repr(self, cosmo_cls, cosmo): + """Test method ``.__repr__()``.""" + super().test_repr(cosmo_cls, cosmo) + + assert repr(cosmo) == ( + "FlatwCDM(name='ABCMeta', H0=, Om0=0.27, " + "Tcmb0=, Neff=3.04, m_nu=, " + "Ob0=0.03, w0=-0.5)" + ) + + # =============================================================== + # Usage Tests + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy is not installed") + @pytest.mark.parametrize( + ("args", "kwargs", "expected"), + [ + ( # no relativistic species + (75.0, 0.25), + {"w0": -1.05, "Tcmb0": 0.0}, + [3216.8296894, 5117.2097601, 6317.05995437, 7149.68648536] * u.Mpc, + ), + ( # massless neutrinos + (75.0, 0.25), + {"w0": -0.95, "Tcmb0": 3.0, "Neff": 3, "m_nu": u.Quantity(0.0, u.eV)}, + [3143.56537758, 5000.32196494, 6184.11444601, 7009.80166062] * u.Mpc, + ), + ( # massive neutrinos + (75.0, 0.25), + {"w0": -0.9, "Tcmb0": 3.0, "Neff": 3, "m_nu": u.Quantity(10.0, u.eV)}, + [2337.76035371, 3372.1971387, 3988.71362289, 4409.40817174] * u.Mpc, + ), + ], + ) + def test_comoving_distance_example(self, cosmo_cls, args, kwargs, expected): + """Test :meth:`astropy.cosmology.LambdaCDM.comoving_distance`. + + These do not come from external codes -- they are just internal checks to make + sure nothing changes if we muck with the distance calculators. + """ + super().test_comoving_distance_example(cosmo_cls, args, kwargs, expected) + + +############################################################################## +# Miscellaneous +# TODO: these should be better integrated into the new test framework + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +def test_de_densityscale(): + cosmo = wCDM(H0=70, Om0=0.3, Ode0=0.60, w0=-0.5) + + z = np.array([0.1, 0.2, 0.5, 1.5, 2.5]) + assert u.allclose( + cosmo.de_density_scale(z), + [1.15369, 1.31453, 1.83712, 3.95285, 6.5479], + rtol=1e-4, + ) + + assert u.allclose(cosmo.de_density_scale(3), cosmo.de_density_scale(3.0), rtol=1e-7) + assert u.allclose( + cosmo.de_density_scale([1, 2, 3]), + cosmo.de_density_scale([1.0, 2.0, 3.0]), + rtol=1e-7, + ) diff --git a/astropy/cosmology/_src/tests/flrw/test_w0wacdm.py b/astropy/cosmology/_src/tests/flrw/test_w0wacdm.py new file mode 100644 index 000000000000..beb596ee87b5 --- /dev/null +++ b/astropy/cosmology/_src/tests/flrw/test_w0wacdm.py @@ -0,0 +1,284 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +"""Testing :mod:`astropy.cosmology.w0wacdm`.""" + +import numpy as np +import pytest + +import astropy.units as u +from astropy.cosmology import Flatw0waCDM, Planck18, w0waCDM +from astropy.cosmology._src.parameter import Parameter +from astropy.cosmology._src.tests.test_core import ParameterTestMixin +from astropy.tests.helper import assert_quantity_allclose +from astropy.utils.compat.optional_deps import HAS_SCIPY + +from .conftest import filter_keys_from_items +from .test_base import FlatFLRWMixinTest, FLRWTest +from .test_w0cdm import Parameterw0TestMixin + +############################################################################## +# TESTS +############################################################################## + + +class ParameterwaTestMixin(ParameterTestMixin): + """Tests for `astropy.cosmology.Parameter` wa on a Cosmology. + + wa is a descriptor, which are tested by mixin, here with ``TestFLRW``. + These tests expect dicts ``_cls_args`` and ``cls_kwargs`` which give the + args and kwargs for the cosmology class, respectively. See ``TestFLRW``. + """ + + def test_wa(self, cosmo_cls, cosmo): + """Test Parameter ``wa``.""" + # on the class + wa = cosmo_cls.parameters["wa"] + assert isinstance(wa, Parameter) + assert "Negative derivative" in wa.__doc__ + assert wa.unit is None + assert wa.default == 0.0 + + # on the instance + assert cosmo.wa is cosmo.__dict__["wa"] + assert cosmo.wa == self.cls_kwargs["wa"] + + def test_init_wa(self, cosmo_cls, ba): + """Test initialization for values of ``wa``.""" + # test that it works with units + ba.arguments["wa"] = ba.arguments["wa"] << u.one # ensure units + cosmo = cosmo_cls(*ba.args, **ba.kwargs) + assert cosmo.wa == ba.arguments["wa"] + + # also without units + ba.arguments["wa"] = ba.arguments["wa"].value # strip units + cosmo = cosmo_cls(*ba.args, **ba.kwargs) + assert cosmo.wa == ba.arguments["wa"] + + # must be dimensionless + ba.arguments["wa"] = 10 * u.km + with pytest.raises(TypeError): + cosmo_cls(*ba.args, **ba.kwargs) + + +class Testw0waCDM(FLRWTest, Parameterw0TestMixin, ParameterwaTestMixin): + """Test :class:`astropy.cosmology.w0waCDM`.""" + + def setup_class(self): + """Setup for testing.""" + super().setup_class(self) + self.cls = w0waCDM + self.cls_kwargs.update(w0=-1, wa=-0.5) + + # =============================================================== + # Method & Attribute Tests + + def test_clone_change_param(self, cosmo): + """Test method ``.clone()`` changing a(many) Parameter(s).""" + super().test_clone_change_param(cosmo) + + # `w` params + c = cosmo.clone(w0=0.1, wa=0.2) + assert c.w0 == 0.1 + assert c.wa == 0.2 + for n, v in filter_keys_from_items(c.parameters, ("w0", "wa")): + v_expect = getattr(cosmo, n) + assert_quantity_allclose(v, v_expect, atol=1e-4 * getattr(v, "unit", 1)) + + # @pytest.mark.parametrize("z", valid_zs) # TODO! recompute comparisons below + def test_w(self, cosmo): + """Test :meth:`astropy.cosmology.w0waCDM.w`.""" + # super().test_w(cosmo, z) + + assert u.allclose(cosmo.w(1.0), -1.25) + assert u.allclose( + cosmo.w([0.0, 0.5, 1.0, 1.5, 2.3]), + [-1, -1.16666667, -1.25, -1.3, -1.34848485], + ) + + def test_repr(self, cosmo_cls, cosmo): + """Test method ``.__repr__()``.""" + assert repr(cosmo) == ( + "w0waCDM(name='ABCMeta', H0=, Om0=0.27, " + "Ode0=0.73, Tcmb0=, Neff=3.04, " + "m_nu=, Ob0=0.03, w0=-1.0, wa=-0.5)" + ) + + # =============================================================== + # Usage Tests + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy is not installed") + @pytest.mark.parametrize( + ("args", "kwargs", "expected"), + [ + ( # no relativistic species + (75.0, 0.3, 0.6), + {"w0": -0.9, "wa": 0.1, "Tcmb0": 0.0}, + [2937.7807638, 4572.59950903, 5611.52821924, 6339.8549956] * u.Mpc, + ), + ( # massless neutrinos + (75.0, 0.25, 0.5), + { + "w0": -0.9, + "wa": 0.1, + "Tcmb0": 3.0, + "Neff": 3, + "m_nu": u.Quantity(0.0, u.eV), + }, + [2907.34722624, 4539.01723198, 5593.51611281, 6342.3228444] * u.Mpc, + ), + ( # massive neutrinos + (75.0, 0.25, 0.5), + { + "w0": -0.9, + "wa": 0.1, + "Tcmb0": 3.0, + "Neff": 3, + "m_nu": u.Quantity(10.0, u.eV), + }, + [2507.18336722, 3633.33231695, 4292.44746919, 4736.35404638] * u.Mpc, + ), + ], + ) + def test_comoving_distance_example(self, cosmo_cls, args, kwargs, expected): + """Test :meth:`astropy.cosmology.LambdaCDM.comoving_distance`. + + These do not come from external codes -- they are just internal checks to make + sure nothing changes if we muck with the distance calculators. + """ + super().test_comoving_distance_example(cosmo_cls, args, kwargs, expected) + + +# ----------------------------------------------------------------------------- + + +class TestFlatw0waCDM(FlatFLRWMixinTest, Testw0waCDM): + """Test :class:`astropy.cosmology.Flatw0waCDM`.""" + + def setup_class(self): + """Setup for testing.""" + super().setup_class(self) + self.cls = Flatw0waCDM + self.cls_kwargs.update(w0=-1, wa=-0.5) + + def test_repr(self, cosmo_cls, cosmo): + """Test method ``.__repr__()``.""" + super().test_repr(cosmo_cls, cosmo) + + assert repr(cosmo) == ( + "Flatw0waCDM(name='ABCMeta', H0=, Om0=0.27, " + "Tcmb0=, Neff=3.04, m_nu=, " + "Ob0=0.03, w0=-1.0, wa=-0.5)" + ) + + # =============================================================== + # Usage Tests + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy is not installed") + @pytest.mark.parametrize( + ("args", "kwargs", "expected"), + [ + ( # no relativistic species + (75.0, 0.25), + {"w0": -0.95, "wa": 0.15, "Tcmb0": 0.0}, + [3123.29892781, 4956.15204302, 6128.15563818, 6948.26480378] * u.Mpc, + ), + ( # massless neutrinos + (75.0, 0.25), + { + "w0": -0.95, + "wa": 0.15, + "Tcmb0": 3.0, + "Neff": 3, + "m_nu": u.Quantity(0.0, u.eV), + }, + [3122.92671907, 4955.03768936, 6126.25719576, 6945.61856513] * u.Mpc, + ), + ( # massive neutrinos + (75.0, 0.25), + { + "w0": -0.95, + "wa": 0.15, + "Tcmb0": 3.0, + "Neff": 3, + "m_nu": u.Quantity(10.0, u.eV), + }, + [2337.70072701, 3372.13719963, 3988.6571093, 4409.35399673] * u.Mpc, + ), + ], + ) + def test_comoving_distance_example(self, cosmo_cls, args, kwargs, expected): + """Test :meth:`astropy.cosmology.LambdaCDM.comoving_distance`. + + These do not come from external codes -- they are just internal checks to make + sure nothing changes if we muck with the distance calculators. + """ + super().test_comoving_distance_example(cosmo_cls, args, kwargs, expected) + + +############################################################################## +# Comparison to Other Codes + + +@pytest.mark.skipif(not HAS_SCIPY, reason="requires scipy.") +def test_varyde_lumdist_mathematica(): + """Tests a few varying dark energy EOS models against a Mathematica computation.""" + z = np.array([0.2, 0.4, 0.9, 1.2]) + + # w0wa models + cosmo = w0waCDM(H0=70, Om0=0.2, Ode0=0.8, w0=-1.1, wa=0.2, Tcmb0=0.0) + assert u.allclose( + cosmo.luminosity_distance(z), + [1004.0, 2268.62, 6265.76, 9061.84] * u.Mpc, + rtol=1e-4, + ) + assert u.allclose(cosmo.de_density_scale(0.0), 1.0, rtol=1e-5) + assert u.allclose( + cosmo.de_density_scale([0.0, 0.5, 1.5]), + [1.0, 0.9246310669529021, 0.9184087000251957], + ) + + cosmo = w0waCDM(H0=70, Om0=0.3, Ode0=0.7, w0=-0.9, wa=0.0, Tcmb0=0.0) + assert u.allclose( + cosmo.luminosity_distance(z), + [971.667, 2141.67, 5685.96, 8107.41] * u.Mpc, + rtol=1e-4, + ) + + cosmo = w0waCDM(H0=70, Om0=0.3, Ode0=0.7, w0=-0.9, wa=-0.5, Tcmb0=0.0) + assert u.allclose( + cosmo.luminosity_distance(z), + [974.087, 2157.08, 5783.92, 8274.08] * u.Mpc, + rtol=1e-4, + ) + + +############################################################################## +# Miscellaneous +# TODO: these should be better integrated into the new test framework + + +def test_equality(): + """Test equality and equivalence.""" + # mismatched signatures, both directions. + newcosmo = w0waCDM(**Planck18.parameters, Ode0=0.6) + assert newcosmo != Planck18 + assert Planck18 != newcosmo + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +def test_de_densityscale(): + cosmo = w0waCDM(H0=70, Om0=0.3, Ode0=0.70, w0=-1, wa=-0.5) + + z = np.array([0.1, 0.2, 0.5, 1.5, 2.5]) + assert u.allclose( + cosmo.de_density_scale(z), + [0.9934201, 0.9767912, 0.897450, 0.622236, 0.4458753], + rtol=1e-4, + ) + + assert u.allclose(cosmo.de_density_scale(3), cosmo.de_density_scale(3.0), rtol=1e-7) + assert u.allclose( + cosmo.de_density_scale([1, 2, 3]), + cosmo.de_density_scale([1.0, 2.0, 3.0]), + rtol=1e-7, + ) diff --git a/astropy/cosmology/_src/tests/flrw/test_w0wzcdm.py b/astropy/cosmology/_src/tests/flrw/test_w0wzcdm.py new file mode 100644 index 000000000000..ba6267c2e9d4 --- /dev/null +++ b/astropy/cosmology/_src/tests/flrw/test_w0wzcdm.py @@ -0,0 +1,303 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +"""Testing :mod:`astropy.cosmology.w0wzcdm`.""" + +import numpy as np +import pytest + +import astropy.units as u +from astropy.cosmology import Flatw0wzCDM, w0wzCDM +from astropy.cosmology._src.parameter import Parameter +from astropy.cosmology._src.tests.test_core import ParameterTestMixin, make_valid_zs +from astropy.utils.compat.optional_deps import HAS_SCIPY + +from .conftest import filter_keys_from_items +from .test_base import FlatFLRWMixinTest, FLRWTest +from .test_w0cdm import Parameterw0TestMixin + +############################################################################## +# PARAMETERS + +COMOVING_DISTANCE_EXAMPLE_KWARGS = {"w0": -0.9, "wz": 0.1, "Tcmb0": 0.0} + +valid_zs = make_valid_zs(max_z=400)[-1] + + +############################################################################## +# TESTS +############################################################################## + + +class ParameterwzTestMixin(ParameterTestMixin): + """Tests for `astropy.cosmology.Parameter` wz on a Cosmology. + + wz is a descriptor, which are tested by mixin, here with ``TestFLRW``. + These tests expect dicts ``_cls_args`` and ``cls_kwargs`` which give the + args and kwargs for the cosmology class, respectively. See ``TestFLRW``. + """ + + def test_wz(self, cosmo_cls, cosmo): + """Test Parameter ``wz``.""" + # on the class + wz = cosmo_cls.parameters["wz"] + assert isinstance(wz, Parameter) + assert "Derivative of the dark energy" in wz.__doc__ + assert wz.unit is None + assert wz.default == 0.0 + + # on the instance + assert cosmo.wz is cosmo.__dict__["wz"] + assert cosmo.wz == self.cls_kwargs["wz"] + + def test_init_wz(self, cosmo_cls, ba): + """Test initialization for values of ``wz``.""" + # test that it works with units + ba.arguments["wz"] = ba.arguments["wz"] << u.one # ensure units + cosmo = cosmo_cls(*ba.args, **ba.kwargs) + assert cosmo.wz == ba.arguments["wz"] + + # also without units + ba.arguments["wz"] = ba.arguments["wz"].value # strip units + cosmo = cosmo_cls(*ba.args, **ba.kwargs) + assert cosmo.wz == ba.arguments["wz"] + + # must be dimensionless + ba.arguments["wz"] = 10 * u.km + with pytest.raises(TypeError): + cosmo_cls(*ba.args, **ba.kwargs) + + +class Testw0wzCDM(FLRWTest, Parameterw0TestMixin, ParameterwzTestMixin): + """Test :class:`astropy.cosmology.w0wzCDM`.""" + + def setup_class(self): + """Setup for testing.""" + super().setup_class(self) + self.cls = w0wzCDM + self.cls_kwargs.update(w0=-1, wz=0.5) + + # =============================================================== + # Method & Attribute Tests + + def test_clone_change_param(self, cosmo): + """Test method ``.clone()`` changing a(many) Parameter(s).""" + super().test_clone_change_param(cosmo) + + # `w` params + c = cosmo.clone(w0=0.1, wz=0.2) + assert c.w0 == 0.1 + assert c.wz == 0.2 + for n, v in filter_keys_from_items(c.parameters, ("w0", "wz")): + assert u.allclose(v, getattr(cosmo, n), atol=1e-4 * getattr(v, "unit", 1)) + + # @pytest.mark.parametrize("z", valid_zs) # TODO! recompute comparisons below + def test_w(self, cosmo): + """Test :meth:`astropy.cosmology.w0wzCDM.w`.""" + # super().test_w(cosmo, z) + + assert u.allclose(cosmo.w(1.0), -0.5) + assert u.allclose( + cosmo.w([0.0, 0.5, 1.0, 1.5, 2.3]), [-1.0, -0.75, -0.5, -0.25, 0.15] + ) + + def test_repr(self, cosmo_cls, cosmo): + """Test method ``.__repr__()``.""" + assert repr(cosmo) == ( + "w0wzCDM(name='ABCMeta', H0=, Om0=0.27, " + "Ode0=0.73, Tcmb0=, Neff=3.04, " + "m_nu=, Ob0=0.03, w0=-1.0, wz=0.5)" + ) + + # --------------------------------------------------------------- + + @pytest.mark.parametrize("z", valid_zs) + def test_Otot(self, cosmo, z): + """Test :meth:`astropy.cosmology.w0wzCDM.Otot`. + + This is tested in the base class, but we need to override it here because + this class is quite unstable. + """ + super().test_Otot(cosmo, z) + + def test_Otot_overflow(self, cosmo): + """Test :meth:`astropy.cosmology.w0wzCDM.Otot` for overflow.""" + with ( + np.errstate(invalid="ignore", over="warn"), + pytest.warns(RuntimeWarning, match="overflow encountered in exp"), + ): + cosmo.Otot(1e3) + + # =============================================================== + # I/O Tests + + @pytest.mark.filterwarnings("ignore:overflow encountered") + def test_toformat_model(self, cosmo, to_format, method_name): + """Test cosmology -> astropy.model.""" + super().test_toformat_model(cosmo, to_format, method_name) + + # =============================================================== + # Usage Tests + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy is not installed") + @pytest.mark.parametrize( + ("args", "kwargs", "expected"), + [ + ( # no relativistic species + (75.0, 0.3, 0.6), + {}, + [2934.20187523, 4559.94636182, 5590.71080419, 6312.66783729] * u.Mpc, + ), + ( # massless neutrinos + (75.0, 0.25, 0.5), + {"Tcmb0": 3.0, "Neff": 3, "m_nu": 0 * u.eV}, + [2904.47062713, 4528.59073707, 5575.95892989, 6318.98689566] * u.Mpc, + ), + ( # massive neutrinos + (75.0, 0.25, 0.5), + {"Tcmb0": 3.0, "Neff": 4, "m_nu": 5 * u.eV}, + [2613.84726408, 3849.66574595, 4585.51172509, 5085.16795412] * u.Mpc, + ), + ], + ) + def test_comoving_distance_example(self, cosmo_cls, args, kwargs, expected): + """Test :meth:`astropy.cosmology.LambdaCDM.comoving_distance`. + + These do not come from external codes -- they are just internal checks to make + sure nothing changes if we muck with the distance calculators. + """ + super().test_comoving_distance_example( + cosmo_cls, args, {**COMOVING_DISTANCE_EXAMPLE_KWARGS, **kwargs}, expected + ) + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy is not installed") + def test_comoving_distance_mathematica(self, cosmo_cls): + """Test with Mathematica example. + + This test should be updated as the code changes. + + :: + In[1]:= {Om0, w0, wz, H0, c}={0.3,-0.9, 0.2, 70, 299792.458}; + c/H0 NIntegrate[1/Sqrt[Om0*(1+z)^3+(1-Om0)(1+z)^(3(1+w0-wz)) Exp[3 *wz*z]],{z, 0, 0.5}] + Out[1]= 1849.75 + """ + assert u.allclose( + cosmo_cls(H0=70, Om0=0.3, w0=-0.9, wz=0.2, Ode0=0.7).comoving_distance(0.5), + 1849.75 * u.Mpc, + rtol=1e-4, + ) + + +class TestFlatw0wzCDM(FlatFLRWMixinTest, Testw0wzCDM): + """Test :class:`astropy.cosmology.Flatw0wzCDM`.""" + + def setup_class(self): + """Setup for testing.""" + super().setup_class(self) + self.cls = Flatw0wzCDM + + def test_repr(self, cosmo_cls, cosmo): + """Test method ``.__repr__()``.""" + super().test_repr(cosmo_cls, cosmo) + + assert repr(cosmo) == ( + "Flatw0wzCDM(name='ABCMeta', H0=, Om0=0.27, " + "Tcmb0=, Neff=3.04, m_nu=, " + "Ob0=0.03, w0=-1.0, wz=0.5)" + ) + + # --------------------------------------------------------------- + + @pytest.mark.parametrize("z", valid_zs) + def test_Otot(self, cosmo, z): + """Test :meth:`astropy.cosmology.Flatw0wzCDM.Otot`. + + This is tested in the base class, but we need to override it here because + this class is quite unstable. + """ + super().test_Otot(cosmo, z) + + def test_Otot_overflow(self, cosmo): + """Test :meth:`astropy.cosmology.Flatw0wzCDM.Otot` for NOT overflowing.""" + cosmo.Otot(1e5) + + # --------------------------------------------------------------- + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy is not installed") + @pytest.mark.parametrize( + ("args", "kwargs", "expected"), + [ + ( # no relativistic species + (75.0, 0.3), + {}, + [3004.55645039, 4694.15295565, 5760.90038238, 6504.07869144] * u.Mpc, + ), + ( # massless neutrinos + (75.0, 0.25), + {"Tcmb0": 3.0, "Neff": 3, "m_nu": 0 * u.eV}, + [3086.14574034, 4885.09170925, 6035.4563298, 6840.89215656] * u.Mpc, + ), + ( # massive neutrinos + (75.0, 0.25), + {"Tcmb0": 3.0, "Neff": 4, "m_nu": 5 * u.eV}, + [2510.44035219, 3683.87910326, 4389.97760294, 4873.33577288] * u.Mpc, + ), + ], + ) + def test_comoving_distance_example(self, cosmo_cls, args, kwargs, expected): + """Test :meth:`astropy.cosmology.LambdaCDM.comoving_distance`. + + These do not come from external codes -- they are just internal checks to make + sure nothing changes if we muck with the distance calculators. + """ + super().test_comoving_distance_example( + cosmo_cls, args, {**COMOVING_DISTANCE_EXAMPLE_KWARGS, **kwargs}, expected + ) + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy is not installed") + def test_comoving_distance_mathematica(self, cosmo_cls): + """Test with Mathematica example. + + This test should be updated as the code changes. + + :: + In[1]:= {Om0, w0, wz, H0, c}={0.3,-0.9, 0.2, 70, 299792.458}; + c/H0 NIntegrate[1/Sqrt[Om0*(1+z)^3+(1-Om0)(1+z)^(3(1+w0-wz)) Exp[3 *wz*z]],{z, 0, 0.5}] + Out[1]= 1849.75 + """ + assert u.allclose( + cosmo_cls(H0=70, Om0=0.3, w0=-0.9, wz=0.2).comoving_distance(0.5), + 1849.75 * u.Mpc, + rtol=1e-4, + ) + + +############################################################################## +# Miscellaneous +# TODO: these should be better integrated into the new test framework + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +def test_de_densityscale(): + cosmo = w0wzCDM(H0=70, Om0=0.3, Ode0=0.50, w0=-1, wz=0.5) + + z = np.array([0.1, 0.2, 0.5, 1.5, 2.5]) + assert u.allclose( + cosmo.de_density_scale(z), + [1.00705953, 1.02687239, 1.15234885, 2.40022841, 6.49384982], + rtol=1e-4, + ) + + assert u.allclose(cosmo.de_density_scale(3), cosmo.de_density_scale(3.0), rtol=1e-7) + assert u.allclose( + cosmo.de_density_scale([1, 2, 3]), + cosmo.de_density_scale([1.0, 2.0, 3.0]), + rtol=1e-7, + ) + + # Flat tests + cosmo = w0wzCDM(H0=70, Om0=0.3, Ode0=0.7, w0=-1, wz=0.5) + flatcosmo = Flatw0wzCDM(H0=70, Om0=0.3, w0=-1, wz=0.5) + + assert u.allclose( + cosmo.de_density_scale(z), flatcosmo.de_density_scale(z), rtol=1e-4 + ) diff --git a/astropy/cosmology/_src/tests/flrw/test_wpwazpcdm.py b/astropy/cosmology/_src/tests/flrw/test_wpwazpcdm.py new file mode 100644 index 000000000000..125e4ccaf183 --- /dev/null +++ b/astropy/cosmology/_src/tests/flrw/test_wpwazpcdm.py @@ -0,0 +1,310 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +"""Testing :mod:`astropy.cosmology.wpwazpcdm`.""" + +import numpy as np +import pytest + +import astropy.cosmology.units as cu +import astropy.units as u +from astropy.cosmology import FlatwpwaCDM, wpwaCDM +from astropy.cosmology._src.parameter import Parameter +from astropy.cosmology._src.tests.test_core import ParameterTestMixin +from astropy.tests.helper import assert_quantity_allclose +from astropy.utils.compat.optional_deps import HAS_SCIPY + +from .conftest import filter_keys_from_items +from .test_base import FlatFLRWMixinTest, FLRWTest +from .test_w0wacdm import ParameterwaTestMixin + +############################################################################## +# PARAMETERS + +COMOVING_DISTANCE_EXAMPLE_KWARGS = {"wp": -0.9, "zp": 0.5, "wa": 0.1, "Tcmb0": 0.0} + + +############################################################################## +# TESTS +############################################################################## + + +class ParameterwpTestMixin(ParameterTestMixin): + """Tests for `astropy.cosmology.Parameter` wp on a Cosmology. + + wp is a descriptor, which are tested by mixin, here with ``TestFLRW``. + These tests expect dicts ``_cls_args`` and ``cls_kwargs`` which give the + args and kwargs for the cosmology class, respectively. See ``TestFLRW``. + """ + + def test_wp(self, cosmo_cls, cosmo): + """Test Parameter ``wp``.""" + # on the class + wp = cosmo_cls.parameters["wp"] + assert isinstance(wp, Parameter) + assert "at the pivot" in wp.__doc__ + assert wp.unit is None + assert wp.default == -1.0 + + # on the instance + assert cosmo.wp is cosmo.__dict__["wp"] + assert cosmo.wp == self.cls_kwargs["wp"] + + def test_init_wp(self, cosmo_cls, ba): + """Test initialization for values of ``wp``.""" + # test that it works with units + ba.arguments["wp"] = ba.arguments["wp"] << u.one # ensure units + cosmo = cosmo_cls(*ba.args, **ba.kwargs) + assert cosmo.wp == ba.arguments["wp"] + + # also without units + ba.arguments["wp"] = ba.arguments["wp"].value # strip units + cosmo = cosmo_cls(*ba.args, **ba.kwargs) + assert cosmo.wp == ba.arguments["wp"] + + # must be dimensionless + ba.arguments["wp"] = 10 * u.km + with pytest.raises(TypeError): + cosmo_cls(*ba.args, **ba.kwargs) + + +class ParameterzpTestMixin(ParameterTestMixin): + """Tests for `astropy.cosmology.Parameter` zp on a Cosmology. + + zp is a descriptor, which are tested by mixin, here with ``TestFLRW``. + These tests expect dicts ``_cls_args`` and ``cls_kwargs`` which give the + args and kwargs for the cosmology class, respectively. See ``TestFLRW``. + """ + + def test_zp(self, cosmo_cls, cosmo): + """Test Parameter ``zp``.""" + # on the class + zp = cosmo_cls.parameters["zp"] + assert isinstance(zp, Parameter) + assert "pivot redshift" in zp.__doc__ + assert zp.unit == cu.redshift + assert zp.default == 0.0 + + # on the instance + assert cosmo.zp is cosmo.__dict__["zp"] + assert cosmo.zp == self.cls_kwargs["zp"] << cu.redshift + + def test_init_zp(self, cosmo_cls, ba): + """Test initialization for values of ``zp``.""" + # test that it works with units + ba.arguments["zp"] = ba.arguments["zp"] << u.one # ensure units + cosmo = cosmo_cls(*ba.args, **ba.kwargs) + assert cosmo.zp == ba.arguments["zp"] + + # also without units + ba.arguments["zp"] = ba.arguments["zp"].value # strip units + cosmo = cosmo_cls(*ba.args, **ba.kwargs) + assert cosmo.zp.value == ba.arguments["zp"] + + # must be dimensionless + ba.arguments["zp"] = 10 * u.km + with pytest.raises(u.UnitConversionError): + cosmo_cls(*ba.args, **ba.kwargs) + + +class TestwpwaCDM( + FLRWTest, ParameterwpTestMixin, ParameterwaTestMixin, ParameterzpTestMixin +): + """Test :class:`astropy.cosmology.wpwaCDM`.""" + + def setup_class(self): + """Setup for testing.""" + super().setup_class(self) + self.cls = wpwaCDM + self.cls_kwargs.update(wp=-0.9, wa=0.2, zp=0.5) + + # =============================================================== + # Method & Attribute Tests + + def test_clone_change_param(self, cosmo): + """Test method ``.clone()`` changing a(many) Parameter(s).""" + super().test_clone_change_param(cosmo) + + # `w` params + c = cosmo.clone(wp=0.1, wa=0.2, zp=14) + assert c.wp == 0.1 + assert c.wa == 0.2 + assert c.zp == 14 + for n, v in filter_keys_from_items(c.parameters, ("wp", "wa", "zp")): + v_expect = getattr(cosmo, n) + assert_quantity_allclose(v, v_expect, atol=1e-4 * getattr(v, "unit", 1)) + + # @pytest.mark.parametrize("z", valid_zs) # TODO! recompute comparisons below + def test_w(self, cosmo): + """Test :meth:`astropy.cosmology.wpwaCDM.w`.""" + # super().test_w(cosmo, z) + + assert u.allclose(cosmo.w(0.5), -0.9) + assert u.allclose( + cosmo.w([0.1, 0.2, 0.5, 1.5, 2.5, 11.5]), + [-0.94848485, -0.93333333, -0.9, -0.84666667, -0.82380952, -0.78266667], + ) + + def test_repr(self, cosmo_cls, cosmo): + """Test method ``.__repr__()``.""" + assert repr(cosmo) == ( + "wpwaCDM(name='ABCMeta', H0=, Om0=0.27," + " Ode0=0.73, Tcmb0=, Neff=3.04," + " m_nu=, Ob0=0.03, wp=-0.9, wa=0.2," + " zp=)" + ) + + # =============================================================== + # Usage Tests + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy required for this test.") + @pytest.mark.parametrize( + ("args", "kwargs", "expected"), + [ + ( # no relativistic species + (75.0, 0.3, 0.6), + {}, + [2954.68975298, 4599.83254834, 5643.04013201, 6373.36147627] * u.Mpc, + ), + ( # massless neutrinos + (75.0, 0.25, 0.5), + {"zp": 0.4, "Tcmb0": 3.0, "Neff": 3, "m_nu": 0 * u.eV}, + [2919.00656215, 4558.0218123, 5615.73412391, 6366.10224229] * u.Mpc, + ), + ( # massive neutrinos + (75.0, 0.25, 0.5), + {"zp": 1.0, "Tcmb0": 3.0, "Neff": 4, "m_nu": 5 * u.eV}, + [2629.48489827, 3874.13392319, 4614.31562397, 5116.51184842] * u.Mpc, + ), + # FLAT: these match the tests in TestFlatwpwaCDM, except Ode0 is set manually. + ( # no relativistic species + (75.0, 0.3, 0.7), + {}, + [3030.70481348, 4745.82435272, 5828.73710847, 6582.60454542] * u.Mpc, + ), + ( # massless neutrinos + (75.0, 0.25, 0.75), + {"zp": 0.4, "Tcmb0": 3.0, "Neff": 3, "m_nu": 0 * u.eV}, + [3113.62199365, 4943.28425668, 6114.45491003, 6934.07461377] * u.Mpc, + ), + ( # massive neutrinos + (75.0, 0.25, 0.2458794183661), # to make Ok0 = 0, Otot0 = 1 + {"zp": 1.0, "Tcmb0": 3.0, "Neff": 4, "m_nu": 5 * u.eV}, + [2517.08634022, 3694.21111754, 4402.17802962, 4886.65787948] * u.Mpc, + ), + ], + ) + def test_comoving_distance_example(self, cosmo_cls, args, kwargs, expected): + """Test :meth:`astropy.cosmology.LambdaCDM.comoving_distance`. + + These do not come from external codes -- they are just internal checks to make + sure nothing changes if we muck with the distance calculators. + """ + super().test_comoving_distance_example( + cosmo_cls, args, {**COMOVING_DISTANCE_EXAMPLE_KWARGS, **kwargs}, expected + ) + + +class TestFlatwpwaCDM(FlatFLRWMixinTest, TestwpwaCDM): + """Test :class:`astropy.cosmology.FlatwpwaCDM`.""" + + def setup_class(self): + """Setup for testing.""" + super().setup_class(self) + self.cls = FlatwpwaCDM + + def test_repr(self, cosmo_cls, cosmo): + """Test method ``.__repr__()``.""" + super().test_repr(cosmo_cls, cosmo) + + assert repr(cosmo) == ( + "FlatwpwaCDM(name='ABCMeta', H0=, Om0=0.27," + " Tcmb0=, Neff=3.04, m_nu=," + " Ob0=0.03, wp=-0.9, wa=0.2, zp=)" + ) + + @pytest.mark.skipif(not HAS_SCIPY, reason="scipy required for this test.") + @pytest.mark.parametrize( + ("args", "kwargs", "expected"), + [ + ( # no relativistic species + (75.0, 0.3), + {}, + [3030.70481348, 4745.82435272, 5828.73710847, 6582.60454542] * u.Mpc, + ), + ( # massless neutrinos + (75.0, 0.25), + {"zp": 0.4, "wa": 0.1, "Tcmb0": 3.0, "Neff": 3, "m_nu": 0.0 * u.eV}, + [3113.62199365, 4943.28425668, 6114.45491003, 6934.07461377] * u.Mpc, + ), + ( # massive neutrinos + (75.0, 0.25), + {"zp": 1.0, "Tcmb0": 3.0, "Neff": 4, "m_nu": 5 * u.eV}, + [2517.08634022, 3694.21111754, 4402.17802962, 4886.65787948] * u.Mpc, + ), + ], + ) + def test_comoving_distance_example(self, cosmo_cls, args, kwargs, expected): + """Test :meth:`astropy.cosmology.LambdaCDM.comoving_distance`. + + These do not come from external codes -- they are just internal checks to make + sure nothing changes if we muck with the distance calculators. + """ + super().test_comoving_distance_example( + cosmo_cls, args, {**COMOVING_DISTANCE_EXAMPLE_KWARGS, **kwargs}, expected + ) + + +############################################################################### +# Comparison to Other Codes + + +@pytest.mark.skipif(not HAS_SCIPY, reason="requires scipy.") +def test_varyde_lumdist_mathematica(): + """Tests a few varying dark energy EOS models against a Mathematica computation.""" + z = np.array([0.2, 0.4, 0.9, 1.2]) + + # wpwa models + cosmo = wpwaCDM(H0=70, Om0=0.2, Ode0=0.8, wp=-1.1, wa=0.2, zp=0.5, Tcmb0=0.0) + assert u.allclose( + cosmo.luminosity_distance(z), + [1010.81, 2294.45, 6369.45, 9218.95] * u.Mpc, + rtol=1e-4, + ) + + cosmo = wpwaCDM(H0=70, Om0=0.2, Ode0=0.8, wp=-1.1, wa=0.2, zp=0.9, Tcmb0=0.0) + assert u.allclose( + cosmo.luminosity_distance(z), + [1013.68, 2305.3, 6412.37, 9283.33] * u.Mpc, + rtol=1e-4, + ) + + +############################################################################## +# Miscellaneous +# TODO: these should be better integrated into the new test framework + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +def test_de_densityscale(): + cosmo = wpwaCDM(H0=70, Om0=0.3, Ode0=0.70, wp=-0.9, wa=0.2, zp=0.5) + + z = np.array([0.1, 0.2, 0.5, 1.5, 2.5]) + assert u.allclose( + cosmo.de_density_scale(z), + [1.012246048, 1.0280102, 1.087439, 1.324988, 1.565746], + rtol=1e-4, + ) + + assert u.allclose(cosmo.de_density_scale(3), cosmo.de_density_scale(3.0), rtol=1e-7) + assert u.allclose( + cosmo.de_density_scale([1, 2, 3]), + cosmo.de_density_scale([1.0, 2.0, 3.0]), + rtol=1e-7, + ) + + # Flat tests + cosmo = wpwaCDM(H0=70, Om0=0.3, Ode0=0.70, wp=-0.9, wa=0.2, zp=0.5) + flatcosmo = FlatwpwaCDM(H0=70, Om0=0.3, wp=-0.9, wa=0.2, zp=0.5) + assert u.allclose( + cosmo.de_density_scale(z), flatcosmo.de_density_scale(z), rtol=1e-7 + ) diff --git a/astropy/utils/compat/numpy/lib/__init__.py b/astropy/cosmology/_src/tests/funcs/__init__.py similarity index 100% rename from astropy/utils/compat/numpy/lib/__init__.py rename to astropy/cosmology/_src/tests/funcs/__init__.py diff --git a/astropy/cosmology/_src/tests/funcs/test_comparison.py b/astropy/cosmology/_src/tests/funcs/test_comparison.py new file mode 100644 index 000000000000..8ecb0cd55f2a --- /dev/null +++ b/astropy/cosmology/_src/tests/funcs/test_comparison.py @@ -0,0 +1,346 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +"""Tests for :mod:`astropy.cosmology.comparison`""" + +import re + +import numpy as np +import pytest + +from astropy.cosmology import Cosmology, FlatCosmologyMixin, Planck18, cosmology_equal +from astropy.cosmology._src.funcs.comparison import ( + _CANT_BROADCAST, + _cosmology_not_equal, + _CosmologyWrapper, + _parse_format, + _parse_formats, +) +from astropy.cosmology._src.tests.io.base import ToFromTestMixinBase +from astropy.cosmology.io import convert_registry + + +class ComparisonFunctionTestBase(ToFromTestMixinBase): + """Tests for cosmology comparison functions. + + This class inherits from + `astropy.cosmology._src.tests.io.base.ToFromTestMixinBase` because the cosmology + comparison functions all have a kwarg ``format`` that allow the arguments to + be converted to a |Cosmology| using the ``to_format`` architecture. + + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must be + inherited in a subclass. + """ + + @pytest.fixture(scope="class") + @classmethod + def cosmo(cls): + return Planck18 + + @pytest.fixture(scope="class") + @classmethod + def cosmo_eqvxflat(cls, cosmo): + if isinstance(cosmo, FlatCosmologyMixin): + return cosmo.nonflat + + pytest.skip( + "cosmology is not flat, so does not have an equivalent non-flat cosmology." + ) + + @pytest.fixture( + scope="class", + params=sorted( + {k for k, _ in convert_registry._readers.keys()} - {"astropy.cosmology"} + ), + ) + @classmethod + def format(cls, request): + return request.param + + @pytest.fixture(scope="class") + @classmethod + def xfail_cant_autoidentify(cls, format): + """`pytest.fixture` form of method ``can_autoidentify`.""" + if not cls.can_autodentify(format): + pytest.xfail("cannot autoidentify") + + @pytest.fixture(scope="class") + @classmethod + def converted(cls, to_format, format): + if format == "astropy.model": # special case Model + return to_format(format, method="comoving_distance") + return to_format(format) + + @pytest.fixture(scope="class") + @classmethod + def pert_cosmo(cls, cosmo): + # change one parameter + p, v = next(iter(cosmo.parameters.items())) + return cosmo.clone( + **{p: v * 1.0001 if v != 0 else 0.001 * getattr(v, "unit", 1)} + ) + + @pytest.fixture(scope="class") + @classmethod + def pert_cosmo_eqvxflat(cls, pert_cosmo): + if isinstance(pert_cosmo, FlatCosmologyMixin): + return pert_cosmo.nonflat + + pytest.skip( + "cosmology is not flat, so does not have an equivalent non-flat cosmology." + ) + + @pytest.fixture(scope="class") + @classmethod + def pert_converted(cls, pert_cosmo, format): + if format == "astropy.model": # special case Model + return pert_cosmo.to_format(format, method="comoving_distance") + return pert_cosmo.to_format(format) + + +class Test_parse_format(ComparisonFunctionTestBase): + """Test functions ``_parse_format``.""" + + @pytest.fixture(scope="class") + @classmethod + def converted(cls, to_format, format): + if format == "astropy.model": # special case Model + return to_format(format, method="comoving_distance") + + converted = to_format(format) + + # Some raise a segfault! TODO: figure out why + if isinstance(converted, _CANT_BROADCAST): + converted = _CosmologyWrapper(converted) + + return converted + + # ======================================================================== + + def test_shortcut(self, cosmo): + """Test the already-a-cosmology shortcut.""" + # A Cosmology + for fmt in (None, True, False, "astropy.cosmology"): + assert _parse_format(cosmo, fmt) is cosmo, f"{fmt} failed" + + # A Cosmology, but improperly formatted + # see ``test_parse_format_error_wrong_format``. + + def test_convert(self, converted, format, cosmo): + """Test converting a cosmology-like object""" + out = _parse_format(converted, format) + + assert isinstance(out, Cosmology) + assert out == cosmo + + def test_parse_format_error_wrong_format(self, cosmo): + """ + Test ``_parse_format`` errors when given a Cosmology object and format + is not compatible. + """ + with pytest.raises( + ValueError, match=re.escape("for parsing a Cosmology, 'format'") + ): + _parse_format(cosmo, "mapping") + + def test_parse_format_error_noncosmology_cant_convert(self): + """ + Test ``_parse_format`` errors when given a non-Cosmology object + and format is `False`. + """ + notacosmo = object() + + with pytest.raises(TypeError, match=re.escape("if 'format' is False")): + _parse_format(notacosmo, False) + + def test_parse_format_vectorized(self, cosmo, format, converted): + # vectorized on cosmos + out = _parse_format([cosmo, cosmo], None) + assert len(out) == 2 + assert np.all(out == cosmo) + + # vectorized on formats + out = _parse_format(cosmo, [None, None]) + assert len(out) == 2 + assert np.all(out == cosmo) + + # more complex broadcast + out = _parse_format( + [[cosmo, converted], [converted, cosmo]], [[None, format], [format, None]] + ) + assert out.shape == (2, 2) + assert np.all(out == cosmo) + + def test_parse_formats_vectorized(self, cosmo): + # vectorized on cosmos + out = _parse_formats(cosmo, cosmo, format=None) + assert len(out) == 2 + assert np.all(out == cosmo) + + # does NOT vectorize on formats + with pytest.raises(ValueError, match="operands could not be broadcast"): + _parse_formats(cosmo, format=[None, None]) + + +class Test_cosmology_equal(ComparisonFunctionTestBase): + """Test :func:`astropy.cosmology.comparison.cosmology_equal`""" + + def test_cosmology_equal_simple(self, cosmo, pert_cosmo): + # equality + assert cosmology_equal(cosmo, cosmo) is True + + # not equal to perturbed cosmology + assert cosmology_equal(cosmo, pert_cosmo) is False + + def test_cosmology_equal_equivalent( + self, cosmo, cosmo_eqvxflat, pert_cosmo, pert_cosmo_eqvxflat + ): + # now need to check equivalent, but not equal, cosmologies. + assert cosmology_equal(cosmo, cosmo_eqvxflat, allow_equivalent=True) is True + assert cosmology_equal(cosmo, cosmo_eqvxflat, allow_equivalent=False) is False + + assert ( + cosmology_equal(pert_cosmo, pert_cosmo_eqvxflat, allow_equivalent=True) + is True + ) + assert ( + cosmology_equal(pert_cosmo, pert_cosmo_eqvxflat, allow_equivalent=False) + is False + ) + + def test_cosmology_equal_too_many_cosmo(self, cosmo): + with pytest.raises( + TypeError, match="cosmology_equal takes 2 positional arguments" + ): + cosmology_equal(cosmo, cosmo, cosmo) + + def test_cosmology_equal_format_error(self, cosmo, converted): + # Not converting `converted` + with pytest.raises(TypeError): + cosmology_equal(cosmo, converted) + + with pytest.raises(TypeError): + cosmology_equal(cosmo, converted, format=False) + + def test_cosmology_equal_format_auto( + self, cosmo, converted, xfail_cant_autoidentify + ): + # These tests only run if the format can autoidentify. + assert cosmology_equal(cosmo, converted, format=None) is True + assert cosmology_equal(cosmo, converted, format=True) is True + + def test_cosmology_equal_format_specify( + self, cosmo, format, converted, pert_converted + ): + # equality + assert cosmology_equal(cosmo, converted, format=[None, format]) is True + assert cosmology_equal(converted, cosmo, format=[format, None]) is True + + # non-equality + assert cosmology_equal(cosmo, pert_converted, format=[None, format]) is False + + def test_cosmology_equal_equivalent_format_specify( + self, cosmo, format, converted, cosmo_eqvxflat + ): + # specifying the format + assert ( + cosmology_equal( + cosmo_eqvxflat, converted, format=[None, format], allow_equivalent=True + ) + is True + ) + assert ( + cosmology_equal( + converted, cosmo_eqvxflat, format=[format, None], allow_equivalent=True + ) + is True + ) + + +class Test_cosmology_not_equal(ComparisonFunctionTestBase): + """Test :func:`astropy.cosmology.comparison._cosmology_not_equal`""" + + def test_cosmology_not_equal_simple(self, cosmo, pert_cosmo): + # equality + assert _cosmology_not_equal(cosmo, cosmo) is False + + # not equal to perturbed cosmology + assert _cosmology_not_equal(cosmo, pert_cosmo) is True + + def test_cosmology_not_equal_too_many_cosmo(self, cosmo): + with pytest.raises(TypeError, match="_cosmology_not_equal takes 2 positional"): + _cosmology_not_equal(cosmo, cosmo, cosmo) + + def test_cosmology_not_equal_equivalent( + self, cosmo, cosmo_eqvxflat, pert_cosmo, pert_cosmo_eqvxflat + ): + # now need to check equivalent, but not equal, cosmologies. + assert ( + _cosmology_not_equal(cosmo, cosmo_eqvxflat, allow_equivalent=False) is True + ) + assert ( + _cosmology_not_equal(cosmo, cosmo_eqvxflat, allow_equivalent=True) is False + ) + + assert ( + _cosmology_not_equal( + pert_cosmo, pert_cosmo_eqvxflat, allow_equivalent=False + ) + is True + ) + assert ( + _cosmology_not_equal(pert_cosmo, pert_cosmo_eqvxflat, allow_equivalent=True) + is False + ) + + def test_cosmology_not_equal_format_error(self, cosmo, converted): + # Not converting `converted` + with pytest.raises(TypeError): + _cosmology_not_equal(cosmo, converted) + + with pytest.raises(TypeError): + _cosmology_not_equal(cosmo, converted, format=False) + + def test_cosmology_not_equal_format_auto( + self, cosmo, pert_converted, xfail_cant_autoidentify + ): + assert _cosmology_not_equal(cosmo, pert_converted, format=None) is True + assert _cosmology_not_equal(cosmo, pert_converted, format=True) is True + + def test_cosmology_not_equal_format_specify( + self, cosmo, format, converted, pert_converted + ): + # specifying the format + assert ( + _cosmology_not_equal(cosmo, pert_converted, format=[None, format]) is True + ) + assert ( + _cosmology_not_equal(pert_converted, cosmo, format=[format, None]) is True + ) + + # equality + assert _cosmology_not_equal(cosmo, converted, format=[None, format]) is False + + def test_cosmology_not_equal_equivalent_format_specify( + self, cosmo, format, converted, cosmo_eqvxflat + ): + # specifying the format + assert ( + _cosmology_not_equal( + cosmo_eqvxflat, converted, format=[None, format], allow_equivalent=False + ) + is True + ) + assert ( + _cosmology_not_equal( + cosmo_eqvxflat, converted, format=[None, format], allow_equivalent=True + ) + is False + ) + + assert ( + _cosmology_not_equal( + converted, cosmo_eqvxflat, format=[format, None], allow_equivalent=True + ) + is False + ) diff --git a/astropy/cosmology/_src/tests/funcs/test_funcs.py b/astropy/cosmology/_src/tests/funcs/test_funcs.py new file mode 100644 index 000000000000..98578b1d84b8 --- /dev/null +++ b/astropy/cosmology/_src/tests/funcs/test_funcs.py @@ -0,0 +1,426 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import inspect +import sys +from contextlib import nullcontext +from io import StringIO + +import numpy as np +import pytest + +from astropy import units as u +from astropy.cosmology import ( + WMAP1, + WMAP3, + WMAP5, + WMAP7, + WMAP9, + CosmologyError, + FlatLambdaCDM, + Flatw0waCDM, + FlatwCDM, + LambdaCDM, + Planck13, + Planck15, + Planck18, + w0waCDM, + w0wzCDM, + wCDM, + wpwaCDM, + z_at_value, +) +from astropy.units import allclose +from astropy.utils.compat.optional_deps import HAS_SCIPY +from astropy.utils.exceptions import AstropyUserWarning + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +def test_z_at_value_scalar(): + # These are tests of expected values, and hence have less precision + # than the roundtrip tests below (test_z_at_value_roundtrip); + # here we have to worry about the cosmological calculations + # giving slightly different values on different architectures, + # there we are checking internal consistency on the same architecture + # and so can be more demanding + cosmo = Planck13 + assert allclose(z_at_value(cosmo.age, 2 * u.Gyr), 3.19812268, rtol=1e-6) + assert allclose(z_at_value(cosmo.lookback_time, 7 * u.Gyr), 0.795198375, rtol=1e-6) + assert allclose(z_at_value(cosmo.distmod, 46 * u.mag), 1.991389168, rtol=1e-6) + assert allclose( + z_at_value(cosmo.luminosity_distance, 1e4 * u.Mpc), 1.36857907, rtol=1e-6 + ) + assert allclose( + z_at_value(cosmo.luminosity_distance, 26.037193804 * u.Gpc, ztol=1e-10), + 3, + rtol=1e-9, + ) + assert allclose( + z_at_value(cosmo.angular_diameter_distance, 1500 * u.Mpc, zmax=2), + 0.681277696, + rtol=1e-6, + ) + assert allclose( + z_at_value(cosmo.angular_diameter_distance, 1500 * u.Mpc, zmin=2.5), + 3.7914908, + rtol=1e-6, + ) + + # test behavior when the solution is outside z limits (should + # raise a CosmologyError) + with ( + pytest.raises(CosmologyError), + pytest.warns(AstropyUserWarning, match="fval is not bracketed"), + ): + z_at_value(cosmo.angular_diameter_distance, 1500 * u.Mpc, zmax=0.5) + + with ( + pytest.raises(CosmologyError), + pytest.warns(AstropyUserWarning, match="fval is not bracketed"), + np.errstate(over="ignore"), + ): + z_at_value(cosmo.angular_diameter_distance, 1500 * u.Mpc, zmin=4.0) + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +class Test_ZatValue: + def setup_class(self): + self.cosmo = Planck13 + + def test_broadcast_arguments(self): + """Test broadcast of arguments.""" + # broadcasting main argument + assert allclose( + z_at_value(self.cosmo.age, [2, 7] * u.Gyr), + [3.1981206134773115, 0.7562044333305182], + rtol=1e-6, + ) + + # basic broadcast of secondary arguments + assert allclose( + z_at_value( + self.cosmo.angular_diameter_distance, + 1500 * u.Mpc, + zmin=[0, 2.5], + zmax=[2, 4], + ), + [0.681277696, 3.7914908], + rtol=1e-6, + ) + + # more interesting broadcast + assert allclose( + z_at_value( + self.cosmo.angular_diameter_distance, + 1500 * u.Mpc, + zmin=[[0, 2.5]], + zmax=[2, 4], + ), + [[0.681277696, 3.7914908]], + rtol=1e-6, + ) + + def test_broadcast_bracket(self): + """`bracket` has special requirements.""" + # start with an easy one + assert allclose( + z_at_value(self.cosmo.age, 2 * u.Gyr, bracket=None), + 3.1981206134773115, + rtol=1e-6, + ) + + # now actually have a bracket + assert allclose( + z_at_value(self.cosmo.age, 2 * u.Gyr, bracket=[0, 4]), + 3.1981206134773115, + rtol=1e-6, + ) + + # now a bad length + with pytest.raises(ValueError, match="sequence"): + z_at_value(self.cosmo.age, 2 * u.Gyr, bracket=[0, 4, 4, 5]) + + # now the wrong dtype : an ndarray, but not an object array + with pytest.raises(TypeError, match="dtype"): + z_at_value(self.cosmo.age, 2 * u.Gyr, bracket=np.array([0, 4])) + + # now an object array of brackets + bracket = np.array([[0, 4], [0, 3, 4]], dtype=object) + assert allclose( + z_at_value(self.cosmo.age, 2 * u.Gyr, bracket=bracket), + [3.1981206134773115, 3.1981206134773115], + rtol=1e-6, + ) + + def test_bad_broadcast(self): + """Shapes mismatch as expected""" + with pytest.raises(ValueError, match="broadcast"): + z_at_value( + self.cosmo.angular_diameter_distance, + 1500 * u.Mpc, + zmin=[0, 2.5, 0.1], + zmax=[2, 4], + ) + + def test_scalar_input_to_output(self): + """Test scalar input returns a scalar.""" + z = z_at_value( + self.cosmo.angular_diameter_distance, 1500 * u.Mpc, zmin=0, zmax=2 + ) + assert isinstance(z, u.Quantity) + assert z.dtype == np.float64 + assert z.shape == () + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +def test_z_at_value_verbose(monkeypatch): + cosmo = Planck13 + + # Test the "verbose" flag. Since this uses "print", need to mod stdout + mock_stdout = StringIO() + monkeypatch.setattr(sys, "stdout", mock_stdout) + + resx = z_at_value(cosmo.age, 2 * u.Gyr, verbose=True) + assert str(resx.value) in mock_stdout.getvalue() # test "verbose" prints res + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +@pytest.mark.parametrize("method", ["Brent", "Golden", "Bounded"]) +def test_z_at_value_bracketed(method): + """ + Test 2 solutions for angular diameter distance by not constraining zmin, zmax, + but setting `bracket` on the appropriate side of the turning point z. + Setting zmin / zmax should override `bracket`. + """ + cosmo = Planck13 + + if method == "Bounded": + with pytest.warns(AstropyUserWarning, match="fval is not bracketed"): + z = z_at_value(cosmo.angular_diameter_distance, 1500 * u.Mpc, method=method) + if z > 1.6: + z = 3.7914908 + bracket = (0.9, 1.5) + else: + z = 0.6812777 + bracket = (1.6, 2.0) + with ( + pytest.warns(UserWarning, match="Option 'bracket' is ignored"), + pytest.warns(AstropyUserWarning, match="fval is not bracketed"), + ): + assert allclose( + z_at_value( + cosmo.angular_diameter_distance, + 1500 * u.Mpc, + method=method, + bracket=bracket, + ), + z, + rtol=1e-6, + ) + else: + assert allclose( + z_at_value( + cosmo.angular_diameter_distance, + 1500 * u.Mpc, + method=method, + bracket=(0.3, 1.0), + ), + 0.6812777, + rtol=1e-6, + ) + assert allclose( + z_at_value( + cosmo.angular_diameter_distance, + 1500 * u.Mpc, + method=method, + bracket=(2.0, 4.0), + ), + 3.7914908, + rtol=1e-6, + ) + assert allclose( + z_at_value( + cosmo.angular_diameter_distance, + 1500 * u.Mpc, + method=method, + bracket=(0.1, 1.5), + ), + 0.6812777, + rtol=1e-6, + ) + assert allclose( + z_at_value( + cosmo.angular_diameter_distance, + 1500 * u.Mpc, + method=method, + bracket=(0.1, 1.0, 2.0), + ), + 0.6812777, + rtol=1e-6, + ) + with pytest.warns(AstropyUserWarning, match=r"fval is not bracketed"): + assert allclose( + z_at_value( + cosmo.angular_diameter_distance, + 1500 * u.Mpc, + method=method, + bracket=(0.9, 1.5), + ), + 0.6812777, + rtol=1e-6, + ) + assert allclose( + z_at_value( + cosmo.angular_diameter_distance, + 1500 * u.Mpc, + method=method, + bracket=(1.6, 2.0), + ), + 3.7914908, + rtol=1e-6, + ) + assert allclose( + z_at_value( + cosmo.angular_diameter_distance, + 1500 * u.Mpc, + method=method, + bracket=(1.6, 2.0), + zmax=1.6, + ), + 0.6812777, + rtol=1e-6, + ) + assert allclose( + z_at_value( + cosmo.angular_diameter_distance, + 1500 * u.Mpc, + method=method, + bracket=(0.9, 1.5), + zmin=1.5, + ), + 3.7914908, + rtol=1e-6, + ) + + if method == "Bounded": + ctx_bracket = pytest.warns( + UserWarning, match="Option 'bracket' is ignored by method Bounded" + ) + else: + ctx_bracket = nullcontext() + + with ( + pytest.raises(CosmologyError), + pytest.warns(AstropyUserWarning, match="fval is not bracketed"), + ctx_bracket, + ): + z_at_value( + cosmo.angular_diameter_distance, + 1500 * u.Mpc, + method=method, + bracket=(3.9, 5.0), + zmin=4.0, + ) + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +@pytest.mark.parametrize("method", ["Brent", "Golden", "Bounded"]) +def test_z_at_value_unconverged(method): + """ + Test warnings on non-converged solution when setting `maxfun` to too small iteration number - + only 'Bounded' returns status value and specific message. + """ + cosmo = Planck18 + ztol = {"Brent": [1e-4, 1e-4], "Golden": [1e-3, 1e-2], "Bounded": [1e-3, 1e-1]} + + if method == "Bounded": + ctx = pytest.warns( + AstropyUserWarning, + match="Solver returned 1: Maximum number of function calls reached", + ) + else: + ctx = pytest.warns(AstropyUserWarning, match="Solver returned None") + + with ctx: + z0 = z_at_value( + cosmo.angular_diameter_distance, 1 * u.Gpc, zmax=2, maxfun=13, method=method + ) + with ctx: + z1 = z_at_value( + cosmo.angular_diameter_distance, 1 * u.Gpc, zmin=2, maxfun=13, method=method + ) + + assert allclose(z0, 0.32442, rtol=ztol[method][0]) + assert allclose(z1, 8.18551, rtol=ztol[method][1]) + + +@pytest.mark.skipif(not HAS_SCIPY, reason="test requires scipy") +@pytest.mark.parametrize( + "cosmo", + [ + Planck13, + Planck15, + Planck18, + WMAP1, + WMAP3, + WMAP5, + WMAP7, + WMAP9, + LambdaCDM, + FlatLambdaCDM, + wpwaCDM, + w0wzCDM, + wCDM, + FlatwCDM, + w0waCDM, + Flatw0waCDM, + ], +) +def test_z_at_value_roundtrip(cosmo): + """ + Calculate values from a known redshift, and then check that + z_at_value returns the right answer. + """ + z = 0.5 + + # Skip Ok, w, de_density_scale because in the Planck cosmologies + # they are redshift independent and hence uninvertable, + # *_distance_z1z2 methods take multiple arguments, so require + # special handling + # clone is not a redshift-dependent method + # nu_relative_density is not redshift-dependent in the WMAP cosmologies + skip = ( + "Ok", + "Otot", + "angular_diameter_distance_z1z2", + "clone", + "is_equivalent", + "de_density_scale", + "w", + ) + if str(cosmo.name).startswith("WMAP"): + skip += ("nu_relative_density",) + + methods = inspect.getmembers(cosmo, predicate=inspect.ismethod) + + for name, func in methods: + if name.startswith("_") or name in skip: + continue + fval = func(z) + # we need zmax here to pick the right solution for + # angular_diameter_distance and related methods. + # Be slightly more generous with rtol than the default 1e-8 + # used in z_at_value + got = z_at_value(func, fval, bracket=[0.3, 1.0], ztol=1e-12) + assert allclose(got, z, rtol=2e-11), f"Round-trip testing {name} failed" + + # Test distance functions between two redshifts; only for realizations + if isinstance(getattr(cosmo, "name", None), str): + z2 = 2.0 + func_z1z2 = [ + lambda z1: cosmo.comoving_distance(z1, z2), + lambda z1: cosmo._comoving_transverse_distance_z1z2(z1, z2), + lambda z1: cosmo.angular_diameter_distance(z1, z2), + ] + for func in func_z1z2: + fval = func(z) + assert allclose(z, z_at_value(func, fval, zmax=1.5, ztol=1e-12), rtol=2e-11) diff --git a/astropy/cosmology/_src/tests/helper.py b/astropy/cosmology/_src/tests/helper.py new file mode 100644 index 000000000000..7519304ed788 --- /dev/null +++ b/astropy/cosmology/_src/tests/helper.py @@ -0,0 +1,107 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +""" +This module provides the tools used to internally run the cosmology test suite +from the installed astropy. It makes use of the |pytest| testing framework. +""" + +__all__ = ("clean_registry", "get_redshift_methods") + +import inspect + +import pytest + +from astropy.cosmology._src import core + +############################################################################### +# FUNCTIONS + + +def get_redshift_methods( + cosmology, + include_deprecated: bool = False, + include_private: bool = True, + include_z2: bool = True, +) -> set[str]: + """Get redshift methods from a cosmology. + + Parameters + ---------- + cosmology : |Cosmology| class or instance + include_deprecated : bool, optional + Whether to include deprecated methods, i.e. methods with a + ``__deprecated__`` attribute. Default is False. + include_private : bool, optional + Whether to include private methods, i.e. starts with an underscore. + Default is True. + include_z2 : bool, optional + Whether to include methods that are functions of 2 (or more) redshifts, + not the more common 1 redshift argument. Default is True. + + Returns + ------- + set[str] + The names of the redshift methods on `cosmology`, satisfying + `include_deprecated`, `include_private` and `include_z2`. + """ + # Get all the method names, optionally sieving out private methods + methods = set() + for n in dir(cosmology): + try: # get method, some will error on ABCs + m = getattr(cosmology, n) + except NotImplementedError: + continue + + # Add anything callable, optionally excluding private and deprecated methods. + if ( + callable(m) + and (not n.startswith("_") or include_private) + and (not hasattr(m, "__deprecated__") or include_deprecated) + ): + methods.add(n) + + # Sieve out incompatible methods. + # The index to check for redshift depends on whether cosmology is a class + # or instance and does/doesn't include 'self'. + iz1 = int(isinstance(cosmology, type)) + for n in tuple(methods): + try: + sig = inspect.signature(getattr(cosmology, n)) + except ValueError: # Remove non-introspectable methods. + methods.discard(n) + continue + else: + params = list(sig.parameters.keys()) + + # Remove non redshift methods: + if len(params) <= iz1: # Check there are enough arguments. + methods.discard(n) + elif len(params) >= iz1 + 1 and not params[iz1].startswith( + "z" + ): # First non-self arg is z. + methods.discard(n) + # If methods with 2 z args are not allowed, the following arg is checked. + elif ( + not include_z2 + and (len(params) >= iz1 + 2) + and params[iz1 + 1].startswith("z") + ): + methods.discard(n) + + return methods + + +############################################################################### +# FIXTURES + + +@pytest.fixture +def clean_registry(): + """`pytest.fixture` for clearing and restoring ``_COSMOLOGY_CLASSES``.""" + # TODO! with monkeypatch instead for thread safety. + ORIGINAL_COSMOLOGY_CLASSES = core._COSMOLOGY_CLASSES + core._COSMOLOGY_CLASSES = {} # set as empty dict + + yield core._COSMOLOGY_CLASSES + + core._COSMOLOGY_CLASSES = ORIGINAL_COSMOLOGY_CLASSES diff --git a/astropy/utils/compat/numpy/tests/__init__.py b/astropy/cosmology/_src/tests/io/__init__.py similarity index 100% rename from astropy/utils/compat/numpy/tests/__init__.py rename to astropy/cosmology/_src/tests/io/__init__.py diff --git a/astropy/cosmology/_src/tests/io/base.py b/astropy/cosmology/_src/tests/io/base.py new file mode 100644 index 000000000000..26c3385ba6eb --- /dev/null +++ b/astropy/cosmology/_src/tests/io/base.py @@ -0,0 +1,215 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import pytest + +import astropy.units as u +from astropy.cosmology import Cosmology, Parameter, realizations +from astropy.cosmology import units as cu +from astropy.cosmology._src.core import _COSMOLOGY_CLASSES, dataclass_decorator +from astropy.cosmology.realizations import available + +cosmo_instances = [getattr(realizations, name) for name in available] + + +############################################################################## + + +class IOTestBase: + """Base class for Cosmology I/O tests. + + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. Subclasses must define a :func:`pytest.fixture` + ``cosmo`` that returns/yields an instance of a |Cosmology|. + See ``TestCosmology`` for an example. + """ + + +class ToFromTestMixinBase(IOTestBase): + """Tests for a Cosmology[To/From]Format with some ``format``. + + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. Subclasses must define a :func:`pytest.fixture` + ``cosmo`` that returns/yields an instance of a |Cosmology|. + See ``TestCosmology`` for an example. + """ + + @pytest.fixture(scope="class") + @classmethod + def from_format(cls): + """Convert to Cosmology using ``Cosmology.from_format()``.""" + return Cosmology.from_format + + @pytest.fixture(scope="class") + @classmethod + def to_format(cls, cosmo): + """Convert Cosmology instance using ``.to_format()``.""" + return cosmo.to_format + + @staticmethod + def can_autodentify(format): + """Check whether a format can auto-identify.""" + return format in Cosmology.from_format.registry._identifiers + + +class ReadWriteTestMixinBase(IOTestBase): + """Tests for a Cosmology[Read/Write]. + + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. Subclasses must define a :func:`pytest.fixture` + ``cosmo`` that returns/yields an instance of a |Cosmology|. + See ``TestCosmology`` for an example. + """ + + @pytest.fixture(scope="class") + @classmethod + def read(cls): + """Read Cosmology instance using ``Cosmology.read()``.""" + return Cosmology.read + + @pytest.fixture(scope="class") + @classmethod + def write(cls, cosmo): + """Write Cosmology using ``.write()``.""" + return cosmo.write + + @pytest.fixture + def add_cu(self): + """Add :mod:`astropy.cosmology.units` to the enabled units.""" + # TODO! autoenable 'cu' if cosmology is imported? + with u.add_enabled_units(cu): + yield + + +############################################################################## + + +class IODirectTestBase(IOTestBase): + """Directly test Cosmology I/O functions. + + These functions are not public API and are discouraged from public use, in + favor of the I/O methods on |Cosmology|. They are tested b/c they are used + internally and because some tests for the methods on |Cosmology| don't need + to be run in the |Cosmology| class's large test matrix. + + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. + """ + + @pytest.fixture(scope="class", autouse=True) + @classmethod + def setup(cls): + """Setup and teardown for tests.""" + + @dataclass_decorator + class CosmologyWithKwargs(Cosmology): + Tcmb0: Parameter = Parameter(default=0, unit=u.K) + + def __init__( + self, Tcmb0=0, name="cosmology with kwargs", meta=None, **kwargs + ): + super().__init__(name=name, meta=meta) + self.__dict__["Tcmb0"] = Tcmb0 << u.K + + yield # run tests + + # pop CosmologyWithKwargs from registered classes + # but don't error b/c it can fail in parallel + _COSMOLOGY_CLASSES.pop(CosmologyWithKwargs.__qualname__, None) + + @pytest.fixture(scope="class", params=cosmo_instances) + @classmethod + def cosmo(cls, request): + """Cosmology instance.""" + if isinstance(request.param, str): # CosmologyWithKwargs + return _COSMOLOGY_CLASSES[request.param](Tcmb0=3) + return request.param + + @pytest.fixture(scope="class") + @classmethod + def cosmo_cls(cls, cosmo): + """Cosmology classes.""" + return cosmo.__class__ + + +class ToFromDirectTestBase(IODirectTestBase, ToFromTestMixinBase): + """Directly test ``to/from_``. + + These functions are not public API and are discouraged from public use, in + favor of ``Cosmology.to/from_format(..., format="")``. They are + tested because they are used internally and because some tests for the + methods on |Cosmology| don't need to be run in the |Cosmology| class's + large test matrix. + + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. + + Subclasses should have an attribute ``functions`` which is a dictionary + containing two items: ``"to"=`` and + ``"from"=``. + """ + + @pytest.fixture(scope="class") + @classmethod + def from_format(cls): + """Convert to Cosmology using function ``from``.""" + + def use_from_format(*args, **kwargs): + kwargs.pop("format", None) # specific to Cosmology.from_format + return cls.functions["from"](*args, **kwargs) + + return use_from_format + + @pytest.fixture(scope="class") + @classmethod + def to_format(cls, cosmo): + """Convert Cosmology to format using function ``to``.""" + + def use_to_format(*args, **kwargs): + return cls.functions["to"](cosmo, *args, **kwargs) + + return use_to_format + + +class ReadWriteDirectTestBase(IODirectTestBase, ToFromTestMixinBase): + """Directly test ``read/write_``. + + These functions are not public API and are discouraged from public use, in + favor of ``Cosmology.read/write(..., format="")``. They are tested + because they are used internally and because some tests for the + methods on |Cosmology| don't need to be run in the |Cosmology| class's + large test matrix. + + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. + + Subclasses should have an attribute ``functions`` which is a dictionary + containing two items: ``"read"=`` and + ``"write"=``. + """ + + @pytest.fixture(scope="class") + @classmethod + def read(cls): + """Read Cosmology from file using function ``read``.""" + + def use_read(*args, **kwargs): + kwargs.pop("format", None) # specific to Cosmology.from_format + return cls.functions["read"](*args, **kwargs) + + return use_read + + @pytest.fixture(scope="class") + @classmethod + def write(cls, cosmo): + """Write Cosmology to file using function ``write``.""" + + def use_write(*args, **kwargs): + return cls.functions["write"](cosmo, *args, **kwargs) + + return use_write diff --git a/astropy/cosmology/_src/tests/io/test_.py b/astropy/cosmology/_src/tests/io/test_.py new file mode 100644 index 000000000000..695e11fa0840 --- /dev/null +++ b/astropy/cosmology/_src/tests/io/test_.py @@ -0,0 +1,33 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +"""Test that all expected methods are present, before I/O tests import. + +This file is weirdly named so that it's the first test of I/O. +""" + +from astropy.cosmology.io import convert_registry, readwrite_registry + + +def test_expected_readwrite_io(): + """Test that ONLY the expected I/O is registered.""" + + got = {k for k, _ in readwrite_registry._readers.keys()} + expected = {"ascii.ecsv", "ascii.html", "ascii.mrt"} + + assert got == expected + + +def test_expected_convert_io(): + """Test that ONLY the expected I/O is registered.""" + + got = {k for k, _ in convert_registry._readers.keys()} + expected = { + "astropy.cosmology", + "mapping", + "astropy.model", + "astropy.row", + "astropy.table", + "yaml", + } + + assert got == expected diff --git a/astropy/cosmology/_src/tests/io/test_connect.py b/astropy/cosmology/_src/tests/io/test_connect.py new file mode 100644 index 000000000000..72e042b0b42b --- /dev/null +++ b/astropy/cosmology/_src/tests/io/test_connect.py @@ -0,0 +1,303 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import inspect +import sys + +import pytest + +from astropy import cosmology +from astropy.cosmology import Cosmology, w0wzCDM +from astropy.cosmology._src.tests.io import ( + test_cosmology, + test_ecsv, + test_html, + test_json, + test_latex, + test_mapping, + test_model, + test_mrt, + test_row, + test_table, + test_yaml, +) +from astropy.cosmology.io import readwrite_registry +from astropy.table import QTable, Row +from astropy.utils.compat.optional_deps import HAS_BS4 + +############################################################################### +# SETUP + +cosmo_instances = cosmology.realizations.available + +# Collect the registered read/write formats. +# (format, supports_metadata, has_all_required_dependencies) +readwrite_formats = [ + ("ascii.ecsv", True, True), + ("ascii.html", False, HAS_BS4), + ("ascii.latex", False, True), + ("ascii.mrt", False, True), + ("json", True, True), + ("latex", False, True), +] + + +# Collect all the registered to/from formats. Unfortunately this is NOT +# automatic since the output format class is not stored on the registry. +# (format, data type) +tofrom_formats = [ + ("mapping", dict), + ("yaml", str), + ("astropy.cosmology", Cosmology), + ("astropy.row", Row), + ("astropy.table", QTable), +] + + +############################################################################### + + +class ReadWriteTestMixin( + test_ecsv.ReadWriteECSVTestMixin, + test_html.ReadWriteHTMLTestMixin, + test_json.ReadWriteJSONTestMixin, + test_latex.WriteLATEXTestMixin, + test_mrt.ReadWriteMRTTestMixin, +): + """ + Tests for a CosmologyRead/Write on a |Cosmology|. + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. Subclasses must define a :func:`pytest.fixture` + ``cosmo`` that returns/yields an instance of a |Cosmology|. + See ``TestReadWriteCosmology`` or ``TestCosmology`` for examples. + """ + + @pytest.mark.parametrize("format, metaio, has_deps", readwrite_formats) + def test_readwrite_complete_info(self, cosmo, tmp_path, format, metaio, has_deps): + """ + Test writing from an instance and reading from the base class. + This requires full information. + The round-tripped metadata can be in a different order, so the + OrderedDict must be converted to a dict before testing equality. + """ + if not has_deps: + pytest.skip("missing a dependency") + if (format, Cosmology) not in readwrite_registry._readers: + pytest.xfail(f"no read method is registered for format {format!r}") + + fname = tmp_path / f"{cosmo.name}.{format}" + cosmo.write(fname, format=format) + + # Also test kwarg "overwrite" + assert fname.is_file() + with pytest.raises(IOError): + cosmo.write(fname, format=format, overwrite=False) + + assert fname.exists() # overwrite file existing file + cosmo.write(fname, format=format, overwrite=True) + + # Read back + got = Cosmology.read(fname, format=format) + + assert got == cosmo + assert (not metaio) ^ (dict(got.meta) == dict(cosmo.meta)) + + @pytest.mark.parametrize("format, metaio, has_deps", readwrite_formats) + def test_readwrite_from_subclass_complete_info( + self, cosmo_cls, cosmo, tmp_path, format, metaio, has_deps + ): + """ + Test writing from an instance and reading from that class, when there's + full information saved. + """ + if not has_deps: + pytest.skip("missing a dependency") + if (format, Cosmology) not in readwrite_registry._readers: + pytest.xfail(f"no read method is registered for format {format!r}") + + fname = str(tmp_path / f"{cosmo.name}.{format}") + cosmo.write(fname, format=format) + + # read with the same class that wrote. + got = cosmo_cls.read(fname, format=format) + assert got == cosmo + assert (not metaio) ^ (dict(got.meta) == dict(cosmo.meta)) + + # this should be equivalent to + got = Cosmology.read(fname, format=format, cosmology=cosmo_cls) + assert got == cosmo + assert (not metaio) ^ (dict(got.meta) == dict(cosmo.meta)) + + # and also + got = Cosmology.read(fname, format=format, cosmology=cosmo_cls.__qualname__) + assert got == cosmo + assert (not metaio) ^ (dict(got.meta) == dict(cosmo.meta)) + + +class TestCosmologyReadWrite(ReadWriteTestMixin): + """Test the classes CosmologyRead/Write.""" + + @pytest.fixture(scope="class", params=cosmo_instances) + @classmethod + def cosmo(cls, request): + return getattr(cosmology.realizations, request.param) + + @pytest.fixture(scope="class") + @classmethod + def cosmo_cls(cls, cosmo): + return cosmo.__class__ + + # ============================================================== + + @pytest.mark.parametrize("format, _, has_deps", readwrite_formats) + def test_write_methods_have_explicit_kwarg_overwrite(self, format, _, has_deps): + if not has_deps: + pytest.skip("missing a dependency") + if (format, Cosmology) not in readwrite_registry._readers: + pytest.xfail(f"no read method is registered for format {format!r}") + + writer = readwrite_registry.get_writer(format, Cosmology) + # test in signature + sig = inspect.signature(writer) + assert "overwrite" in sig.parameters + + # also in docstring + if not sys.flags.optimize: + assert "overwrite : bool" in writer.__doc__ + + @pytest.mark.parametrize("format, _, has_deps", readwrite_formats) + def test_readwrite_reader_class_mismatch( + self, cosmo, tmp_path, format, _, has_deps + ): + """Test when the reader class doesn't match the file.""" + if not has_deps: + pytest.skip("missing a dependency") + if (format, Cosmology) not in readwrite_registry._readers: + pytest.xfail(f"no read method is registered for format {format!r}") + + fname = tmp_path / f"{cosmo.name}.{format}" + cosmo.write(fname, format=format) + + # class mismatch + # when reading directly + with pytest.raises(TypeError, match="missing 1 required"): + w0wzCDM.read(fname, format=format) + + with pytest.raises(TypeError, match="missing 1 required"): + Cosmology.read(fname, format=format, cosmology=w0wzCDM) + + # when specifying the class + with pytest.raises(ValueError, match="`cosmology` must be either"): + w0wzCDM.read(fname, format=format, cosmology="FlatLambdaCDM") + + +############################################################################### +# To/From_Format Tests + + +class ToFromFormatTestMixin( + test_cosmology.ToFromCosmologyTestMixin, + test_mapping.ToFromMappingTestMixin, + test_model.ToFromModelTestMixin, + test_row.ToFromRowTestMixin, + test_table.ToFromTableTestMixin, + test_yaml.ToFromYAMLTestMixin, +): + """ + Tests for a Cosmology[To/From]Format on a |Cosmology|. + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. Subclasses must define a :func:`pytest.fixture` + ``cosmo`` that returns/yields an instance of a |Cosmology|. + See ``TestCosmology`` for an example. + """ + + @pytest.mark.parametrize("format, totype", tofrom_formats) + def test_tofromformat_complete_info( + self, cosmo, format, totype, xfail_if_not_registered_with_yaml + ): + """Read tests happen later.""" + # test to_format + obj = cosmo.to_format(format) + assert isinstance(obj, totype) + + # test from_format + got = Cosmology.from_format(obj, format=format) + + # Test autodetect, if enabled + if self.can_autodentify(format): + got2 = Cosmology.from_format(obj) + assert got2 == got # internal consistency + + assert got == cosmo # external consistency + assert got.meta == cosmo.meta + + @pytest.mark.parametrize("format, totype", tofrom_formats) + def test_fromformat_subclass_complete_info( + self, cosmo_cls, cosmo, format, totype, xfail_if_not_registered_with_yaml + ): + """ + Test transforming an instance and parsing from that class, when there's + full information available. + Partial information tests are handled in the Mixin super classes. + """ + # test to_format + obj = cosmo.to_format(format) + assert isinstance(obj, totype) + + # read with the same class that wrote. + got = cosmo_cls.from_format(obj, format=format) + + if self.can_autodentify(format): + got2 = Cosmology.from_format(obj) # and autodetect + assert got2 == got # internal consistency + + assert got == cosmo # external consistency + assert got.meta == cosmo.meta + + # this should be equivalent to + got = Cosmology.from_format(obj, format=format, cosmology=cosmo_cls) + assert got == cosmo + assert got.meta == cosmo.meta + + # and also + got = Cosmology.from_format( + obj, format=format, cosmology=cosmo_cls.__qualname__ + ) + assert got == cosmo + assert got.meta == cosmo.meta + + +class TestCosmologyToFromFormat(ToFromFormatTestMixin): + """Test Cosmology[To/From]Format classes.""" + + @pytest.fixture(scope="class", params=cosmo_instances) + @classmethod + def cosmo(cls, request): + return getattr(cosmology.realizations, request.param) + + @pytest.fixture(scope="class") + @classmethod + def cosmo_cls(cls, cosmo): + return cosmo.__class__ + + # ============================================================== + + @pytest.mark.parametrize("format_type", tofrom_formats) + def test_fromformat_class_mismatch(self, cosmo, format_type): + format, totype = format_type + + # test to_format + obj = cosmo.to_format(format) + assert isinstance(obj, totype) + + # class mismatch + with pytest.raises(TypeError): + w0wzCDM.from_format(obj, format=format) + + with pytest.raises(TypeError): + Cosmology.from_format(obj, format=format, cosmology=w0wzCDM) + + # when specifying the class + with pytest.raises(ValueError, match="`cosmology` must be either"): + w0wzCDM.from_format(obj, format=format, cosmology="FlatLambdaCDM") diff --git a/astropy/cosmology/_src/tests/io/test_cosmology.py b/astropy/cosmology/_src/tests/io/test_cosmology.py new file mode 100644 index 000000000000..6d1bbfade320 --- /dev/null +++ b/astropy/cosmology/_src/tests/io/test_cosmology.py @@ -0,0 +1,57 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import pytest + +from astropy.cosmology._src.io.builtin.cosmology import from_cosmology, to_cosmology + +from .base import IODirectTestBase, ToFromTestMixinBase + +############################################################################### + + +class ToFromCosmologyTestMixin(ToFromTestMixinBase): + """ + Tests for a Cosmology[To/From]Format with ``format="astropy.cosmology"``. + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. Subclasses must define a :func:`pytest.fixture` + ``cosmo`` that returns/yields an instance of a |Cosmology|. + See ``TestCosmology`` for an example. + """ + + def test_to_cosmology_default(self, cosmo, to_format): + """Test cosmology -> cosmology.""" + newcosmo = to_format("astropy.cosmology") + assert newcosmo is cosmo + + def test_from_not_cosmology(self, cosmo, from_format): + """Test incorrect type in ``Cosmology``.""" + with pytest.raises(TypeError): + from_format("NOT A COSMOLOGY", format="astropy.cosmology") + + def test_from_cosmology_default(self, cosmo, from_format): + """Test cosmology -> cosmology.""" + newcosmo = from_format(cosmo) + assert newcosmo is cosmo + + @pytest.mark.parametrize("format", [True, False, None, "astropy.cosmology"]) + def test_is_equivalent_to_cosmology(self, cosmo, to_format, format): + """Test :meth:`astropy.cosmology.Cosmology.is_equivalent`. + + This test checks that Cosmology equivalency can be extended to any + Python object that can be converted to a Cosmology -- in this case + a Cosmology! Since it's the identity conversion, the cosmology is + always equivalent to itself, regardless of ``format``. + """ + obj = to_format("astropy.cosmology") + assert obj is cosmo + + is_equiv = cosmo.is_equivalent(obj, format=format) + assert is_equiv is True # equivalent to self + + +class TestToFromCosmology(IODirectTestBase, ToFromCosmologyTestMixin): + """Directly test ``to/from_cosmology``.""" + + def setup_class(self): + self.functions = {"to": to_cosmology, "from": from_cosmology} diff --git a/astropy/cosmology/_src/tests/io/test_ecsv.py b/astropy/cosmology/_src/tests/io/test_ecsv.py new file mode 100644 index 000000000000..1de3dafec8c3 --- /dev/null +++ b/astropy/cosmology/_src/tests/io/test_ecsv.py @@ -0,0 +1,230 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import pytest + +from astropy.cosmology._src.core import _COSMOLOGY_CLASSES +from astropy.cosmology._src.io.builtin.ecsv import read_ecsv, write_ecsv +from astropy.table import QTable, Table, vstack + +from .base import ReadWriteDirectTestBase, ReadWriteTestMixinBase + +############################################################################### + + +class ReadWriteECSVTestMixin(ReadWriteTestMixinBase): + """ + Tests for a Cosmology[Read/Write] with ``format="ascii.ecsv"``. + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. Subclasses must define a :func:`pytest.fixture` + ``cosmo`` that returns/yields an instance of a |Cosmology|. + See ``TestCosmology`` for an example. + """ + + def test_to_ecsv_bad_index(self, read, write, tmp_path): + """Test if argument ``index`` is incorrect""" + fp = tmp_path / "test_to_ecsv_bad_index.ecsv" + + write(fp, format="ascii.ecsv") + + # single-row table and has a non-0/None index + with pytest.raises(IndexError, match="index 2 out of range"): + read(fp, index=2, format="ascii.ecsv") + + # string index where doesn't match + with pytest.raises(KeyError, match="No matches found for key"): + read(fp, index="row 0", format="ascii.ecsv") + + # ----------------------- + + def test_to_ecsv_failed_cls(self, write, tmp_path): + """Test failed table type.""" + fp = tmp_path / "test_to_ecsv_failed_cls.ecsv" + + with pytest.raises(TypeError, match="'cls' must be"): + write(fp, format="ascii.ecsv", cls=list) + + @pytest.mark.parametrize("tbl_cls", [QTable, Table]) + def test_to_ecsv_cls(self, write, tbl_cls, tmp_path): + fp = tmp_path / "test_to_ecsv_cls.ecsv" + write(fp, format="ascii.ecsv", cls=tbl_cls) + + # ----------------------- + + @pytest.mark.parametrize("in_meta", [True, False]) + def test_to_ecsv_in_meta(self, cosmo_cls, write, in_meta, tmp_path, add_cu): + """Test where the cosmology class is placed.""" + fp = tmp_path / "test_to_ecsv_in_meta.ecsv" + write(fp, format="ascii.ecsv", cosmology_in_meta=in_meta) + + # if it's in metadata, it's not a column. And vice versa. + tbl = QTable.read(fp) + if in_meta: + assert tbl.meta["cosmology"] == cosmo_cls.__qualname__ + assert "cosmology" not in tbl.colnames # not also a column + else: + assert tbl["cosmology"][0] == cosmo_cls.__qualname__ + assert "cosmology" not in tbl.meta + + # ----------------------- + + def test_readwrite_ecsv_instance( + self, cosmo_cls, cosmo, read, write, tmp_path, add_cu + ): + """Test cosmology -> ascii.ecsv -> cosmology.""" + fp = tmp_path / "test_readwrite_ecsv_instance.ecsv" + + # ------------ + # To Table + + write(fp, format="ascii.ecsv") + + # some checks on the saved file + tbl = QTable.read(fp) + assert tbl.meta["cosmology"] == cosmo_cls.__qualname__ + assert tbl["name"] == cosmo.name + + # ------------ + # From Table + + tbl["mismatching"] = "will error" + tbl.write(fp, format="ascii.ecsv", overwrite=True) + + # tests are different if the last argument is a **kwarg + if cosmo._init_has_kwargs: + got = read(fp, format="ascii.ecsv") + + assert got.__class__ is cosmo_cls + assert got.name == cosmo.name + assert "mismatching" not in got.meta + + return # don't continue testing + + # read with mismatching parameters errors + with pytest.raises(TypeError, match="there are unused parameters"): + read(fp, format="ascii.ecsv") + + # unless mismatched are moved to meta + got = read(fp, format="ascii.ecsv", move_to_meta=True) + assert got == cosmo + assert got.meta["mismatching"] == "will error" + + # it won't error if everything matches up + tbl.remove_column("mismatching") + tbl.write(fp, format="ascii.ecsv", overwrite=True) + got = read(fp, format="ascii.ecsv") + assert got == cosmo + + # and it will also work if the cosmology is a class + # Note this is not the default output of ``write``. + tbl.meta["cosmology"] = _COSMOLOGY_CLASSES[tbl.meta["cosmology"]] + got = read(fp, format="ascii.ecsv") + assert got == cosmo + + # also it auto-identifies 'format' + got = read(fp) + assert got == cosmo + + def test_readwrite_ecsv_renamed_columns( + self, cosmo_cls, cosmo, read, write, tmp_path, add_cu + ): + """Test rename argument to read/write.""" + fp = tmp_path / "test_readwrite_ecsv_rename.ecsv" + rename = {"name": "cosmo_name"} + + write(fp, format="ascii.ecsv", rename=rename) + + tbl = QTable.read(fp, format="ascii.ecsv") + + assert "name" not in tbl.colnames + assert "cosmo_name" in tbl.colnames + + # Errors if reading + with pytest.raises( + TypeError, match="there are unused parameters {'cosmo_name':" + ): + read(fp) + + # Roundtrips + inv_rename = {v: k for k, v in rename.items()} + got = read(fp, rename=inv_rename) + assert got == cosmo + + def test_readwrite_ecsv_subclass_partial_info( + self, cosmo_cls, cosmo, read, write, tmp_path, add_cu + ): + """ + Test writing from an instance and reading from that class. + This works with missing information. + """ + fp = tmp_path / "test_read_ecsv_subclass_partial_info.ecsv" + + # test write + write(fp, format="ascii.ecsv") + + # partial information + tbl = QTable.read(fp) + tbl.meta.pop("cosmology", None) + del tbl["Tcmb0"] + tbl.write(fp, overwrite=True) + + # read with the same class that wrote fills in the missing info with + # the default value + got = cosmo_cls.read(fp, format="ascii.ecsv") + got2 = read(fp, format="ascii.ecsv", cosmology=cosmo_cls) + got3 = read(fp, format="ascii.ecsv", cosmology=cosmo_cls.__qualname__) + + assert (got == got2) and (got2 == got3) # internal consistency + + # not equal, because Tcmb0 is changed, which also changes m_nu + assert got != cosmo + assert got.Tcmb0 == cosmo_cls.parameters["Tcmb0"].default + assert got.clone(name=cosmo.name, Tcmb0=cosmo.Tcmb0, m_nu=cosmo.m_nu) == cosmo + # but the metadata is the same + assert got.meta == cosmo.meta + + def test_readwrite_ecsv_mutlirow(self, cosmo, read, write, tmp_path, add_cu): + """Test if table has multiple rows.""" + fp = tmp_path / "test_readwrite_ecsv_mutlirow.ecsv" + + # Make + cosmo1 = cosmo.clone(name="row 0") + cosmo2 = cosmo.clone(name="row 2") + tbl = vstack( + [c.to_format("astropy.table") for c in (cosmo1, cosmo, cosmo2)], + metadata_conflicts="silent", + ) + tbl.write(fp, format="ascii.ecsv") + + # ------------ + # From Table + + # it will error on a multi-row table + with pytest.raises(ValueError, match="need to select a specific row"): + read(fp, format="ascii.ecsv") + + # unless the index argument is provided + got = read(fp, index=1, format="ascii.ecsv") + assert got == cosmo + + # the index can be a string + got = read(fp, index=cosmo.name, format="ascii.ecsv") + assert got == cosmo + + # it's better if the table already has an index + # this will be identical to the previous ``got`` + tbl.add_index("name") + got2 = read(fp, index=cosmo.name, format="ascii.ecsv") + assert got2 == cosmo + + +class TestReadWriteECSV(ReadWriteDirectTestBase, ReadWriteECSVTestMixin): + """ + Directly test ``read/write_ecsv``. + These are not public API and are discouraged from use, in favor of + ``Cosmology.read/write(..., format="ascii.ecsv")``, but should be + tested regardless b/c they are used internally. + """ + + def setup_class(self): + self.functions = {"read": read_ecsv, "write": write_ecsv} diff --git a/astropy/cosmology/_src/tests/io/test_html.py b/astropy/cosmology/_src/tests/io/test_html.py new file mode 100644 index 000000000000..30478bcfb1be --- /dev/null +++ b/astropy/cosmology/_src/tests/io/test_html.py @@ -0,0 +1,261 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import pytest + +import astropy.units as u +from astropy.cosmology._src.io.builtin.html import ( + _FORMAT_TABLE, + read_html_table, + write_html_table, +) +from astropy.table import QTable, Table, vstack +from astropy.utils.compat.optional_deps import HAS_BS4 + +from .base import ReadWriteDirectTestBase, ReadWriteTestMixinBase + +############################################################################### + + +class ReadWriteHTMLTestMixin(ReadWriteTestMixinBase): + """ + Tests for a Cosmology[Read/Write] with ``format="ascii.html"``. + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. Subclasses must define a :func:`pytest.fixture` + ``cosmo`` that returns/yields an instance of a |Cosmology|. + See ``TestCosmology`` for an example. + """ + + @pytest.mark.skipif(not HAS_BS4, reason="requires beautifulsoup4") + def test_to_html_table_bad_index(self, read, write, tmp_path): + """Test if argument ``index`` is incorrect""" + fp = tmp_path / "test_to_html_table_bad_index.html" + + write(fp, format="ascii.html") + + # single-row table and has a non-0/None index + with pytest.raises(IndexError, match="index 2 out of range"): + read(fp, index=2, format="ascii.html") + + # string index where doesn't match + with pytest.raises(KeyError, match="No matches found for key"): + read(fp, index="row 0", format="ascii.html") + + # ----------------------- + + @pytest.mark.skipif(not HAS_BS4, reason="requires beautifulsoup4") + def test_to_html_table_failed_cls(self, write, tmp_path): + """Test failed table type.""" + fp = tmp_path / "test_to_html_table_failed_cls.html" + + with pytest.raises(TypeError, match="'cls' must be"): + write(fp, format="ascii.html", cls=list) + + @pytest.mark.parametrize("tbl_cls", [QTable, Table]) + @pytest.mark.skipif(not HAS_BS4, reason="requires beautifulsoup4") + def test_to_html_table_cls(self, write, tbl_cls, tmp_path): + fp = tmp_path / "test_to_html_table_cls.html" + write(fp, format="ascii.html", cls=tbl_cls) + + # ----------------------- + + @pytest.mark.skipif(not HAS_BS4, reason="requires beautifulsoup4") + def test_readwrite_html_table_instance( + self, cosmo_cls, cosmo, read, write, tmp_path, add_cu + ): + """Test cosmology -> ascii.html -> cosmology.""" + fp = tmp_path / "test_readwrite_html_table_instance.html" + + # ------------ + # To Table + + write(fp, format="ascii.html") + + # some checks on the saved file + tbl = QTable.read(fp) + # assert tbl.meta["cosmology"] == cosmo_cls.__qualname__ # metadata read not implemented + assert tbl["name"] == cosmo.name + + # ------------ + # From Table + + tbl["mismatching"] = "will error" + tbl.write(fp, format="ascii.html", overwrite=True) + + # tests are different if the last argument is a **kwarg + if cosmo._init_has_kwargs: + got = read(fp, format="ascii.html") + + assert got.__class__ is cosmo_cls + assert got.name == cosmo.name + # assert "mismatching" not in got.meta # metadata read not implemented + + return # don't continue testing + + # read with mismatching parameters errors + with pytest.raises(TypeError, match="there are unused parameters"): + read(fp, format="ascii.html") + + # unless mismatched are moved to meta + got = read(fp, format="ascii.html", move_to_meta=True) + assert got == cosmo + # assert got.meta["mismatching"] == "will error" # metadata read not implemented + + # it won't error if everything matches up + tbl.remove_column("mismatching") + tbl.write(fp, format="ascii.html", overwrite=True) + got = read(fp, format="ascii.html") + assert got == cosmo + + # and it will also work if the cosmology is a class + # Note this is not the default output of ``write``. + # tbl.meta["cosmology"] = _COSMOLOGY_CLASSES[tbl.meta["cosmology"]] # + # metadata read not implemented + got = read(fp, format="ascii.html") + assert got == cosmo + + got = read(fp) + assert got == cosmo + + @pytest.mark.skipif(not HAS_BS4, reason="requires beautifulsoup4") + def test_rename_html_table_columns(self, read, write, tmp_path): + """Tests renaming columns""" + fp = tmp_path / "test_rename_html_table_columns.html" + + write(fp, format="ascii.html", latex_names=True) + + tbl = QTable.read(fp) + + # asserts each column name has not been reverted yet + # For now, Cosmology class and name are stored in first 2 slots + for column_name in tbl.colnames[2:]: + assert column_name in _FORMAT_TABLE.values() + + cosmo = read(fp, format="ascii.html") + converted_tbl = cosmo.to_format("astropy.table") + + # asserts each column name has been reverted + # cosmology name is still stored in first slot + for column_name in converted_tbl.colnames[1:]: + assert column_name in _FORMAT_TABLE.keys() + + @pytest.mark.skipif(not HAS_BS4, reason="requires beautifulsoup4") + @pytest.mark.parametrize("latex_names", [True, False]) + def test_readwrite_html_subclass_partial_info( + self, cosmo_cls, cosmo, read, write, latex_names, tmp_path, add_cu + ): + """ + Test writing from an instance and reading from that class. + This works with missing information. + """ + fp = tmp_path / "test_read_html_subclass_partial_info.html" + + # test write + write(fp, format="ascii.html", latex_names=latex_names) + + # partial information + tbl = QTable.read(fp) + + # tbl.meta.pop("cosmology", None) # metadata not implemented + cname = "$$T_{0}$$" if latex_names else "Tcmb0" + del tbl[cname] # format is not converted to original units + tbl.write(fp, overwrite=True) + + # read with the same class that wrote fills in the missing info with + # the default value + got = cosmo_cls.read(fp, format="ascii.html") + got2 = read(fp, format="ascii.html", cosmology=cosmo_cls) + got3 = read(fp, format="ascii.html", cosmology=cosmo_cls.__qualname__) + + assert (got == got2) and (got2 == got3) # internal consistency + + # not equal, because Tcmb0 is changed, which also changes m_nu + assert got != cosmo + assert got.Tcmb0 == cosmo_cls.parameters["Tcmb0"].default + assert got.clone(name=cosmo.name, Tcmb0=cosmo.Tcmb0, m_nu=cosmo.m_nu) == cosmo + # but the metadata is the same + # assert got.meta == cosmo.meta # metadata read not implemented + + @pytest.mark.skipif(not HAS_BS4, reason="requires beautifulsoup4") + def test_readwrite_html_mutlirow(self, cosmo, read, write, tmp_path, add_cu): + """Test if table has multiple rows.""" + fp = tmp_path / "test_readwrite_html_mutlirow.html" + + # Make + cosmo1 = cosmo.clone(name="row 0") + cosmo2 = cosmo.clone(name="row 2") + table = vstack( + [c.to_format("astropy.table") for c in (cosmo1, cosmo, cosmo2)], + metadata_conflicts="silent", + ) + + cosmo_cls = type(cosmo) + assert cosmo is not None + + for n, col in zip(table.colnames, table.itercols()): + if n not in cosmo_cls.parameters: + continue + param = cosmo_cls.parameters[n] + if param.unit in (None, u.one): + continue + # Replace column with unitless version + table.replace_column(n, (col << param.unit).value, copy=False) + + table.write(fp, format="ascii.html") + + # ------------ + # From Table + + # it will error on a multi-row table + with pytest.raises(ValueError, match="need to select a specific row"): + read(fp, format="ascii.html") + + # unless the index argument is provided + got = cosmo_cls.read(fp, index=1, format="ascii.html") + # got = read(fp, index=1, format="ascii.html") + assert got == cosmo + + # the index can be a string + got = cosmo_cls.read(fp, index=cosmo.name, format="ascii.html") + assert got == cosmo + + # it's better if the table already has an index + # this will be identical to the previous ``got`` + table.add_index("name") + got2 = cosmo_cls.read(fp, index=cosmo.name, format="ascii.html") + assert got2 == cosmo + + +class TestReadWriteHTML(ReadWriteDirectTestBase, ReadWriteHTMLTestMixin): + """ + Directly test ``read/write_html``. + These are not public API and are discouraged from use, in favor of + ``Cosmology.read/write(..., format="ascii.html")``, but should be + tested regardless b/c they are used internally. + """ + + def setup_class(self): + self.functions = {"read": read_html_table, "write": write_html_table} + + @pytest.mark.skipif(not HAS_BS4, reason="requires beautifulsoup4") + def test_rename_direct_html_table_columns(self, read, write, tmp_path): + """Tests renaming columns""" + + fp = tmp_path / "test_rename_html_table_columns.html" + + write(fp, format="ascii.html", latex_names=True) + + tbl = QTable.read(fp) + + # asserts each column name has not been reverted yet + for column_name in tbl.colnames[2:]: + # for now, Cosmology as metadata and name is stored in first 2 slots + assert column_name in _FORMAT_TABLE.values() + + cosmo = read(fp, format="ascii.html") + converted_tbl = cosmo.to_format("astropy.table") + + # asserts each column name has been reverted + for column_name in converted_tbl.colnames[1:]: + # for now now, metadata is still stored in first slot + assert column_name in _FORMAT_TABLE.keys() diff --git a/astropy/cosmology/_src/tests/io/test_json.py b/astropy/cosmology/_src/tests/io/test_json.py new file mode 100644 index 000000000000..4f04f6329a82 --- /dev/null +++ b/astropy/cosmology/_src/tests/io/test_json.py @@ -0,0 +1,167 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import json +import os +from pathlib import Path + +import pytest + +import astropy.units as u +from astropy.cosmology import Cosmology +from astropy.cosmology import units as cu +from astropy.cosmology.io import readwrite_registry + +from .base import ReadWriteDirectTestBase, ReadWriteTestMixinBase + +############################################################################### + + +def read_json(filename, **kwargs): + """Read JSON. + + Parameters + ---------- + filename : str | bytes | os.PathLike + **kwargs + Keyword arguments into :meth:`~astropy.cosmology.Cosmology.from_format` + + Returns + ------- + `~astropy.cosmology.Cosmology` instance + """ + # read + if isinstance(filename, (str, bytes, os.PathLike)): + data = Path(filename).read_text() + else: # file-like : this also handles errors in dumping + data = filename.read() + + mapping = json.loads(data) # parse json mappable to dict + + # deserialize Quantity + with u.add_enabled_units(cu.redshift): + for k, v in mapping.items(): + if isinstance(v, dict) and "value" in v and "unit" in v: + mapping[k] = u.Quantity(v["value"], v["unit"]) + for k, v in mapping.get("meta", {}).items(): # also the metadata + if isinstance(v, dict) and "value" in v and "unit" in v: + mapping["meta"][k] = u.Quantity(v["value"], v["unit"]) + + return Cosmology.from_format(mapping, format="mapping", **kwargs) + + +def write_json(cosmology, file, *, overwrite=False): + """Write Cosmology to JSON. + + Parameters + ---------- + cosmology : `astropy.cosmology.Cosmology` subclass instance + file : path-like or file-like + overwrite : bool (optional, keyword-only) + """ + data = cosmology.to_format("mapping") # start by turning into dict + data["cosmology"] = data["cosmology"].__qualname__ + + # serialize Quantity + for k, v in data.items(): + if isinstance(v, u.Quantity): + data[k] = {"value": v.value.tolist(), "unit": str(v.unit)} + for k, v in data.get("meta", {}).items(): # also serialize the metadata + if isinstance(v, u.Quantity): + data["meta"][k] = {"value": v.value.tolist(), "unit": str(v.unit)} + + # check that file exists and whether to overwrite. + file = Path(file) + if file.exists() and not overwrite: + raise OSError(f"{file} exists. Set 'overwrite' to write over.") + with file.open("w") as write_file: + json.dump(data, write_file) + + +def json_identify(origin, filepath, fileobj, *args, **kwargs): + return filepath is not None and filepath.endswith(".json") + + +############################################################################### + + +class ReadWriteJSONTestMixin(ReadWriteTestMixinBase): + """ + Tests for a Cosmology[Read/Write] with ``format="json"``. + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. Subclasses must define a :func:`pytest.fixture` + ``cosmo`` that returns/yields an instance of a |Cosmology|. + See ``TestCosmology`` for an example. + """ + + @pytest.fixture(scope="class", autouse=True) + @classmethod + def register_and_unregister_json(cls): + """Setup & teardown for JSON read/write tests.""" + # Register + readwrite_registry.register_reader("json", Cosmology, read_json, force=True) + readwrite_registry.register_writer("json", Cosmology, write_json, force=True) + readwrite_registry.register_identifier( + "json", Cosmology, json_identify, force=True + ) + + yield # Run all tests in class + + # Unregister + readwrite_registry.unregister_reader("json", Cosmology) + readwrite_registry.unregister_writer("json", Cosmology) + readwrite_registry.unregister_identifier("json", Cosmology) + + # ======================================================================== + + def test_readwrite_json_subclass_partial_info( + self, cosmo_cls, cosmo, read, write, tmp_path, add_cu + ): + """ + Test writing from an instance and reading from that class. + This works with missing information. + """ + fp = tmp_path / "test_readwrite_json_subclass_partial_info.json" + + # test write + cosmo.write(fp, format="json") + + # partial information + with fp.open() as file: + L = file.readline() + L = ( + L[: L.index('"cosmology":')] + L[L.index(", ") + 2 :] + ) # remove cosmology : #203 + i = L.index('"Tcmb0":') # delete Tcmb0 + L = ( + L[:i] + L[L.index(", ", L.index(", ", i) + 1) + 2 :] + ) # second occurrence : #203 + + tempfname = tmp_path / f"{cosmo.name}_temp.json" + tempfname.write_text("".join(L)) + # read with the same class that wrote fills in the missing info with + # the default value + got = cosmo_cls.read(tempfname, format="json") + got2 = read(tempfname, format="json", cosmology=cosmo_cls) + got3 = read(tempfname, format="json", cosmology=cosmo_cls.__qualname__) + + assert (got == got2) and (got2 == got3) # internal consistency + + # not equal, because Tcmb0 is changed, which also changes m_nu + assert got != cosmo + assert got.Tcmb0 == cosmo_cls.parameters["Tcmb0"].default + assert got.clone(name=cosmo.name, Tcmb0=cosmo.Tcmb0, m_nu=cosmo.m_nu) == cosmo + # but the metadata is the same + assert got.meta == cosmo.meta + + +class TestReadWriteJSON(ReadWriteDirectTestBase, ReadWriteJSONTestMixin): + """ + Directly test ``read/write_json``. + These are not public API and are discouraged from use, in favor of + ``Cosmology.read/write(..., format="json")``, but should be + tested regardless b/c they are used internally. + """ + + def setup_class(self): + self.functions = {"read": read_json, "write": write_json} diff --git a/astropy/cosmology/_src/tests/io/test_latex.py b/astropy/cosmology/_src/tests/io/test_latex.py new file mode 100644 index 000000000000..2f73cc2080be --- /dev/null +++ b/astropy/cosmology/_src/tests/io/test_latex.py @@ -0,0 +1,86 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import pytest + +from astropy.cosmology._src.io.builtin.latex import _FORMAT_TABLE, write_latex +from astropy.io.registry.base import IORegistryError +from astropy.table import QTable, Table + +from .base import ReadWriteDirectTestBase, ReadWriteTestMixinBase + + +class WriteLATEXTestMixin(ReadWriteTestMixinBase): + """ + Tests for a Cosmology[Write] with ``format="latex"``. + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. Subclasses must define a :func:`pytest.fixture` + ``cosmo`` that returns/yields an instance of a |Cosmology|. + See ``TestCosmology`` for an example. + """ + + def test_to_latex_failed_cls(self, write, tmp_path): + """Test failed table type.""" + fp = tmp_path / "test_to_latex_failed_cls.tex" + + with pytest.raises(TypeError, match="'cls' must be"): + write(fp, cls=list) + + @pytest.mark.parametrize("tbl_cls", [QTable, Table]) + def test_to_latex_cls(self, write, tbl_cls, tmp_path): + fp = tmp_path / "test_to_latex_cls.tex" + write(fp, cls=tbl_cls) + + def test_latex_columns(self, write, tmp_path): + fp = tmp_path / "test_rename_latex_columns.tex" + write(fp, latex_names=True) + tbl = QTable.read(fp) + # asserts each column name has not been reverted yet + # For now, Cosmology class and name are stored in first 2 slots + for column_name in tbl.colnames[2:]: + assert column_name in _FORMAT_TABLE.values() + + def test_write_latex_invalid_path(self, write): + """Test passing an invalid path""" + invalid_fp = "" + with pytest.raises(FileNotFoundError, match="No such file or directory"): + write(invalid_fp, format="ascii.latex") + + def test_write_latex_false_overwrite(self, write, tmp_path): + """Test to write a LaTeX file without overwriting an existing file""" + # Test that passing an invalid path to write_latex() raises a IOError + fp = tmp_path / "test_write_latex_false_overwrite.tex" + write(fp) + with pytest.raises(OSError, match="overwrite=True"): + write(fp, overwrite=False) + + def test_write_latex_unsupported_format(self, write, tmp_path): + """Test for unsupported format""" + fp = tmp_path / "test_write_latex_unsupported_format.tex" + invalid_format = "unsupported" + with pytest.raises((ValueError, IORegistryError)) as exc_info: + pytest.raises(ValueError, match="format must be 'ascii.latex'") + pytest.raises(IORegistryError, match="No writer defined for format") + write(fp, format=invalid_format) + + +class TestReadWriteLaTex(ReadWriteDirectTestBase, WriteLATEXTestMixin): + """ + Directly test ``write_latex``. + These are not public API and are discouraged from use, in favor of + ``Cosmology.write(..., format="latex")``, but should be + tested regardless b/c they are used internally. + """ + + def setup_class(self): + self.functions = {"write": write_latex} + + def test_rename_direct_latex_columns(self, write, tmp_path): + """Tests renaming columns""" + fp = tmp_path / "test_rename_latex_columns.tex" + write(fp, latex_names=True) + tbl = QTable.read(fp) + # asserts each column name has not been reverted yet + for column_name in tbl.colnames[2:]: + # for now, Cosmology as metadata and name is stored in first 2 slots + assert column_name in _FORMAT_TABLE.values() diff --git a/astropy/cosmology/_src/tests/io/test_mapping.py b/astropy/cosmology/_src/tests/io/test_mapping.py new file mode 100644 index 000000000000..10067efceb51 --- /dev/null +++ b/astropy/cosmology/_src/tests/io/test_mapping.py @@ -0,0 +1,240 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +from collections import OrderedDict + +import numpy as np +import pytest + +from astropy.cosmology import Cosmology +from astropy.cosmology._src.io.builtin.mapping import from_mapping, to_mapping + +from .base import ToFromDirectTestBase, ToFromTestMixinBase + +############################################################################### + + +class ToFromMappingTestMixin(ToFromTestMixinBase): + """Tests for a Cosmology[To/From]Format with ``format="mapping"``. + + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. Subclasses must define a :func:`pytest.fixture` + ``cosmo`` that returns/yields an instance of a |Cosmology|. + See ``TestCosmology`` for an example. + """ + + def test_to_mapping_default(self, cosmo, to_format): + """Test default usage of Cosmology -> mapping.""" + m = to_format("mapping") + keys = tuple(m.keys()) + + assert isinstance(m, dict) + # Check equality of all expected items + assert keys[0] == "cosmology" + assert m.pop("cosmology") is cosmo.__class__ + assert keys[1] == "name" + assert m.pop("name") == cosmo.name + for i, (k, v) in enumerate(cosmo.parameters.items(), start=2): + assert keys[i] == k + np.testing.assert_array_equal(m.pop(k), v) + assert keys[-1] == "meta" + assert m.pop("meta") == cosmo.meta + + # No unexpected items + assert not m + + def test_to_mapping_wrong_cls(self, to_format): + """Test incorrect argument ``cls`` in ``to_mapping()``.""" + with pytest.raises(TypeError, match="'cls' must be"): + to_format("mapping", cls=list) + + @pytest.mark.parametrize("map_cls", [dict, OrderedDict]) + def test_to_mapping_cls(self, to_format, map_cls): + """Test argument ``cls`` in ``to_mapping()``.""" + m = to_format("mapping", cls=map_cls) + assert isinstance(m, map_cls) # test type + + def test_to_mapping_cosmology_as_str(self, cosmo_cls, to_format): + """Test argument ``cosmology_as_str`` in ``to_mapping()``.""" + default = to_format("mapping") + + # Cosmology is the class + m = to_format("mapping", cosmology_as_str=False) + assert isinstance(m["cosmology"], type) + assert cosmo_cls is m["cosmology"] + + assert m == default # False is the default option + + # Cosmology is a string + m = to_format("mapping", cosmology_as_str=True) + assert isinstance(m["cosmology"], str) + assert m["cosmology"] == cosmo_cls.__qualname__ # Correct class + assert tuple(m.keys())[0] == "cosmology" # Stayed at same index + + def test_tofrom_mapping_cosmology_as_str(self, cosmo, to_format, from_format): + """Test roundtrip with ``cosmology_as_str=True``. + + The test for the default option (`False`) is in ``test_tofrom_mapping_instance``. + """ + m = to_format("mapping", cosmology_as_str=True) + + got = from_format(m, format="mapping") + assert got == cosmo + assert got.meta == cosmo.meta + + def test_to_mapping_move_from_meta(self, to_format): + """Test argument ``move_from_meta`` in ``to_mapping()``.""" + default = to_format("mapping") + + # Metadata is 'separate' from main mapping + m = to_format("mapping", move_from_meta=False) + assert "meta" in m.keys() + assert not any(k in m for k in m["meta"]) # Not added to main + + assert m == default # False is the default option + + # Metadata is mixed into main mapping. + m = to_format("mapping", move_from_meta=True) + assert "meta" not in m.keys() + assert all(k in m for k in default["meta"]) # All added to main + # The parameters take precedence over the metadata + assert all(np.array_equal(v, m[k]) for k, v in default.items() if k != "meta") + + def test_tofrom_mapping_move_tofrom_meta(self, cosmo, to_format, from_format): + """Test roundtrip of ``move_from/to_meta`` in ``to/from_mapping()``.""" + # Metadata is mixed into main mapping. + m = to_format("mapping", move_from_meta=True) + # (Just adding something to ensure there's 'metadata') + m["mismatching"] = "will error" + + # (Tests are different if the last argument is a **kwarg) + if cosmo._init_has_kwargs: + got = from_format(m, format="mapping") + + assert got.name == cosmo.name + assert "mismatching" not in got.meta + + return # don't continue testing + + # Reading with mismatching parameters errors... + with pytest.raises(TypeError, match="there are unused parameters"): + from_format(m, format="mapping") + + # unless mismatched are moved to meta. + got = from_format(m, format="mapping", move_to_meta=True) + assert got == cosmo # (Doesn't check metadata) + assert got.meta["mismatching"] == "will error" + + def test_to_mapping_rename_conflict(self, cosmo, to_format): + """Test ``rename`` in ``to_mapping()``.""" + to_rename = {"name": "name", "H0": "H_0"} + match = ( + "'renames' values must be disjoint from 'map' keys, " + "the common keys are: {'name'}" + ) + with pytest.raises(ValueError, match=match): + to_format("mapping", rename=to_rename) + + def test_from_mapping_rename_conflict(self, cosmo, to_format, from_format): + """Test ``rename`` in `from_mapping()``.""" + m = to_format("mapping") + + match = ( + "'renames' values must be disjoint from 'map' keys, " + "the common keys are: {'name'}" + ) + with pytest.raises(ValueError, match=match): + from_format(m, format="mapping", rename={"name": "name", "H0": "H_0"}) + + def test_tofrom_mapping_rename_roundtrip(self, cosmo, to_format, from_format): + """Test roundtrip in ``to/from_mapping()`` with ``rename``.""" + to_rename = {"name": "cosmo_name"} + m = to_format("mapping", rename=to_rename) + + assert "name" not in m + assert "cosmo_name" in m + + # Wrong names = error + with pytest.raises( + TypeError, match="there are unused parameters {'cosmo_name':" + ): + from_format(m, format="mapping") + + # Roundtrip. correct names = success + from_rename = {v: k for k, v in to_rename.items()} + got = from_format(m, format="mapping", rename=from_rename) + assert got == cosmo + + # ----------------------------------------------------- + + def test_from_not_mapping(self, cosmo, from_format): + """Test incorrect map type in ``from_mapping()``.""" + with pytest.raises((TypeError, ValueError)): + from_format("NOT A MAP", format="mapping") + + def test_from_mapping_default(self, cosmo, to_format, from_format): + """Test (cosmology -> Mapping) -> cosmology.""" + m = to_format("mapping") + + # Read from exactly as given. + got = from_format(m, format="mapping") + assert got == cosmo + assert got.meta == cosmo.meta + + # Reading auto-identifies 'format' + got = from_format(m) + assert got == cosmo + assert got.meta == cosmo.meta + + def test_fromformat_subclass_partial_info_mapping(self, cosmo): + """ + Test writing from an instance and reading from that class. + This works with missing information. + """ + m = cosmo.to_format("mapping") + + # partial information + m.pop("cosmology", None) + m.pop("Tcmb0", None) + + # read with the same class that wrote fills in the missing info with + # the default value + got = cosmo.__class__.from_format(m, format="mapping") + got2 = Cosmology.from_format(m, format="mapping", cosmology=cosmo.__class__) + got3 = Cosmology.from_format( + m, format="mapping", cosmology=cosmo.__class__.__qualname__ + ) + + assert (got == got2) and (got2 == got3) # internal consistency + + # not equal, because Tcmb0 is changed, which also changes m_nu + assert got != cosmo + assert got.Tcmb0 == cosmo.__class__.parameters["Tcmb0"].default + assert got.clone(name=cosmo.name, Tcmb0=cosmo.Tcmb0, m_nu=cosmo.m_nu) == cosmo + # but the metadata is the same + assert got.meta == cosmo.meta + + @pytest.mark.parametrize("format", [True, False, None, "mapping"]) + def test_is_equivalent_to_mapping(self, cosmo, to_format, format): + """Test :meth:`astropy.cosmology.Cosmology.is_equivalent`. + + This test checks that Cosmology equivalency can be extended to any + Python object that can be converted to a Cosmology -- in this case + a mapping. + """ + obj = to_format("mapping") + assert not isinstance(obj, Cosmology) + + is_equiv = cosmo.is_equivalent(obj, format=format) + assert is_equiv is (format is not False) + + +class TestToFromMapping(ToFromDirectTestBase, ToFromMappingTestMixin): + """Directly test ``to/from_mapping``.""" + + def setup_class(self): + self.functions = {"to": to_mapping, "from": from_mapping} + + @pytest.mark.skip("N/A") + def test_fromformat_subclass_partial_info_mapping(self): + """This test does not apply to the direct functions.""" diff --git a/astropy/cosmology/_src/tests/io/test_model.py b/astropy/cosmology/_src/tests/io/test_model.py new file mode 100644 index 000000000000..4fb0a64a93ef --- /dev/null +++ b/astropy/cosmology/_src/tests/io/test_model.py @@ -0,0 +1,182 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import inspect +import random + +import numpy as np +import pytest + +from astropy.cosmology import Cosmology, w0wzCDM +from astropy.cosmology._src.io.builtin.model import ( + _CosmologyModel, + from_model, + to_model, +) +from astropy.cosmology._src.tests.helper import get_redshift_methods +from astropy.modeling.models import Gaussian1D +from astropy.utils.compat.optional_deps import HAS_SCIPY + +from .base import ToFromDirectTestBase, ToFromTestMixinBase + +############################################################################### + + +class ToFromModelTestMixin(ToFromTestMixinBase): + """Tests for a Cosmology[To/From]Format with ``format="astropy.model"``. + + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. Subclasses must define a :func:`pytest.fixture` + ``cosmo`` that returns/yields an instance of a |Cosmology|. + See ``TestCosmologyToFromFormat`` or ``TestCosmology`` for examples. + """ + + @pytest.fixture(scope="class") + @classmethod + def method_name(cls, cosmo): + # get methods, ignoring private and dunder + methods = get_redshift_methods(cosmo, include_private=False, include_z2=True) + + # dynamically detect ABC and optional dependencies + for n in tuple(methods): + params = inspect.signature(getattr(cosmo, n)).parameters.keys() + + ERROR_SEIVE = (NotImplementedError, ValueError) + # # ABC can't introspect for good input + if not HAS_SCIPY: + ERROR_SEIVE = ERROR_SEIVE + (ModuleNotFoundError,) + + args = np.arange(len(params)) + 1 + try: + getattr(cosmo, n)(*args) + except ERROR_SEIVE: + methods.discard(n) + except TypeError: + # w0wzCDM has numerical instabilities when evaluating at z->inf + # TODO: a more robust fix in w0wzCDM itself would be better + if isinstance(cosmo, w0wzCDM): + methods.discard(n) + + # TODO! pytest doesn't currently allow multiple yields (`cosmo`) so + # testing with 1 random method + # yield from methods + return random.choice(tuple(methods)) if methods else None + + # =============================================================== + + def test_fromformat_model_wrong_cls(self, from_format): + """Test when Model is not the correct class.""" + model = Gaussian1D(amplitude=10, mean=14) + + with pytest.raises(AttributeError): + from_format(model) + + def test_toformat_model_not_method(self, to_format): + """Test when method is not a method.""" + with pytest.raises(AttributeError): + to_format("astropy.model", method="this is definitely not a method.") + + def test_toformat_model_not_callable(self, to_format): + """Test when method is actually an attribute.""" + with pytest.raises(ValueError): + to_format("astropy.model", method="name") + + def test_toformat_model(self, cosmo, to_format, method_name): + """Test cosmology -> astropy.model.""" + if method_name is None: # no test if no method + return + + model = to_format("astropy.model", method=method_name) + assert isinstance(model, _CosmologyModel) + + # Parameters + expect = tuple(k for k, v in cosmo.parameters.items() if v is not None) + assert model.param_names == expect + + # scalar result + args = np.arange(model.n_inputs) + 1 + + got = model.evaluate(*args) + expected = getattr(cosmo, method_name)(*args) + assert np.all(got == expected) + + got = model(*args) + expected = getattr(cosmo, method_name)(*args) + np.testing.assert_allclose(got, expected) + + # vector result + if "scalar" not in method_name: + args = (np.ones((model.n_inputs, 3)).T + np.arange(model.n_inputs)).T + + got = model.evaluate(*args) + expected = getattr(cosmo, method_name)(*args) + assert np.all(got == expected) + + got = model(*args) + expected = getattr(cosmo, method_name)(*args) + np.testing.assert_allclose(got, expected) + + def test_tofromformat_model_instance( + self, cosmo_cls, cosmo, method_name, to_format, from_format + ): + """Test cosmology -> astropy.model -> cosmology.""" + if method_name is None: # no test if no method + return + + # ------------ + # To Model + # this also serves as a test of all added methods / attributes + # in _CosmologyModel. + + model = to_format("astropy.model", method=method_name) + + assert isinstance(model, _CosmologyModel) + assert model.cosmology_class is cosmo_cls + assert model.cosmology == cosmo + assert model.method_name == method_name + + # ------------ + # From Model + + # it won't error if everything matches up + got = from_format(model, format="astropy.model") + assert got == cosmo + assert set(cosmo.meta.keys()).issubset(got.meta.keys()) + # Note: model adds parameter attributes to the metadata + + # also it auto-identifies 'format' + got = from_format(model) + assert got == cosmo + assert set(cosmo.meta.keys()).issubset(got.meta.keys()) + + def test_fromformat_model_subclass_partial_info(self) -> None: + """ + Test writing from an instance and reading from that class. + This works with missing information. + + There's no partial information with a Model + """ + + @pytest.mark.parametrize("format", [True, False, None, "astropy.model"]) + def test_is_equivalent_to_model(self, cosmo, method_name, to_format, format): + """Test :meth:`astropy.cosmology.Cosmology.is_equivalent`. + + This test checks that Cosmology equivalency can be extended to any + Python object that can be converted to a Cosmology -- in this case + a model. + """ + if method_name is None: # no test if no method + return + + obj = to_format("astropy.model", method=method_name) + assert not isinstance(obj, Cosmology) + + is_equiv = cosmo.is_equivalent(obj, format=format) + assert is_equiv is (format is not False) + + +class TestToFromModel(ToFromDirectTestBase, ToFromModelTestMixin): + """Directly test ``to/from_model``.""" + + def setup_class(self): + self.functions = {"to": to_model, "from": from_model} diff --git a/astropy/cosmology/_src/tests/io/test_mrt.py b/astropy/cosmology/_src/tests/io/test_mrt.py new file mode 100644 index 000000000000..d8224ec8abe0 --- /dev/null +++ b/astropy/cosmology/_src/tests/io/test_mrt.py @@ -0,0 +1,201 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +# THIRD PARTY + +import pytest + +from astropy.cosmology import Planck18 +from astropy.cosmology._src.io.builtin.mrt import ( + read_mrt, + write_mrt, +) +from astropy.table import QTable, Table + +from .base import ReadWriteDirectTestBase, ReadWriteTestMixinBase + + +class ReadWriteMRTTestMixin(ReadWriteTestMixinBase): + """ + Tests for a Cosmology[Read/Write] with ``format="mrt"``. + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. Subclasses must define a :func:`pytest.fixture` + ``cosmo`` that returns/yields an instance of a |Cosmology|. + See ``TestCosmology`` for an example. + """ + + def test_to_mrt_bad_index(self, read, write, tmp_path): + """Test if argument ``index`` is incorrect""" + fp = tmp_path / "test_to_mrt_bad_index.mrt" + + write(fp, format="ascii.mrt") + + # single-row table and has a non-0/None index + with pytest.raises(IndexError, match="index 2 out of range"): + read(fp, index=2, format="ascii.mrt") + + # string index where doesn't match + with pytest.raises(KeyError, match="No matches found for key"): + read(fp, index="row 0", format="ascii.mrt") + + # ----------------------- + + def test_to_mrt_failed_cls(self, write, tmp_path): + """Test failed table type.""" + fp = tmp_path / "test_to_mrt_failed_cls.mrt" + + with pytest.raises(TypeError, match="'cls' must be"): + write(fp, format="ascii.mrt", cls=list) + + # ----------------------- + + @pytest.mark.parametrize("tbl_cls", [QTable, Table]) + def test_to_mrt_cls(self, write, tbl_cls, tmp_path): + fp = tmp_path / "test_to_mrt_cls.mrt" + write(fp, format="ascii.mrt", cls=tbl_cls) + + # ----------------------- + + def test_readwrite_mrt_instance(self, cosmo_cls, cosmo, read, write, tmp_path): + """Test cosmology -> ascii.mrt -> cosmology.""" + fp = tmp_path / "test_readwrite_mrt_instance.mrt" + + # ------------ + # To Table + + write(fp, format="ascii.mrt") + + # some checks on the saved file + tbl = Table.read(fp, format="ascii.mrt") + assert tbl["name"] == cosmo.name + + # ------------ + # From Table + + tbl["mismatching"] = "will error" + tbl.write(fp, format="ascii.mrt", overwrite=True) + + # tests are different if the last argument is a **kwarg + if cosmo._init_has_kwargs: + got = read(fp, format="ascii.mrt") + + assert got.__class__ is cosmo_cls + assert got.name == cosmo.name + assert "mismatching" not in got.meta + + return # don't continue testing + + # read with mismatching parameters errors + with pytest.raises(TypeError, match="there are unused parameters"): + read(fp, format="ascii.mrt") + + # unless mismatched are moved to meta + got = read(fp, format="ascii.mrt", move_to_meta=True) + assert got == cosmo + assert got.meta["mismatching"] == "will error" + + # it won't error if everything matches up + tbl.remove_column("mismatching") + tbl.write(fp, format="ascii.mrt", overwrite=True) + got = read(fp, format="ascii.mrt") + assert got == cosmo + + # ----------------------- + + def test_readwrite_mrt_subclass_partial_info( + self, cosmo_cls, cosmo, read, write, tmp_path + ): + """ + Test writing from an instance and reading from that class. + This works with missing information. + """ + fp = tmp_path / "test_read_mrt_subclass_partial_info.mrt" + + # test write + write(fp, format="ascii.mrt") + + # partial information + tbl = Table.read(fp, format="ascii.mrt") + del tbl["Tcmb0"] + tbl.write(fp, format="ascii.mrt", overwrite=True) + + # read with the same class that wrote fills in the missing info with + # the default value + got = cosmo_cls.read(fp, format="ascii.mrt") + got2 = read(fp, format="ascii.mrt", cosmology=cosmo_cls) + + assert got == got2 # internal consistency + + # not equal, because Tcmb0 is changed, which also changes m_nu + assert got != cosmo + assert got.Tcmb0 == cosmo_cls.parameters["Tcmb0"].default + assert got.clone(name=cosmo.name, Tcmb0=cosmo.Tcmb0, m_nu=cosmo.m_nu) == cosmo + + +class WriteMRTTestMixin(ReadWriteTestMixinBase): + def test_to_mrt_failed_cls(self, write, tmp_path): + """Test failed table type.""" + fp = tmp_path / "test_to_mrt_failed_cls.mrt" + + with pytest.raises(TypeError, match="'cls' must be"): + write(fp, format="ascii.mrt", cls=list) + + @pytest.mark.parametrize("tbl_cls", [QTable, Table]) + def test_to_mrt_cls(self, write, tbl_cls, tmp_path): + fp = tmp_path / "test_to_mrt_cls.mrt" + write(fp, format="ascii.mrt", cls=tbl_cls) + + def test_to_mrt_bad_index(self, read, write, tmp_path): + """Test if argument ``index`` is incorrect""" + fp = tmp_path / "test_to_mrt_bad_index.mrt" + + write(fp, format="ascii.mrt") + + # single-row table and has a non-0/None index + with pytest.raises(IndexError, match="index 2 out of range"): + read(fp, index=2, format="ascii.mrt") + + # string index where doesn't match + with pytest.raises(KeyError, match="No matches found for key"): + read(fp, index="row 0", format="ascii.mrt") + + def test_write_mrt_invalid_path(self, write): + """Test passing an invalid path""" + invalid_fp = "" + with pytest.raises(FileNotFoundError, match="No such file or directory"): + write(invalid_fp, format="ascii.mrt") + + def test_readwrite_mrt_instance(self, cosmo_cls, cosmo, read, write, tmp_path): + fp = tmp_path / "test_readwrite_mrt_instance.mrt" + write(fp, format="ascii.mrt") + tb1 = Table.read(fp, format="ascii.mrt") + assert tb1["name"] == cosmo.name + + +class TestReadWriteMRT(ReadWriteDirectTestBase, WriteMRTTestMixin): + """ + Directly test ``read/write_mrt``. + These are not public API and are discouraged from use, in favor of + ``Cosmology.read/write(..., format="mrt")``, but should be + tested regardless b/c they are used internally. + """ + + def setup_class(self): + self.functions = {"read": read_mrt, "write": write_mrt} + + +def test_write_mrt_invalid_format(tmp_path): + """Test passing an invalid format""" + fp = tmp_path / "test_write_mrt_invalid_format.mrt" + with pytest.raises(ValueError, match="format must be 'ascii.mrt'"): + write_mrt(Planck18, fp, format="ascii.ecsv") + + +def test_read_mrt_invalid_format(tmp_path): + """Test read_mrt with invalid format parameter.""" + fp = tmp_path / "test_read_mrt_invalid_format.mrt" + Planck18.write(fp, format="ascii.mrt") + + # Test that passing a different format raises ValueError + with pytest.raises(ValueError, match="format must be 'ascii.mrt'"): + read_mrt(fp, format="ascii.ecsv") diff --git a/astropy/cosmology/_src/tests/io/test_row.py b/astropy/cosmology/_src/tests/io/test_row.py new file mode 100644 index 000000000000..9bdcaf062cf6 --- /dev/null +++ b/astropy/cosmology/_src/tests/io/test_row.py @@ -0,0 +1,145 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import pytest + +from astropy.cosmology._src.core import _COSMOLOGY_CLASSES, Cosmology +from astropy.cosmology._src.io.builtin.row import from_row, to_row +from astropy.table import Row + +from .base import ToFromDirectTestBase, ToFromTestMixinBase + +############################################################################### + + +class ToFromRowTestMixin(ToFromTestMixinBase): + """ + Tests for a Cosmology[To/From]Format with ``format="astropy.row"``. + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. Subclasses must define a :func:`pytest.fixture` + ``cosmo`` that returns/yields an instance of a |Cosmology|. + See ``TestCosmologyToFromFormat`` or ``TestCosmology`` for examples. + """ + + @pytest.mark.parametrize("in_meta", [True, False]) + def test_to_row_in_meta(self, cosmo_cls, cosmo, in_meta): + """Test where the cosmology class is placed.""" + row = cosmo.to_format("astropy.row", cosmology_in_meta=in_meta) + + # if it's in metadata, it's not a column. And vice versa. + if in_meta: + assert row.meta["cosmology"] == cosmo_cls.__qualname__ + assert "cosmology" not in row.colnames # not also a column + else: + assert row["cosmology"] == cosmo_cls.__qualname__ + assert "cosmology" not in row.meta + + # ----------------------- + + def test_from_not_row(self, cosmo, from_format): + """Test not passing a Row to the Row parser.""" + with pytest.raises(AttributeError): + from_format("NOT A ROW", format="astropy.row") + + def test_tofrom_row_instance(self, cosmo, to_format, from_format): + """Test cosmology -> astropy.row -> cosmology.""" + # ------------ + # To Row + + row = to_format("astropy.row") + assert isinstance(row, Row) + assert row["cosmology"] == cosmo.__class__.__qualname__ + assert row["name"] == cosmo.name + + # ------------ + # From Row + + row.table["mismatching"] = "will error" + + # tests are different if the last argument is a **kwarg + if cosmo._init_has_kwargs: + got = from_format(row, format="astropy.row") + + assert got.__class__ is cosmo.__class__ + assert got.name == cosmo.name + assert "mismatching" not in got.meta + + return # don't continue testing + + # read with mismatching parameters errors + with pytest.raises(TypeError, match="there are unused parameters"): + from_format(row, format="astropy.row") + + # unless mismatched are moved to meta + got = from_format(row, format="astropy.row", move_to_meta=True) + assert got == cosmo + assert got.meta["mismatching"] == "will error" + + # it won't error if everything matches up + row.table.remove_column("mismatching") + got = from_format(row, format="astropy.row") + assert got == cosmo + + # and it will also work if the cosmology is a class + # Note this is not the default output of ``to_format``. + cosmology = _COSMOLOGY_CLASSES[row["cosmology"]] + row.table.remove_column("cosmology") + row.table["cosmology"] = cosmology + got = from_format(row, format="astropy.row") + assert got == cosmo + + # also it auto-identifies 'format' + got = from_format(row) + assert got == cosmo + + def test_tofrom_row_rename(self, cosmo, to_format, from_format): + """Test renaming columns in row.""" + rename = {"name": "cosmo_name"} + row = to_format("astropy.row", rename=rename) + + assert "name" not in row.colnames + assert "cosmo_name" in row.colnames + + # Error if just reading + with pytest.raises(TypeError, match="there are unused parameters"): + from_format(row) + + # Roundtrip + inv_rename = {v: k for k, v in rename.items()} + got = from_format(row, rename=inv_rename) + assert got == cosmo + + def test_fromformat_row_subclass_partial_info(self, cosmo: Cosmology) -> None: + """ + Test writing from an instance and reading from that class. + This works with missing information. + + There are no partial info options + """ + + @pytest.mark.parametrize("format", [True, False, None, "astropy.row"]) + def test_is_equivalent_to_row(self, cosmo, to_format, format): + """Test :meth:`astropy.cosmology.Cosmology.is_equivalent`. + + This test checks that Cosmology equivalency can be extended to any + Python object that can be converted to a Cosmology -- in this case + a Row. + """ + obj = to_format("astropy.row") + assert not isinstance(obj, Cosmology) + + is_equiv = cosmo.is_equivalent(obj, format=format) + assert is_equiv is (format is not False) + + +class TestToFromRow(ToFromDirectTestBase, ToFromRowTestMixin): + """ + Directly test ``to/from_row``. + These are not public API and are discouraged from use, in favor of + ``Cosmology.to/from_format(..., format="astropy.row")``, but should be + tested regardless b/c 3rd party packages might use these in their Cosmology + I/O. Also, it's cheap to test. + """ + + def setup_class(self): + self.functions = {"to": to_row, "from": from_row} diff --git a/astropy/cosmology/_src/tests/io/test_table.py b/astropy/cosmology/_src/tests/io/test_table.py new file mode 100644 index 000000000000..1380839c6c78 --- /dev/null +++ b/astropy/cosmology/_src/tests/io/test_table.py @@ -0,0 +1,258 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import pytest + +from astropy.cosmology import Cosmology +from astropy.cosmology._src.core import _COSMOLOGY_CLASSES +from astropy.cosmology._src.io.builtin.table import from_table, to_table +from astropy.table import QTable, Table, vstack + +from .base import ToFromDirectTestBase, ToFromTestMixinBase + +############################################################################### + + +class ToFromTableTestMixin(ToFromTestMixinBase): + """ + Tests for a Cosmology[To/From]Format with ``format="astropy.table"``. + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. Subclasses must define a :func:`pytest.fixture` + ``cosmo`` that returns/yields an instance of a |Cosmology|. + See ``TestCosmology`` for an example. + """ + + def test_to_table_bad_index(self, from_format, to_format): + """Test if argument ``index`` is incorrect""" + tbl = to_format("astropy.table") + + # single-row table and has a non-0/None index + with pytest.raises(IndexError, match="index 2 out of range"): + from_format(tbl, index=2, format="astropy.table") + + # string index where doesn't match + with pytest.raises(KeyError, match="No matches found for key"): + from_format(tbl, index="row 0", format="astropy.table") + + # ----------------------- + + def test_to_table_failed_cls(self, to_format): + """Test failed table type.""" + with pytest.raises(TypeError, match="'cls' must be"): + to_format("astropy.table", cls=list) + + @pytest.mark.parametrize("tbl_cls", [QTable, Table]) + def test_to_table_cls(self, to_format, tbl_cls): + tbl = to_format("astropy.table", cls=tbl_cls) + assert isinstance(tbl, tbl_cls) # test type + + # ----------------------- + + @pytest.mark.parametrize("in_meta", [True, False]) + def test_to_table_in_meta(self, cosmo_cls, to_format, in_meta): + """Test where the cosmology class is placed.""" + tbl = to_format("astropy.table", cosmology_in_meta=in_meta) + + # if it's in metadata, it's not a column. And vice versa. + if in_meta: + assert tbl.meta["cosmology"] == cosmo_cls.__qualname__ + assert "cosmology" not in tbl.colnames # not also a column + else: + assert tbl["cosmology"][0] == cosmo_cls.__qualname__ + assert "cosmology" not in tbl.meta + + # ----------------------- + + def test_to_table(self, cosmo_cls, cosmo, to_format): + """Test cosmology -> astropy.table.""" + tbl = to_format("astropy.table") + + # Test properties of Table. + assert isinstance(tbl, QTable) + assert tbl.meta["cosmology"] == cosmo_cls.__qualname__ + assert tbl["name"] == cosmo.name + assert tbl.indices # indexed + + # Test each Parameter column has expected information. + for n, P in cosmo_cls.parameters.items(): + col = tbl[n] # Column + + # Compare the two + assert col.info.name == P.name + assert col.info.description == P.__doc__ + assert col.info.meta == (cosmo.meta.get(n) or {}) + + # ----------------------- + + def test_from_not_table(self, cosmo, from_format): + """Test not passing a Table to the Table parser.""" + with pytest.raises((TypeError, ValueError)): + from_format("NOT A TABLE", format="astropy.table") + + def test_tofrom_table_instance(self, cosmo_cls, cosmo, from_format, to_format): + """Test cosmology -> astropy.table -> cosmology.""" + tbl = to_format("astropy.table") + + # add information + tbl["mismatching"] = "will error" + + # tests are different if the last argument is a **kwarg + if cosmo._init_has_kwargs: + got = from_format(tbl, format="astropy.table") + + assert got.__class__ is cosmo_cls + assert got.name == cosmo.name + assert "mismatching" not in got.meta + + return # don't continue testing + + # read with mismatching parameters errors + with pytest.raises(TypeError, match="there are unused parameters"): + from_format(tbl, format="astropy.table") + + # unless mismatched are moved to meta + got = from_format(tbl, format="astropy.table", move_to_meta=True) + assert got == cosmo + assert got.meta["mismatching"] == "will error" + + # it won't error if everything matches up + tbl.remove_column("mismatching") + got = from_format(tbl, format="astropy.table") + assert got == cosmo + + # and it will also work if the cosmology is a class + # Note this is not the default output of ``to_format``. + tbl.meta["cosmology"] = _COSMOLOGY_CLASSES[tbl.meta["cosmology"]] + got = from_format(tbl, format="astropy.table") + assert got == cosmo + + # also it auto-identifies 'format' + got = from_format(tbl) + assert got == cosmo + + def test_fromformat_table_subclass_partial_info( + self, cosmo_cls, cosmo, from_format, to_format + ): + """ + Test writing from an instance and reading from that class. + This works with missing information. + """ + # test to_format + tbl = to_format("astropy.table") + assert isinstance(tbl, QTable) + + # partial information + tbl.meta.pop("cosmology", None) + del tbl["Tcmb0"] + + # read with the same class that wrote fills in the missing info with + # the default value + got = cosmo_cls.from_format(tbl, format="astropy.table") + got2 = from_format(tbl, format="astropy.table", cosmology=cosmo_cls) + got3 = from_format( + tbl, format="astropy.table", cosmology=cosmo_cls.__qualname__ + ) + + assert (got == got2) and (got2 == got3) # internal consistency + + # not equal, because Tcmb0 is changed, which also changes m_nu + assert got != cosmo + assert got.Tcmb0 == cosmo_cls.parameters["Tcmb0"].default + assert got.clone(name=cosmo.name, Tcmb0=cosmo.Tcmb0, m_nu=cosmo.m_nu) == cosmo + # but the metadata is the same + assert got.meta == cosmo.meta + + @pytest.mark.parametrize("add_index", [True, False]) + def test_tofrom_table_mutlirow(self, cosmo_cls, cosmo, from_format, add_index): + """Test if table has multiple rows.""" + # ------------ + # To Table + + cosmo1 = cosmo.clone(name="row 0") + cosmo2 = cosmo.clone(name="row 2") + tbl = vstack( + [c.to_format("astropy.table") for c in (cosmo1, cosmo, cosmo2)], + metadata_conflicts="silent", + ) + + assert isinstance(tbl, QTable) + assert tbl.meta["cosmology"] == cosmo_cls.__qualname__ + assert tbl[1]["name"] == cosmo.name + + # whether to add an index. `from_format` can work with or without. + if add_index: + tbl.add_index("name", unique=True) + + # ------------ + # From Table + + # it will error on a multi-row table + with pytest.raises(ValueError, match="need to select a specific row"): + from_format(tbl, format="astropy.table") + + # unless the index argument is provided + got = from_format(tbl, index=1, format="astropy.table") + assert got == cosmo + + # the index can be a string + got = from_format(tbl, index=cosmo.name, format="astropy.table") + assert got == cosmo + + # when there's more than one cosmology found + tbls = vstack([tbl, tbl], metadata_conflicts="silent") + with pytest.raises(ValueError, match="more than one"): + from_format(tbls, index=cosmo.name, format="astropy.table") + + def test_tofrom_table_rename(self, cosmo, to_format, from_format): + """Test renaming columns in row.""" + rename = {"name": "cosmo_name"} + table = to_format("astropy.table", rename=rename) + + assert "name" not in table.colnames + assert "cosmo_name" in table.colnames + + # Error if just reading + with pytest.raises(TypeError, match="there are unused parameters"): + from_format(table) + + # Roundtrip + inv_rename = {v: k for k, v in rename.items()} + got = from_format(table, rename=inv_rename) + assert got == cosmo + + def test_from_table_renamed_index_column(self, cosmo, to_format, from_format): + """Test reading from a table with a renamed index column.""" + cosmo1 = cosmo.clone(name="row 0") + cosmo2 = cosmo.clone(name="row 2") + tbl = vstack( + [c.to_format("astropy.table") for c in (cosmo1, cosmo, cosmo2)], + metadata_conflicts="silent", + ) + tbl.rename_column("name", "cosmo_name") + + inv_rename = {"cosmo_name": "name"} + newcosmo = from_format( + tbl, index="row 0", rename=inv_rename, format="astropy.table" + ) + assert newcosmo == cosmo1 + + @pytest.mark.parametrize("format", [True, False, None, "astropy.table"]) + def test_is_equivalent_to_table(self, cosmo, to_format, format): + """Test :meth:`astropy.cosmology.Cosmology.is_equivalent`. + + This test checks that Cosmology equivalency can be extended to any + Python object that can be converted to a Cosmology -- in this case + a |Table|. + """ + obj = to_format("astropy.table") + assert not isinstance(obj, Cosmology) + + is_equiv = cosmo.is_equivalent(obj, format=format) + assert is_equiv is (format is not False) + + +class TestToFromTable(ToFromDirectTestBase, ToFromTableTestMixin): + """Directly test ``to/from_table``.""" + + def setup_class(self): + self.functions = {"to": to_table, "from": from_table} diff --git a/astropy/cosmology/_src/tests/io/test_yaml.py b/astropy/cosmology/_src/tests/io/test_yaml.py new file mode 100644 index 000000000000..4ddfc8f6f6be --- /dev/null +++ b/astropy/cosmology/_src/tests/io/test_yaml.py @@ -0,0 +1,184 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import pytest + +import astropy.units as u +from astropy.cosmology import Cosmology, FlatLambdaCDM, Planck18 +from astropy.cosmology import units as cu +from astropy.cosmology._src.io.builtin.yaml import ( + from_yaml, + to_yaml, + yaml_constructor, + yaml_representer, +) +from astropy.io.misc.yaml import AstropyDumper, dump, load + +from .base import ToFromDirectTestBase, ToFromTestMixinBase + +############################################################################## +# Test Serializer + + +def test_yaml_representer(): + """Test :func:`~astropy.cosmology._src.io.builtin.yaml.yaml_representer`.""" + # test function `representer` + representer = yaml_representer("!astropy.cosmology.LambdaCDM") + assert callable(representer) + + # test the normal method of dumping to YAML + yml = dump(Planck18) + assert isinstance(yml, str) + assert yml.startswith("!astropy.cosmology.FlatLambdaCDM") + + +def test_yaml_constructor(): + """Test :func:`~astropy.cosmology._src.io.builtin.yaml.yaml_constructor`.""" + # test function `constructor` + constructor = yaml_constructor(FlatLambdaCDM) + assert callable(constructor) + + # it's too hard to manually construct a node, so we only test dump/load + # this is also a good round-trip test + yml = dump(Planck18) + with u.add_enabled_units(cu): # needed for redshift units + cosmo = load(yml) + assert isinstance(cosmo, FlatLambdaCDM) + assert cosmo == Planck18 + assert cosmo.meta == Planck18.meta + + +############################################################################## +# Test Unified I/O + + +class ToFromYAMLTestMixin(ToFromTestMixinBase): + """ + Tests for a Cosmology[To/From]Format with ``format="yaml"``. + This class will not be directly called by :mod:`pytest` since its name does + not begin with ``Test``. To activate the contained tests this class must + be inherited in a subclass. Subclasses must define a :func:`pytest.fixture` + ``cosmo`` that returns/yields an instance of a |Cosmology|. + See ``TestCosmologyToFromFormat`` or ``TestCosmology`` for examples. + """ + + @pytest.fixture + def xfail_if_not_registered_with_yaml(self, cosmo_cls): + """ + YAML I/O only works on registered classes. So the thing to check is + if this class is registered. If not, :func:`pytest.xfail` this test. + Some of the tests define custom cosmologies. They are not registered. + """ + if cosmo_cls not in AstropyDumper.yaml_representers: + pytest.xfail( + f"Cosmologies of type {cosmo_cls} are not registered with YAML." + ) + + # =============================================================== + + def test_to_yaml(self, cosmo_cls, to_format, xfail_if_not_registered_with_yaml): + """Test cosmology -> YAML.""" + yml = to_format("yaml") + + assert isinstance(yml, str) # test type + assert yml.startswith("!" + ".".join(cosmo_cls.__module__.split(".")[:2])) + # e.g. "astropy.cosmology" for built-in cosmologies, or "__main__" for the test + # SubCosmology class defined in ``astropy.cosmology._src.tests.test_core``. + + def test_from_yaml_default( + self, cosmo, to_format, from_format, xfail_if_not_registered_with_yaml + ): + """Test cosmology -> YAML -> cosmology.""" + yml = to_format("yaml") + + got = from_format(yml, format="yaml") # (cannot autoidentify) + + assert got.name == cosmo.name + assert got.meta == cosmo.meta + + # it won't error if everything matches up + got = from_format(yml, format="yaml") + assert got == cosmo + assert got.meta == cosmo.meta + + # auto-identify test moved because it doesn't work. + # see test_from_yaml_autoidentify + + def test_from_yaml_autoidentify( + self, cosmo, to_format, from_format, xfail_if_not_registered_with_yaml + ): + """As a non-path string, it does NOT auto-identifies 'format'. + + TODO! this says there should be different types of I/O registries. + not just hacking object conversion on top of file I/O. + """ + assert self.can_autodentify("yaml") is False + + # Showing the specific error. The str is interpreted as a file location + # but is too long a file name. + yml = to_format("yaml") + with pytest.raises((FileNotFoundError, OSError)): # OSError in Windows + from_format(yml) + + # # TODO! this is a challenging test to write. It's also unlikely to happen. + # def test_fromformat_subclass_partial_info_yaml(self, cosmo): + # """ + # Test writing from an instance and reading from that class. + # This works with missing information. + # """ + + # ----------------------------------------------------- + + @pytest.mark.parametrize("format", [True, False, None]) + def test_is_equivalent_to_yaml( + self, cosmo, to_format, format, xfail_if_not_registered_with_yaml + ): + """Test :meth:`astropy.cosmology.Cosmology.is_equivalent`. + + This test checks that Cosmology equivalency can be extended to any + Python object that can be converted to a Cosmology -- in this case + a YAML string. YAML can't be identified without "format" specified. + """ + obj = to_format("yaml") + assert not isinstance(obj, Cosmology) + + is_equiv = cosmo.is_equivalent(obj, format=format) + assert is_equiv is False + + def test_is_equivalent_to_yaml_specify_format( + self, cosmo, to_format, xfail_if_not_registered_with_yaml + ): + """Test :meth:`astropy.cosmology.Cosmology.is_equivalent`. + + Same as ``test_is_equivalent_to_yaml`` but with ``format="yaml"``. + """ + assert cosmo.is_equivalent(to_format("yaml"), format="yaml") is True + + +class TestToFromYAML(ToFromDirectTestBase, ToFromYAMLTestMixin): + """ + Directly test ``to/from_yaml``. + These are not public API and are discouraged from use, in favor of + ``Cosmology.to/from_format(..., format="yaml")``, but should be tested + regardless b/c 3rd party packages might use these in their Cosmology I/O. + Also, it's cheap to test. + """ + + def setup_class(self): + """Set up fixtures to use ``to/from_yaml``, not the I/O abstractions.""" + self.functions = {"to": to_yaml, "from": from_yaml} + + @pytest.fixture(scope="class", autouse=True) + @classmethod + def setup(cls): + """ + Setup and teardown for tests. + This overrides from super because `ToFromDirectTestBase` adds a custom + Cosmology ``CosmologyWithKwargs`` that is not registered with YAML. + """ + return # run tests + + def test_from_yaml_autoidentify(self, cosmo, to_format, from_format): + """ + If directly calling the function there's no auto-identification. + So this overrides the test from `ToFromYAMLTestMixin` + """ diff --git a/astropy/vo/client/__init__.py b/astropy/cosmology/_src/tests/parameter/__init__.py similarity index 100% rename from astropy/vo/client/__init__.py rename to astropy/cosmology/_src/tests/parameter/__init__.py diff --git a/astropy/cosmology/_src/tests/parameter/conftest.py b/astropy/cosmology/_src/tests/parameter/conftest.py new file mode 100644 index 000000000000..4fd6dcdefd16 --- /dev/null +++ b/astropy/cosmology/_src/tests/parameter/conftest.py @@ -0,0 +1,6 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +"""Configure the tests for :mod:`astropy._src.cosmology.parameter`.""" + +from astropy.cosmology._src.tests.helper import clean_registry # noqa: F401 +from astropy.tests.helper import pickle_protocol # noqa: F401 diff --git a/astropy/cosmology/_src/tests/parameter/test_descriptors.py b/astropy/cosmology/_src/tests/parameter/test_descriptors.py new file mode 100644 index 000000000000..610396d36ce6 --- /dev/null +++ b/astropy/cosmology/_src/tests/parameter/test_descriptors.py @@ -0,0 +1,129 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +"""Testing :mod:`astropy.cosmology._src.parameter.descriptor`.""" + +from types import MappingProxyType +from typing import ClassVar + +import pytest + +from astropy.cosmology import Cosmology, Parameter +from astropy.cosmology._src.parameter import ParametersAttribute, all_parameters + + +class Obj: + """Example class with a ParametersAttribute.""" + + # Attributes that will be accessed by ParametersAttribute when called an instance + # of this class. On a Cosmology these would be the Parameter objects. + a: ClassVar[int] = 1 + b: ClassVar[int] = 2 + c: ClassVar[int] = 3 + # The class attribute that is accessed by ParametersAttribute when called on the + # class. On a Cosmology this would be the mapping of Parameter objects. + # Here it is just the names of the attributes that will be accessed by the + # ParametersAttribute to better distinguish between the class and instance + # attributes. + _attr_map: ClassVar[tuple[str, ...]] = ("a", "b", "c") + + # The ParametersAttribute descriptor. This will return a mapping of the values of + # the attributes listed in ``_attr_map`` when called on an instance of this class. + # When called on the class, it will return ``_attr_map`` itself. + attr = ParametersAttribute(attr_name="_attr_map") + + +class TestParametersAttribute: + """Test the descriptor ``ParametersAttribute``.""" + + def test_init(self) -> None: + """Test constructing a ParametersAttribute.""" + # Proper construction + attr = ParametersAttribute("attr_name") + assert attr.attr_name == "attr_name" + + # Improper construction + # There isn't type checking on the attr_name, so this is allowed, but will fail + # later when the descriptor is used. + attr = ParametersAttribute(1) # type: ignore[arg-type] + assert attr.attr_name == 1 + + def test_get_from_class(self) -> None: + """Test the descriptor ``__get__`` from the class.""" + assert Obj.attr == ("a", "b", "c") + + def test_get_from_instance(self) -> None: + """Test the descriptor ``__get__``.""" + obj = Obj() # Construct an instance for the attribute `attr`. + assert isinstance(obj.attr, MappingProxyType) + assert tuple(obj.attr.keys()) == obj._attr_map + + def test_set_from_instance(self) -> None: + """Test the descriptor ``__set__``.""" + obj = Obj() # Construct an instance for the attribute `attr`. + with pytest.raises(AttributeError, match="cannot set 'attr' of"): + obj.attr = {} + + def test_descriptor_attr_name_not_str(self) -> None: + """Test when ``attr_name`` is not a string and used as a descriptor. + + This is a regression test for #15882. + """ + + class Obj2(Obj): + attr = ParametersAttribute(attr_name=None) # type: ignore[arg-type] + + obj = Obj2() + with pytest.raises( + TypeError, match=r"attribute name must be string, not 'NoneType'" + ): + _ = obj.attr + + +############################################################################## + + +class ParametersAttributeTestMixin: + """Test the descriptor for ``parameters`` on Cosmology classes. + + This is a mixin class and is mixed into + :class:`~astropy.cosmology._src.tests.test_core.CosmologyTest`. + """ + + @pytest.mark.parametrize("name", ["parameters", "_derived_parameters"]) + def test_parameters_from_class(self, cosmo_cls: type[Cosmology], name: str) -> None: + """Test descriptor ``parameters`` accessed from the class.""" + # test presence + assert hasattr(cosmo_cls, name) + # test Parameter is a MappingProxyType + parameters = getattr(cosmo_cls, name) + assert isinstance(parameters, MappingProxyType) + # Test items + assert all(isinstance(p, Parameter) for p in parameters.values()) + assert set(parameters) == { + k + for k, v in all_parameters(cosmo_cls).items() + if v.derived == ("derived" in name) + } + + @pytest.mark.parametrize("name", ["parameters", "_derived_parameters"]) + def test_parameters_from_instance(self, cosmo: Cosmology, name: str) -> None: + """Test descriptor ``parameters`` accessed from the instance.""" + # test presence + assert hasattr(cosmo, name) + # test Parameter is a MappingProxyType + parameters = getattr(cosmo, name) + assert isinstance(parameters, MappingProxyType) + # Test keys + assert set(parameters) == { + k + for k, v in all_parameters(cosmo).items() + if (v.derived == ("derived" in name)) + } + + @pytest.mark.parametrize("name", ["parameters", "_derived_parameters"]) + def test_parameters_cannot_set_on_instance( + self, cosmo: Cosmology, name: str + ) -> None: + """Test descriptor ``parameters`` cannot be set on the instance.""" + with pytest.raises(AttributeError, match=f"cannot assign to field {name!r}"): + setattr(cosmo, name, {}) diff --git a/astropy/cosmology/_src/tests/parameter/test_parameter.py b/astropy/cosmology/_src/tests/parameter/test_parameter.py new file mode 100644 index 000000000000..a1646a694578 --- /dev/null +++ b/astropy/cosmology/_src/tests/parameter/test_parameter.py @@ -0,0 +1,447 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +"""Testing :mod:`astropy.cosmology._src.parameter`.""" + +from collections.abc import Callable + +import numpy as np +import pytest + +import astropy.units as u +from astropy.cosmology import Cosmology, Parameter +from astropy.cosmology._src.core import _COSMOLOGY_CLASSES, dataclass_decorator +from astropy.cosmology._src.parameter.converter import ( + _REGISTRY_FVALIDATORS, + validate_with_unit, +) +from astropy.cosmology._src.parameter.core import MISSING + +############################################################################## + + +def test_registry_validators(): + """Test :class:`astropy.cosmology.Parameter` attributes on class.""" + # _registry_validators + assert isinstance(_REGISTRY_FVALIDATORS, dict) + assert all(isinstance(k, str) for k in _REGISTRY_FVALIDATORS.keys()) + assert all(callable(v) for v in _REGISTRY_FVALIDATORS.values()) + + +class Test_Parameter: + """Test :class:`astropy.cosmology.Parameter` not on a cosmology.""" + + @pytest.mark.parametrize( + "kwargs", + [ + {}, + dict( + default=1.0, + fvalidate="float", + doc="DOCSTRING", + unit="km", + equivalencies=[u.mass_energy()], + derived=True, + ), + ], + ) + def test_Parameter_init(self, kwargs): + """Test :class:`astropy.cosmology.Parameter` instantiation.""" + unit = kwargs.get("unit") + + param = Parameter(**kwargs) + assert param.default == kwargs.get("default", MISSING) + assert param.fvalidate is _REGISTRY_FVALIDATORS.get( + kwargs.get("fvalidate"), validate_with_unit + ) + assert param.doc == kwargs.get("doc") + assert param.unit is (u.Unit(unit) if unit is not None else None) + assert param.equivalencies == kwargs.get("equivalencies", []) + assert param.derived is kwargs.get("derived", False) + assert param.name == "name not initialized" + + def test_Parameter_default(self): + """Test :attr:`astropy.cosmology.Parameter.default`.""" + parameter = Parameter() + assert parameter.default is MISSING + assert repr(parameter.default) == "" + + +class ParameterTestMixin: + """Tests for a :class:`astropy.cosmology.Parameter` on a Cosmology. + + :class:`astropy.cosmology.Parameter` is a descriptor and this test suite + tests descriptors by class inheritance, so ``ParameterTestMixin`` is mixed + into ``TestCosmology`` (tests :class:`astropy.cosmology.Cosmology`). + """ + + @pytest.fixture + def parameter(self, cosmo_cls): + """Cosmological Parameters""" + yield from cosmo_cls.parameters.values() + + @pytest.fixture + def all_parameter(self, cosmo_cls): + """Cosmological All Parameter instances""" + # just return one parameter at random + n = set(cosmo_cls._parameters_all).pop() + try: + yield cosmo_cls.parameters[n] + except KeyError: + yield cosmo_cls._derived_parameters[n] + + # =============================================================== + # Method Tests + + def test_Parameter_instance_attributes(self, all_parameter): + """Test :class:`astropy.cosmology.Parameter` attributes from init.""" + assert hasattr(all_parameter, "fvalidate") + assert callable(all_parameter.fvalidate) + + assert hasattr(all_parameter, "__doc__") + + # Parameter + assert hasattr(all_parameter, "_unit") + assert hasattr(all_parameter, "equivalencies") + assert hasattr(all_parameter, "derived") + + # __set_name__ + assert hasattr(all_parameter, "name") + + def test_Parameter_fvalidate(self, all_parameter): + """Test :attr:`astropy.cosmology.Parameter.fvalidate`.""" + assert hasattr(all_parameter, "fvalidate") + assert callable(all_parameter.fvalidate) + assert hasattr(all_parameter, "_fvalidate_in") + assert isinstance(all_parameter._fvalidate_in, (str, Callable)) + + def test_Parameter_name(self, all_parameter): + """Test :attr:`astropy.cosmology.Parameter.name`.""" + assert hasattr(all_parameter, "name") + assert isinstance(all_parameter.name, str) + + def test_Parameter_unit(self, all_parameter): + """Test :attr:`astropy.cosmology.Parameter.unit`.""" + assert hasattr(all_parameter, "unit") + assert isinstance(all_parameter.unit, (u.UnitBase, type(None))) + assert all_parameter.unit is all_parameter._unit + + def test_Parameter_equivalencies(self, all_parameter): + """Test :attr:`astropy.cosmology.Parameter.equivalencies`.""" + assert hasattr(all_parameter, "equivalencies") + assert isinstance(all_parameter.equivalencies, (list, u.Equivalency)) + + def test_Parameter_derived(self, cosmo_cls, all_parameter): + """Test :attr:`astropy.cosmology.Parameter.derived`.""" + assert hasattr(all_parameter, "derived") + assert isinstance(all_parameter.derived, bool) + + # test value + assert all_parameter.derived is (all_parameter.name not in cosmo_cls.parameters) + + def test_Parameter_default(self, cosmo_cls, all_parameter): + """Test :attr:`astropy.cosmology.Parameter.default`.""" + assert hasattr(all_parameter, "default") + assert all_parameter.default is MISSING or isinstance( + all_parameter.default, (type(None), int, float, u.Quantity) + ) + + # ------------------------------------------- + # descriptor methods + + def test_Parameter_descriptor_get(self, cosmo_cls, cosmo, all_parameter): + """Test :attr:`astropy.cosmology.Parameter.__get__`.""" + # from class + np.testing.assert_array_equal( + getattr(cosmo_cls, all_parameter.name).default, all_parameter.default + ) + + # from instance + parameter = getattr(cosmo, all_parameter.name) + assert np.all(parameter == cosmo.__dict__[all_parameter.name]) + + def test_Parameter_descriptor_set(self, cosmo, all_parameter): + """Test :attr:`astropy.cosmology.Parameter.__set__`.""" + # test it's already set + assert all_parameter.name in cosmo.__dict__ + + # ------------------------------------------- + # validate value + # tested later. + + # =============================================================== + # Usage Tests + + def test_Parameter_listed(self, cosmo_cls, all_parameter): + """Test each `astropy.cosmology.Parameter` attached to Cosmology.""" + # just double check that each entry is a Parameter + assert isinstance(all_parameter, Parameter) + + # the reverse: check that if it is a Parameter, it's listed. + if all_parameter.derived: + assert all_parameter.name in cosmo_cls._derived_parameters + else: + assert all_parameter.name in cosmo_cls.parameters + + +# ======================================================================== + + +class TestParameter(ParameterTestMixin): + """ + Test `astropy.cosmology.Parameter` directly. Adds a lot of specific tests + that wouldn't be covered by the per-cosmology tests. + """ + + def setup_class(self): + theparam = Parameter( + default=15, + doc="Description of example parameter.", + unit=u.m, + equivalencies=u.mass_energy(), + ) + + @dataclass_decorator + class Example1(Cosmology): + param: Parameter = theparam.clone() + + @property + def is_flat(self) -> bool: + return super().is_flat() + + # with validator + @dataclass_decorator + class Example2(Example1): + param: Parameter = theparam.clone(default=15 * u.m) + + @param.validator + def param(self, param, value): + return value.to(u.km) + + # attributes + self.classes = {"Example1": Example1, "Example2": Example2} + + def teardown_class(self): + for cls in self.classes.values(): + _COSMOLOGY_CLASSES.pop(cls.__qualname__, None) + + @pytest.fixture(scope="class", params=["Example1", "Example2"]) + @classmethod + def cosmo_cls(cls, request): + """Cosmology class.""" + return cls.classes[request.param] + + @pytest.fixture(scope="class") + @classmethod + def cosmo(cls, cosmo_cls): + """Cosmology instance""" + return cosmo_cls() + + @pytest.fixture(scope="class") + @classmethod + def param(cls, cosmo_cls): + """Get Parameter 'param' from cosmology class.""" + return cosmo_cls.parameters["param"] + + @pytest.fixture(scope="class") + @classmethod + def param_cls(cls, param): + """Get Parameter class from cosmology class.""" + return type(param) + + # ============================================================== + + def test_Parameter_instance_attributes(self, param): + """Test :class:`astropy.cosmology.Parameter` attributes from init.""" + super().test_Parameter_instance_attributes(param) + + # property + assert param.__doc__ == "Description of example parameter." + + # custom from init + assert param.unit == u.m + assert param.equivalencies == u.mass_energy() + assert param.derived == np.False_ + + # custom from set_name + assert param.name == "param" + + def test_Parameter_fvalidate(self, cosmo, param): + """Test :attr:`astropy.cosmology.Parameter.fvalidate`.""" + super().test_Parameter_fvalidate(param) + + value = param.fvalidate(cosmo, param, 1000 * u.m) + assert value == 1 * u.km + + def test_Parameter_name(self, param): + """Test :attr:`astropy.cosmology.Parameter.name`.""" + super().test_Parameter_name(param) + + assert param.name == "param" + + def test_Parameter_unit(self, param): + """Test :attr:`astropy.cosmology.Parameter.unit`.""" + super().test_Parameter_unit(param) + + assert param.unit == u.m + + def test_Parameter_equivalencies(self, param): + """Test :attr:`astropy.cosmology.Parameter.equivalencies`.""" + super().test_Parameter_equivalencies(param) + + assert param.equivalencies == u.mass_energy() + + def test_Parameter_derived(self, cosmo_cls, param): + """Test :attr:`astropy.cosmology.Parameter.derived`.""" + super().test_Parameter_derived(cosmo_cls, param) + + assert param.derived is False + + # ------------------------------------------- + # descriptor methods + + def test_Parameter_descriptor_get(self, cosmo_cls, cosmo, param): + """Test :meth:`astropy.cosmology.Parameter.__get__`.""" + super().test_Parameter_descriptor_get(cosmo_cls, cosmo, param) + + # from instance + value = getattr(cosmo, param.name) + assert value == 15 * u.m + + # ------------------------------------------- + # validation + + def test_Parameter_validator(self, param): + """Test :meth:`astropy.cosmology.Parameter.validator`.""" + for k in _REGISTRY_FVALIDATORS: + newparam = param.validator(k) + assert newparam.fvalidate == _REGISTRY_FVALIDATORS[k] + + # error for non-registered str + with pytest.raises(ValueError, match="`fvalidate`, if str"): + Parameter(fvalidate="NOT REGISTERED") + + # error if wrong type + with pytest.raises(TypeError, match="`fvalidate` must be a function or"): + Parameter(fvalidate=object()) + + def test_Parameter_validate(self, cosmo, param): + """Test :meth:`astropy.cosmology.Parameter.validate`.""" + value = param.validate(cosmo, 1000 * u.m) + + # whether has custom validator + if param.fvalidate is _REGISTRY_FVALIDATORS["default"]: + assert value.unit == u.m + assert value.value == 1000 + else: + assert value.unit == u.km + assert value.value == 1 + + def test_Parameter_register_validator(self, param_cls): + """Test :meth:`astropy.cosmology.Parameter.register_validator`.""" + # already registered + with pytest.raises(KeyError, match="validator 'default' already"): + param_cls.register_validator("default", None) + + # validator not None + def notnonefunc(x): + return x + + try: + validator = param_cls.register_validator("newvalidator", notnonefunc) + assert validator is notnonefunc + finally: + _REGISTRY_FVALIDATORS.pop("newvalidator", None) + + # used as decorator + try: + + @param_cls.register_validator("newvalidator") + def func(cosmology, param, value): + return value + + assert _REGISTRY_FVALIDATORS["newvalidator"] is func + finally: + _REGISTRY_FVALIDATORS.pop("newvalidator", None) + + # ------------------------------------------- + + def test_Parameter_clone(self, param): + """Test :meth:`astropy.cosmology.Parameter.clone`.""" + # this implicitly relies on `__eq__` testing properly. Which is tested. + + # basic test that nothing changes + assert param.clone() == param + assert param.clone() is not param # but it's not a 'singleton' + + # passing kwargs will change stuff + newparam = param.clone(unit="km/(yr sr)") + assert newparam.unit == u.km / u.yr / u.sr + assert param.unit != u.km / u.yr / u.sr # original is unchanged + + # expected failure for not-an-argument + with pytest.raises(TypeError): + param.clone(not_a_valid_parameter=True) + + # ------------------------------------------- + + def test_Parameter_equality(self): + """Test Parameter equality. + + Determined from the processed initialization args (including defaults). + """ + p1 = Parameter(unit="km / (s Mpc)") + p2 = Parameter(unit="km / (s Mpc)") + assert p1 == p2 + + # not equal parameters + p3 = Parameter(unit="km / s") + assert p3 != p1 + + # misc + assert p1 != 2 # show doesn't error + + # ------------------------------------------- + + def test_Parameter_repr(self, cosmo_cls, param): + """Test Parameter repr.""" + r = repr(param) + + assert "Parameter(" in r + for subs in ( + "derived=False", + 'unit=Unit("m")', + 'equivalencies=[(Unit("kg"), Unit("J")', + "doc='Description of example parameter.'", + ): + assert subs in r, subs + + # `fvalidate` is a little tricker b/c one of them is custom! + if param.fvalidate in _REGISTRY_FVALIDATORS.values(): # not custom + assert "fvalidate='default'" in r + else: + assert "fvalidate=<" in r # Some function, don't care about details. + + def test_Parameter_repr_roundtrip(self, param): + """Test ``eval(repr(Parameter))`` can round trip to ``Parameter``.""" + P = Parameter(doc="A description of this parameter.", derived=True) + NP = eval(repr(P)) # Evaluate string representation back into a param. + + assert P == NP + + # ======================================================================== + + def test_make_from_Parameter(self, cosmo_cls, clean_registry): + """Test the parameter creation process. Uses ``__set__``.""" + + @dataclass_decorator + class Example(cosmo_cls): + param: Parameter = Parameter(unit=u.eV, equivalencies=u.mass_energy()) + + @property + def is_flat(self) -> bool: + return super().is_flat() + + assert Example(1).param == 1 * u.eV + assert Example(1 * u.eV).param == 1 * u.eV + assert Example(1 * u.J).param == (1 * u.J).to(u.eV) + assert Example(1 * u.kg).param == (1 * u.kg).to(u.eV, u.mass_energy()) diff --git a/astropy/cosmology/_src/tests/parameter/test_utils.py b/astropy/cosmology/_src/tests/parameter/test_utils.py new file mode 100644 index 000000000000..18e2004d8cdc --- /dev/null +++ b/astropy/cosmology/_src/tests/parameter/test_utils.py @@ -0,0 +1,26 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + + +from astropy.cosmology._src.parameter import Parameter, all_parameters + + +def test_all_parameters(): + """Test :func:`astropy.cosmology._src.utils.all_parameters`.""" + + class ClassA: + a = 1 + b = 2 + + got = all_parameters(ClassA) + assert got == {} + + class ClassB(ClassA): + H0: Parameter = Parameter( + doc="Hubble constant at z=0.", + unit="km/(s Mpc)", + fvalidate="scalar", + ) + + got = all_parameters(ClassB) + assert got.keys() == {"H0"} + assert all(isinstance(p, Parameter) for p in got.values()) diff --git a/astropy/cosmology/_src/tests/test_core.py b/astropy/cosmology/_src/tests/test_core.py new file mode 100644 index 000000000000..6f3dcc262150 --- /dev/null +++ b/astropy/cosmology/_src/tests/test_core.py @@ -0,0 +1,507 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +"""Testing :mod:`astropy.cosmology.core`.""" + +import abc +import inspect +import pickle + +import numpy as np +import pytest +from numpy.typing import NDArray + +import astropy.cosmology.units as cu +import astropy.units as u +from astropy.cosmology import Cosmology, FlatCosmologyMixin, Parameter +from astropy.cosmology._src.core import _COSMOLOGY_CLASSES, dataclass_decorator +from astropy.cosmology._src.tests.io.test_connect import ( + ReadWriteTestMixin, + ToFromFormatTestMixin, +) +from astropy.table import Column, QTable, Table + +from .parameter.test_descriptors import ( + ParametersAttributeTestMixin, +) +from .parameter.test_parameter import ParameterTestMixin + +############################################################################## +# SETUP / TEARDOWN + + +def make_valid_zs(max_z: float = 1e5) -> tuple[list, NDArray[float], list, list]: + """Make a list of valid redshifts for testing.""" + # scalar + scalar_zs = [ + 0, + 1, + min(1100, max_z), # interesting times + # FIXME! np.inf breaks some funcs. 0 * inf is an error + np.float64(min(3300, max_z)), # different type + 2 * cu.redshift, + 3 * u.one, # compatible units + ] + # array + _zarr = np.linspace(0, min(1e5, max_z), num=20) + array_zs = [ + _zarr, # numpy + _zarr.tolist(), # pure python + Column(_zarr), # table-like + _zarr * cu.redshift, # Quantity + ] + return scalar_zs, _zarr, array_zs, scalar_zs + array_zs + + +scalar_zs, z_arr, array_zs, valid_zs = make_valid_zs() + +invalid_zs = [ + (None, TypeError), # wrong type + # Wrong units (the TypeError is for the cython, which can differ) + (4 * u.MeV, (u.UnitConversionError, TypeError)), # scalar + ([0, 1] * u.m, (u.UnitConversionError, TypeError)), # array +] + + +@dataclass_decorator +class SubCosmology(Cosmology): + """Defined here to be serializable.""" + + H0: Parameter = Parameter(unit="km/(s Mpc)") + Tcmb0: Parameter = Parameter(default=0 * u.K, unit=u.K) + m_nu: Parameter = Parameter(default=0 * u.eV, unit=u.eV) + + @property + def is_flat(self) -> bool: + return super().is_flat() + + +############################################################################## +# TESTS +############################################################################## + + +class MetaTestMixin: + """Tests for a :class:`astropy.utils.metadata.MetaData` on a Cosmology.""" + + def test_meta_on_class(self, cosmo_cls): + assert cosmo_cls.meta is None + + def test_meta_on_instance(self, cosmo): + assert isinstance(cosmo.meta, dict) # test type + # value set at initialization + assert cosmo.meta == self.cls_kwargs.get("meta", {}) + + def test_meta_mutable(self, cosmo): + """The metadata is NOT immutable on a cosmology""" + key = next(iter(cosmo.meta.keys())) # select some key + cosmo.meta[key] = cosmo.meta.pop(key) # will error if immutable + + +class CosmologyTest( + ParameterTestMixin, + ParametersAttributeTestMixin, + MetaTestMixin, + ReadWriteTestMixin, + ToFromFormatTestMixin, + metaclass=abc.ABCMeta, +): + """Test subclasses of :class:`astropy.cosmology.Cosmology`.""" + + @abc.abstractmethod + def setup_class(self): + """Setup for testing.""" + + def teardown_class(self): + pass + + @property + def cls_args(self): + return tuple(self._cls_args.values()) + + @pytest.fixture(scope="class") + @classmethod + def cosmo_cls(cls): + """The Cosmology class as a :func:`pytest.fixture`.""" + return cls.cls + + @pytest.fixture(scope="function") # ensure not cached. + def ba(self): + """Return filled `inspect.BoundArguments` for cosmology.""" + ba = inspect.signature(self.cls).bind(*self.cls_args, **self.cls_kwargs) + ba.apply_defaults() + return ba + + @pytest.fixture(scope="class") + @classmethod + def cosmo(cls, cosmo_cls): + """The cosmology instance with which to test.""" + # `cls_args` is a @property on the test class, which doesn't fire + # when accessed via cls (only via self), so dereference _cls_args + # directly here. + cls_args = tuple(cls._cls_args.values()) + ba = inspect.signature(cls.cls).bind(*cls_args, **cls.cls_kwargs) + ba.apply_defaults() + return cosmo_cls(*ba.args, **ba.kwargs) + + # =============================================================== + # Method & Attribute Tests + + # --------------------------------------------------------------- + # class-level + + def test_init_subclass(self, cosmo_cls): + """Test creating subclasses registers classes and manages Parameters.""" + + # ----------------------------------------------------------- + # Normal subclass creation + + class InitSubclassTest(cosmo_cls): + pass + + # test parameters + assert InitSubclassTest.parameters == cosmo_cls.parameters + + # test and cleanup registry + registrant = _COSMOLOGY_CLASSES.pop(InitSubclassTest.__qualname__) + assert registrant is InitSubclassTest + + # ----------------------------------------------------------- + # Skip + + class UnRegisteredSubclassTest(cosmo_cls): + @classmethod + def _register_cls(cls): + """Override to not register.""" + + assert UnRegisteredSubclassTest.parameters == cosmo_cls.parameters + assert UnRegisteredSubclassTest.__qualname__ not in _COSMOLOGY_CLASSES + + # --------------------------------------------------------------- + # instance-level + + def test_init(self, cosmo_cls): + """Test initialization.""" + # Cosmology only does name and meta, but this subclass adds H0 & Tcmb0. + cosmo = cosmo_cls(*self.cls_args, name="test_init", meta={"m": 1}) + assert cosmo.name == "test_init" + assert cosmo.meta["m"] == 1 + + # if meta is None, it is changed to a dict + cosmo = cosmo_cls(*self.cls_args, name="test_init", meta=None) + assert cosmo.meta == {} + + def test_name(self, cosmo): + """Test property ``name``.""" + assert cosmo.name is None or isinstance(cosmo.name, str) # type + assert cosmo.name == self.cls_kwargs["name"] # test has expected value + + def test_name_immutable(self, cosmo): + """The name field should be immutable.""" + match = "cannot assign to field 'name'" + with pytest.raises(AttributeError, match=match): + cosmo.name = None + + def test_name_on_cls(self, cosmo_cls): + """Test accessing :attr:`~astropy.cosmology.Cosmology.name` from the class.""" + assert cosmo_cls.name is None + + @abc.abstractmethod + def test_is_flat(self, cosmo_cls, cosmo): + """Test property ``is_flat``.""" + + # ------------------------------------------------ + # clone + + def test_clone_identical(self, cosmo): + """Test method ``.clone()`` if no (kw)args.""" + assert cosmo.clone() is cosmo + + def test_clone_name(self, cosmo): + """Test method ``.clone()`` name argument.""" + # test changing name. clone treats 'name' differently (see next test) + c = cosmo.clone(name="cloned cosmo") + assert c.name == "cloned cosmo" # changed + # show name is the only thing changed + object.__setattr__(c, "name", cosmo.name) # first change name back + assert c == cosmo + assert c.meta == cosmo.meta + + # now change a different parameter and see how 'name' changes + c = cosmo.clone(meta={"test_clone_name": True}) + assert c.name == cosmo.name + " (modified)" + + def test_clone_meta(self, cosmo): + """Test method ``.clone()`` meta argument: updates meta, doesn't clear.""" + # start with no change + c = cosmo.clone(meta=None) + assert c.meta == cosmo.meta + + # add something + c = cosmo.clone(meta=dict(test_clone_meta=True)) + assert c.meta["test_clone_meta"] is True + c.meta.pop("test_clone_meta") # remove from meta + assert c.meta == cosmo.meta # now they match + + def test_clone_change_param(self, cosmo): + """ + Test method ``.clone()`` changing a(many) Parameter(s). + Nothing here b/c no Parameters. + """ + + def test_clone_fail_unexpected_arg(self, cosmo): + """Test when ``.clone()`` gets an unexpected argument.""" + with pytest.raises(TypeError, match="unexpected keyword argument"): + cosmo.clone(not_an_arg=4) + + def test_clone_fail_positional_arg(self, cosmo): + with pytest.raises(TypeError, match="1 positional argument"): + cosmo.clone(None) + + # --------------------------------------------------------------- + # comparison methods + + def test_is_equivalent(self, cosmo): + """Test :meth:`astropy.cosmology.Cosmology.is_equivalent`.""" + # to self + assert cosmo.is_equivalent(cosmo) + + # same class, different instance + newclone = cosmo.clone(name="test_is_equivalent") + assert cosmo.is_equivalent(newclone) + assert newclone.is_equivalent(cosmo) + + # different class and not convertible to Cosmology. + assert not cosmo.is_equivalent(2) + + def test_equality(self, cosmo): + """Test method ``.__eq__().""" + # wrong class + assert (cosmo != 2) and (2 != cosmo) + # correct + assert cosmo == cosmo + # different name <= not equal, but equivalent + newcosmo = cosmo.clone(name="test_equality") + assert (cosmo != newcosmo) and (newcosmo != cosmo) + assert cosmo.__equiv__(newcosmo) and newcosmo.__equiv__(cosmo) + + # ------------------------------------------------ + + @pytest.mark.parametrize("in_meta", [True, False]) + @pytest.mark.parametrize("table_cls", [Table, QTable]) + def test_astropy_table(self, cosmo, table_cls, in_meta): + """Test ``astropy.table.Table(cosmology)``.""" + tbl = table_cls(cosmo, cosmology_in_meta=in_meta) + + assert isinstance(tbl, table_cls) + # the name & all parameters are columns + for n in ("name", *cosmo.parameters): + assert n in tbl.colnames + assert np.all(tbl[n] == getattr(cosmo, n)) + # check if Cosmology is in metadata or a column + if in_meta: + assert tbl.meta["cosmology"] == cosmo.__class__.__qualname__ + assert "cosmology" not in tbl.colnames + else: + assert "cosmology" not in tbl.meta + assert tbl["cosmology"][0] == cosmo.__class__.__qualname__ + # the metadata is transferred + for k, v in cosmo.meta.items(): + assert np.all(tbl.meta[k] == v) + + # =============================================================== + # Usage Tests + + def test_immutability(self, cosmo): + """ + Test immutability of cosmologies. + The metadata is mutable: see ``test_meta_mutable``. + """ + for n in (*cosmo.parameters, *cosmo._derived_parameters): + with pytest.raises(AttributeError): + setattr(cosmo, n, getattr(cosmo, n)) + + def test_pickle_class(self, cosmo_cls, pickle_protocol): + """Test classes can pickle and unpickle.""" + # pickle and unpickle + f = pickle.dumps(cosmo_cls, protocol=pickle_protocol) + unpickled = pickle.loads(f) + + # test equality + assert unpickled == cosmo_cls + + def test_pickle_instance(self, cosmo, pickle_protocol): + """Test instances can pickle and unpickle.""" + # pickle and unpickle + f = pickle.dumps(cosmo, protocol=pickle_protocol) + with u.add_enabled_units(cu): + unpickled = pickle.loads(f) + + assert unpickled == cosmo + assert unpickled.meta == cosmo.meta + + +class TestCosmology(CosmologyTest): + """Test :class:`astropy.cosmology.Cosmology`. + + Subclasses should define tests for: + + - ``test_clone_change_param()`` + - ``test_repr()`` + """ + + def setup_class(self): + """ + Setup for testing. + Cosmology should not be instantiated, so tests are done on a subclass. + """ + # make sure SubCosmology is known + _COSMOLOGY_CLASSES["SubCosmology"] = SubCosmology + + self.cls = SubCosmology + self._cls_args = dict( + H0=70 * (u.km / u.s / u.Mpc), Tcmb0=2.7 * u.K, m_nu=0.6 * u.eV + ) + self.cls_kwargs = dict(name=self.__class__.__name__, meta={"a": "b"}) + + def teardown_class(self): + """Teardown for testing.""" + super().teardown_class(self) + _COSMOLOGY_CLASSES.pop("SubCosmology", None) + + # =============================================================== + # Method & Attribute Tests + + def test_is_flat(self, cosmo_cls, cosmo): + """Test property ``is_flat``. It's an ABC.""" + with pytest.raises(NotImplementedError, match="is_flat is not implemented"): + cosmo.is_flat + + +# ----------------------------------------------------------------------------- + + +class FlatCosmologyMixinTest: + """Tests for :class:`astropy.cosmology.core.FlatCosmologyMixin` subclasses. + + The test suite structure mirrors the implementation of the tested code. + Just like :class:`astropy.cosmology.FlatCosmologyMixin` is an abstract + base class (ABC) that cannot be used by itself, so too is this corresponding + test class an ABC mixin. + + E.g to use this class:: + + class TestFlatSomeCosmology(FlatCosmologyMixinTest, TestSomeCosmology): + ... + """ + + def test_nonflat_class_(self, cosmo_cls, cosmo): + """Test :attr:`astropy.cosmology.core.FlatCosmologyMixin.nonflat_cls`.""" + # Test it's a method on the class + assert issubclass(cosmo_cls, cosmo_cls.__nonflatclass__) + + # It also works from the instance. # TODO! as a "metaclassmethod" + assert issubclass(cosmo_cls, cosmo.__nonflatclass__) + + # Maybe not the most robust test, but so far all Flat classes have the + # name of their parent class. + assert cosmo.__nonflatclass__.__name__ in cosmo_cls.__name__ + + def test_is_flat(self, cosmo_cls, cosmo): + """Test property ``is_flat``.""" + super().test_is_flat(cosmo_cls, cosmo) + + # it's always True + assert cosmo.is_flat is True + + def test_nonflat(self, cosmo): + """Test :attr:`astropy.cosmology.core.FlatCosmologyMixin.nonflat`.""" + assert cosmo.nonflat.is_equivalent(cosmo) + assert cosmo.is_equivalent(cosmo.nonflat) + + # ------------------------------------------------ + # clone + + def test_clone_to_nonflat_equivalent(self, cosmo): + """Test method ``.clone()``to_nonflat argument.""" + # just converting the class + nc = cosmo.clone(to_nonflat=True) + assert isinstance(nc, cosmo.__nonflatclass__) + assert nc == cosmo.nonflat + + @abc.abstractmethod + def test_clone_to_nonflat_change_param(self, cosmo): + """ + Test method ``.clone()`` changing a(many) Parameter(s). No parameters + are changed here because FlatCosmologyMixin has no Parameters. + See class docstring for why this test method exists. + """ + # send to non-flat + nc = cosmo.clone(to_nonflat=True) + assert isinstance(nc, cosmo.__nonflatclass__) + assert nc == cosmo.nonflat + + # ------------------------------------------------ + + def test_is_equivalent(self, cosmo): + """Test :meth:`astropy.cosmology.core.FlatCosmologyMixin.is_equivalent`. + + Normally this would pass up via super(), but ``__equiv__`` is meant + to be overridden, so we skip super(). + e.g. FlatFLRWMixinTest -> FlatCosmologyMixinTest -> TestCosmology + vs FlatFLRWMixinTest -> FlatCosmologyMixinTest -> TestFLRW -> TestCosmology + """ + CosmologyTest.test_is_equivalent(self, cosmo) + + # See FlatFLRWMixinTest for tests. It's a bit hard here since this class + # is for an ABC. + + # =============================================================== + # Usage Tests + + def test_subclassing(self, cosmo_cls): + """Test when subclassing a flat cosmology.""" + + class SubClass1(cosmo_cls): + pass + + # The classes have the same non-flat parent class + assert SubClass1.__nonflatclass__ is cosmo_cls.__nonflatclass__ + + # A more complex example is when Mixin classes are used. + class Mixin: + pass + + class SubClass2(Mixin, cosmo_cls): + pass + + # The classes have the same non-flat parent class + assert SubClass2.__nonflatclass__ is cosmo_cls.__nonflatclass__ + + # The order of the Mixin should not matter + class SubClass3(cosmo_cls, Mixin): + pass + + # The classes have the same non-flat parent class + assert SubClass3.__nonflatclass__ is cosmo_cls.__nonflatclass__ + + +def test__nonflatclass__multiple_nonflat_inheritance(): + """ + Test :meth:`astropy.cosmology.core.FlatCosmologyMixin.__nonflatclass__` + when there's more than one non-flat class in the inheritance. + """ + + # Define a non-operable minimal subclass of Cosmology. + @dataclass_decorator + class SubCosmology2(Cosmology): + @property + def is_flat(self) -> bool: + return False + + # Now make an ambiguous flat cosmology from the two SubCosmologies + with pytest.raises(TypeError, match="cannot create a consistent non-flat class"): + + class FlatSubCosmology(FlatCosmologyMixin, SubCosmology, SubCosmology2): + @property + def nonflat(self): + pass diff --git a/astropy/cosmology/_src/tests/test_parameters.py b/astropy/cosmology/_src/tests/test_parameters.py new file mode 100644 index 000000000000..21902f82aae1 --- /dev/null +++ b/astropy/cosmology/_src/tests/test_parameters.py @@ -0,0 +1,44 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +from types import MappingProxyType + +import numpy as np +import pytest + +from astropy.cosmology import parameters, realizations + + +def test_realizations_in_dir(): + """Test the realizations are in ``dir`` of :mod:`astropy.cosmology.parameters`.""" + d = dir(parameters) + + assert set(d) == set(parameters.__all__) + for n in parameters.available: + assert n in d + + +@pytest.mark.parametrize("name", parameters.available) +def test_getting_parameters(name): + """ + Test getting 'parameters' and that it is derived from the corresponding + realization. + """ + params = getattr(parameters, name) + + assert isinstance(params, MappingProxyType) + assert params["name"] == name + + # Check parameters have the right keys and values + cosmo = getattr(realizations, name) + assert params["name"] == cosmo.name + assert params["cosmology"] == cosmo.__class__.__qualname__ + # All the cosmology parameters are equal + for k, v in cosmo.parameters.items(): + np.testing.assert_array_equal(params[k], v) + # All the metadata is included. Parameter values take precedence, so only + # checking the keys. + assert set(cosmo.meta.keys()).issubset(params.keys()) + + # Lastly, check the generation process. + m = cosmo.to_format("mapping", cosmology_as_str=True, move_from_meta=True) + assert params == m diff --git a/astropy/cosmology/_src/tests/test_realizations.py b/astropy/cosmology/_src/tests/test_realizations.py new file mode 100644 index 000000000000..bfe7a1352e52 --- /dev/null +++ b/astropy/cosmology/_src/tests/test_realizations.py @@ -0,0 +1,105 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import pickle + +import pytest + +import astropy.cosmology.units as cu +import astropy.units as u +from astropy import cosmology +from astropy.cosmology import parameters, realizations +from astropy.cosmology.realizations import default_cosmology + + +def test_realizations_in_toplevel_dir(): + """Test the realizations are in ``dir`` of :mod:`astropy.cosmology`.""" + d = dir(cosmology) + + assert set(d) == set(cosmology.__all__) + for n in parameters.available: + assert n in d + + +def test_realizations_in_realizations_dir(): + """Test the realizations are in ``dir`` of :mod:`astropy.cosmology.realizations`.""" + d = dir(realizations) + + assert set(d) == set(realizations.__all__) + for n in parameters.available: + assert n in d + + +class Test_default_cosmology: + """Tests for :class:`~astropy.cosmology.realizations.default_cosmology`.""" + + # ----------------------------------------------------- + # Get + + def test_get_current(self): + """Test :meth:`astropy.cosmology.default_cosmology.get` current value.""" + cosmo = default_cosmology.get() + assert cosmo is default_cosmology.validate(default_cosmology._value) + + # ----------------------------------------------------- + # Validate + + def test_validate_fail(self): + """Test :meth:`astropy.cosmology.default_cosmology.validate`.""" + # bad input type + with pytest.raises(TypeError, match="must be a string or Cosmology"): + default_cosmology.validate(TypeError) + + # a not-valid option, but still a str + with pytest.raises(ValueError, match="Unknown cosmology"): + default_cosmology.validate("fail!") + + # a not-valid type + with pytest.raises(TypeError, match="cannot find a Cosmology"): + default_cosmology.validate("available") + + def test_validate_default(self): + """Test method ``validate`` for specific values.""" + value = default_cosmology.validate(None) + assert value is realizations.Planck18 + + @pytest.mark.parametrize("name", parameters.available) + def test_validate_str(self, name): + """Test method ``validate`` for string input.""" + value = default_cosmology.validate(name) + assert value is getattr(realizations, name) + + @pytest.mark.parametrize("name", parameters.available) + def test_validate_cosmo(self, name): + """Test method ``validate`` for cosmology instance input.""" + cosmo = getattr(realizations, name) + value = default_cosmology.validate(cosmo) + assert value is cosmo + + def test_validate_no_default(self): + """Test :meth:`astropy.cosmology.default_cosmology.get` to `None`.""" + cosmo = default_cosmology.validate("no_default") + assert cosmo is None + + +@pytest.mark.parametrize("name", parameters.available) +def test_pickle_builtin_realizations(name, pickle_protocol): + """ + Test in-built realizations can pickle and unpickle. + Also a regression test for #12008. + """ + # get class instance + original = getattr(cosmology, name) + + # pickle and unpickle + f = pickle.dumps(original, protocol=pickle_protocol) + with u.add_enabled_units(cu): + unpickled = pickle.loads(f) + + assert unpickled == original + assert unpickled.meta == original.meta + + # if the units are not enabled, it isn't equal because redshift units + # are not equal. This is a weird, known issue. + unpickled = pickle.loads(f) + assert unpickled == original + assert unpickled.meta != original.meta diff --git a/astropy/cosmology/_src/tests/test_scipy_compat.py b/astropy/cosmology/_src/tests/test_scipy_compat.py new file mode 100644 index 000000000000..5457472250e5 --- /dev/null +++ b/astropy/cosmology/_src/tests/test_scipy_compat.py @@ -0,0 +1,13 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import pytest + +from astropy.cosmology._src.flrw.base import quad +from astropy.utils.compat.optional_deps import HAS_SCIPY + + +@pytest.mark.skipif(HAS_SCIPY, reason="scipy is installed") +def test_optional_deps_functions(): + """Test stand-in functions when optional dependencies not installed.""" + with pytest.raises(ModuleNotFoundError, match="No module named 'scipy.integrate'"): + quad() diff --git a/astropy/cosmology/_src/tests/test_units.py b/astropy/cosmology/_src/tests/test_units.py new file mode 100644 index 000000000000..2c67498f4a9f --- /dev/null +++ b/astropy/cosmology/_src/tests/test_units.py @@ -0,0 +1,372 @@ +"""Testing :mod:`astropy.cosmology.units`.""" + +import pytest + +import astropy.cosmology.units as cu +import astropy.units as u +from astropy.cosmology import Planck13, default_cosmology +from astropy.tests.helper import assert_quantity_allclose +from astropy.utils.compat.optional_deps import HAS_SCIPY + +############################################################################## +# TESTS +############################################################################## + + +def test_littleh(): + """Test :func:`astropy.cosmology.units.with_H0`.""" + H0_70 = 70 * u.km / u.s / u.Mpc + h70dist = 70 * u.Mpc / cu.littleh + + assert_quantity_allclose(h70dist.to(u.Mpc, cu.with_H0(H0_70)), 100 * u.Mpc) + + # make sure using the default cosmology works + cosmodist = default_cosmology.get().H0.value * u.Mpc / cu.littleh + assert_quantity_allclose(cosmodist.to(u.Mpc, cu.with_H0()), 100 * u.Mpc) + + # Now try a luminosity scaling + h1lum = 0.49 * u.Lsun * cu.littleh**-2 + assert_quantity_allclose(h1lum.to(u.Lsun, cu.with_H0(H0_70)), 1 * u.Lsun) + + # And the trickiest one: magnitudes. Using H0=10 here for the round numbers + H0_10 = 10 * u.km / u.s / u.Mpc + # assume the "true" magnitude M = 12. + # Then M - 5*log_10(h) = M + 5 = 17 + withlittlehmag = 17 * (u.mag - u.MagUnit(cu.littleh**2)) + assert_quantity_allclose(withlittlehmag.to(u.mag, cu.with_H0(H0_10)), 12 * u.mag) + + +@pytest.mark.skipif(not HAS_SCIPY, reason="Cosmology needs scipy") +def test_dimensionless_redshift(): + """Test :func:`astropy.cosmology.units.dimensionless_redshift`.""" + z = 3 * cu.redshift + val = 3 * u.one + + # show units not equal + assert z.unit == cu.redshift + assert z.unit != u.one + assert u.get_physical_type(z) == "redshift" + + # test equivalency enabled by default + assert z == val + + # also test that it works for powers + assert (3 * cu.redshift**3) == val + + # and in composite units + assert (3 * u.km / cu.redshift**3) == 3 * u.km + + # test it also works as an equivalency + with u.set_enabled_equivalencies([]): # turn off default equivalencies + assert z.to(u.one, equivalencies=cu.dimensionless_redshift()) == val + + with pytest.raises(ValueError): + z.to(u.one) + + # if this fails, something is really wrong + with u.add_enabled_equivalencies(cu.dimensionless_redshift()): + assert z == val + + +@pytest.mark.skipif(not HAS_SCIPY, reason="Cosmology needs scipy") +def test_redshift_temperature(): + """Test :func:`astropy.cosmology.units.redshift_temperature`.""" + cosmo = Planck13.clone(Tcmb0=3 * u.K) + default_cosmo = default_cosmology.get() + z = 15 * cu.redshift + Tcmb = cosmo.Tcmb(z) + + # 1) Default (without specifying the cosmology) + with default_cosmology.set(cosmo): + equivalency = cu.redshift_temperature() + assert_quantity_allclose(z.to(u.K, equivalency), Tcmb) + assert_quantity_allclose(Tcmb.to(cu.redshift, equivalency), z) + + # showing the answer changes if the cosmology changes + # this test uses the default cosmology + equivalency = cu.redshift_temperature() + assert_quantity_allclose(z.to(u.K, equivalency), default_cosmo.Tcmb(z)) + assert default_cosmo.Tcmb(z) != Tcmb + + # 2) Specifying the cosmology + equivalency = cu.redshift_temperature(cosmo) + assert_quantity_allclose(z.to(u.K, equivalency), Tcmb) + assert_quantity_allclose(Tcmb.to(cu.redshift, equivalency), z) + + # Test `atzkw` + equivalency = cu.redshift_temperature(cosmo, ztol=1e-10) + assert_quantity_allclose(Tcmb.to(cu.redshift, equivalency), z) + + +@pytest.mark.skipif(not HAS_SCIPY, reason="Cosmology needs scipy") +def test_redshift_hubble(): + """Test :func:`astropy.cosmology.units.redshift_hubble`.""" + unit = u.km / u.s / u.Mpc + cosmo = Planck13.clone(H0=100 * unit) + default_cosmo = default_cosmology.get() + z = 15 * cu.redshift + H = cosmo.H(z) + h = H.to_value(u.km / u.s / u.Mpc) / 100 * cu.littleh + + # 1) Default (without specifying the cosmology) + with default_cosmology.set(cosmo): + equivalency = cu.redshift_hubble() + # H + assert_quantity_allclose(z.to(unit, equivalency), H) + assert_quantity_allclose(H.to(cu.redshift, equivalency), z) + # little-h + assert_quantity_allclose(z.to(cu.littleh, equivalency), h) + assert_quantity_allclose(h.to(cu.redshift, equivalency), z) + + # showing the answer changes if the cosmology changes + # this test uses the default cosmology + equivalency = cu.redshift_hubble() + assert_quantity_allclose(z.to(unit, equivalency), default_cosmo.H(z)) + assert default_cosmo.H(z) != H + + # 2) Specifying the cosmology + equivalency = cu.redshift_hubble(cosmo) + # H + assert_quantity_allclose(z.to(unit, equivalency), H) + assert_quantity_allclose(H.to(cu.redshift, equivalency), z) + # little-h + assert_quantity_allclose(z.to(cu.littleh, equivalency), h) + assert_quantity_allclose(h.to(cu.redshift, equivalency), z) + + # Test `atzkw` + equivalency = cu.redshift_hubble(cosmo, ztol=1e-10) + assert_quantity_allclose(H.to(cu.redshift, equivalency), z) # H + assert_quantity_allclose(h.to(cu.redshift, equivalency), z) # little-h + + +@pytest.mark.skipif(not HAS_SCIPY, reason="Cosmology needs scipy") +@pytest.mark.parametrize( + "kind", + [cu.redshift_distance.__defaults__[-1], "comoving", "lookback", "luminosity"], +) +def test_redshift_distance(kind): + """Test :func:`astropy.cosmology.units.redshift_distance`.""" + z = 15 * cu.redshift + d = getattr(Planck13, kind + "_distance")(z) + + equivalency = cu.redshift_distance(cosmology=Planck13, kind=kind) + + # properties of Equivalency + assert equivalency.name[0] == "redshift_distance" + assert equivalency.kwargs[0]["cosmology"] == Planck13 + assert equivalency.kwargs[0]["distance"] == kind + + # roundtrip + assert_quantity_allclose(z.to(u.Mpc, equivalency), d) + assert_quantity_allclose(d.to(cu.redshift, equivalency), z) + + +def test_redshift_distance_wrong_kind(): + """Test :func:`astropy.cosmology.units.redshift_distance` wrong kind.""" + with pytest.raises(ValueError, match="`kind`"): + cu.redshift_distance(kind=None) + + +@pytest.mark.skipif(not HAS_SCIPY, reason="Cosmology needs scipy") +class Test_with_redshift: + """Test `astropy.cosmology.units.with_redshift`.""" + + @pytest.fixture(scope="class") + @classmethod + def cosmo(cls): + """Test cosmology.""" + return Planck13.clone(Tcmb0=3 * u.K) + + # =========================================== + + def test_cosmo_different(self, cosmo): + """The default is different than the test cosmology.""" + default_cosmo = default_cosmology.get() + assert default_cosmo != cosmo # shows changing default + + def test_no_equivalency(self, cosmo): + """Test the equivalency ``with_redshift`` without any enabled.""" + equivalency = cu.with_redshift(distance=None, hubble=False, Tcmb=False) + assert len(equivalency) == 0 + + # ------------------------------------------- + + def test_temperature_off(self, cosmo): + """Test ``with_redshift`` with the temperature off.""" + z = 15 * cu.redshift + err_msg = ( + r"^'redshift' \(redshift\) and 'K' \(temperature\) are not convertible$" + ) + + # 1) Default (without specifying the cosmology) + with default_cosmology.set(cosmo): + equivalency = cu.with_redshift(Tcmb=False) + with pytest.raises(u.UnitConversionError, match=err_msg): + z.to(u.K, equivalency) + + # 2) Specifying the cosmology + equivalency = cu.with_redshift(cosmo, Tcmb=False) + with pytest.raises(u.UnitConversionError, match=err_msg): + z.to(u.K, equivalency) + + def test_temperature(self, cosmo): + """Test temperature equivalency component.""" + default_cosmo = default_cosmology.get() + z = 15 * cu.redshift + Tcmb = cosmo.Tcmb(z) + + # 1) Default (without specifying the cosmology) + with default_cosmology.set(cosmo): + equivalency = cu.with_redshift(Tcmb=True) + assert_quantity_allclose(z.to(u.K, equivalency), Tcmb) + assert_quantity_allclose(Tcmb.to(cu.redshift, equivalency), z) + + # showing the answer changes if the cosmology changes + # this test uses the default cosmology + equivalency = cu.with_redshift(Tcmb=True) + assert_quantity_allclose(z.to(u.K, equivalency), default_cosmo.Tcmb(z)) + assert default_cosmo.Tcmb(z) != Tcmb + + # 2) Specifying the cosmology + equivalency = cu.with_redshift(cosmo, Tcmb=True) + assert_quantity_allclose(z.to(u.K, equivalency), Tcmb) + assert_quantity_allclose(Tcmb.to(cu.redshift, equivalency), z) + + # Test `atzkw` + # this is really just a test that 'atzkw' doesn't fail + equivalency = cu.with_redshift(cosmo, Tcmb=True, atzkw={"ztol": 1e-10}) + assert_quantity_allclose(Tcmb.to(cu.redshift, equivalency), z) + + # ------------------------------------------- + + def test_hubble_off(self, cosmo): + """Test ``with_redshift`` with Hubble off.""" + unit = u.km / u.s / u.Mpc + z = 15 * cu.redshift + err_msg = ( + r"^'redshift' \(redshift\) and 'km / \(Mpc s\)' \(frequency\) are not " + "convertible$" + ) + + # 1) Default (without specifying the cosmology) + with default_cosmology.set(cosmo): + equivalency = cu.with_redshift(hubble=False) + with pytest.raises(u.UnitConversionError, match=err_msg): + z.to(unit, equivalency) + + # 2) Specifying the cosmology + equivalency = cu.with_redshift(cosmo, hubble=False) + with pytest.raises(u.UnitConversionError, match=err_msg): + z.to(unit, equivalency) + + def test_hubble(self, cosmo): + """Test Hubble equivalency component.""" + unit = u.km / u.s / u.Mpc + default_cosmo = default_cosmology.get() + z = 15 * cu.redshift + H = cosmo.H(z) + h = H.to_value(u.km / u.s / u.Mpc) / 100 * cu.littleh + + # 1) Default (without specifying the cosmology) + with default_cosmology.set(cosmo): + equivalency = cu.with_redshift(hubble=True) + # H + assert_quantity_allclose(z.to(unit, equivalency), H) + assert_quantity_allclose(H.to(cu.redshift, equivalency), z) + # little-h + assert_quantity_allclose(z.to(cu.littleh, equivalency), h) + assert_quantity_allclose(h.to(cu.redshift, equivalency), z) + + # showing the answer changes if the cosmology changes + # this test uses the default cosmology + equivalency = cu.with_redshift(hubble=True) + assert_quantity_allclose(z.to(unit, equivalency), default_cosmo.H(z)) + assert default_cosmo.H(z) != H + + # 2) Specifying the cosmology + equivalency = cu.with_redshift(cosmo, hubble=True) + # H + assert_quantity_allclose(z.to(unit, equivalency), H) + assert_quantity_allclose(H.to(cu.redshift, equivalency), z) + # little-h + assert_quantity_allclose(z.to(cu.littleh, equivalency), h) + assert_quantity_allclose(h.to(cu.redshift, equivalency), z) + + # Test `atzkw` + # this is really just a test that 'atzkw' doesn't fail + equivalency = cu.with_redshift(cosmo, hubble=True, atzkw={"ztol": 1e-10}) + assert_quantity_allclose(H.to(cu.redshift, equivalency), z) # H + assert_quantity_allclose(h.to(cu.redshift, equivalency), z) # h + + # ------------------------------------------- + + def test_distance_off(self, cosmo): + """Test ``with_redshift`` with the distance off.""" + z = 15 * cu.redshift + err_msg = r"^'redshift' \(redshift\) and 'Mpc' \(length\) are not convertible$" + + # 1) Default (without specifying the cosmology) + with default_cosmology.set(cosmo): + equivalency = cu.with_redshift(distance=None) + with pytest.raises(u.UnitConversionError, match=err_msg): + z.to(u.Mpc, equivalency) + + # 2) Specifying the cosmology + equivalency = cu.with_redshift(cosmo, distance=None) + with pytest.raises(u.UnitConversionError, match=err_msg): + z.to(u.Mpc, equivalency) + + def test_distance_default(self): + """Test distance equivalency default.""" + z = 15 * cu.redshift + d = default_cosmology.get().comoving_distance(z) + + equivalency = cu.with_redshift() + assert_quantity_allclose(z.to(u.Mpc, equivalency), d) + assert_quantity_allclose(d.to(cu.redshift, equivalency), z) + + def test_distance_wrong_kind(self): + """Test distance equivalency, but the wrong kind.""" + with pytest.raises(ValueError, match="`kind`"): + cu.with_redshift(distance=ValueError) + + @pytest.mark.parametrize("kind", ["comoving", "lookback", "luminosity"]) + def test_distance(self, kind): + """Test distance equivalency.""" + cosmo = Planck13 + z = 15 * cu.redshift + dist = getattr(cosmo, kind + "_distance")(z) + + default_cosmo = default_cosmology.get() + assert default_cosmo != cosmo # shows changing default + + # 1) without specifying the cosmology + with default_cosmology.set(cosmo): + equivalency = cu.with_redshift(distance=kind) + assert_quantity_allclose(z.to(u.Mpc, equivalency), dist) + + # showing the answer changes if the cosmology changes + # this test uses the default cosmology + equivalency = cu.with_redshift(distance=kind) + assert_quantity_allclose( + z.to(u.Mpc, equivalency), getattr(default_cosmo, kind + "_distance")(z) + ) + assert not u.allclose(getattr(default_cosmo, kind + "_distance")(z), dist) + + # 2) Specifying the cosmology + equivalency = cu.with_redshift(cosmo, distance=kind) + assert_quantity_allclose(z.to(u.Mpc, equivalency), dist) + assert_quantity_allclose(dist.to(cu.redshift, equivalency), z) + + # Test atzkw + # this is really just a test that 'atzkw' doesn't fail + equivalency = cu.with_redshift(cosmo, distance=kind, atzkw={"ztol": 1e-10}) + assert_quantity_allclose(dist.to(cu.redshift, equivalency), z) + + +def test_equivalency_context_manager(): + base_registry = u.get_current_unit_registry() + + # check starting with only the dimensionless_redshift equivalency. + assert len(base_registry.equivalencies) == 1 + assert str(base_registry.equivalencies[0][0]) == "redshift" diff --git a/astropy/cosmology/_src/tests/test_utils.py b/astropy/cosmology/_src/tests/test_utils.py new file mode 100644 index 000000000000..beac4b9e92fc --- /dev/null +++ b/astropy/cosmology/_src/tests/test_utils.py @@ -0,0 +1,177 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import numpy as np +import pytest + +import astropy.units as u +from astropy.cosmology._src.utils import ( + aszarr, + deprecated_keywords, + vectorize_redshift_method, +) +from astropy.utils.compat.optional_deps import HAS_PANDAS + +from .test_core import invalid_zs, valid_zs, z_arr + + +def test_vectorize_redshift_method(): + """Test :func:`astropy.cosmology._src.utils.vectorize_redshift_method`.""" + + class Class: + @vectorize_redshift_method + def method(self, z): + return z + + c = Class() + + assert hasattr(c.method, "__vectorized__") + assert isinstance(c.method.__vectorized__, np.vectorize) + + # calling with Number + assert c.method(1) == 1 + assert isinstance(c.method(1), int) + + # calling with a numpy scalar + assert c.method(np.float64(1)) == np.float64(1) + assert isinstance(c.method(np.float64(1)), np.float64) + + # numpy array + assert all(c.method(np.array([1, 2])) == np.array([1, 2])) + assert isinstance(c.method(np.array([1, 2])), np.ndarray) + + # non-scalar + assert all(c.method([1, 2]) == np.array([1, 2])) + assert isinstance(c.method([1, 2]), np.ndarray) + + +# ------------------------------------------------------------------- + + +class Test_aszarr: + @pytest.mark.parametrize( + "z, expect", + list( + zip( + valid_zs, + [0, 1, 1100, np.float64(3300), 2.0, 3.0, z_arr, z_arr, z_arr, z_arr], + ) + ), + ) + def test_valid(self, z, expect): + """Test :func:`astropy.cosmology._src.utils.aszarr`.""" + got = aszarr(z) + assert np.array_equal(got, expect) + + @pytest.mark.parametrize("z, exc", invalid_zs) + def test_invalid(self, z, exc): + """Test :func:`astropy.cosmology._src.utils.aszarr`.""" + with pytest.raises(exc): + aszarr(z) + + @pytest.mark.skipif(not HAS_PANDAS, reason="requires pandas") + def test_pandas(self): + import pandas as pd + + x = pd.Series([1, 2, 3, 4, 5]) + + # Demonstrate Pandas doesn't work with units + assert not isinstance(x * u.km, u.Quantity) + + # Test aszarr works with Pandas + assert isinstance(aszarr(x), np.ndarray) + np.testing.assert_array_equal(aszarr(x), x.values) + + +# ------------------------------------------------------------------- + + +class TestDeprecatedKeywords: + @classmethod + def setup_class(cls): + def noop(a, b, c, d): + # a minimal function that does nothing, + # with multiple positional-or-keywords arguments + return + + cls.base_func = noop + cls.depr_funcs = { + 1: deprecated_keywords("a", since="999.999.999")(noop), + 2: deprecated_keywords("a", "b", since="999.999.999")(noop), + 4: deprecated_keywords("a", "b", "c", "d", since="999.999.999")(noop), + } + + def test_type_safety(self): + dec = deprecated_keywords(b"a", since="999.999.999") + with pytest.raises(TypeError, match=r"names\[0\] must be a string"): + dec(self.base_func) + + dec = deprecated_keywords("a", since=b"999.999.999") + with pytest.raises(TypeError, match=r"since must be a string"): + dec(self.base_func) + + @pytest.mark.parametrize("n_deprecated_keywords", [1, 2, 4]) + def test_no_warn(self, n_deprecated_keywords): + func = self.depr_funcs[n_deprecated_keywords] + func(1, 2, 3, 4) + + @pytest.mark.parametrize( + "n_deprecated_keywords, args, kwargs, match", + [ + pytest.param( + 1, + (), + {"a": 1, "b": 2, "c": 3, "d": 4}, + r"Passing 'a' as keyword is deprecated since", + id="1 deprecation, 1 warn", + ), + pytest.param( + 2, + (1,), + {"b": 2, "c": 3, "d": 4}, + r"Passing 'b' as keyword is deprecated since", + id="2 deprecation, 1 warn", + ), + pytest.param( + 2, + (), + {"a": 1, "b": 2, "c": 3, "d": 4}, + r"Passing \['a', 'b'\] arguments as keywords is deprecated since", + id="2 deprecations, 2 warns", + ), + pytest.param( + 4, + (), + {"a": 1, "b": 2, "c": 3, "d": 4}, + ( + r"Passing \['a', 'b', 'c', 'd'\] arguments as keywords " + "is deprecated since" + ), + id="4 deprecations, 4 warns", + ), + pytest.param( + 4, + (1,), + {"b": 2, "c": 3, "d": 4}, + r"Passing \['b', 'c', 'd'\] arguments as keywords is deprecated since", + id="4 deprecations, 3 warns", + ), + pytest.param( + 4, + (1, 2), + {"c": 3, "d": 4}, + r"Passing \['c', 'd'\] arguments as keywords is deprecated since", + id="4 deprecations, 2 warns", + ), + pytest.param( + 4, + (1, 2, 3), + {"d": 4}, + r"Passing 'd' as keyword is deprecated since", + id="4 deprecations, 1 warn", + ), + ], + ) + def test_warn(self, n_deprecated_keywords, args, kwargs, match): + func = self.depr_funcs[n_deprecated_keywords] + with pytest.warns(FutureWarning, match=match): + func(*args, **kwargs) diff --git a/astropy/cosmology/_src/tests/traits/__init__.py b/astropy/cosmology/_src/tests/traits/__init__.py new file mode 100644 index 000000000000..46dc7ab34bba --- /dev/null +++ b/astropy/cosmology/_src/tests/traits/__init__.py @@ -0,0 +1,3 @@ +"""Test package for cosmology trait tests.""" + +__all__ = [] diff --git a/astropy/cosmology/_src/tests/traits/helper.py b/astropy/cosmology/_src/tests/traits/helper.py new file mode 100644 index 000000000000..58a952463977 --- /dev/null +++ b/astropy/cosmology/_src/tests/traits/helper.py @@ -0,0 +1,17 @@ +import inspect +from collections.abc import Callable + + +def is_positional_only(func: Callable, /, param: str) -> bool: + """Return True if ``param:str`` is a positional-only parameter. + + Parameters + ---------- + func: Callable + Function to check whether parameter `param` is positional-only. + param : str + The name of the parameter in `func` to check. + """ + sig = inspect.signature(func) + p = sig.parameters.get(param) + return p is not None and p.kind == inspect.Parameter.POSITIONAL_ONLY diff --git a/astropy/cosmology/_src/tests/traits/test_trait_baryons.py b/astropy/cosmology/_src/tests/traits/test_trait_baryons.py new file mode 100644 index 000000000000..1a7a349d8e03 --- /dev/null +++ b/astropy/cosmology/_src/tests/traits/test_trait_baryons.py @@ -0,0 +1,25 @@ +import numpy as np +import pytest +from numpy.testing import assert_allclose + +from astropy.cosmology._src.traits.baryons import BaryonComponent + +from .helper import is_positional_only + + +class DummyBaryon(BaryonComponent): + Ob0 = 0.05 + + def inv_efunc(self, z): + return np.ones_like(np.asarray(z)) + + +@pytest.fixture +def dummy_baryon(): + return DummyBaryon() + + +def test_baryon_signature_and_behavior(dummy_baryon): + assert hasattr(BaryonComponent, "Ob") + assert is_positional_only(BaryonComponent.Ob, "z") + assert_allclose(dummy_baryon.Ob(1), 0.05 * (1 + 1) ** 3) diff --git a/astropy/cosmology/_src/tests/traits/test_trait_curvature.py b/astropy/cosmology/_src/tests/traits/test_trait_curvature.py new file mode 100644 index 000000000000..fe50939cb0ac --- /dev/null +++ b/astropy/cosmology/_src/tests/traits/test_trait_curvature.py @@ -0,0 +1,42 @@ +import numpy as np +import pytest +from numpy.testing import assert_allclose + +from astropy.cosmology._src.traits.curvature import CurvatureComponent + +from .helper import is_positional_only + + +class DummyCurvature(CurvatureComponent): + def __init__(self, ok0): + self._ok0 = ok0 + + @property + def Ok0(self): + return self._ok0 + + @property + def is_flat(self): + return self._ok0 == 0 + + def inv_efunc(self, z): + return np.ones_like(np.asarray(z)) + + +@pytest.fixture +def dummy_curvature_zero(): + return DummyCurvature(0.0) + + +@pytest.fixture +def dummy_curvature_nonzero(): + return DummyCurvature(-0.02) + + +def test_curvature_signature_and_behavior( + dummy_curvature_zero, dummy_curvature_nonzero +): + assert hasattr(CurvatureComponent, "Ok") + assert is_positional_only(CurvatureComponent.Ok, "z") + assert_allclose(dummy_curvature_zero.Ok(1), 0.0) + assert_allclose(dummy_curvature_nonzero.Ok(1), -0.02 * (1 + 1) ** 2) diff --git a/astropy/cosmology/_src/tests/traits/test_trait_darkenergy.py b/astropy/cosmology/_src/tests/traits/test_trait_darkenergy.py new file mode 100644 index 000000000000..112a508e42a4 --- /dev/null +++ b/astropy/cosmology/_src/tests/traits/test_trait_darkenergy.py @@ -0,0 +1,48 @@ +import numpy as np +import pytest + +from astropy.cosmology._src.traits.darkenergy import DarkEnergyComponent + +from .helper import is_positional_only + + +class MinimalDarkEnergy(DarkEnergyComponent): + Ode0 = 0.7 + + def w(self, z, /): + return -1.0 + + +class ConcreteDarkEnergy(DarkEnergyComponent): + Ode0 = 0.7 + + def w(self, z, /): + return -1.0 + + def inv_efunc(self, z): + return np.ones_like(np.asarray(z)) + + +@pytest.fixture +def minimal_de(): + return MinimalDarkEnergy() + + +@pytest.fixture +def concrete_de(): + return ConcreteDarkEnergy() + + +def test_darkenergy_signature_and_missing_inv_efunc_raises(minimal_de): + assert hasattr(DarkEnergyComponent, "w") + assert is_positional_only(DarkEnergyComponent.w, "z") + # Ode requires inv_efunc; calling should raise NotImplementedError + with pytest.raises(NotImplementedError): + minimal_de.Ode(1) + + +def test_darkenergy_with_inv_efunc(concrete_de): + pytest.importorskip("scipy") + # For w = -1 (cosmological constant) the density scale is 1, so Ode==Ode0 + val = concrete_de.Ode(1) + assert pytest.approx(val) == 0.7 diff --git a/astropy/cosmology/_src/tests/traits/test_trait_darkmatter.py b/astropy/cosmology/_src/tests/traits/test_trait_darkmatter.py new file mode 100644 index 000000000000..2973e9f246e6 --- /dev/null +++ b/astropy/cosmology/_src/tests/traits/test_trait_darkmatter.py @@ -0,0 +1,25 @@ +import numpy as np +import pytest +from numpy.testing import assert_allclose + +from astropy.cosmology._src.traits.darkmatter import DarkMatterComponent + +from .helper import is_positional_only + + +class DummyDarkMatter(DarkMatterComponent): + Odm0 = 0.25 + + def inv_efunc(self, z): + return np.ones_like(np.asarray(z)) + + +@pytest.fixture +def dummy_darkmatter(): + return DummyDarkMatter() + + +def test_darkmatter_signature_and_behavior(dummy_darkmatter): + assert hasattr(DarkMatterComponent, "Odm") + assert is_positional_only(DarkMatterComponent.Odm, "z") + assert_allclose(dummy_darkmatter.Odm(1), 0.25 * (1 + 1) ** 3) diff --git a/astropy/cosmology/_src/tests/traits/test_trait_hubble.py b/astropy/cosmology/_src/tests/traits/test_trait_hubble.py new file mode 100644 index 000000000000..6be7fae7ea2c --- /dev/null +++ b/astropy/cosmology/_src/tests/traits/test_trait_hubble.py @@ -0,0 +1,31 @@ +import numpy as np +import pytest + +import astropy.units as u +from astropy.cosmology._src.traits.hubble import HubbleParameter +from astropy.tests.helper import assert_quantity_allclose + + +class DummyHubble(HubbleParameter): + H0 = 70 * u.km / (u.s * u.Mpc) + + def efunc(self, z): + return np.ones_like(np.asarray(z)) + + def inv_efunc(self, z): + return np.ones_like(np.asarray(z)) + + +@pytest.fixture +def dummy_hubble(): + return DummyHubble() + + +def test_hubble_H_and_properties(dummy_hubble): + h = dummy_hubble + H1 = h.H(1) + assert isinstance(H1, u.Quantity) + assert_quantity_allclose(H1, 70 * u.km / (u.s * u.Mpc)) + # h property + assert hasattr(h, "h") + assert isinstance(h.h, (float, np.floating)) diff --git a/astropy/cosmology/_src/tests/traits/test_trait_matter.py b/astropy/cosmology/_src/tests/traits/test_trait_matter.py new file mode 100644 index 000000000000..4fb30cbf891f --- /dev/null +++ b/astropy/cosmology/_src/tests/traits/test_trait_matter.py @@ -0,0 +1,56 @@ +import numpy as np +import pytest +from numpy.testing import assert_allclose + +from astropy.cosmology._src.traits.matter import MatterComponent + +from .helper import is_positional_only + + +class DummyMatter(MatterComponent): + Om0 = 0.3 + + def inv_efunc(self, z): + return np.ones_like(np.asarray(z)) + + +class ZeroMatter(MatterComponent): + Om0 = 0.0 + + def inv_efunc(self, z): + return np.ones_like(np.asarray(z)) + + +@pytest.fixture +def dummy_matter(): + return DummyMatter() + + +@pytest.fixture +def zero_matter(): + return ZeroMatter() + + +def test_matter_exists_and_signature(): + assert hasattr(MatterComponent, "Om") + assert is_positional_only(MatterComponent.Om, "z") + + +def test_matter_scalar_array_quantity_behavior(dummy_matter): + d = dummy_matter + # keyword should raise TypeError because z is positional-only + with pytest.raises(TypeError): + d.Om(z=1) + + # scalar + assert_allclose(d.Om(1), 0.3 * (1 + 1) ** 3) + + # array + zin = np.array([0.0, 1.0]) + out = d.Om(zin) + assert out.shape == zin.shape + assert_allclose(out, 0.3 * (1 + zin) ** 3) + + +def test_matter_zero_case(zero_matter): + assert_allclose(zero_matter.Om(1), 0.0) diff --git a/astropy/cosmology/_src/tests/traits/test_trait_neutrino.py b/astropy/cosmology/_src/tests/traits/test_trait_neutrino.py new file mode 100644 index 000000000000..946532aa0a88 --- /dev/null +++ b/astropy/cosmology/_src/tests/traits/test_trait_neutrino.py @@ -0,0 +1,39 @@ +import numpy as np +import pytest + +import astropy.units as u +from astropy.cosmology._src.traits.neutrino import NeutrinoComponent +from astropy.tests.helper import assert_quantity_allclose + + +class DummyNeutrino(NeutrinoComponent): + Tcmb0 = 2.7255 * u.K + Ogamma0 = 5e-5 + + @property + def has_massive_nu(self): + return False + + @property + def Onu0(self): + return 0.22710731766 * 3.046 * self.Ogamma0 + + def nu_relative_density(self, z): + return 0.22710731766 * 3.046 * np.ones_like(np.asarray(z)) + + def Ogamma(self, z): + return self.Ogamma0 * (np.asarray(z) + 1.0) ** 4 + + +@pytest.fixture +def dummy_neutrino(): + return DummyNeutrino() + + +def test_neutrino_onu_and_tnu_basic(dummy_neutrino): + d = dummy_neutrino + out = d.Onu(1) + # scalar-like + assert np.asarray(out).shape == () + # Tnu scales as (1+z) + assert_quantity_allclose(d.Tnu(1), d.Tnu0 * (1 + 1)) diff --git a/astropy/cosmology/_src/tests/traits/test_trait_photoncomponent.py b/astropy/cosmology/_src/tests/traits/test_trait_photoncomponent.py new file mode 100644 index 000000000000..008dacbbb627 --- /dev/null +++ b/astropy/cosmology/_src/tests/traits/test_trait_photoncomponent.py @@ -0,0 +1,25 @@ +import numpy as np +import pytest + +from astropy.cosmology._src.traits.photoncomponent import PhotonComponent +from astropy.tests.helper import assert_quantity_allclose + +from .helper import is_positional_only + + +class DummyPhoton(PhotonComponent): + Ogamma0 = 1e-4 + + def inv_efunc(self, z): + return np.ones_like(np.asarray(z)) + + +@pytest.fixture +def dummy_photon(): + return DummyPhoton() + + +def test_photon_signature_and_behavior(dummy_photon): + assert hasattr(PhotonComponent, "Ogamma") + assert is_positional_only(PhotonComponent.Ogamma, "z") + assert_quantity_allclose(dummy_photon.Ogamma(1), 1e-4 * (1 + 1) ** 4) diff --git a/astropy/cosmology/_src/tests/traits/test_trait_rhocrit.py b/astropy/cosmology/_src/tests/traits/test_trait_rhocrit.py new file mode 100644 index 000000000000..8f77eaeaa8f6 --- /dev/null +++ b/astropy/cosmology/_src/tests/traits/test_trait_rhocrit.py @@ -0,0 +1,24 @@ +import numpy as np +import pytest + +import astropy.units as u +from astropy.cosmology._src.traits.rhocrit import CriticalDensity +from astropy.tests.helper import assert_quantity_allclose + + +class DummyRho(CriticalDensity): + critical_density0 = 1.0 * u.kg / (u.m**3) + + def efunc(self, z): + return np.ones_like(np.asarray(z)) + + +@pytest.fixture +def dummy_rho(): + return DummyRho() + + +def test_critical_density_returns_quantity(dummy_rho): + rho = dummy_rho.critical_density(1) + assert isinstance(rho, u.Quantity) + assert_quantity_allclose(rho, 1.0 * u.kg / (u.m**3)) diff --git a/astropy/cosmology/_src/tests/traits/test_trait_scale_factor.py b/astropy/cosmology/_src/tests/traits/test_trait_scale_factor.py new file mode 100644 index 000000000000..c4d3d8b9f6e9 --- /dev/null +++ b/astropy/cosmology/_src/tests/traits/test_trait_scale_factor.py @@ -0,0 +1,39 @@ +import pytest +from numpy.testing import assert_allclose + +from astropy.cosmology._src.traits.scale_factor import ScaleFactor + +from .helper import is_positional_only + + +class DummyScale(ScaleFactor): + pass + + +@pytest.fixture +def dummy_scale(): + return DummyScale() + + +def test_scale_factor_behavior_and_signature(dummy_scale): + s = dummy_scale + assert hasattr(ScaleFactor, "scale_factor") + # basic value + assert_allclose(s.scale_factor(1), 1.0 / (1 + 1)) + + # scale_factor0 default + assert s.scale_factor0 == 1 << s.scale_factor0.unit + + # positional-only API + assert is_positional_only(ScaleFactor.scale_factor, "z") + + # passing as keyword should raise TypeError (positional-only) + with pytest.raises(TypeError): + s.scale_factor(z=1) + + # array input returns same-shaped array + import numpy as np + + arr = np.array([0.0, 1.0, 2.0]) + out = s.scale_factor(arr) + assert out.shape == arr.shape diff --git a/astropy/cosmology/_src/tests/traits/test_trait_tcmb.py b/astropy/cosmology/_src/tests/traits/test_trait_tcmb.py new file mode 100644 index 000000000000..c2ae729b9246 --- /dev/null +++ b/astropy/cosmology/_src/tests/traits/test_trait_tcmb.py @@ -0,0 +1,22 @@ +import pytest + +import astropy.units as u +from astropy.cosmology._src.traits.tcmb import TemperatureCMB +from astropy.tests.helper import assert_quantity_allclose + + +class DummyTcmb(TemperatureCMB): + Tcmb0 = 2.7 * u.K + + +@pytest.fixture +def dummy_tcmb(): + return DummyTcmb() + + +def test_tcmb_behavior_and_signature(dummy_tcmb): + t = dummy_tcmb + assert hasattr(TemperatureCMB, "Tcmb") + tout = t.Tcmb(1) + assert tout.unit == u.K + assert_quantity_allclose(tout, 2.7 * u.K * (1 + 1)) diff --git a/astropy/cosmology/_src/tests/traits/test_trait_totalcomponent.py b/astropy/cosmology/_src/tests/traits/test_trait_totalcomponent.py new file mode 100644 index 000000000000..cc7bbaf11aca --- /dev/null +++ b/astropy/cosmology/_src/tests/traits/test_trait_totalcomponent.py @@ -0,0 +1,24 @@ +import numpy as np +import pytest +from numpy.testing import assert_allclose + +from astropy.cosmology._src.traits.totalcomponent import TotalComponent + + +class DummyTotal(TotalComponent): + @property + def Otot0(self): + return 1.0 + + def Otot(self, z, /): + z = np.asarray(z) + return np.ones_like(z) + + +@pytest.fixture +def dummy_total(): + return DummyTotal() + + +def test_totalcomponent_minimal_impl(dummy_total): + assert_allclose(dummy_total.Otot(1), 1.0) diff --git a/astropy/cosmology/_src/traits/__init__.py b/astropy/cosmology/_src/traits/__init__.py new file mode 100644 index 000000000000..15f18a692603 --- /dev/null +++ b/astropy/cosmology/_src/traits/__init__.py @@ -0,0 +1,18 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""Astropy Cosmology. **NOT public API**. + +The public API is provided by `astropy.cosmology.traits`. +""" + +from .baryons import * +from .curvature import * +from .darkenergy import * +from .darkmatter import * +from .hubble import * +from .matter import * +from .neutrino import * +from .photoncomponent import * +from .rhocrit import * +from .scale_factor import * +from .tcmb import * +from .totalcomponent import * diff --git a/astropy/cosmology/_src/traits/baryons.py b/astropy/cosmology/_src/traits/baryons.py new file mode 100644 index 000000000000..91c0215ee018 --- /dev/null +++ b/astropy/cosmology/_src/traits/baryons.py @@ -0,0 +1,47 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""Baryon component.""" + +__all__ = ("BaryonComponent",) + +from collections.abc import Callable +from typing import Any + +import numpy as np +from numpy.typing import ArrayLike, NDArray + +from astropy.cosmology._src.typing import FArray +from astropy.cosmology._src.utils import aszarr +from astropy.units import Quantity + + +class BaryonComponent: + """The cosmology has attributes and methods for the baryon density.""" + + Ob0: float | np.floating + """Omega baryons: density of baryonic matter in units of the critical density at z=0.""" + + inv_efunc: Callable[[NDArray[Any]], NDArray[Any]] + + def Ob(self, z: Quantity | ArrayLike, /) -> FArray: + """Return the density parameter for baryonic matter at redshift ``z``. + + Parameters + ---------- + z : Quantity-like ['redshift'], array-like + Input redshift. + + .. versionchanged:: 7.0 + Passing z as a keyword argument is deprecated. + + .. versionchanged:: 8.0 + z must be a positional argument. + + Returns + ------- + Ob : ndarray + The density of baryonic matter relative to the critical density at + each redshift. + + """ + z = aszarr(z) + return self.Ob0 * (z + 1.0) ** 3 * self.inv_efunc(z) ** 2 diff --git a/astropy/cosmology/_src/traits/curvature.py b/astropy/cosmology/_src/traits/curvature.py new file mode 100644 index 000000000000..bce1e84d3ba4 --- /dev/null +++ b/astropy/cosmology/_src/traits/curvature.py @@ -0,0 +1,90 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""Global Curvature. + +This is private API. See `~astropy.cosmology.traits` for public API. + +""" + +__all__ = ("CurvatureComponent",) + +import abc + +import numpy as np +from numpy.typing import ArrayLike, NDArray + +from astropy.cosmology._src.utils import aszarr +from astropy.units import Quantity + + +class CurvatureComponent: + """The object has attributes and methods related to the global curvature. + + This is a trait class; it is not meant to be instantiated directly, but + instead to be used as a mixin to other classes. + + """ + + @property + @abc.abstractmethod + def Ok0(self) -> float | np.floating: + """Omega curvature; the effective curvature density/critical density at z=0.""" + raise NotImplementedError + + @property + @abc.abstractmethod + def is_flat(self) -> bool: + """Return `bool`; `True` if the cosmology is globally flat.""" + raise NotImplementedError + + def Ok(self, z: Quantity | ArrayLike, /) -> NDArray[np.floating]: + """Return the equivalent density parameter for curvature at redshift ``z``. + + Parameters + ---------- + z : Quantity-like ['redshift'], array-like + Input redshift. + + .. versionchanged:: 7.0 + Passing z as a keyword argument is deprecated. + + .. versionchanged:: 8.0 + z must be a positional argument. + + Returns + ------- + Ok : ndarray + The equivalent density parameter for curvature at each redshift. + + .. versionchanged:: 7.2 + Always returns a numpy object, never a `float`. + + Examples + -------- + >>> import numpy as np + >>> from astropy.cosmology import Planck18, units as cu + + >>> Planck18.Ok(2) + array(0.) + + >>> Planck18.Ok([1, 2]) + array([0., 0.]) + + >>> Planck18.Ok(np.array([2])) + array([0.]) + + >>> Planck18.Ok(2 * cu.redshift) + array(0.) + + >>> cosmo = Planck18.clone(Ode0=0.71, to_nonflat=True) + + >>> cosmo.Ok0 + np.float64(-0.021153694455455927) + + >>> cosmo.Ok(100) + np.float64(-0.0006557825253017665) + + """ + z = aszarr(z) + if self.Ok0 == 0: # Common enough to be worth checking explicitly + return np.zeros(getattr(z, "shape", ())) + return self.Ok0 * (z + 1.0) ** 2 * self.inv_efunc(z) ** 2 diff --git a/astropy/cosmology/_src/traits/darkenergy.py b/astropy/cosmology/_src/traits/darkenergy.py new file mode 100644 index 000000000000..ed59dc2dfdab --- /dev/null +++ b/astropy/cosmology/_src/traits/darkenergy.py @@ -0,0 +1,153 @@ +__all__ = ("DarkEnergyComponent",) + +from abc import abstractmethod +from math import exp, log + +import numpy as np +from numpy.typing import ArrayLike + +from astropy.cosmology._src.scipy_compat import quad +from astropy.cosmology._src.typing import FArray +from astropy.cosmology._src.utils import aszarr +from astropy.units import Quantity + + +class DarkEnergyComponent: + # Subclasses should use `Parameter` to make this a parameter of the cosmology. + Ode0: float | np.floating + """Omega dark energy; dark energy density/critical density at z=0.""" + + @abstractmethod + def w(self, z: Quantity | ArrayLike, /) -> FArray: + r"""The dark energy equation of state. + + Parameters + ---------- + z : Quantity-like ['redshift'], array-like + Input redshift. + + .. versionchanged:: 7.0 + Passing z as a keyword argument is deprecated. + + .. versionchanged:: 8.0 + z must be a positional argument. + + Returns + ------- + w : ndarray or float + The dark energy equation of state. + `float` if scalar input. + + Notes + ----- + The dark energy equation of state is defined as + :math:`w(z) = P(z)/\rho(z)`, where :math:`P(z)` is the pressure at + redshift z and :math:`\rho(z)` is the density at redshift z, both in + units where c=1. + + This must be overridden by subclasses. + """ + raise NotImplementedError("w(z) is not implemented") + + def _w_integrand(self, ln1pz: float | FArray, /) -> FArray: + """Internal convenience function for w(z) integral (eq. 5 of [1]_). + + Parameters + ---------- + ln1pz : `~numbers.Number` or scalar ndarray, positional-only + Assumes scalar input, since this should only be called inside an + integral. + + .. versionchanged:: 7.0 + The argument is positional-only. + + References + ---------- + .. [1] Linder, E. (2003). Exploring the Expansion History of the + Universe. Phys. Rev. Lett., 90, 091301. + """ + return 1.0 + self.w(exp(ln1pz) - 1.0) + + def de_density_scale(self, z: Quantity | ArrayLike, /) -> FArray: + r"""Evaluates the redshift dependence of the dark energy density. + + Parameters + ---------- + z : Quantity-like ['redshift'], array-like + Input redshift. + + .. versionchanged:: 7.0 + Passing z as a keyword argument is deprecated. + + .. versionchanged:: 8.0 + z must be a positional argument. + + Returns + ------- + I : ndarray or float + The scaling of the energy density of dark energy with redshift. + Returns `float` if the input is scalar. + + Notes + ----- + The scaling factor, I, is defined by :math:`\rho(z) = \rho_0 I`, + and is given by + + .. math:: + + I = \exp \left( 3 \int_{a}^1 \frac{ da^{\prime} }{ a^{\prime} } + \left[ 1 + w\left( a^{\prime} \right) \right] \right) + + The actual integral used is rewritten from [1]_ to be in terms of z. + + It will generally helpful for subclasses to overload this method if + the integral can be done analytically for the particular dark + energy equation of state that they implement. + + References + ---------- + .. [1] Linder, E. (2003). Exploring the Expansion History of the + Universe. Phys. Rev. Lett., 90, 091301. + """ + # This allows for an arbitrary w(z) following eq (5) of + # Linder 2003, PRL 90, 91301. The code here evaluates + # the integral numerically. However, most popular + # forms of w(z) are designed to make this integral analytic, + # so it is probably a good idea for subclasses to overload this + # method if an analytic form is available. + z = aszarr(z) + ival = ( + quad(self._w_integrand, 0, log(z + 1.0))[0] # scalar + if z.ndim == 0 + else np.asarray([quad(self._w_integrand, 0, log(1 + _z))[0] for _z in z]) + ) + return np.exp(3 * ival) + + def Ode(self, z: Quantity | ArrayLike, /) -> FArray: + """Return the density parameter for dark energy at redshift ``z``. + + Parameters + ---------- + z : Quantity-like ['redshift'], array-like + Input redshift. + + .. versionchanged:: 7.0 + Passing z as a keyword argument is deprecated. + + .. versionchanged:: 8.0 + z must be a positional argument. + + Returns + ------- + Ode : ndarray + The density of dark energy relative to the critical density at each + redshift. + """ + z = aszarr(z) + if self.Ode0 == 0: # Common enough to be worth checking explicitly + return np.zeros_like(z) + # Ensure self.inv_efunc is implemented by the main class + if not hasattr(self, "inv_efunc") or not callable(self.inv_efunc): + msg = "The main class must implement an 'inv_efunc(z)' method." + raise NotImplementedError(msg) + return self.Ode0 * self.de_density_scale(z) * self.inv_efunc(z) ** 2 diff --git a/astropy/cosmology/_src/traits/darkmatter.py b/astropy/cosmology/_src/traits/darkmatter.py new file mode 100644 index 000000000000..2e5f47b5ca0f --- /dev/null +++ b/astropy/cosmology/_src/traits/darkmatter.py @@ -0,0 +1,50 @@ +"""Trait for dark matter component of cosmology.""" + +__all__ = ("DarkMatterComponent",) + +from collections.abc import Callable +from typing import Any + +import numpy as np +from numpy.typing import ArrayLike, NDArray + +from astropy.cosmology._src.typing import FArray +from astropy.cosmology._src.utils import aszarr +from astropy.units import Quantity + + +class DarkMatterComponent: + """The cosmology has attributes and methods for the dark matter density. + + This trait provides an ``Odm`` method that returns the dark matter + density parameter (i.e., total matter minus baryons) at redshift ``z``. + """ + + Odm0: float | np.floating + """Omega dark matter: dark matter density/critical density at z=0.""" + + inv_efunc: Callable[[NDArray[Any]], NDArray[Any]] + + def Odm(self, z: Quantity | ArrayLike, /) -> FArray: + """Return the density parameter for dark matter at redshift ``z``. + + Parameters + ---------- + z : Quantity-like ['redshift'], array-like + Input redshift. + + .. versionchanged:: 7.0 + Passing z as a keyword argument is deprecated. + + .. versionchanged:: 8.0 + z must be a positional argument. + + Returns + ------- + Odm : ndarray + The density of dark matter relative to the critical density at + each redshift. + + """ + z = aszarr(z) + return self.Odm0 * (z + 1.0) ** 3 * self.inv_efunc(z) ** 2 diff --git a/astropy/cosmology/_src/traits/hubble.py b/astropy/cosmology/_src/traits/hubble.py new file mode 100644 index 000000000000..94b18e2b2c9f --- /dev/null +++ b/astropy/cosmology/_src/traits/hubble.py @@ -0,0 +1,60 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""Hubble parameter trait. + +This is private API. See `~astropy.cosmology.traits` for public API. +""" + +__all__ = ("HubbleParameter",) + +from collections.abc import Callable +from functools import cached_property +from typing import Any + +import numpy as np +from numpy.typing import ArrayLike, NDArray + +import astropy.units as u +from astropy import constants as const +from astropy.cosmology._src.typing import FArray +from astropy.units import Quantity + + +class HubbleParameter: + """The object has attributes and methods for the Hubble parameter.""" + + H0: Quantity + """Hubble Parameter at redshift 0.""" + + efunc: Callable[[Any], NDArray[Any]] + + inv_efunc: Callable[[Any], FArray | float] + + def H(self, z: Quantity | ArrayLike, /) -> Quantity: + """Hubble parameter at redshift ``z``. + + Parameters + ---------- + z : Quantity-like ['redshift'], array-like + Input redshift. + + Returns + ------- + H : Quantity ['frequency'] + Hubble parameter at each input redshift. + """ + return self.H0 * self.efunc(z) + + @cached_property + def h(self) -> np.floating: + """Dimensionless Hubble constant: h = H_0 / 100 [km/sec/Mpc].""" + return self.H0.to_value("km/(s Mpc)") / 100.0 + + @cached_property + def hubble_time(self) -> u.Quantity: + """Hubble time.""" + return (1 / self.H0).to(u.Gyr) + + @cached_property + def hubble_distance(self) -> u.Quantity: + """Hubble distance.""" + return (const.c / self.H0).to(u.Mpc) diff --git a/astropy/cosmology/_src/traits/matter.py b/astropy/cosmology/_src/traits/matter.py new file mode 100644 index 000000000000..0f6c19a0f240 --- /dev/null +++ b/astropy/cosmology/_src/traits/matter.py @@ -0,0 +1,45 @@ +"""Matter component.""" + +import numpy as np +from numpy.typing import ArrayLike + +from astropy.cosmology._src.typing import FArray +from astropy.cosmology._src.utils import aszarr +from astropy.units import Quantity + +__all__ = ("MatterComponent",) + + +class MatterComponent: + """The cosmology has attributes and methods for the matter density.""" + + Om0: float | np.floating + """Omega matter; matter density/critical density at z=0.""" + + def Om(self, z: Quantity | ArrayLike, /) -> FArray: + """Return the density parameter for non-relativistic matter at redshift ``z``. + + Parameters + ---------- + z : Quantity-like ['redshift'], array-like + Input redshift. + + .. versionchanged:: 7.0 + Passing z as a keyword argument is deprecated. + + .. versionchanged:: 8.0 + z must be a positional argument. + + Returns + ------- + Om : ndarray + The density of non-relativistic matter relative to the critical + density at each redshift. + + Notes + ----- + This does not include neutrinos, even if non-relativistic at the + redshift of interest. + """ + z = aszarr(z) + return self.Om0 * (z + 1.0) ** 3 * self.inv_efunc(z) ** 2 diff --git a/astropy/cosmology/_src/traits/neutrino.py b/astropy/cosmology/_src/traits/neutrino.py new file mode 100644 index 000000000000..390501fb2687 --- /dev/null +++ b/astropy/cosmology/_src/traits/neutrino.py @@ -0,0 +1,258 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +r"""Neutrino component trait. + +This is private API. See `~astropy.cosmology.traits` for public API. +""" + +__all__ = ["NeutrinoComponent"] + +from abc import abstractmethod +from collections.abc import Callable +from functools import cached_property +from typing import Final + +import numpy as np +from numpy.typing import ArrayLike + +from astropy.cosmology._src.typing import FArray +from astropy.cosmology._src.utils import aszarr, deprecated_keywords +from astropy.units import Quantity + +# Physics constants for neutrino calculations +TEMP_NEUTRINO: Final = 0.7137658555036082 # (4/11)^1/3 +NEUTRINO_FERMI_DIRAC_CORRECTION: Final = 0.22710731766 # 7/8 (4/11)^4/3 + + +############################################################################## + + +class NeutrinoComponent: + r"""The cosmology has attributes and methods for the neutrino density. + + This trait handles both massless neutrinos (relativistic, radiation-like) + and massive neutrinos (with complex evolution). + + This is an abstract trait. Subclasses must implement: + + - `has_massive_nu` (property): Whether there are massive neutrinos + - `Onu0` (property): Neutrino density parameter at z=0 + - `nu_relative_density` (method): Neutrino-to-photon density ratio at redshift z + + The parent class must provide ``Tcmb0`` (CMB temperature) and ``Ogamma`` + (method to compute photon density at redshift z). + + Notes + ----- + The density in neutrinos is given by: + + .. math:: + + \rho_{\nu} \left(a\right) = 0.2271 \, N_{eff} \, + f\left(m_{\nu} a / T_{\nu 0} \right) \, + \rho_{\\gamma} \left( a \right) + + where + + .. math:: + + f \left(y\right) = \frac{120}{7 \pi^4} + \int_0^{\\infty} \, dx \frac{x^2 \\sqrt{x^2 + y^2}} + {e^x + 1} + + assuming that all neutrino species have the same mass. + + If they have different masses, a similar term is calculated for each + one. + + Note that ``f`` has the asymptotic behavior :math:`f(0) = 1`. This + method returns :math:`0.2271 f` using an analytical fitting formula + (Komatsu et al., 2011), ApJS, 192, 18. + + The neutrino density evolution depends on whether neutrinos are massive or massless: + + - **Massless neutrinos**: Behave like radiation with density scaling as (1+z)^4. + The density is simply proportional to the photon density with a constant ratio + determined by Neff and Fermi-Dirac statistics. + + - **Massive neutrinos**: Have complex evolution that transitions from relativistic + (radiation-like) at early times to non-relativistic (matter-like) at late times. + The implementation typically uses the Komatsu fitting formula (Komatsu + et al., 2011) for computational efficiency. + + References + ---------- + Komatsu et al. (2011), "Seven-Year Wilkinson Microwave Anisotropy Probe + (WMAP) Observations: Cosmological Interpretation", ApJS, 192, 18. + + Examples + -------- + >>> import numpy as np + >>> import astropy.units as u + >>> from astropy.cosmology.traits import NeutrinoComponent + >>> NEUTRINO_FERMI_DIRAC_CORRECTION = 0.22710731766 # 7/8 (4/11)^4/3 + >>> + >>> class ExampleNeutrinoCosmology(NeutrinoComponent): + ... def __init__(self): + ... self.Tcmb0 = 2.7255 * u.K + ... self.Neff = 3.046 + ... self.Ogamma0 = 5e-5 + ... @property + ... def has_massive_nu(self): + ... return False + ... @property + ... def Onu0(self): + ... return NEUTRINO_FERMI_DIRAC_CORRECTION * self.Neff * self.Ogamma0 + ... def nu_relative_density(self, z): + ... return NEUTRINO_FERMI_DIRAC_CORRECTION * self.Neff * np.ones_like(np.asarray(z)) + ... def Ogamma(self, z): + ... return self.Ogamma0 * (np.asarray(z) + 1.0) ** 4 + """ + + # Type annotations for dependencies (provided by parent class) + Tcmb0: Quantity + Ogamma: Callable[[ArrayLike], FArray] + + @property + @abstractmethod + def has_massive_nu(self) -> bool: + """Does this cosmology have at least one massive neutrino species? + + Returns + ------- + has_massive_nu : bool + True if at least one neutrino species has non-zero mass. + + Notes + ----- + Subclasses must implement this property. + """ + + @property + @abstractmethod + def Onu0(self) -> float: + """Omega nu; the density/critical density of neutrinos at z=0. + + Returns + ------- + Onu0 : float + The density parameter for neutrinos at z=0. + + Notes + ----- + Subclasses must implement this property. + """ + + @abstractmethod + def nu_relative_density(self, z: Quantity | ArrayLike) -> FArray: + r"""Neutrino density function relative to the energy density in photons. + + Parameters + ---------- + z : Quantity-like ['redshift'], array-like + Input redshift. + + Returns + ------- + f : array + The neutrino density scaling factor relative to the density in + photons at each redshift. + + Notes + ----- + Subclasses must implement this method. For massless neutrinos, this + should return a constant. For massive neutrinos, this should use an + appropriate fitting formula (e.g., Komatsu et al. 2011). + """ + + @cached_property + def Tnu0(self) -> Quantity: + """Temperature of the neutrino background as |Quantity| at z=0. + + Returns + ------- + Tnu0 : Quantity ['temperature'] + The neutrino temperature at z=0 in Kelvin. + + Notes + ----- + The neutrino temperature is related to the CMB temperature by: + + .. math:: + + T_{\\nu 0} = \\left(\\frac{4}{11}\\right)^{1/3} T_{CMB} + + This comes from the decoupling of neutrinos before electron-positron + annihilation. See Weinberg 'Cosmology' p 154 eq (3.1.21). + """ + # The constant in front is (4/11)^1/3 -- see any cosmology book for an + # explanation -- for example, Weinberg 'Cosmology' p 154 eq (3.1.21). + return TEMP_NEUTRINO * self.Tcmb0 + + @deprecated_keywords("z", since="7.0") + def Onu(self, z: Quantity | ArrayLike) -> FArray: + r"""Return the density parameter for neutrinos at redshift ``z``. + + Parameters + ---------- + z : Quantity-like ['redshift'], array-like + Input redshift. + + .. versionchanged:: 7.0 + Passing z as a keyword argument is deprecated. + + Returns + ------- + Onu : ndarray + The energy density of neutrinos relative to the critical density at + each redshift. Note that this includes their kinetic energy (if + they have mass), so it is not equal to the commonly used + :math:`\sum \frac{m_{\nu}}{94 eV}`, which does not include + kinetic energy. + + Notes + ----- + The neutrino density parameter evolves with redshift according to: + + .. math:: + + \\Omega_{\\nu}(z) = \\Omega_{\\gamma}(z) \\times f(z) + + where f(z) is the neutrino-to-photon density ratio computed by + nu_relative_density(z). For massless neutrinos, f(z) is constant. + For massive neutrinos, f(z) evolves as neutrinos transition from + relativistic to non-relativistic. + """ + z = aszarr(z) + if self.Onu0 == 0: # Common enough to be worth checking explicitly + return np.zeros_like(z) + return self.Ogamma(z) * self.nu_relative_density(z) + + @deprecated_keywords("z", since="7.0") + def Tnu(self, z: Quantity | ArrayLike) -> Quantity: + """Return the neutrino temperature at redshift ``z``. + + Parameters + ---------- + z : Quantity-like ['redshift'], array-like + Input redshift. + + .. versionchanged:: 7.0 + Passing z as a keyword argument is deprecated. + + Returns + ------- + Tnu : Quantity ['temperature'] + The temperature of the cosmic neutrino background in K. + + Notes + ----- + The neutrino temperature scales with redshift as: + + .. math:: + + T_{\\nu}(z) = T_{\\nu 0} (1 + z) + + This simple scaling applies to both massless and massive neutrinos, + as the temperature depends only on the expansion of the universe. + """ + return self.Tnu0 * (aszarr(z) + 1.0) diff --git a/astropy/cosmology/_src/traits/photoncomponent.py b/astropy/cosmology/_src/traits/photoncomponent.py new file mode 100644 index 000000000000..d06490a2353b --- /dev/null +++ b/astropy/cosmology/_src/traits/photoncomponent.py @@ -0,0 +1,46 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""Photon component.""" + +__all__ = ("PhotonComponent",) + +from collections.abc import Callable +from typing import Any + +import numpy as np +from numpy.typing import ArrayLike, NDArray + +from astropy.cosmology._src.typing import FArray +from astropy.cosmology._src.utils import aszarr +from astropy.units import Quantity + + +class PhotonComponent: + """The cosmology has attributes and methods for the photon density.""" + + Ogamma0: float | np.floating + """Omega gamma; the density/critical density of photons at z=0.""" + + inv_efunc: Callable[[NDArray[Any]], NDArray[Any]] + + def Ogamma(self, z: Quantity | ArrayLike, /) -> FArray: + """Return the density parameter for photons at redshift ``z``. + + Parameters + ---------- + z : Quantity-like ['redshift'], array-like + Input redshift. + + .. versionchanged:: 7.0 + Passing z as a keyword argument is deprecated. + + .. versionchanged:: 8.0 + z must be a positional argument. + + Returns + ------- + Ogamma : array + The energy density of photons relative to the critical density at + each redshift. + """ + z = aszarr(z) + return self.Ogamma0 * (z + 1.0) ** 4 * self.inv_efunc(z) ** 2 diff --git a/astropy/cosmology/_src/traits/rhocrit.py b/astropy/cosmology/_src/traits/rhocrit.py new file mode 100644 index 000000000000..feb555017733 --- /dev/null +++ b/astropy/cosmology/_src/traits/rhocrit.py @@ -0,0 +1,41 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""Critical density component.""" + +__all__ = ("CriticalDensity",) + +from collections.abc import Callable +from typing import Any + +from numpy.typing import ArrayLike, NDArray + +from astropy.units import Quantity + + +class CriticalDensity: + """The object has attributes and methods for the critical density.""" + + critical_density0: Quantity + """Critical density at redshift 0.""" + + efunc: Callable[[Any], NDArray[Any]] + + def critical_density(self, z: Quantity | ArrayLike, /) -> Quantity: + """Critical density in grams per cubic cm at redshift ``z``. + + Parameters + ---------- + z : Quantity-like ['redshift'], array-like + Input redshift. + + .. versionchanged:: 7.0 + Passing z as a keyword argument is deprecated. + + .. versionchanged:: 8.0 + z must be a positional argument. + + Returns + ------- + rho : Quantity ['mass density'] + Critical density at each input redshift. + """ + return self.critical_density0 * self.efunc(z) ** 2 diff --git a/astropy/cosmology/_src/traits/scale_factor.py b/astropy/cosmology/_src/traits/scale_factor.py new file mode 100644 index 000000000000..737e2c0a4474 --- /dev/null +++ b/astropy/cosmology/_src/traits/scale_factor.py @@ -0,0 +1,57 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""Scale factor. + +This is private API. See `~astropy.cosmology.traits` for public API. + +""" + +__all__ = ("ScaleFactor",) + + +from numpy.typing import ArrayLike + +import astropy.units as u +from astropy.cosmology._src.utils import aszarr + + +class ScaleFactor: + """The trait for computing the cosmological scale factor. + + The scale factor is defined as :math:`a = a_0 / (1 + z)`. + + """ + + @property + def scale_factor0(self) -> u.Quantity: + r"""Scale factor at redshift 0. + + The scale factor is defined as :math:`a = a_0 / (1 + z)`. The common convention + is to set :math:`a_0 = 1`. However, in some cases, like in some old CMB papers, + :math:`a_0` is used to normalize `a` to be a convenient number at the redshift + of interest for that paper. Explicitly using :math:`a_0` in both calculation and + code avoids ambiguity. + """ + return 1 << u.one + + def scale_factor(self, z: u.Quantity | ArrayLike, /) -> u.Quantity: + """Compute the scale factor at redshift ``z``. + + The scale factor is defined as :math:`a = a_0 / (1 + z)`. + + Parameters + ---------- + z : Quantity-like ['redshift'] | array-like + Input redshift. + + .. versionchanged:: 7.0 + Passing z as a keyword argument is deprecated. + + .. versionchanged:: 8.0 + z must be a positional argument. + + Returns + ------- + |Quantity| + Scale factor at each input redshift. + """ + return self.scale_factor0 / (aszarr(z) + 1) diff --git a/astropy/cosmology/_src/traits/tcmb.py b/astropy/cosmology/_src/traits/tcmb.py new file mode 100644 index 000000000000..97830c92d869 --- /dev/null +++ b/astropy/cosmology/_src/traits/tcmb.py @@ -0,0 +1,59 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""CMB Temperature. + +This is private API. See `~astropy.cosmology.traits` for public API. + +""" + +__all__ = ("TemperatureCMB",) + + +from numpy.typing import ArrayLike + +from astropy.cosmology._src.utils import aszarr +from astropy.units import Quantity + + +class TemperatureCMB: + """The trait for computing the cosmological background temperature.""" + + Tcmb0: Quantity + """Temperature of the CMB at z=0.""" + + def Tcmb(self, z: Quantity | ArrayLike, /) -> Quantity: + """Compute the CMB temperature at redshift ``z``. + + Parameters + ---------- + z : Quantity-like ['redshift'], array-like + Input redshift. + + .. versionchanged:: 7.0 + Passing z as a keyword argument is deprecated. + + .. versionchanged:: 8.0 + z must be a positional argument. + + Returns + ------- + Tcmb : Quantity ['temperature'] + The temperature of the CMB. + + Examples + -------- + >>> import astropy.units as u + >>> from astropy.cosmology import Planck18, units as cu + + >>> Planck18.Tcmb(u.Quantity([0.5, 1.0], cu.redshift)) + + + >>> Planck18.Tcmb(u.Quantity(0.5, '')) + + + >>> Planck18.Tcmb(0.5) + + + >>> Planck18.Tcmb([0.5, 1.0]) + + """ + return self.Tcmb0 * (aszarr(z) + 1.0) diff --git a/astropy/cosmology/_src/traits/totalcomponent.py b/astropy/cosmology/_src/traits/totalcomponent.py new file mode 100644 index 000000000000..d72f387b9329 --- /dev/null +++ b/astropy/cosmology/_src/traits/totalcomponent.py @@ -0,0 +1,42 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +__all__ = ("TotalComponent",) + +from abc import abstractmethod + +import numpy as np +from numpy.typing import ArrayLike + +from astropy.cosmology._src.typing import FArray +from astropy.units import Quantity + + +class TotalComponent: + """The cosmology has attributes and methods for the total density. + + This trait has the abstract ``Otot`` method that returns the total density + parameter at redshift ``z``. It should be the sum of all other components. + """ + + @property + @abstractmethod + def Otot0(self) -> float | np.floating: + """Omega total; the total density/critical density at z=0.""" + raise NotImplementedError # pragma: no cover + + @abstractmethod + def Otot(self, z: Quantity | ArrayLike, /) -> FArray: + """The total density parameter at redshift ``z``. + + Parameters + ---------- + z : Quantity-like ['redshift'], array-like + Input redshifts. + + Returns + ------- + Otot : array + The total density relative to the critical density at each + redshift. + """ + raise NotImplementedError # pragma: no cover diff --git a/astropy/cosmology/_src/typing.py b/astropy/cosmology/_src/typing.py new file mode 100644 index 000000000000..9003f3bdf067 --- /dev/null +++ b/astropy/cosmology/_src/typing.py @@ -0,0 +1,22 @@ +"""Static typing for :mod:`astropy.cosmology`. PRIVATE API.""" +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +__all__ = ("CosmoMeta", "FArray", "_CosmoT") + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar + +import numpy as np +from numpy.typing import NDArray + +if TYPE_CHECKING: + import astropy.cosmology + +_CosmoT = TypeVar("_CosmoT", bound="astropy.cosmology.Cosmology") +"""Type variable for :class:`~astropy.cosmology.Cosmology` and subclasses.""" + +CosmoMeta: TypeAlias = Mapping[Any, Any] +"""Type alias for cosmology metadata.""" + +FArray: TypeAlias = NDArray[np.floating] +"""Type alias for numpy array of floating dtype.""" diff --git a/astropy/cosmology/_src/units.py b/astropy/cosmology/_src/units.py new file mode 100644 index 000000000000..997c6f9bf0c3 --- /dev/null +++ b/astropy/cosmology/_src/units.py @@ -0,0 +1,35 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""Cosmological units.""" + +__all__ = ("littleh", "redshift") + +from typing import Final + +import astropy.units as u + +_ns = globals() + + +# This is not formally a unit, but is used in that way in many contexts, and +# an appropriate equivalency is only possible if it's treated as a unit. +redshift: Final = u.def_unit( + ["redshift"], + prefixes=False, + namespace=_ns, + doc="Cosmological redshift.", + format={"latex": r""}, +) +u.def_physical_type(redshift, "redshift") + +# This is not formally a unit, but is used in that way in many contexts, and +# an appropriate equivalency is only possible if it's treated as a unit (see +# https://arxiv.org/pdf/1308.4150.pdf for more) +# Also note that h or h100 or h_100 would be a better name, but they either +# conflict or have numbers in them, which is disallowed +littleh: Final = u.def_unit( + ["littleh"], + namespace=_ns, + prefixes=False, + doc='Reduced/"dimensionless" Hubble constant', + format={"latex": r"h_{100}"}, +) diff --git a/astropy/cosmology/_src/units_equivalencies.py b/astropy/cosmology/_src/units_equivalencies.py new file mode 100644 index 000000000000..f4374c33ff4a --- /dev/null +++ b/astropy/cosmology/_src/units_equivalencies.py @@ -0,0 +1,369 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""Cosmological unit equivalencies.""" + +__all__ = ( + # redshift equivalencies + "dimensionless_redshift", + "redshift_distance", + "redshift_hubble", + "redshift_temperature", + # other equivalencies + "with_H0", + "with_redshift", +) + + +import sys +from typing import TYPE_CHECKING, Any, Literal, TypeAlias, Union + +import astropy.units as u +from astropy.cosmology._src.funcs.optimize import _ZAtValueKWArgs + +from .default import default_cosmology +from .funcs.optimize import z_at_value +from .units import littleh, redshift + +if TYPE_CHECKING: + import astropy.cosmology + +if sys.version_info < (3, 12): + _UnpackZAtValueKWArgs = Any +else: + from typing import Unpack + + _UnpackZAtValueKWArgs: TypeAlias = Unpack[_ZAtValueKWArgs] + + +__doctest_requires__ = {("with_redshift", "redshift_distance"): ["scipy"]} + + +def dimensionless_redshift() -> u.Equivalency: + """Allow redshift to be 1-to-1 equivalent to dimensionless. + + It is special compared to other equivalency pairs in that it allows + this independent of the power to which the redshift is raised, and + independent of whether it is part of a more complicated unit. It is + similar to u.dimensionless_angles() in this respect. + """ + return u.Equivalency([(redshift, None)], "dimensionless_redshift") + + +def redshift_distance( + cosmology: Union["astropy.cosmology.Cosmology", str, None] = None, + kind: Literal["comoving", "lookback", "luminosity"] = "comoving", + **atzkw: _UnpackZAtValueKWArgs, +) -> u.Equivalency: + """Convert quantities between redshift and distance. + + Care should be taken to not misinterpret a relativistic, gravitational, etc + redshift as a cosmological one. + + Parameters + ---------- + cosmology : `~astropy.cosmology.Cosmology`, str, or None, optional + A cosmology realization or built-in cosmology's name (e.g. 'Planck18'). + If None, will use the default cosmology + (controlled by |default_cosmology|). + kind : {'comoving', 'lookback', 'luminosity'}, optional + The distance type for the Equivalency. + Note this does NOT include the angular diameter distance as this + distance measure is not monotonic. + **atzkw + keyword arguments for :func:`~astropy.cosmology.z_at_value`, which is used to + convert distance to redshift. + + Returns + ------- + `~astropy.units.equivalencies.Equivalency` + Equivalency between redshift and temperature. + + Raises + ------ + `~astropy.cosmology.CosmologyError` + If the distance corresponds to a redshift that is larger than ``zmax``. + Exception + See :func:`~astropy.cosmology.z_at_value` for possible exceptions, e.g. if the + distance maps to a redshift that is larger than ``zmax``, the maximum redshift. + + Examples + -------- + >>> import astropy.units as u + >>> import astropy.cosmology.units as cu + >>> from astropy.cosmology import WMAP9 + + >>> z = 1100 * cu.redshift + >>> d = z.to(u.Mpc, cu.redshift_distance(WMAP9, kind="comoving")) + >>> d # doctest: +FLOAT_CMP + + + The reverse operation is also possible, though not always as simple. To convert a + very large distance to a redshift it might be necessary to specify a large enough + ``zmax`` value. See :func:`~astropy.cosmology.z_at_value` for details. + + >>> d.to(cu.redshift, cu.redshift_distance(WMAP9, kind="comoving", zmax=1200)) # doctest: +FLOAT_CMP + + """ + # get cosmology: None -> default and process str / class + cosmology = cosmology if cosmology is not None else default_cosmology.get() + with default_cosmology.set(cosmology): # if already cosmo, passes through + cosmology = default_cosmology.get() + + allowed_kinds = ("comoving", "lookback", "luminosity") + if kind not in allowed_kinds: + raise ValueError(f"`kind` is not one of {allowed_kinds}") + + method = getattr(cosmology, kind + "_distance") + + def z_to_distance(z): + """Redshift to distance.""" + return method(z) + + def distance_to_z(d): + """Distance to redshift.""" + return z_at_value(method, d << u.Mpc, **atzkw) + + return u.Equivalency( + [(redshift, u.Mpc, z_to_distance, distance_to_z)], + "redshift_distance", + {"cosmology": cosmology, "distance": kind}, + ) + + +def redshift_hubble( + cosmology: Union["astropy.cosmology.Cosmology", str, None] = None, + **atzkw: _UnpackZAtValueKWArgs, +) -> u.Equivalency: + """Convert quantities between redshift and Hubble parameter and little-h. + + Care should be taken to not misinterpret a relativistic, gravitational, etc + redshift as a cosmological one. + + Parameters + ---------- + cosmology : `~astropy.cosmology.Cosmology`, str, or None, optional + A cosmology realization or built-in cosmology's name (e.g. 'Planck18'). + If None, will use the default cosmology + (controlled by |default_cosmology|). + **atzkw + keyword arguments for :func:`~astropy.cosmology.z_at_value` + + Returns + ------- + `~astropy.units.equivalencies.Equivalency` + Equivalency between redshift and Hubble parameter and little-h unit. + + Examples + -------- + >>> import astropy.units as u + >>> import astropy.cosmology.units as cu + >>> from astropy.cosmology import WMAP9 + + >>> z = 1100 * cu.redshift + >>> equivalency = cu.redshift_hubble(WMAP9) # construct equivalency + + >>> z.to(u.km / u.s / u.Mpc, equivalency) # doctest: +FLOAT_CMP + + + >>> z.to(cu.littleh, equivalency) # doctest: +FLOAT_CMP + + """ + # get cosmology: None -> default and process str / class + cosmology = cosmology if cosmology is not None else default_cosmology.get() + with default_cosmology.set(cosmology): # if already cosmo, passes through + cosmology = default_cosmology.get() + + def z_to_hubble(z): + """Redshift to Hubble parameter.""" + return cosmology.H(z) + + def hubble_to_z(H): + """Hubble parameter to redshift.""" + return z_at_value(cosmology.H, H << (u.km / u.s / u.Mpc), **atzkw) + + def z_to_littleh(z): + """Redshift to :math:`h`-unit Quantity.""" + return z_to_hubble(z).to_value(u.km / u.s / u.Mpc) / 100 * littleh + + def littleh_to_z(h): + """:math:`h`-unit Quantity to redshift.""" + return hubble_to_z(h * 100) + + return u.Equivalency( + [ + (redshift, u.km / u.s / u.Mpc, z_to_hubble, hubble_to_z), + (redshift, littleh, z_to_littleh, littleh_to_z), + ], + "redshift_hubble", + {"cosmology": cosmology}, + ) + + +def redshift_temperature( + cosmology: Union["astropy.cosmology.Cosmology", str, None] = None, + **atzkw: _UnpackZAtValueKWArgs, +) -> u.Equivalency: + """Convert quantities between redshift and CMB temperature. + + Care should be taken to not misinterpret a relativistic, gravitational, etc + redshift as a cosmological one. + + Parameters + ---------- + cosmology : `~astropy.cosmology.Cosmology`, str, or None, optional + A cosmology realization or built-in cosmology's name (e.g. 'Planck18'). + If None, will use the default cosmology + (controlled by |default_cosmology|). + **atzkw + keyword arguments for :func:`~astropy.cosmology.z_at_value` + + Returns + ------- + `~astropy.units.equivalencies.Equivalency` + Equivalency between redshift and temperature. + + Examples + -------- + >>> import astropy.units as u + >>> import astropy.cosmology.units as cu + >>> from astropy.cosmology import WMAP9 + + >>> z = 1100 * cu.redshift + >>> z.to(u.K, cu.redshift_temperature(WMAP9)) + + """ + # get cosmology: None -> default and process str / class + cosmology = cosmology if cosmology is not None else default_cosmology.get() + with default_cosmology.set(cosmology): # if already cosmo, passes through + cosmology = default_cosmology.get() + + def z_to_Tcmb(z): + return cosmology.Tcmb(z) + + def Tcmb_to_z(T): + return z_at_value(cosmology.Tcmb, T << u.K, **atzkw) + + return u.Equivalency( + [(redshift, u.K, z_to_Tcmb, Tcmb_to_z)], + "redshift_temperature", + {"cosmology": cosmology}, + ) + + +def with_redshift( + cosmology: Union["astropy.cosmology.Cosmology", str, None] = None, + *, + distance: Literal["comoving", "lookback", "luminosity"] = "comoving", + hubble: bool = True, + Tcmb: bool = True, + atzkw: _ZAtValueKWArgs | None = None, +) -> u.Equivalency: + """Convert quantities between measures of cosmological distance. + + Note: by default all equivalencies are on and must be explicitly turned off. + Care should be taken to not misinterpret a relativistic, gravitational, etc + redshift as a cosmological one. + + Parameters + ---------- + cosmology : `~astropy.cosmology.Cosmology`, str, or None, optional + A cosmology realization or built-in cosmology's name (e.g. 'Planck18'). + If `None`, will use the default cosmology + (controlled by |default_cosmology|). + + distance : {'comoving', 'lookback', 'luminosity'} or None (optional, keyword-only) + The type of distance equivalency to create or `None`. + Default is 'comoving'. + hubble : bool (optional, keyword-only) + Whether to create a Hubble parameter <-> redshift equivalency, using + ``Cosmology.H``. Default is `True`. + Tcmb : bool (optional, keyword-only) + Whether to create a CMB temperature <-> redshift equivalency, using + ``Cosmology.Tcmb``. Default is `True`. + + atzkw : dict or None (optional, keyword-only) + keyword arguments for :func:`~astropy.cosmology.z_at_value` + + Returns + ------- + `~astropy.units.equivalencies.Equivalency` + With equivalencies between redshift and distance / Hubble / temperature. + + Examples + -------- + >>> import astropy.units as u + >>> import astropy.cosmology.units as cu + >>> from astropy.cosmology import WMAP9 + + >>> equivalency = cu.with_redshift(WMAP9) + >>> z = 1100 * cu.redshift + + Redshift to (comoving) distance: + + >>> z.to(u.Mpc, equivalency) # doctest: +FLOAT_CMP + + + Redshift to the Hubble parameter: + + >>> z.to(u.km / u.s / u.Mpc, equivalency) # doctest: +FLOAT_CMP + + + >>> z.to(cu.littleh, equivalency) # doctest: +FLOAT_CMP + + + Redshift to CMB temperature: + + >>> z.to(u.K, equivalency) + + """ + # get cosmology: None -> default and process str / class + cosmology = cosmology if cosmology is not None else default_cosmology.get() + with default_cosmology.set(cosmology): # if already cosmo, passes through + cosmology = default_cosmology.get() + + atzkw = atzkw if atzkw is not None else {} + equivs: list[u.Equivalency] = [] # will append as built + + # Hubble <-> Redshift + if hubble: + equivs.extend(redshift_hubble(cosmology, **atzkw)) + + # CMB Temperature <-> Redshift + if Tcmb: + equivs.extend(redshift_temperature(cosmology, **atzkw)) + + # Distance <-> Redshift, but need to choose which distance + if distance is not None: + equivs.extend(redshift_distance(cosmology, kind=distance, **atzkw)) + + # ----------- + return u.Equivalency( + equivs, + "with_redshift", + {"cosmology": cosmology, "distance": distance, "hubble": hubble, "Tcmb": Tcmb}, + ) + + +# =================================================================== + + +def with_H0(H0: u.Quantity | None = None) -> u.Equivalency: + """Convert between quantities with little-h and the equivalent physical units. + + Parameters + ---------- + H0 : None or Quantity ['frequency'] + The value of the Hubble constant to assume. If a |Quantity|, will assume the + quantity *is* ``H0``. If `None` (default), use the ``H0`` attribute from + |default_cosmology|. + + References + ---------- + For an illuminating discussion on why you may or may not want to use + little-h at all, see https://arxiv.org/pdf/1308.4150.pdf + """ + if H0 is None: + H0 = default_cosmology.get().H0 + + h100_val_unit = u.Unit(100 / (H0.to_value((u.km / u.s) / u.Mpc)) * littleh) + + return u.Equivalency([(h100_val_unit, None)], "with_H0", kwargs={"H0": H0}) diff --git a/astropy/cosmology/_src/utils.py b/astropy/cosmology/_src/utils.py new file mode 100644 index 000000000000..b40042bf08b4 --- /dev/null +++ b/astropy/cosmology/_src/utils.py @@ -0,0 +1,128 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +__all__: list[str] = [] # nothing is publicly scoped + +import functools +from collections.abc import Callable +from numbers import Number +from typing import Any, Final, ParamSpec, Protocol, TypeAlias, TypeVar + +import numpy as np +from numpy.typing import ArrayLike, NDArray + +from astropy.units import Quantity + +# isort: split +import astropy.cosmology._src.units as cu + +from .signature_deprecations import _depr_kws_wrap + +P = ParamSpec("P") +R = TypeVar("R") + + +def vectorize_redshift_method(func=None, nin=1): + """Vectorize a method of redshift(s). + + Parameters + ---------- + func : callable or None + method to wrap. If `None` returns a :func:`functools.partial` + with ``nin`` loaded. + nin : int + Number of positional redshift arguments. + + Returns + ------- + wrapper : callable + :func:`functools.wraps` of ``func`` where the first ``nin`` + arguments are converted from |Quantity| to :class:`numpy.ndarray`. + """ + # allow for pie-syntax & setting nin + if func is None: + return functools.partial(vectorize_redshift_method, nin=nin) + + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + """Wrapper converting arguments to numpy-compatible inputs. + + :func:`functools.wraps` of ``func`` where the first ``nin`` arguments are + converted from |Quantity| to `numpy.ndarray` or scalar. + """ + # process inputs + # TODO! quantity-aware vectorization can simplify this. + zs = [ + z if not isinstance(z, Quantity) else z.to_value(cu.redshift) + for z in args[:nin] + ] + # scalar inputs + if all(isinstance(z, (Number, np.generic)) for z in zs): + return func(self, *zs, *args[nin:], **kwargs) + # non-scalar. use vectorized func + return wrapper.__vectorized__(self, *zs, *args[nin:], **kwargs) + + wrapper.__vectorized__ = np.vectorize(func) # attach vectorized function + # TODO! use frompyfunc when can solve return type errors + + return wrapper + + +# =================================================================== + +ScalarTypes: TypeAlias = Number | np.generic +SCALAR_TYPES: Final = (float, int, np.generic, Number) # arranged for speed + + +class HasShape(Protocol): + shape: tuple[int, ...] + + +def aszarr( + z: Quantity | NDArray[Any] | ArrayLike | ScalarTypes | HasShape, / +) -> NDArray[Any]: + """Redshift as an Array duck type. + + Allows for any ndarray ducktype by checking for attribute "shape". + """ + # Scalars + if isinstance(z, SCALAR_TYPES): + return np.asarray(z) + + # Quantities. We do this before checking for normal ndarray because Quantity is a + # subclass of ndarray. + elif isinstance(z, Quantity): + return z.to_value(cu.redshift)[...] + + # Arrays + elif isinstance(z, np.ndarray): + return z + + return Quantity(z, cu.redshift, copy=None, subok=True).view(np.ndarray) + + +# =================================================================== + + +def deprecated_keywords( + *kws: str, since: str | tuple[str, ...] +) -> Callable[[Callable[P, R]], Callable[P, R]]: + """Deprecate calling one or more arguments as keywords. + + Parameters + ---------- + *kws: str + Names of the arguments that will become positional-only. + + since : str, float, or tuple of str or float + The release at which the old argument became deprecated. Can be a single + version (e.g., "7.0" or 7.0) or a tuple of versions for multiple arguments. + """ + return functools.partial(_depr_kws, kws=kws, since=since) + + +def _depr_kws( + func: Callable[P, R], /, kws: tuple[str, ...], since: str | tuple[str, ...] +) -> Callable[P, R]: + wrapper = _depr_kws_wrap(func, kws, since) + functools.update_wrapper(wrapper, func) + return wrapper diff --git a/astropy/cosmology/core.py b/astropy/cosmology/core.py deleted file mode 100644 index 4385962dd4c3..000000000000 --- a/astropy/cosmology/core.py +++ /dev/null @@ -1,3071 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -from __future__ import (absolute_import, division, print_function, - unicode_literals) - -from ..extern import six - -import sys -from math import sqrt, pi, exp, log, floor -from abc import ABCMeta, abstractmethod - -import numpy as np - -from .. import constants as const -from .. import units as u -from ..utils import isiterable, deprecated -from ..utils.state import ScienceState, ScienceStateAlias - -from . import parameters - -# Originally authored by Andrew Becker (becker@astro.washington.edu), -# and modified by Neil Crighton (neilcrighton@gmail.com) and Roban -# Kramer (robanhk@gmail.com). - -# Many of these adapted from Hogg 1999, astro-ph/9905116 -# and Linder 2003, PRL 90, 91301 - -__all__ = ["FLRW", "LambdaCDM", "FlatLambdaCDM", "wCDM", "FlatwCDM", - "Flatw0waCDM", "w0waCDM", "wpwaCDM", "w0wzCDM", "WMAP5", "WMAP7", - "WMAP9", "Planck13", "Planck15", "default_cosmology"] - -__doctest_requires__ = {'*': ['scipy.integrate']} - -# Notes about speeding up integrals: -# --------------------------------- -# The supplied cosmology classes use a few tricks to speed -# up distance and time integrals. It is not necessary for -# anyone subclassing FLRW to use these tricks -- but if they -# do, such calculations may be a lot faster. -# The first, more basic, idea is that, in many cases, it's a big deal to -# provide explicit formulae for inv_efunc rather than simply -# setting up de_energy_scale -- assuming there is a nice expression. -# As noted above, almost all of the provided classes do this, and -# that template can pretty much be followed directly with the appropriate -# formula changes. -# The second, and more advanced, option is to also explicitly -# provide a scalar only version of inv_efunc. This results in a fairly -# large speedup (>10x in most cases) in the distance and age integrals, -# because testing whether the inputs are iterable or pure scalars turns out -# to be rather expensive. If somebody making a new subclass wants to pursue -# this optimization, the key thing is to explicitly set the -# instance variables self._inv_efunc_scalar and self._inv_efunc_scalar_args -# in the constructor for the subclass, where the latter are all the -# arguments except z to _inv_efunc_scalar. -# Again, the provided classes do use this optimization, and in fact go -# even further and provide optimizations for no radiation, and for radiation -# with massless neutrinos. Consult the subclasses for details. -# -# However, the important point is that it is -not- necessary to do this. - -# Some conversion constants -- useful to compute them once here -# and reuse in the initialization rather than have every object do them -# Note that the call to cgs is actually extremely expensive, -# so we actually skip using the units package directly, and -# hardwire the conversion from mks to cgs. This assumes that constants -# will always return mks by default -- if this is made faster for simple -# cases like this, it should be changed back. -# Note that the unit tests should catch it if this happens -H0units_to_invs = (u.km / (u.s * u.Mpc)).to(1.0 / u.s) -sec_to_Gyr = u.s.to(u.Gyr) -# const in critical density in cgs units (g cm^-3) -critdens_const = 3. / (8. * pi * const.G.value * 1000) -arcsec_in_radians = pi / (3600. * 180) -arcmin_in_radians = pi / (60. * 180) -# Radiation parameter over c^2 in cgs (g cm^-3 K^-4) -a_B_c2 = 4e-3 * const.sigma_sb.value / const.c.value ** 3 -# Boltzmann constant in eV / K -kB_evK = const.k_B.to(u.eV / u.K) - - -class CosmologyError(Exception): - pass - - -class Cosmology(object): - """ Placeholder for when a more general Cosmology class is - implemented. """ - - -@six.add_metaclass(ABCMeta) -class FLRW(Cosmology): - """ A class describing an isotropic and homogeneous - (Friedmann-Lemaitre-Robertson-Walker) cosmology. - - This is an abstract base class -- you can't instantiate - examples of this class, but must work with one of its - subclasses such as `LambdaCDM` or `wCDM`. - - Parameters - ---------- - - H0 : float or scalar `~astropy.units.Quantity` - Hubble constant at z = 0. If a float, must be in [km/sec/Mpc] - - Om0 : float - Omega matter: density of non-relativistic matter in units of the - critical density at z=0. Note that this does not include - massive neutrinos. - - Ode0 : float - Omega dark energy: density of dark energy in units of the critical - density at z=0. - - Tcmb0 : float or scalar `~astropy.units.Quantity` - Temperature of the CMB z=0. If a float, must be in [K]. Default: 2.725. - Setting this to zero will turn off both photons and neutrinos (even - massive ones) - - Neff : float - Effective number of Neutrino species. Default 3.04. - - m_nu : `~astropy.units.Quantity` - Mass of each neutrino species. If this is a scalar Quantity, then all - neutrino species are assumed to have that mass. Otherwise, the mass of - each species. The actual number of neutrino species (and hence the - number of elements of m_nu if it is not scalar) must be the floor of - Neff. Usually this means you must provide three neutrino masses unless - you are considering something like a sterile neutrino. - - name : str - Optional name for this cosmological object. - - Ob0 : float - Omega baryons: density of baryonic matter in units of the critical - density at z=0. - - Notes - ----- - Class instances are static -- you can't change the values - of the parameters. That is, all of the attributes above are - read only. - """ - def __init__(self, H0, Om0, Ode0, Tcmb0=2.725, Neff=3.04, - m_nu=u.Quantity(0.0, u.eV), name=None, Ob0=None): - - # all densities are in units of the critical density - self._Om0 = float(Om0) - if self._Om0 < 0.0: - raise ValueError("Matter density can not be negative") - self._Ode0 = float(Ode0) - if Ob0 is not None: - self._Ob0 = float(Ob0) - if self._Ob0 < 0.0: - raise ValueError("Baryonic density can not be negative") - if self._Ob0 > self._Om0: - raise ValueError("Baryonic density can not be larger than " - "total matter density") - self._Odm0 = self._Om0 - self._Ob0 - else: - self._Ob0 = None - self._Odm0 = None - - self._Neff = float(Neff) - if self._Neff < 0.0: - raise ValueError("Effective number of neutrinos can " - "not be negative") - self.name = name - - # Tcmb may have units - self._Tcmb0 = u.Quantity(Tcmb0, unit=u.K, dtype=np.float) - if not self._Tcmb0.isscalar: - raise ValueError("Tcmb0 is a non-scalar quantity") - - # Hubble parameter at z=0, km/s/Mpc - self._H0 = u.Quantity(H0, unit=u.km / u.s / u.Mpc, dtype=np.float) - if not self._H0.isscalar: - raise ValueError("H0 is a non-scalar quantity") - - # 100 km/s/Mpc * h = H0 (so h is dimensionless) - self._h = self._H0.value / 100. - # Hubble distance - self._hubble_distance = (const.c / self._H0).to(u.Mpc) - # H0 in s^-1; don't use units for speed - H0_s = self._H0.value * H0units_to_invs - # Hubble time; again, avoiding units package for speed - self._hubble_time = u.Quantity(sec_to_Gyr / H0_s, u.Gyr) - - # critical density at z=0 (grams per cubic cm) - cd0value = critdens_const * H0_s ** 2 - self._critical_density0 = u.Quantity(cd0value, u.g / u.cm ** 3) - - # Load up neutrino masses. Note: in Py2.x, floor is floating - self._nneutrinos = int(floor(self._Neff)) - - # We are going to share Neff between the neutrinos equally. - # In detail this is not correct, but it is a standard assumption - # because properly calculating it is a) complicated b) depends - # on the details of the massive neutrinos (e.g., their weak - # interactions, which could be unusual if one is considering sterile - # neutrinos) - self._massivenu = False - if self._nneutrinos > 0 and self._Tcmb0.value > 0: - self._neff_per_nu = self._Neff / self._nneutrinos - - # We can't use the u.Quantity constructor as we do above - # because it doesn't understand equivalencies - if not isinstance(m_nu, u.Quantity): - raise ValueError("m_nu must be a Quantity") - - m_nu = m_nu.to(u.eV, equivalencies=u.mass_energy()) - - # Now, figure out if we have massive neutrinos to deal with, - # and, if so, get the right number of masses - # It is worth the effort to keep track of massless ones separately - # (since they are quite easy to deal with, and a common use case - # is to set only one neutrino to have mass) - if m_nu.isscalar: - # Assume all neutrinos have the same mass - if m_nu.value == 0: - self._nmasslessnu = self._nneutrinos - self._nmassivenu = 0 - else: - self._massivenu = True - self._nmasslessnu = 0 - self._nmassivenu = self._nneutrinos - self._massivenu_mass = (m_nu.value * - np.ones(self._nneutrinos)) - else: - # Make sure we have the right number of masses - # -unless- they are massless, in which case we cheat a little - if m_nu.value.min() < 0: - raise ValueError("Invalid (negative) neutrino mass" - " encountered") - if m_nu.value.max() == 0: - self._nmasslessnu = self._nneutrinos - self._nmassivenu = 0 - else: - self._massivenu = True - if len(m_nu) != self._nneutrinos: - errstr = "Unexpected number of neutrino masses" - raise ValueError(errstr) - # Segregate out the massless ones - self._nmasslessnu = len(np.nonzero(m_nu.value == 0)[0]) - self._nmassivenu = self._nneutrinos - self._nmasslessnu - w = np.nonzero(m_nu.value > 0)[0] - self._massivenu_mass = m_nu[w] - - # Compute photon density, Tcmb, neutrino parameters - # Tcmb0=0 removes both photons and neutrinos, is handled - # as a special case for efficiency - if self._Tcmb0.value > 0: - # Compute photon density from Tcmb - self._Ogamma0 = a_B_c2 * self._Tcmb0.value ** 4 /\ - self._critical_density0.value - - # Compute Neutrino temperature - # The constant in front is (4/11)^1/3 -- see any - # cosmology book for an explanation -- for example, - # Weinberg 'Cosmology' p 154 eq (3.1.21) - self._Tnu0 = 0.7137658555036082 * self._Tcmb0 - - # Compute Neutrino Omega and total relativistic component - # for massive neutrinos - if self._massivenu: - nu_y = self._massivenu_mass / (kB_evK * self._Tnu0) - self._nu_y = nu_y.value - self._Onu0 = self._Ogamma0 * self.nu_relative_density(0) - else: - # This case is particularly simple, so do it directly - # The 0.2271... is 7/8 (4/11)^(4/3) -- the temperature - # bit ^4 (blackbody energy density) times 7/8 for - # FD vs. BE statistics. - self._Onu0 = 0.22710731766 * self._Neff * self._Ogamma0 - - else: - self._Ogamma0 = 0.0 - self._Tnu0 = u.Quantity(0.0, u.K) - self._Onu0 = 0.0 - - # Compute curvature density - self._Ok0 = 1.0 - self._Om0 - self._Ode0 - self._Ogamma0 - self._Onu0 - - # Subclasses should override this reference if they provide - # more efficient scalar versions of inv_efunc. - self._inv_efunc_scalar = self.inv_efunc - self._inv_efunc_scalar_args = () - - def _namelead(self): - """ Helper function for constructing __repr__""" - if self.name is None: - return "{0}(".format(self.__class__.__name__) - else: - return "{0}(name=\"{1}\", ".format(self.__class__.__name__, - self.name) - - def __repr__(self): - retstr = "{0}H0={1:.3g}, Om0={2:.3g}, Ode0={3:.3g}, "\ - "Tcmb0={4:.4g}, Neff={5:.3g}, m_nu={6}, "\ - "Ob0={7:s})" - return retstr.format(self._namelead(), self._H0, self._Om0, self._Ode0, - self._Tcmb0, self._Neff, self.m_nu, - _float_or_none(self._Ob0)) - - # Set up a set of properties for H0, Om0, Ode0, Ok0, etc. for user access. - # Note that we don't let these be set (so, obj.Om0 = value fails) - - @property - def H0(self): - """ Return the Hubble constant as an `~astropy.units.Quantity` at z=0""" - return self._H0 - - @property - def Om0(self): - """ Omega matter; matter density/critical density at z=0""" - return self._Om0 - - @property - def Ode0(self): - """ Omega dark energy; dark energy density/critical density at z=0""" - return self._Ode0 - - @property - def Ob0(self): - """ Omega baryon; baryonic matter density/critical density at z=0""" - return self._Ob0 - - @property - def Odm0(self): - """ Omega dark matter; dark matter density/critical density at z=0""" - return self._Odm0 - - @property - def Ok0(self): - """ Omega curvature; the effective curvature density/critical density - at z=0""" - return self._Ok0 - - @property - def Tcmb0(self): - """ Temperature of the CMB as `~astropy.units.Quantity` at z=0""" - return self._Tcmb0 - - @property - def Tnu0(self): - """ Temperature of the neutrino background as `~astropy.units.Quantity` at z=0""" - return self._Tnu0 - - @property - def Neff(self): - """ Number of effective neutrino species""" - return self._Neff - - @property - def has_massive_nu(self): - """ Does this cosmology have at least one massive neutrino species?""" - if self._Tnu0.value == 0: - return False - return self._massivenu - - @property - def m_nu(self): - """ Mass of neutrino species""" - if self._Tnu0.value == 0: - return None - if not self._massivenu: - # Only massless - return u.Quantity(np.zeros(self._nmasslessnu), u.eV, - dtype=np.float) - if self._nmasslessnu == 0: - # Only massive - return u.Quantity(self._massivenu_mass, u.eV, - dtype=np.float) - # A mix -- the most complicated case - numass = np.append(np.zeros(self._nmasslessnu), - self._massivenu_mass.value) - return u.Quantity(numass, u.eV, dtype=np.float) - - @property - def h(self): - """ Dimensionless Hubble constant: h = H_0 / 100 [km/sec/Mpc]""" - return self._h - - @property - def hubble_time(self): - """ Hubble time as `~astropy.units.Quantity`""" - return self._hubble_time - - @property - def hubble_distance(self): - """ Hubble distance as `~astropy.units.Quantity`""" - return self._hubble_distance - - @property - def critical_density0(self): - """ Critical density as `~astropy.units.Quantity` at z=0""" - return self._critical_density0 - - @property - def Ogamma0(self): - """ Omega gamma; the density/critical density of photons at z=0""" - return self._Ogamma0 - - @property - def Onu0(self): - """ Omega nu; the density/critical density of neutrinos at z=0""" - return self._Onu0 - - def clone(self, **kwargs): - """ Returns a copy of this object, potentially with some changes. - - Returns - ------- - newcos : Subclass of FLRW - A new instance of this class with the specified changes. - - Notes - ----- - This assumes that the values of all constructor arguments - are available as properties, which is true of all the provided - subclasses but may not be true of user-provided ones. You can't - change the type of class, so this can't be used to change between - flat and non-flat. If no modifications are requested, then - a reference to this object is returned. - - Examples - -------- - To make a copy of the Planck13 cosmology with a different Omega_m - and a new name: - - >>> from astropy.cosmology import Planck13 - >>> newcos = Planck13.clone(name="Modified Planck 2013", Om0=0.35) - """ - - # Quick return check, taking advantage of the - # immutability of cosmological objects - if len(kwargs) == 0: - return self - - # Get constructor arguments - import inspect - arglist = inspect.getargspec(self.__init__).args - - # Build the dictionary of values used to construct this - # object. This -assumes- every argument to __init__ has a - # property. This is true of all the classes we provide, but - # maybe a user won't do that. So at least try to have a useful - # error message. - argdict = {} - for arg in arglist[1:]: # Skip self, which should always be first - try: - val = getattr(self, arg) - argdict[arg] = val - except AttributeError: - # We didn't find a property -- complain usefully - errstr = "Object did not have property corresponding "\ - "to constructor argument '%s'; perhaps it is a "\ - "user provided subclass that does not do so" - raise AttributeError(errstr % arg) - - # Now substitute in new arguments - for newarg in kwargs: - if newarg not in argdict: - errstr = "User provided argument '%s' not found in "\ - "constructor for this object" - raise AttributeError(errstr % newarg) - argdict[newarg] = kwargs[newarg] - - return self.__class__(**argdict) - - @abstractmethod - def w(self, z): - """ The dark energy equation of state. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - w : ndarray, or float if input scalar - The dark energy equation of state - - Notes - ----- - The dark energy equation of state is defined as - :math:`w(z) = P(z)/\\rho(z)`, where :math:`P(z)` is the - pressure at redshift z and :math:`\\rho(z)` is the density - at redshift z, both in units where c=1. - - This must be overridden by subclasses. - """ - raise NotImplementedError("w(z) is not implemented") - - def Om(self, z): - """ Return the density parameter for non-relativistic matter - at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - Om : ndarray, or float if input scalar - The density of non-relativistic matter relative to the critical - density at each redshift. - - Notes - ----- - This does not include neutrinos, even if non-relativistic - at the redshift of interest; see `Onu`. - """ - - if isiterable(z): - z = np.asarray(z) - return self._Om0 * (1. + z) ** 3 * self.inv_efunc(z) ** 2 - - def Ob(self, z): - """ Return the density parameter for baryonic matter at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - Ob : ndarray, or float if input scalar - The density of baryonic matter relative to the critical density at - each redshift. - - Raises - ------ - ValueError - If Ob0 is None. - """ - - if self._Ob0 is None: - raise ValueError("Baryon density not set for this cosmology") - if isiterable(z): - z = np.asarray(z) - return self._Ob0 * (1. + z) ** 3 * self.inv_efunc(z) ** 2 - - def Odm(self, z): - """ Return the density parameter for dark matter at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - Odm : ndarray, or float if input scalar - The density of non-relativistic dark matter relative to the critical - density at each redshift. - - Raises - ------ - ValueError - If Ob0 is None. - Notes - ----- - This does not include neutrinos, even if non-relativistic - at the redshift of interest. - """ - - if self._Odm0 is None: - raise ValueError("Baryonic density not set for this cosmology, " - "unclear meaning of dark matter density") - if isiterable(z): - z = np.asarray(z) - return self._Odm0 * (1. + z) ** 3 * self.inv_efunc(z) ** 2 - - def Ok(self, z): - """ Return the equivalent density parameter for curvature - at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - Ok : ndarray, or float if input scalar - The equivalent density parameter for curvature at each redshift. - """ - - if isiterable(z): - z = np.asarray(z) - # Common enough case to be worth checking explicitly - if self._Ok0 == 0: - return np.zeros(np.asanyarray(z).shape, dtype=np.float) - else: - if self._Ok0 == 0: - return 0.0 - - return self._Ok0 * (1. + z) ** 2 * self.inv_efunc(z) ** 2 - - def Ode(self, z): - """ Return the density parameter for dark energy at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - Ode : ndarray, or float if input scalar - The density of non-relativistic matter relative to the critical - density at each redshift. - """ - - if isiterable(z): - z = np.asarray(z) - # Common case worth checking - if self._Ode0 == 0: - return np.zeros(np.asanyarray(z).shape, dtype=np.float) - else: - if self._Ode0 == 0: - return 0.0 - - return self._Ode0 * self.de_density_scale(z) * self.inv_efunc(z) ** 2 - - def Ogamma(self, z): - """ Return the density parameter for photons at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - Ogamma : ndarray, or float if input scalar - The energy density of photons relative to the critical - density at each redshift. - """ - - if isiterable(z): - z = np.asarray(z) - return self._Ogamma0 * (1. + z) ** 4 * self.inv_efunc(z) ** 2 - - def Onu(self, z): - """ Return the density parameter for massless neutrinos at - redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - Onu : ndarray, or float if input scalar - The energy density of neutrinos relative to the critical - density at each redshift. Note that this includes their - kinetic energy (if they have mass), so it is not equal to - the commonly used :math:`\\sum \\frac{m_{\\nu}}{94 eV}`, - which does not include kinetic energy. - """ - - if isiterable(z): - z = np.asarray(z) - if self._Onu0 == 0: - return np.zeros(np.asanyarray(z).shape, dtype=np.float) - else: - if self._Onu0 == 0: - return 0.0 - - return self.Ogamma(z) * self.nu_relative_density(z) - - def Tcmb(self, z): - """ Return the CMB temperature at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - Tcmb : `~astropy.units.Quantity` - The temperature of the CMB in K. - """ - - if isiterable(z): - z = np.asarray(z) - return self._Tcmb0 * (1. + z) - - def Tnu(self, z): - """ Return the neutrino temperature at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - Tnu : `~astropy.units.Quantity` - The temperature of the cosmic neutrino background in K. - """ - - if isiterable(z): - z = np.asarray(z) - return self._Tnu0 * (1. + z) - - def nu_relative_density(self, z): - """ Neutrino density function relative to the energy density in - photons. - - Parameters - ---------- - z : array like - Redshift - - Returns - ------- - f : ndarray, or float if z is scalar - The neutrino density scaling factor relative to the density - in photons at each redshift - - Notes - ----- - The density in neutrinos is given by - - .. math:: - - \\rho_{\\nu} \\left(a\\right) = 0.2271 \\, N_{eff} \\, - f\\left(m_{\\nu} a / T_{\\nu 0} \\right) \\, - \\rho_{\\gamma} \\left( a \\right) - - where - - .. math:: - - f \\left(y\\right) = \\frac{120}{7 \\pi^4} - \\int_0^{\\infty} \\, dx \\frac{x^2 \\sqrt{x^2 + y^2}} - {e^x + 1} - - assuming that all neutrino species have the same mass. - If they have different masses, a similar term is calculated - for each one. Note that f has the asymptotic behavior :math:`f(0) = 1`. - This method returns :math:`0.2271 f` using an - analytical fitting formula given in Komatsu et al. 2011, ApJS 192, 18. - """ - - # See Komatsu et al. 2011, eq 26 and the surrounding discussion - # However, this is modified to handle multiple neutrino masses - # by computing the above for each mass, then summing - prefac = 0.22710731766 # 7/8 (4/11)^4/3 -- see any cosmo book - - # The massive and massless contribution must be handled separately - # But check for common cases first - if not self._massivenu: - if np.isscalar(z): - return prefac * self._Neff - else: - return prefac * self._Neff *\ - np.ones(np.asanyarray(z).shape, dtype=np.float) - - p = 1.83 - invp = 1.0 / p - if np.isscalar(z): - curr_nu_y = self._nu_y / (1.0 + z) # only includes massive ones - rel_mass_per = (1.0 + (0.3173 * curr_nu_y) ** p) ** invp - rel_mass = rel_mass_per.sum() + self._nmasslessnu - else: - z = np.asarray(z) - curr_nu_y = self._nu_y / (1. + np.expand_dims(z, axis=-1)) - rel_mass_per = (1. + (0.3173 * curr_nu_y) ** p) ** invp - rel_mass = rel_mass_per.sum(-1) + self._nmasslessnu - - return prefac * self._neff_per_nu * rel_mass - - def _w_integrand(self, ln1pz): - """ Internal convenience function for w(z) integral.""" - - # See Linder 2003, PRL 90, 91301 eq (5) - # Assumes scalar input, since this should only be called - # inside an integral - - z = exp(ln1pz) - 1.0 - return 1.0 + self.w(z) - - def de_density_scale(self, z): - """ Evaluates the redshift dependence of the dark energy density. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - I : ndarray, or float if input scalar - The scaling of the energy density of dark energy with redshift. - - Notes - ----- - The scaling factor, I, is defined by :math:`\\rho(z) = \\rho_0 I`, - and is given by - - .. math:: - - I = \\exp \\left( 3 \int_{a}^1 \\frac{ da^{\\prime} }{ a^{\\prime} } - \\left[ 1 + w\\left( a^{\\prime} \\right) \\right] \\right) - - It will generally helpful for subclasses to overload this method if - the integral can be done analytically for the particular dark - energy equation of state that they implement. - """ - - # This allows for an arbitrary w(z) following eq (5) of - # Linder 2003, PRL 90, 91301. The code here evaluates - # the integral numerically. However, most popular - # forms of w(z) are designed to make this integral analytic, - # so it is probably a good idea for subclasses to overload this - # method if an analytic form is available. - # - # The integral we actually use (the one given in Linder) - # is rewritten in terms of z, so looks slightly different than the - # one in the documentation string, but it's the same thing. - - from scipy.integrate import quad - - if isiterable(z): - z = np.asarray(z) - ival = np.array([quad(self._w_integrand, 0, log(1 + redshift))[0] - for redshift in z]) - return np.exp(3 * ival) - else: - ival = quad(self._w_integrand, 0, log(1 + z))[0] - return exp(3 * ival) - - def efunc(self, z): - """ Function used to calculate H(z), the Hubble parameter. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - E : ndarray, or float if input scalar - The redshift scaling of the Hubble constant. - - Notes - ----- - The return value, E, is defined such that :math:`H(z) = H_0 E`. - - It is not necessary to override this method, but if de_density_scale - takes a particularly simple form, it may be advantageous to. - """ - - if isiterable(z): - z = np.asarray(z) - - Om0, Ode0, Ok0 = self._Om0, self._Ode0, self._Ok0 - if self._massivenu: - Or = self._Ogamma0 * (1 + self.nu_relative_density(z)) - else: - Or = self._Ogamma0 + self._Onu0 - zp1 = 1.0 + z - - return np.sqrt(zp1 ** 2 * ((Or * zp1 + Om0) * zp1 + Ok0) + - Ode0 * self.de_density_scale(z)) - - def inv_efunc(self, z): - """Inverse of efunc. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - E : ndarray, or float if input scalar - The redshift scaling of the inverse Hubble constant. - """ - - # Avoid the function overhead by repeating code - if isiterable(z): - z = np.asarray(z) - Om0, Ode0, Ok0 = self._Om0, self._Ode0, self._Ok0 - if self._massivenu: - Or = self._Ogamma0 * (1 + self.nu_relative_density(z)) - else: - Or = self._Ogamma0 + self._Onu0 - zp1 = 1.0 + z - - return (zp1 ** 2 * ((Or * zp1 + Om0) * zp1 + Ok0) + - Ode0 * self.de_density_scale(z))**(-0.5) - - def _lookback_time_integrand_scalar(self, z): - """ Integrand of the lookback time. - - Parameters - ---------- - z : float - Input redshift. - - Returns - ------- - I : float - The integrand for the lookback time - - References - ---------- - Eqn 30 from Hogg 1999. - """ - - args = self._inv_efunc_scalar_args - return self._inv_efunc_scalar(z, *args) / (1.0 + z) - - @deprecated(since=1.1, alternative='lookback_time_integrand') - def _tfunc(self, z): - """ Integrand of the lookback time. - - Parameters - ---------- - z : float or array-like - Input redshift. - - Returns - ------- - I : float or array - The integrand for the lookback time - - References - ---------- - Eqn 30 from Hogg 1999. - """ - - if isiterable(z): - zp1 = 1.0 + np.asarray(z) - else: - zp1 = 1. + z - - return self.inv_efunc(z) / zp1 - - def lookback_time_integrand(self, z): - """ Integrand of the lookback time. - - Parameters - ---------- - z : float or array-like - Input redshift. - - Returns - ------- - I : float or array - The integrand for the lookback time - - References - ---------- - Eqn 30 from Hogg 1999. - """ - - if isiterable(z): - zp1 = 1.0 + np.asarray(z) - else: - zp1 = 1. + z - - return self.inv_efunc(z) / zp1 - - def _abs_distance_integrand_scalar(self, z): - """ Integrand of the absorption distance. - - Parameters - ---------- - z : float - Input redshift. - - Returns - ------- - X : float - The integrand for the absorption distance - - References - ---------- - See Hogg 1999 section 11. - """ - - args = self._inv_efunc_scalar_args - return (1.0 + z) ** 2 * self._inv_efunc_scalar(z, *args) - - @deprecated(since=1.1, alternative='abs_distance_integrand') - def _xfunc(self, z): - """ Integrand of the absorption distance. - - Parameters - ---------- - z : float or array - Input redshift. - - Returns - ------- - X : float or array - The integrand for the absorption distance - - References - ---------- - See Hogg 1999 section 11. - """ - - if isiterable(z): - zp1 = 1.0 + np.asarray(z) - else: - zp1 = 1. + z - return zp1 ** 2 * self.inv_efunc(z) - - def abs_distance_integrand(self, z): - """ Integrand of the absorption distance. - - Parameters - ---------- - z : float or array - Input redshift. - - Returns - ------- - X : float or array - The integrand for the absorption distance - - References - ---------- - See Hogg 1999 section 11. - """ - - if isiterable(z): - zp1 = 1.0 + np.asarray(z) - else: - zp1 = 1. + z - return zp1 ** 2 * self.inv_efunc(z) - - def H(self, z): - """ Hubble parameter (km/s/Mpc) at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - H : `~astropy.units.Quantity` - Hubble parameter at each input redshift. - """ - - return self._H0 * self.efunc(z) - - def scale_factor(self, z): - """ Scale factor at redshift ``z``. - - The scale factor is defined as :math:`a = 1 / (1 + z)`. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - a : ndarray, or float if input scalar - Scale factor at each input redshift. - """ - - if isiterable(z): - z = np.asarray(z) - - return 1. / (1. + z) - - def lookback_time(self, z): - """ Lookback time in Gyr to redshift ``z``. - - The lookback time is the difference between the age of the - Universe now and the age at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. Must be 1D or scalar - - Returns - ------- - t : `~astropy.units.Quantity` - Lookback time in Gyr to each input redshift. - - See Also - -------- - z_at_value : Find the redshift corresponding to a lookback time. - """ - - from scipy.integrate import quad - f = lambda red: quad(self._lookback_time_integrand_scalar, 0, red)[0] - return self._hubble_time * vectorize_if_needed(f, z) - - def lookback_distance(self, z): - """ - The lookback distance is the light travel time distance to a given - redshift. It is simply c * lookback_time. It may be used to calculate - the proper distance between two redshifts, e.g. for the mean free path - to ionizing radiation. - - Parameters - ---------- - z : array-like - Input redshifts. Must be 1D or scalar - - Returns - ------- - d : `~astropy.units.Quantity` - Lookback distance in Mpc - """ - return (self.lookback_time(z) * const.c).to(u.Mpc) - - def age(self, z): - """ Age of the universe in Gyr at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. Must be 1D or scalar. - - Returns - ------- - t : `~astropy.units.Quantity` - The age of the universe in Gyr at each input redshift. - - See Also - -------- - z_at_value : Find the redshift corresponding to an age. - """ - - from scipy.integrate import quad - f = lambda red: quad(self._lookback_time_integrand_scalar, - red, np.inf)[0] - return self._hubble_time * vectorize_if_needed(f, z) - - def critical_density(self, z): - """ Critical density in grams per cubic cm at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - rho : `~astropy.units.Quantity` - Critical density in g/cm^3 at each input redshift. - """ - - return self._critical_density0 * (self.efunc(z)) ** 2 - - def comoving_distance(self, z): - """ Comoving line-of-sight distance in Mpc at a given - redshift. - - The comoving distance along the line-of-sight between two - objects remains constant with time for objects in the Hubble - flow. - - Parameters - ---------- - z : array-like - Input redshifts. Must be 1D or scalar. - - Returns - ------- - d : ndarray, or float if input scalar - Comoving distance in Mpc to each input redshift. - """ - - from scipy.integrate import quad - f = lambda red: quad(self._inv_efunc_scalar, 0, red, - args=self._inv_efunc_scalar_args)[0] - return self._hubble_distance * vectorize_if_needed(f, z) - - def comoving_transverse_distance(self, z): - """ Comoving transverse distance in Mpc at a given redshift. - - This value is the transverse comoving distance at redshift ``z`` - corresponding to an angular separation of 1 radian. This is - the same as the comoving distance if omega_k is zero (as in - the current concordance lambda CDM model). - - Parameters - ---------- - z : array-like - Input redshifts. Must be 1D or scalar. - - Returns - ------- - d : `~astropy.units.Quantity` - Comoving transverse distance in Mpc at each input redshift. - - Notes - ----- - This quantity also called the 'proper motion distance' in some - texts. - """ - - Ok0 = self._Ok0 - dc = self.comoving_distance(z) - if Ok0 == 0: - return dc - sqrtOk0 = sqrt(abs(Ok0)) - dh = self._hubble_distance - if Ok0 > 0: - return dh / sqrtOk0 * np.sinh(sqrtOk0 * dc.value / dh.value) - else: - return dh / sqrtOk0 * np.sin(sqrtOk0 * dc.value / dh.value) - - def angular_diameter_distance(self, z): - """ Angular diameter distance in Mpc at a given redshift. - - This gives the proper (sometimes called 'physical') transverse - distance corresponding to an angle of 1 radian for an object - at redshift ``z``. - - Weinberg, 1972, pp 421-424; Weedman, 1986, pp 65-67; Peebles, - 1993, pp 325-327. - - Parameters - ---------- - z : array-like - Input redshifts. Must be 1D or scalar. - - Returns - ------- - d : `~astropy.units.Quantity` - Angular diameter distance in Mpc at each input redshift. - """ - - if isiterable(z): - z = np.asarray(z) - - return self.comoving_transverse_distance(z) / (1. + z) - - def luminosity_distance(self, z): - """ Luminosity distance in Mpc at redshift ``z``. - - This is the distance to use when converting between the - bolometric flux from an object at redshift ``z`` and its - bolometric luminosity. - - Parameters - ---------- - z : array-like - Input redshifts. Must be 1D or scalar. - - Returns - ------- - d : `~astropy.units.Quantity` - Luminosity distance in Mpc at each input redshift. - - See Also - -------- - z_at_value : Find the redshift corresponding to a luminosity distance. - - References - ---------- - Weinberg, 1972, pp 420-424; Weedman, 1986, pp 60-62. - """ - - if isiterable(z): - z = np.asarray(z) - - return (1. + z) * self.comoving_transverse_distance(z) - - def angular_diameter_distance_z1z2(self, z1, z2): - """ Angular diameter distance between objects at 2 redshifts. - Useful for gravitational lensing. - - Parameters - ---------- - z1, z2 : array-like, shape (N,) - Input redshifts. z2 must be large than z1. - - Returns - ------- - d : `~astropy.units.Quantity`, shape (N,) or single if input scalar - The angular diameter distance between each input redshift - pair. - - Raises - ------ - CosmologyError - If omega_k is < 0. - - Notes - ----- - This method only works for flat or open curvature - (omega_k >= 0). - """ - - # does not work for negative curvature - Ok0 = self._Ok0 - if Ok0 < 0: - raise CosmologyError('Ok0 must be >= 0 to use this method.') - - outscalar = False - if not isiterable(z1) and not isiterable(z2): - outscalar = True - - z1 = np.atleast_1d(z1) - z2 = np.atleast_1d(z2) - - if z1.size != z2.size: - raise ValueError('z1 and z2 must be the same size.') - - if (z1 > z2).any(): - raise ValueError('z2 must greater than z1') - - dm1 = self.comoving_transverse_distance(z1).value - dm2 = self.comoving_transverse_distance(z2).value - dh_2 = self._hubble_distance.value ** 2 - - if Ok0 == 0: - # Common case worth checking - out = (dm2 - dm1) / (1. + z2) - else: - out = ((dm2 * np.sqrt(1. + Ok0 * dm1 ** 2 / dh_2) - - dm1 * np.sqrt(1. + Ok0 * dm2 ** 2 / dh_2)) / - (1. + z2)) - - if outscalar: - return u.Quantity(out[0], u.Mpc) - - return u.Quantity(out, u.Mpc) - - def absorption_distance(self, z): - """ Absorption distance at redshift ``z``. - - This is used to calculate the number of objects with some - cross section of absorption and number density intersecting a - sightline per unit redshift path. - - Parameters - ---------- - z : array-like - Input redshifts. Must be 1D or scalar. - - Returns - ------- - d : float or ndarray - Absorption distance (dimensionless) at each input redshift. - - References - ---------- - Hogg 1999 Section 11. (astro-ph/9905116) - Bahcall, John N. and Peebles, P.J.E. 1969, ApJ, 156L, 7B - """ - - from scipy.integrate import quad - f = lambda red: quad(self._abs_distance_integrand_scalar, 0, red)[0] - return vectorize_if_needed(f, z) - - def distmod(self, z): - """ Distance modulus at redshift ``z``. - - The distance modulus is defined as the (apparent magnitude - - absolute magnitude) for an object at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. Must be 1D or scalar. - - Returns - ------- - distmod : `~astropy.units.Quantity` - Distance modulus at each input redshift, in magnitudes - - See Also - -------- - z_at_value : Find the redshift corresponding to a distance modulus. - """ - - # Remember that the luminosity distance is in Mpc - # Abs is necessary because in certain obscure closed cosmologies - # the distance modulus can be negative -- which is okay because - # it enters as the square. - val = 5. * np.log10(abs(self.luminosity_distance(z).value)) + 25.0 - return u.Quantity(val, u.mag) - - def comoving_volume(self, z): - """ Comoving volume in cubic Mpc at redshift ``z``. - - This is the volume of the universe encompassed by redshifts less - than ``z``. For the case of omega_k = 0 it is a sphere of radius - `comoving_distance` but it is less intuitive - if omega_k is not 0. - - Parameters - ---------- - z : array-like - Input redshifts. Must be 1D or scalar. - - Returns - ------- - V : `~astropy.units.Quantity` - Comoving volume in :math:`Mpc^3` at each input redshift. - """ - - Ok0 = self._Ok0 - if Ok0 == 0: - return 4. / 3. * pi * self.comoving_distance(z) ** 3 - - dh = self._hubble_distance.value # .value for speed - dm = self.comoving_transverse_distance(z).value - term1 = 4. * pi * dh ** 3 / (2. * Ok0) * u.Mpc ** 3 - term2 = dm / dh * np.sqrt(1 + Ok0 * (dm / dh) ** 2) - term3 = sqrt(abs(Ok0)) * dm / dh - - if Ok0 > 0: - return term1 * (term2 - 1. / sqrt(abs(Ok0)) * np.arcsinh(term3)) - else: - return term1 * (term2 - 1. / sqrt(abs(Ok0)) * np.arcsin(term3)) - - def differential_comoving_volume(self, z): - """Differential comoving volume at redshift z. - - Useful for calculating the effective comoving volume. - For example, allows for integration over a comoving volume - that has a sensitivity function that changes with redshift. - The total comoving volume is given by integrating - differential_comoving_volume to redshift z - and multiplying by a solid angle. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - dV : `~astropy.units.Quantity` - Differential comoving volume per redshift per steradian at - each input redshift.""" - dh = self._hubble_distance - da = self.angular_diameter_distance(z) - zp1 = 1.0 + z - return dh * (zp1 ** 2.0 * da ** 2.0) / u.Quantity(self.efunc(z), - u.steradian) - - def kpc_comoving_per_arcmin(self, z): - """ Separation in transverse comoving kpc corresponding to an - arcminute at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. Must be 1D or scalar. - - Returns - ------- - d : `~astropy.units.Quantity` - The distance in comoving kpc corresponding to an arcmin at each - input redshift. - """ - return (self.comoving_transverse_distance(z).to(u.kpc) * - arcmin_in_radians / u.arcmin) - - def kpc_proper_per_arcmin(self, z): - """ Separation in transverse proper kpc corresponding to an - arcminute at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. Must be 1D or scalar. - - Returns - ------- - d : `~astropy.units.Quantity` - The distance in proper kpc corresponding to an arcmin at each - input redshift. - """ - return (self.angular_diameter_distance(z).to(u.kpc) * - arcmin_in_radians / u.arcmin) - - def arcsec_per_kpc_comoving(self, z): - """ Angular separation in arcsec corresponding to a comoving kpc - at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. Must be 1D or scalar. - - Returns - ------- - theta : `~astropy.units.Quantity` - The angular separation in arcsec corresponding to a comoving kpc - at each input redshift. - """ - return u.arcsec / (self.comoving_transverse_distance(z).to(u.kpc) * - arcsec_in_radians) - - def arcsec_per_kpc_proper(self, z): - """ Angular separation in arcsec corresponding to a proper kpc at - redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. Must be 1D or scalar. - - Returns - ------- - theta : `~astropy.units.Quantity` - The angular separation in arcsec corresponding to a proper kpc - at each input redshift. - """ - return u.arcsec / (self.angular_diameter_distance(z).to(u.kpc) * - arcsec_in_radians) - - -class LambdaCDM(FLRW): - """FLRW cosmology with a cosmological constant and curvature. - - This has no additional attributes beyond those of FLRW. - - Parameters - ---------- - - H0 : float or `~astropy.units.Quantity` - Hubble constant at z = 0. If a float, must be in [km/sec/Mpc] - - Om0 : float - Omega matter: density of non-relativistic matter in units of the - critical density at z=0. - - Ode0 : float - Omega dark energy: density of the cosmological constant in units of the - critical density at z=0. - - Tcmb0 : float or `~astropy.units.Quantity` - Temperature of the CMB z=0. If a float, must be in [K]. Default: 2.725. - - Neff : float - Effective number of Neutrino species. Default 3.04. - - m_nu : `~astropy.units.Quantity` - Mass of each neutrino species. If this is a scalar Quantity, then all - neutrino species are assumed to have that mass. Otherwise, the mass of - each species. The actual number of neutrino species (and hence the - number of elements of m_nu if it is not scalar) must be the floor of - Neff. Usually this means you must provide three neutrino masses unless - you are considering something like a sterile neutrino. - - name : str - Optional name for this cosmological object. - - Examples - -------- - >>> from astropy.cosmology import LambdaCDM - >>> cosmo = LambdaCDM(H0=70, Om0=0.3, Ode0=0.7) - - The comoving distance in Mpc at redshift z: - - >>> z = 0.5 - >>> dc = cosmo.comoving_distance(z) - """ - - def __init__(self, H0, Om0, Ode0, Tcmb0=2.725, Neff=3.04, - m_nu=u.Quantity(0.0, u.eV), name=None, Ob0=None): - - FLRW.__init__(self, H0, Om0, Ode0, Tcmb0, Neff, m_nu, name=name, - Ob0=Ob0) - - # Please see "Notes about speeding up integrals" for discussion - # about what is being done here. - if self._Tcmb0.value == 0: - self._inv_efunc_scalar = self._lcdm_inv_efunc_norel - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, self._Ok0) - elif not self._massivenu: - self._inv_efunc_scalar = self._lcdm_inv_efunc_nomnu - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, self._Ok0, - self._Ogamma0 + self._Onu0) - else: - self._inv_efunc_scalar = self._lcdm_inv_efunc - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, self._Ok0, - self._Ogamma0, - self.nu_relative_density) - - def w(self, z): - """Returns dark energy equation of state at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - w : ndarray, or float if input scalar - The dark energy equation of state - - Notes - ------ - The dark energy equation of state is defined as - :math:`w(z) = P(z)/\\rho(z)`, where :math:`P(z)` is the - pressure at redshift z and :math:`\\rho(z)` is the density - at redshift z, both in units where c=1. Here this is - :math:`w(z) = -1`. - """ - - if np.isscalar(z): - return -1.0 - else: - return -1.0 * np.ones(np.asanyarray(z).shape, dtype=np.float) - - def de_density_scale(self, z): - """ Evaluates the redshift dependence of the dark energy density. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - I : ndarray, or float if input scalar - The scaling of the energy density of dark energy with redshift. - - Notes - ----- - The scaling factor, I, is defined by :math:`\\rho(z) = \\rho_0 I`, - and in this case is given by :math:`I = 1`. - """ - - if np.isscalar(z): - return 1. - else: - return np.ones(np.asanyarray(z).shape, dtype=np.float) - - def efunc(self, z): - """ Function used to calculate H(z), the Hubble parameter. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - E : ndarray, or float if input scalar - The redshift scaling of the Hubble constant. - - Notes - ----- - The return value, E, is defined such that :math:`H(z) = H_0 E`. - """ - - if isiterable(z): - z = np.asarray(z) - - # We override this because it takes a particularly simple - # form for a cosmological constant - Om0, Ode0, Ok0 = self._Om0, self._Ode0, self._Ok0 - if self._massivenu: - Or = self._Ogamma0 * (1. + self.nu_relative_density(z)) - else: - Or = self._Ogamma0 + self._Onu0 - zp1 = 1.0 + z - - return np.sqrt(zp1 ** 2 * ((Or * zp1 + Om0) * zp1 + Ok0) + Ode0) - - def inv_efunc(self, z): - r""" Function used to calculate :math:`\frac{1}{H_z}`. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - E : ndarray, or float if input scalar - The inverse redshift scaling of the Hubble constant. - - Notes - ----- - The return value, E, is defined such that :math:`H_z = H_0 / - E`. - """ - - if isiterable(z): - z = np.asarray(z) - Om0, Ode0, Ok0 = self._Om0, self._Ode0, self._Ok0 - if self._massivenu: - Or = self._Ogamma0 * (1 + self.nu_relative_density(z)) - else: - Or = self._Ogamma0 + self._Onu0 - zp1 = 1.0 + z - - return (zp1 ** 2 * ((Or * zp1 + Om0) * zp1 + Ok0) + Ode0)**(-0.5) - - # The stuff below here for this class is -not- something - # you need to overload for your own classes. It is done - # purely for efficiency reasons. - @staticmethod - def _lcdm_inv_efunc_norel(z, Om0, Ode0, Ok0): - opz = 1.0 + z - return (opz**2 * (opz * Om0 + Ok0) + Ode0)**(-0.5) - - @staticmethod - def _lcdm_inv_efunc_nomnu(z, Om0, Ode0, Ok0, Or0): - opz = 1.0 + z - return ((((opz * Or0 + Om0) * opz) + Ok0) * opz**2 + Ode0)**(-0.5) - - @staticmethod - def _lcdm_inv_efunc(z, Om0, Ode0, Ok0, Ogamma0, nufunc): - Or0 = Ogamma0 * (1. + nufunc(z)) - opz = 1.0 + z - return ((((opz * Or0 + Om0) * opz) + Ok0) * opz**2 + Ode0)**(-0.5) - - -class FlatLambdaCDM(LambdaCDM): - """FLRW cosmology with a cosmological constant and no curvature. - - This has no additional attributes beyond those of FLRW. - - Parameters - ---------- - H0 : float or `~astropy.units.Quantity` - Hubble constant at z = 0. If a float, must be in [km/sec/Mpc] - - Om0 : float - Omega matter: density of non-relativistic matter in units of the - critical density at z=0. - - Tcmb0 : float or `~astropy.units.Quantity` - Temperature of the CMB z=0. If a float, must be in [K]. Default: 2.725. - - Neff : float - Effective number of Neutrino species. Default 3.04. - - m_nu : `~astropy.units.Quantity` - Mass of each neutrino species. If this is a scalar Quantity, then all - neutrino species are assumed to have that mass. Otherwise, the mass of - each species. The actual number of neutrino species (and hence the - number of elements of m_nu if it is not scalar) must be the floor of - Neff. Usually this means you must provide three neutrino masses unless - you are considering something like a sterile neutrino. - - name : str - Optional name for this cosmological object. - - Examples - -------- - >>> from astropy.cosmology import FlatLambdaCDM - >>> cosmo = FlatLambdaCDM(H0=70, Om0=0.3) - - The comoving distance in Mpc at redshift z: - - >>> z = 0.5 - >>> dc = cosmo.comoving_distance(z) - """ - - def __init__(self, H0, Om0, Tcmb0=2.725, Neff=3.04, - m_nu=u.Quantity(0.0, u.eV), name=None, Ob0=None): - - LambdaCDM.__init__(self, H0, Om0, 0.0, Tcmb0, Neff, m_nu, name=name, - Ob0=Ob0) - # Do some twiddling after the fact to get flatness - self._Ode0 = 1.0 - self._Om0 - self._Ogamma0 - self._Onu0 - self._Ok0 = 0.0 - - # Please see "Notes about speeding up integrals" for discussion - # about what is being done here. - if self._Tcmb0.value == 0: - self._inv_efunc_scalar = self._flcdm_inv_efunc_norel - self._inv_efunc_scalar_args = (self._Om0, self._Ode0) - elif not self._massivenu: - self._inv_efunc_scalar = self._flcdm_inv_efunc_nomnu - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, - self._Ogamma0 + self._Onu0) - else: - self._inv_efunc_scalar = self._flcdm_inv_efunc - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, - self._Ogamma0, - self.nu_relative_density) - - def efunc(self, z): - """ Function used to calculate H(z), the Hubble parameter. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - E : ndarray, or float if input scalar - The redshift scaling of the Hubble constant. - - Notes - ----- - The return value, E, is defined such that :math:`H(z) = H_0 E`. - """ - - if isiterable(z): - z = np.asarray(z) - - # We override this because it takes a particularly simple - # form for a cosmological constant - Om0, Ode0 = self._Om0, self._Ode0 - if self._massivenu: - Or = self._Ogamma0 * (1 + self.nu_relative_density(z)) - else: - Or = self._Ogamma0 + self._Onu0 - zp1 = 1.0 + z - - return np.sqrt(zp1 ** 3 * (Or * zp1 + Om0) + Ode0) - - def inv_efunc(self, z): - """Function used to calculate :math:`\frac{1}{H_z}`. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - E : ndarray, or float if input scalar - The inverse redshift scaling of the Hubble constant. - - Notes - ----- - The return value, E, is defined such that :math:`H_z = H_0 / E`. - """ - - if isiterable(z): - z = np.asarray(z) - Om0, Ode0 = self._Om0, self._Ode0 - if self._massivenu: - Or = self._Ogamma0 * (1. + self.nu_relative_density(z)) - else: - Or = self._Ogamma0 + self._Onu0 - zp1 = 1.0 + z - return (zp1 ** 3 * (Or * zp1 + Om0) + Ode0)**(-0.5) - - def __repr__(self): - retstr = "{0}H0={1:.3g}, Om0={2:.3g}, Tcmb0={3:.4g}, "\ - "Neff={4:.3g}, m_nu={5}, Ob0={6:s})" - return retstr.format(self._namelead(), self._H0, self._Om0, - self._Tcmb0, self._Neff, self.m_nu, - _float_or_none(self._Ob0)) - - # The stuff below here for this class is -not- something - # you need to overload for your own classes. It is done - # purely for efficiency reasons, and results in a 10x speedup - # in distance calculations. - @staticmethod - def _flcdm_inv_efunc_norel(z, Om0, Ode0): - return ((1. + z)**3 * Om0 + Ode0)**(-0.5) - - @staticmethod - def _flcdm_inv_efunc_nomnu(z, Om0, Ode0, Or0): - opz = 1.0 + z - return (opz**3 * (opz * Or0 + Om0) + Ode0)**(-0.5) - - @staticmethod - def _flcdm_inv_efunc(z, Om0, Ode0, Ogamma0, nufunc): - Or0 = Ogamma0 * (1. + nufunc(z)) - opz = 1.0 + z - return (opz**3 * (opz * Or0 + Om0) + Ode0)**(-0.5) - - -class wCDM(FLRW): - """FLRW cosmology with a constant dark energy equation of state - and curvature. - - This has one additional attribute beyond those of FLRW. - - Parameters - ---------- - - H0 : float or `~astropy.units.Quantity` - Hubble constant at z = 0. If a float, must be in [km/sec/Mpc] - - Om0 : float - Omega matter: density of non-relativistic matter in units of the - critical density at z=0. - - Ode0 : float - Omega dark energy: density of dark energy in units of the critical - density at z=0. - - w0 : float - Dark energy equation of state at all redshifts. This is - pressure/density for dark energy in units where c=1. A cosmological - constant has w0=-1.0. - - Tcmb0 : float or `~astropy.units.Quantity` - Temperature of the CMB z=0. If a float, must be in [K]. Default: 2.725. - - Neff : float - Effective number of Neutrino species. Default 3.04. - - m_nu : `~astropy.units.Quantity` - Mass of each neutrino species. If this is a scalar Quantity, then all - neutrino species are assumed to have that mass. Otherwise, the mass of - each species. The actual number of neutrino species (and hence the - number of elements of m_nu if it is not scalar) must be the floor of - Neff. Usually this means you must provide three neutrino masses unless - you are considering something like a sterile neutrino. - - name : str - Optional name for this cosmological object. - - Ob0 : float - Omega baryons: density of baryonic matter in units of the critical - density at z=0. - - Examples - -------- - >>> from astropy.cosmology import wCDM - >>> cosmo = wCDM(H0=70, Om0=0.3, Ode0=0.7, w0=-0.9) - - The comoving distance in Mpc at redshift z: - - >>> z = 0.5 - >>> dc = cosmo.comoving_distance(z) - """ - - def __init__(self, H0, Om0, Ode0, w0=-1., Tcmb0=2.725, - Neff=3.04, m_nu=u.Quantity(0.0, u.eV), name=None, Ob0=None): - - FLRW.__init__(self, H0, Om0, Ode0, Tcmb0, Neff, m_nu, name=name, - Ob0=None) - self._w0 = float(w0) - - # Please see "Notes about speeding up integrals" for discussion - # about what is being done here. - if self._Tcmb0.value == 0: - self._inv_efunc_scalar = self._wcdm_inv_efunc_norel - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, self._Ok0, - self._w0) - elif not self._massivenu: - self._inv_efunc_scalar = self._wcdm_inv_efunc_nomnu - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, self._Ok0, - self._Ogamma0 + self._Onu0, - self._w0) - else: - self._inv_efunc_scalar = self._wcdm_inv_efunc - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, self._Ok0, - self._Ogamma0, - self.nu_relative_density, - self._w0) - - @property - def w0(self): - """ Dark energy equation of state""" - return self._w0 - - def w(self, z): - """Returns dark energy equation of state at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - w : ndarray, or float if input scalar - The dark energy equation of state - - Notes - ------ - The dark energy equation of state is defined as - :math:`w(z) = P(z)/\\rho(z)`, where :math:`P(z)` is the - pressure at redshift z and :math:`\\rho(z)` is the density - at redshift z, both in units where c=1. Here this is - :math:`w(z) = w_0`. - """ - - if np.isscalar(z): - return self._w0 - else: - return self._w0 * np.ones(np.asanyarray(z).shape, dtype=np.float) - - def de_density_scale(self, z): - """ Evaluates the redshift dependence of the dark energy density. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - I : ndarray, or float if input scalar - The scaling of the energy density of dark energy with redshift. - - Notes - ----- - The scaling factor, I, is defined by :math:`\\rho(z) = \\rho_0 I`, - and in this case is given by - :math:`I = \\left(1 + z\\right)^{3\\left(1 + w_0\\right)}` - """ - - if isiterable(z): - z = np.asarray(z) - return (1. + z) ** (3. * (1. + self._w0)) - - def efunc(self, z): - """ Function used to calculate H(z), the Hubble parameter. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - E : ndarray, or float if input scalar - The redshift scaling of the Hubble constant. - - Notes - ----- - The return value, E, is defined such that :math:`H(z) = H_0 E`. - """ - - if isiterable(z): - z = np.asarray(z) - Om0, Ode0, Ok0, w0 = self._Om0, self._Ode0, self._Ok0, self._w0 - if self._massivenu: - Or = self._Ogamma0 * (1. + self.nu_relative_density(z)) - else: - Or = self._Ogamma0 + self._Onu0 - zp1 = 1.0 + z - - return np.sqrt(zp1 ** 2 * ((Or * zp1 + Om0) * zp1 + Ok0) + - Ode0 * zp1 ** (3. * (1. + w0))) - - def inv_efunc(self, z): - r""" Function used to calculate :math:`\frac{1}{H_z}`. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - E : ndarray, or float if input scalar - The inverse redshift scaling of the Hubble constant. - - Notes - ----- - The return value, E, is defined such that :math:`H_z = H_0 / E`. - """ - - if isiterable(z): - z = np.asarray(z) - Om0, Ode0, Ok0, w0 = self._Om0, self._Ode0, self._Ok0, self._w0 - if self._massivenu: - Or = self._Ogamma0 * (1. + self.nu_relative_density(z)) - else: - Or = self._Ogamma0 + self._Onu0 - zp1 = 1.0 + z - - return (zp1 ** 2 * ((Or * zp1 + Om0) * zp1 + Ok0) + - Ode0 * zp1 ** (3. * (1. + w0)))**(-0.5) - - def __repr__(self): - retstr = "{0}H0={1:.3g}, Om0={2:.3g}, Ode0={3:.3g}, w0={4:.3g}, "\ - "Tcmb0={5:.4g}, Neff={6:.3g}, m_nu={7}, Ob0={8:s})" - return retstr.format(self._namelead(), self._H0, self._Om0, - self._Ode0, self._w0, self._Tcmb0, self._Neff, - self.m_nu, _float_or_none(self._Ob0)) - - # Please see "Notes about speeding up integrals" for discussion - # about what is being done here. - @staticmethod - def _wcdm_inv_efunc_norel(z, Om0, Ode0, Ok0, w0): - opz = 1.0 + z - return (opz**2 * (opz * Om0 + Ok0) + - Ode0 * opz**(3. * (1.0 + w0)))**(-0.5) - - @staticmethod - def _wcdm_inv_efunc_nomnu(z, Om0, Ode0, Ok0, Or0, w0): - opz = 1.0 + z - return ((((opz * Or0 + Om0) * opz) + Ok0) * opz**2 + - Ode0 * opz**(3. * (1.0 + w0)))**(-0.5) - - @staticmethod - def _wcdm_inv_efunc(z, Om0, Ode0, Ok0, Ogamma0, nufunc, w0): - Or0 = Ogamma0 * (1. + nufunc(z)) - opz = 1.0 + z - return ((((opz * Or0 + Om0) * opz) + Ok0) * opz**2 + - Ode0 * opz**(3. * (1.0 + w0)))**(-0.5) - - -class FlatwCDM(wCDM): - """FLRW cosmology with a constant dark energy equation of state - and no spatial curvature. - - This has one additional attribute beyond those of FLRW. - - Parameters - ---------- - - H0 : float or `~astropy.units.Quantity` - Hubble constant at z = 0. If a float, must be in [km/sec/Mpc] - - Om0 : float - Omega matter: density of non-relativistic matter in units of the - critical density at z=0. - - w0 : float - Dark energy equation of state at all redshifts. This is - pressure/density for dark energy in units where c=1. A cosmological - constant has w0=-1.0. - - Tcmb0 : float or `~astropy.units.Quantity` - Temperature of the CMB z=0. If a float, must be in [K]. Default: 2.725. - - Neff : float - Effective number of Neutrino species. Default 3.04. - - m_nu : `~astropy.units.Quantity` - Mass of each neutrino species. If this is a scalar Quantity, then all - neutrino species are assumed to have that mass. Otherwise, the mass of - each species. The actual number of neutrino species (and hence the - number of elements of m_nu if it is not scalar) must be the floor of - Neff. Usually this means you must provide three neutrino masses unless - you are considering something like a sterile neutrino. - - name : str - Optional name for this cosmological object. - - Ob0 : float - Omega baryons: density of baryonic matter in units of the critical - density at z=0. - - Examples - -------- - >>> from astropy.cosmology import FlatwCDM - >>> cosmo = FlatwCDM(H0=70, Om0=0.3, w0=-0.9) - - The comoving distance in Mpc at redshift z: - - >>> z = 0.5 - >>> dc = cosmo.comoving_distance(z) - """ - - def __init__(self, H0, Om0, w0=-1., Tcmb0=2.725, - Neff=3.04, m_nu=u.Quantity(0.0, u.eV), name=None, Ob0=None): - - wCDM.__init__(self, H0, Om0, 0.0, w0, Tcmb0, Neff, m_nu, - name=name, Ob0=Ob0) - # Do some twiddling after the fact to get flatness - self._Ode0 = 1.0 - self._Om0 - self._Ogamma0 - self._Onu0 - self._Ok0 = 0.0 - - # Please see "Notes about speeding up integrals" for discussion - # about what is being done here. - if self._Tcmb0.value == 0: - self._inv_efunc_scalar = self._fwcdm_inv_efunc_norel - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, - self._w0) - elif not self._massivenu: - self._inv_efunc_scalar = self._fwcdm_inv_efunc_nomnu - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, - self._Ogamma0 + self._Onu0, - self._w0) - else: - self._inv_efunc_scalar = self._fwcdm_inv_efunc - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, - self._Ogamma0, - self.nu_relative_density, - self._w0) - - def efunc(self, z): - """ Function used to calculate H(z), the Hubble parameter. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - E : ndarray, or float if input scalar - The redshift scaling of the Hubble constant. - - Notes - ----- - The return value, E, is defined such that :math:`H(z) = H_0 E`. - """ - - if isiterable(z): - z = np.asarray(z) - Om0, Ode0, w0 = self._Om0, self._Ode0, self._w0 - if self._massivenu: - Or = self._Ogamma0 * (1. + self.nu_relative_density(z)) - else: - Or = self._Ogamma0 + self._Onu0 - zp1 = 1. + z - - return np.sqrt(zp1 ** 3 * (Or * zp1 + Om0) + - Ode0 * zp1 ** (3. * (1 + w0))) - - def inv_efunc(self, z): - r""" Function used to calculate :math:`\frac{1}{H_z}`. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - E : ndarray, or float if input scalar - The inverse redshift scaling of the Hubble constant. - - Notes - ----- - The return value, E, is defined such that :math:`H_z = H_0 / E`. - """ - - if isiterable(z): - z = np.asarray(z) - Om0, Ode0, w0 = self._Om0, self._Ode0, self._w0 - if self._massivenu: - Or = self._Ogamma0 * (1. + self.nu_relative_density(z)) - else: - Or = self._Ogamma0 + self._Onu0 - zp1 = 1. + z - - return (zp1 ** 3 * (Or * zp1 + Om0) + - Ode0 * zp1 ** (3. * (1. + w0)))**(-0.5) - - def __repr__(self): - retstr = "{0}H0={1:.3g}, Om0={2:.3g}, w0={3:.3g}, Tcmb0={4:.4g}, "\ - "Neff={5:.3g}, m_nu={6}, Ob0={7:s})" - return retstr.format(self._namelead(), self._H0, self._Om0, self._w0, - self._Tcmb0, self._Neff, self.m_nu, - _float_or_none(self._Ob0)) - - # Please see "Notes about speeding up integrals" for discussion - # about what is being done here. - @staticmethod - def _fwcdm_inv_efunc_norel(z, Om0, Ode0, w0): - opz = 1.0 + z - return (opz**3 * Om0 + Ode0 * opz**(3. * (1.0 + w0)))**(-0.5) - - @staticmethod - def _fwcdm_inv_efunc_nomnu(z, Om0, Ode0, Or0, w0): - opz = 1.0 + z - return (opz**3 * (opz * Or0 + Om0) + - Ode0 * opz**(3. * (1.0 + w0)))**(-0.5) - - @staticmethod - def _fwcdm_inv_efunc(z, Om0, Ode0, Ogamma0, nufunc, w0): - Or0 = Ogamma0 * (1. + nufunc(z)) - opz = 1.0 + z - return (opz**3 * (opz * Or0 + Om0) + - Ode0 * opz**(3. * (1.0 + w0)))**(-0.5) - - -class w0waCDM(FLRW): - """FLRW cosmology with a CPL dark energy equation of state and curvature. - - The equation for the dark energy equation of state uses the - CPL form as described in Chevallier & Polarski Int. J. Mod. Phys. - D10, 213 (2001) and Linder PRL 90, 91301 (2003): - :math:`w(z) = w_0 + w_a (1-a) = w_0 + w_a z / (1+z)`. - - Parameters - ---------- - H0 : float or `~astropy.units.Quantity` - Hubble constant at z = 0. If a float, must be in [km/sec/Mpc] - - Om0 : float - Omega matter: density of non-relativistic matter in units of the - critical density at z=0. - - Ode0 : float - Omega dark energy: density of dark energy in units of the critical - density at z=0. - - w0 : float - Dark energy equation of state at z=0 (a=1). This is pressure/density - for dark energy in units where c=1. - - wa : float - Negative derivative of the dark energy equation of state with respect - to the scale factor. A cosmological constant has w0=-1.0 and wa=0.0. - - Tcmb0 : float or `~astropy.units.Quantity` - Temperature of the CMB z=0. If a float, must be in [K]. Default: 2.725. - - Neff : float - Effective number of Neutrino species. Default 3.04. - - m_nu : `~astropy.units.Quantity` - Mass of each neutrino species. If this is a scalar Quantity, then all - neutrino species are assumed to have that mass. Otherwise, the mass of - each species. The actual number of neutrino species (and hence the - number of elements of m_nu if it is not scalar) must be the floor of - Neff. Usually this means you must provide three neutrino masses unless - you are considering something like a sterile neutrino. - - name : str - Optional name for this cosmological object. - - Ob0 : float - Omega baryons: density of baryonic matter in units of the critical - density at z=0. - - Examples - -------- - >>> from astropy.cosmology import w0waCDM - >>> cosmo = w0waCDM(H0=70, Om0=0.3, Ode0=0.7, w0=-0.9, wa=0.2) - - The comoving distance in Mpc at redshift z: - - >>> z = 0.5 - >>> dc = cosmo.comoving_distance(z) - """ - - def __init__(self, H0, Om0, Ode0, w0=-1., wa=0., Tcmb0=2.725, - Neff=3.04, m_nu=u.Quantity(0.0, u.eV), name=None, Ob0=None): - - FLRW.__init__(self, H0, Om0, Ode0, Tcmb0, Neff, m_nu, name=name, - Ob0=Ob0) - self._w0 = float(w0) - self._wa = float(wa) - - # Please see "Notes about speeding up integrals" for discussion - # about what is being done here. - if self._Tcmb0.value == 0: - self._inv_efunc_scalar = self._w0wa_inv_efunc_norel - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, self._Ok0, - self._w0, self._wa) - elif not self._massivenu: - self._inv_efunc_scalar = self._w0wa_inv_efunc_nomnu - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, self._Ok0, - self._Ogamma0 + self._Onu0, - self._w0, self._wa) - else: - self._inv_efunc_scalar = self._w0wa_inv_efunc - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, self._Ok0, - self._Ogamma0, - self.nu_relative_density, - self._w0, self._wa) - - @property - def w0(self): - """ Dark energy equation of state at z=0""" - return self._w0 - - @property - def wa(self): - """ Negative derivative of dark energy equation of state w.r.t. a""" - return self._wa - - def w(self, z): - """Returns dark energy equation of state at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - w : ndarray, or float if input scalar - The dark energy equation of state - - Notes - ------ - The dark energy equation of state is defined as - :math:`w(z) = P(z)/\\rho(z)`, where :math:`P(z)` is the - pressure at redshift z and :math:`\\rho(z)` is the density - at redshift z, both in units where c=1. Here this is - :math:`w(z) = w_0 + w_a (1 - a) = w_0 + w_a \\frac{z}{1+z}`. - """ - - if isiterable(z): - z = np.asarray(z) - - return self._w0 + self._wa * z / (1.0 + z) - - def de_density_scale(self, z): - """ Evaluates the redshift dependence of the dark energy density. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - I : ndarray, or float if input scalar - The scaling of the energy density of dark energy with redshift. - - Notes - ----- - The scaling factor, I, is defined by :math:`\\rho(z) = \\rho_0 I`, - and in this case is given by - - .. math:: - - I = \\left(1 + z\\right)^{3 \\left(1 + w_0 + w_a\\right)} - \exp \\left(-3 w_a \\frac{z}{1+z}\\right) - - """ - if isiterable(z): - z = np.asarray(z) - zp1 = 1.0 + z - return zp1 ** (3 * (1 + self._w0 + self._wa)) * \ - np.exp(-3 * self._wa * z / zp1) - - def __repr__(self): - retstr = "{0}H0={1:.3g}, Om0={2:.3g}, "\ - "Ode0={3:.3g}, w0={4:.3g}, wa={5:.3g}, Tcmb0={6:.4g}, "\ - "Neff={7:.3g}, m_nu={8}, Ob0={9:s})" - return retstr.format(self._namelead(), self._H0, self._Om0, - self._Ode0, self._w0, self._wa, - self._Tcmb0, self._Neff, self.m_nu, - _float_or_none(self._Ob0)) - - # The stuff below here for this class is -not- something - # you need to overload for your own classes. It is done - # purely for efficiency reasons, and results in a 10x speedup - # in distance calculations. - @staticmethod - def _w0wa_inv_efunc_norel(z, Om0, Ode0, Ok0, w0, wa): - opz = 1.0 + z - Odescl = opz**(3. * (1 + w0 + wa)) * exp(-3.0 * wa * z / opz) - return (opz**2 * (opz * Om0 + Ok0) + Ode0 * Odescl)**(-0.5) - - @staticmethod - def _w0wa_inv_efunc_nomnu(z, Om0, Ode0, Ok0, Or0, w0, wa): - opz = 1.0 + z - Odescl = opz**(3. * (1 + w0 + wa)) * exp(-3.0 * wa * z / opz) - return ((((opz * Or0 + Om0) * opz) + Ok0) * opz**2 + - Ode0 * Odescl)**(-0.5) - - @staticmethod - def _w0wa_inv_efunc(z, Om0, Ode0, Ok0, Ogamma0, nufunc, w0, wa): - Or0 = Ogamma0 * (1. + nufunc(z)) - opz = 1.0 + z - Odescl = opz**(3. * (1 + w0 + wa)) * exp(-3.0 * wa * z / opz) - return ((((opz * Or0 + Om0) * opz) + Ok0) * opz**2 + - Ode0 * Odescl)**(-0.5) - - -class Flatw0waCDM(w0waCDM): - """FLRW cosmology with a CPL dark energy equation of state and no - curvature. - - The equation for the dark energy equation of state uses the - CPL form as described in Chevallier & Polarski Int. J. Mod. Phys. - D10, 213 (2001) and Linder PRL 90, 91301 (2003): - :math:`w(z) = w_0 + w_a (1-a) = w_0 + w_a z / (1+z)`. - - Parameters - ---------- - - H0 : float or `~astropy.units.Quantity` - Hubble constant at z = 0. If a float, must be in [km/sec/Mpc] - - Om0 : float - Omega matter: density of non-relativistic matter in units of the - critical density at z=0. - - w0 : float - Dark energy equation of state at z=0 (a=1). This is pressure/density - for dark energy in units where c=1. - - wa : float - Negative derivative of the dark energy equation of state with respect - to the scale factor. A cosmological constant has w0=-1.0 and wa=0.0. - - Tcmb0 : float or `~astropy.units.Quantity` - Temperature of the CMB z=0. If a float, must be in [K]. Default: 2.725. - - Neff : float - Effective number of Neutrino species. Default 3.04. - - m_nu : `~astropy.units.Quantity` - Mass of each neutrino species. If this is a scalar Quantity, then all - neutrino species are assumed to have that mass. Otherwise, the mass of - each species. The actual number of neutrino species (and hence the - number of elements of m_nu if it is not scalar) must be the floor of - Neff. Usually this means you must provide three neutrino masses unless - you are considering something like a sterile neutrino. - - name : str - Optional name for this cosmological object. - - Ob0 : float - Omega baryons: density of baryonic matter in units of the critical - density at z=0. - - Examples - -------- - >>> from astropy.cosmology import Flatw0waCDM - >>> cosmo = Flatw0waCDM(H0=70, Om0=0.3, w0=-0.9, wa=0.2) - - The comoving distance in Mpc at redshift z: - - >>> z = 0.5 - >>> dc = cosmo.comoving_distance(z) - """ - - def __init__(self, H0, Om0, w0=-1., wa=0., Tcmb0=2.725, - Neff=3.04, m_nu=u.Quantity(0.0, u.eV), name=None, Ob0=None): - - w0waCDM.__init__(self, H0, Om0, 0.0, w0=w0, wa=wa, Tcmb0=Tcmb0, - Neff=Neff, m_nu=m_nu, name=name, Ob0=Ob0) - # Do some twiddling after the fact to get flatness - self._Ode0 = 1.0 - self._Om0 - self._Ogamma0 - self._Onu0 - self._Ok0 = 0.0 - - # Please see "Notes about speeding up integrals" for discussion - # about what is being done here. - if self._Tcmb0.value == 0: - self._inv_efunc_scalar = self._fw0wa_inv_efunc_norel - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, - self._w0, self._wa) - elif not self._massivenu: - self._inv_efunc_scalar = self._fw0wa_inv_efunc_nomnu - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, - self._Ogamma0 + self._Onu0, - self._w0, self._wa) - else: - self._inv_efunc_scalar = self._fw0wa_inv_efunc - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, - self._Ogamma0, - self.nu_relative_density, - self._w0, self._wa) - - def __repr__(self): - retstr = "{0}H0={1:.3g}, Om0={2:.3g}, "\ - "w0={3:.3g}, Tcmb0={4:.4g}, Neff={5:.3g}, m_nu={6}, "\ - "Ob0={7:s})" - return retstr.format(self._namelead(), self._H0, self._Om0, self._w0, - self._Tcmb0, self._Neff, self.m_nu, - _float_or_none(self._Ob0)) - - # Please see "Notes about speeding up integrals" for discussion - # about what is being done here. - @staticmethod - def _fw0wa_inv_efunc_norel(z, Om0, Ode0, w0, wa): - opz = 1.0 + z - Odescl = opz**(3. * (1 + w0 + wa)) * exp(-3.0 * wa * z / opz) - return (Om0 * opz**3 + Ode0 * Odescl)**(-0.5) - - @staticmethod - def _fw0wa_inv_efunc_nomnu(z, Om0, Ode0, Or0, w0, wa): - opz = 1.0 + z - Odescl = opz**(3. * (1 + w0 + wa)) * exp(-3.0 * wa * z / opz) - return ((opz * Or0 + Om0) * opz**3 + Ode0 * Odescl)**(-0.5) - - @staticmethod - def _fw0wa_inv_efunc(z, Om0, Ode0, Ogamma0, nufunc, w0, wa): - Or0 = Ogamma0 * (1. + nufunc(z)) - opz = 1.0 + z - Odescl = opz**(3. * (1 + w0 + wa)) * exp(-3.0 * wa * z / opz) - return ((opz * Or0 + Om0) * opz**3 + Ode0 * Odescl)**(-0.5) - - -class wpwaCDM(FLRW): - """FLRW cosmology with a CPL dark energy equation of state, a pivot - redshift, and curvature. - - The equation for the dark energy equation of state uses the - CPL form as described in Chevallier & Polarski Int. J. Mod. Phys. - D10, 213 (2001) and Linder PRL 90, 91301 (2003), but modified - to have a pivot redshift as in the findings of the Dark Energy - Task Force (Albrecht et al. arXiv:0901.0721 (2009)): - :math:`w(a) = w_p + w_a (a_p - a) = w_p + w_a( 1/(1+zp) - 1/(1+z) )`. - - Parameters - ---------- - - H0 : float or `~astropy.units.Quantity` - Hubble constant at z = 0. If a float, must be in [km/sec/Mpc] - - Om0 : float - Omega matter: density of non-relativistic matter in units of the - critical density at z=0. - - Ode0 : float - Omega dark energy: density of dark energy in units of the critical - density at z=0. - - wp : float - Dark energy equation of state at the pivot redshift zp. This is - pressure/density for dark energy in units where c=1. - - wa : float - Negative derivative of the dark energy equation of state with respect - to the scale factor. A cosmological constant has w0=-1.0 and wa=0.0. - - zp : float - Pivot redshift -- the redshift where w(z) = wp - - Tcmb0 : float or `~astropy.units.Quantity` - Temperature of the CMB z=0. If a float, must be in [K]. Default: 2.725. - - Neff : float - Effective number of Neutrino species. Default 3.04. - - m_nu : `~astropy.units.Quantity` - Mass of each neutrino species. If this is a scalar Quantity, then all - neutrino species are assumed to have that mass. Otherwise, the mass of - each species. The actual number of neutrino species (and hence the - number of elements of m_nu if it is not scalar) must be the floor of - Neff. Usually this means you must provide three neutrino masses unless - you are considering something like a sterile neutrino. - - name : str - Optional name for this cosmological object. - - Ob0 : float - Omega baryons: density of baryonic matter in units of the critical - density at z=0. - - Examples - -------- - >>> from astropy.cosmology import wpwaCDM - >>> cosmo = wpwaCDM(H0=70, Om0=0.3, Ode0=0.7, wp=-0.9, wa=0.2, zp=0.4) - - The comoving distance in Mpc at redshift z: - - >>> z = 0.5 - >>> dc = cosmo.comoving_distance(z) - """ - - def __init__(self, H0, Om0, Ode0, wp=-1., wa=0., zp=0, - Tcmb0=2.725, Neff=3.04, m_nu=u.Quantity(0.0, u.eV), - name=None, Ob0=None): - - FLRW.__init__(self, H0, Om0, Ode0, Tcmb0, Neff, m_nu, name=name, - Ob0=Ob0) - self._wp = float(wp) - self._wa = float(wa) - self._zp = float(zp) - - # Please see "Notes about speeding up integrals" for discussion - # about what is being done here. - apiv = 1.0 / (1.0 + self._zp) - if self._Tcmb0.value == 0: - self._inv_efunc_scalar = self._wpwa_inv_efunc_norel - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, self._Ok0, - self._wp, apiv, self._wa) - elif not self._massivenu: - self._inv_efunc_scalar = self._wpwa_inv_efunc_nomnu - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, self._Ok0, - self._Ogamma0 + self._Onu0, - self._wp, apiv, self._wa) - else: - self._inv_efunc_scalar = self._wpwa_inv_efunc - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, self._Ok0, - self._Ogamma0, - self.nu_relative_density, - self._wp, apiv, self._wa) - - @property - def wp(self): - """ Dark energy equation of state at the pivot redshift zp""" - return self._wp - - @property - def wa(self): - """ Negative derivative of dark energy equation of state w.r.t. a""" - return self._wa - - @property - def zp(self): - """ The pivot redshift, where w(z) = wp""" - return self._zp - - def w(self, z): - """Returns dark energy equation of state at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - w : ndarray, or float if input scalar - The dark energy equation of state - - Notes - ------ - The dark energy equation of state is defined as - :math:`w(z) = P(z)/\\rho(z)`, where :math:`P(z)` is the - pressure at redshift z and :math:`\\rho(z)` is the density - at redshift z, both in units where c=1. Here this is - :math:`w(z) = w_p + w_a (a_p - a)` where :math:`a = 1/1+z` - and :math:`a_p = 1 / 1 + z_p`. - """ - - if isiterable(z): - z = np.asarray(z) - - apiv = 1.0 / (1.0 + self._zp) - return self._wp + self._wa * (apiv - 1.0 / (1. + z)) - - def de_density_scale(self, z): - """ Evaluates the redshift dependence of the dark energy density. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - I : ndarray, or float if input scalar - The scaling of the energy density of dark energy with redshift. - - Notes - ----- - The scaling factor, I, is defined by :math:`\\rho(z) = \\rho_0 I`, - and in this case is given by - - .. math:: - - a_p = \\frac{1}{1 + z_p} - - I = \\left(1 + z\\right)^{3 \\left(1 + w_p + a_p w_a\\right)} - \exp \\left(-3 w_a \\frac{z}{1+z}\\right) - """ - - if isiterable(z): - z = np.asarray(z) - zp1 = 1. + z - apiv = 1. / (1. + self._zp) - return zp1 ** (3. * (1. + self._wp + apiv * self._wa)) * \ - np.exp(-3. * self._wa * z / zp1) - - def __repr__(self): - retstr = "{0}H0={1:.3g}, Om0={2:.3g}, Ode0={3:.3g}, wp={4:.3g}, "\ - "wa={5:.3g}, zp={6:.3g}, Tcmb0={7:.4g}, Neff={8:.3g}, "\ - "m_nu={9}, Ob0={10:s})" - return retstr.format(self._namelead(), self._H0, self._Om0, - self._Ode0, self._wp, self._wa, self._zp, - self._Tcmb0, self._Neff, self.m_nu, - _float_or_none(self._Ob0)) - - # The stuff below here for this class is -not- something - # you need to overload for your own classes. It is done - # purely for efficiency reasons, and results in a 10x speedup - # in distance calculations. - @staticmethod - def _wpwa_inv_efunc_norel(z, Om0, Ode0, Ok0, wp, apiv, wa): - opz = 1.0 + z - Odescl = opz**(3. * (1. + wp + apiv * wa)) * exp(-3. * wa * z / opz) - return (opz**2 * (opz * Om0 + Ok0) + Ode0 * Odescl)**(-0.5) - - @staticmethod - def _wpwa_inv_efunc_nomnu(z, Om0, Ode0, Ok0, Or0, wp, apiv, wa): - opz = 1.0 + z - Odescl = opz**(3. * (1. + wp + apiv * wa)) * exp(-3. * wa * z / opz) - return ((((opz * Or0 + Om0) * opz) + Ok0) * opz**2 + - Ode0 * Odescl)**(-0.5) - - @staticmethod - def _wpwa_inv_efunc(z, Om0, Ode0, Ok0, Ogamma0, nufunc, wp, apiv, wa): - Or0 = Ogamma0 * (1. + nufunc(z)) - opz = 1.0 + z - Odescl = opz**(3. * (1. + wp + apiv * wa)) * exp(-3. * wa * z / opz) - return ((((opz * Or0 + Om0) * opz) + Ok0) * opz**2 + - Ode0 * Odescl)**(-0.5) - - -class w0wzCDM(FLRW): - """FLRW cosmology with a variable dark energy equation of state - and curvature. - - The equation for the dark energy equation of state uses the - simple form: :math:`w(z) = w_0 + w_z z`. - - This form is not recommended for z > 1. - - Parameters - ---------- - - H0 : float or `~astropy.units.Quantity` - Hubble constant at z = 0. If a float, must be in [km/sec/Mpc] - - Om0 : float - Omega matter: density of non-relativistic matter in units of the - critical density at z=0. - - Ode0 : float - Omega dark energy: density of dark energy in units of the critical - density at z=0. - - w0 : float - Dark energy equation of state at z=0. This is pressure/density for dark - energy in units where c=1. A cosmological constant has w0=-1.0. - - wz : float - Derivative of the dark energy equation of state with respect to z. - - Tcmb0 : float or `~astropy.units.Quantity` - Temperature of the CMB z=0. If a float, must be in [K]. Default: 2.725. - - Neff : float - Effective number of Neutrino species. Default 3.04. - - m_nu : float or ndarray or `~astropy.units.Quantity` - Mass of each neutrino species, in eV. If this is a float or scalar - Quantity, then all neutrino species are assumed to have that mass. If a - ndarray or array Quantity, then these are the values of the mass of - each species. The actual number of neutrino species (and hence the - number of elements of m_nu if it is not scalar) must be the floor of - Neff. Usually this means you must provide three neutrino masses unless - you are considering something like a sterile neutrino. - - name : str - Optional name for this cosmological object. - - Ob0 : float - Omega baryons: density of baryonic matter in units of the critical - density at z=0. - - Examples - -------- - >>> from astropy.cosmology import w0wzCDM - >>> cosmo = w0wzCDM(H0=70, Om0=0.3, Ode0=0.7, w0=-0.9, wz=0.2) - - The comoving distance in Mpc at redshift z: - - >>> z = 0.5 - >>> dc = cosmo.comoving_distance(z) - """ - - def __init__(self, H0, Om0, Ode0, w0=-1., wz=0., Tcmb0=2.725, - Neff=3.04, m_nu=u.Quantity(0.0, u.eV), name=None, Ob0=None): - - FLRW.__init__(self, H0, Om0, Ode0, Tcmb0, Neff, m_nu, name=name, - Ob0=Ob0) - self._w0 = float(w0) - self._wz = float(wz) - - # Please see "Notes about speeding up integrals" for discussion - # about what is being done here. - if self._Tcmb0.value == 0: - self._inv_efunc_scalar = self._w0wz_inv_efunc_norel - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, self._Ok0, - self._w0, self._wz) - elif not self._massivenu: - self._inv_efunc_scalar = self._w0wz_inv_efunc_nomnu - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, self._Ok0, - self._Ogamma0 + self._Onu0, - self._w0, self._wz) - else: - self._inv_efunc_scalar = self._w0wz_inv_efunc - self._inv_efunc_scalar_args = (self._Om0, self._Ode0, self._Ok0, - self._Ogamma0, - self.nu_relative_density, - self._w0, self._wz) - - @property - def w0(self): - """ Dark energy equation of state at z=0""" - return self._w0 - - @property - def wz(self): - """ Derivative of the dark energy equation of state w.r.t. z""" - return self._wz - - def w(self, z): - """Returns dark energy equation of state at redshift ``z``. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - w : ndarray, or float if input scalar - The dark energy equation of state - - Notes - ------ - The dark energy equation of state is defined as - :math:`w(z) = P(z)/\\rho(z)`, where :math:`P(z)` is the - pressure at redshift z and :math:`\\rho(z)` is the density - at redshift z, both in units where c=1. Here this is given by - :math:`w(z) = w_0 + w_z z`. - """ - - if isiterable(z): - z = np.asarray(z) - - return self._w0 + self._wz * z - - def de_density_scale(self, z): - """ Evaluates the redshift dependence of the dark energy density. - - Parameters - ---------- - z : array-like - Input redshifts. - - Returns - ------- - I : ndarray, or float if input scalar - The scaling of the energy density of dark energy with redshift. - - Notes - ----- - The scaling factor, I, is defined by :math:`\\rho(z) = \\rho_0 I`, - and in this case is given by - - .. math:: - - I = \\left(1 + z\\right)^{3 \\left(1 + w_0 - w_z\\right)} - \exp \\left(-3 w_z z\\right) - """ - - if isiterable(z): - z = np.asarray(z) - zp1 = 1. + z - return zp1 ** (3. * (1. + self._w0 - self._wz)) *\ - np.exp(-3. * self._wz * z) - - def __repr__(self): - retstr = "{0}H0={1:.3g}, Om0={2:.3g}, "\ - "Ode0={3:.3g}, w0={4:.3g}, wz={5:.3g} Tcmb0={6:.4g}, "\ - "Neff={7:.3g}, m_nu={8}, Ob0={9:s})" - return retstr.format(self._namelead(), self._H0, self._Om0, - self._Ode0, self._w0, self._wz, self._Tcmb0, - self._Neff, self.m_nu, _float_or_none(self._Ob0)) - - # Please see "Notes about speeding up integrals" for discussion - # about what is being done here. - @staticmethod - def _w0wz_inv_efunc_norel(z, Om0, Ode0, Ok0, w0, wz): - opz = 1.0 + z - Odescl = opz**(3. * (1. + w0 - wz)) * exp(-3. * wz * z) - return (opz**2 * (opz * Om0 + Ok0) + Ode0 * Odescl)**(-0.5) - - @staticmethod - def _w0wz_inv_efunc_nomnu(z, Om0, Ode0, Ok0, Or0, w0, wz): - opz = 1.0 + z - Odescl = opz**(3. * (1. + w0 - wz)) * exp(-3. * wz * z) - return ((((opz * Or0 + Om0) * opz) + Ok0) * opz**2 + - Ode0 * Odescl)**(-0.5) - - @staticmethod - def _w0wz_inv_efunc(z, Om0, Ode0, Ok0, Ogamma0, nufunc, w0, wz): - Or0 = Ogamma0 * (1. + nufunc(z)) - opz = 1.0 + z - Odescl = opz**(3. * (1. + w0 - wz)) * exp(-3. * wz * z) - return ((((opz * Or0 + Om0) * opz) + Ok0) * opz**2 + - Ode0 * Odescl)**(-0.5) - - -def _float_or_none(x, digits=3): - """ Helper function to format a variable that can be a float or None""" - if x is None: - return str(x) - fmtstr = "{0:.{digits}g}".format(x, digits=digits) - return fmtstr.format(x) - - -def vectorize_if_needed(func, x): - """ Helper function to vectorize functions on array inputs""" - if isiterable(x): - return np.vectorize(func)(x) - else: - return func(x) - - -# Pre-defined cosmologies. This loops over the parameter sets in the -# parameters module and creates a LambdaCDM or FlatLambdaCDM instance -# with the same name as the parameter set in the current module's namespace. -# Note this assumes all the cosmologies in parameters are LambdaCDM, -# which is true at least as of this writing. - -for key in parameters.available: - par = getattr(parameters, key) - if par['flat']: - cosmo = FlatLambdaCDM(par['H0'], par['Om0'], Tcmb0=par['Tcmb0'], - Neff=par['Neff'], - m_nu=u.Quantity(par['m_nu'], u.eV), - name=key, - Ob0=par['Ob0']) - docstr = "%s instance of FlatLambdaCDM cosmology\n\n(from %s)" - cosmo.__doc__ = docstr % (key, par['reference']) - else: - cosmo = LambdaCDM(par['H0'], par['Om0'], par['Ode0'], - Tcmb0=par['Tcmb0'], Neff=par['Neff'], - m_nu=u.Quantity(par['m_nu'], u.eV), name=key, - Ob0=par['Ob0']) - docstr = "%s instance of LambdaCDM cosmology\n\n(from %s)" - cosmo.__doc__ = docstr % (key, par['reference']) - setattr(sys.modules[__name__], key, cosmo) - -# don't leave these variables floating around in the namespace -del key, par, cosmo - -######################################################################### -# The science state below contains the current cosmology. -######################################################################### - - -class default_cosmology(ScienceState): - """ - The default cosmology to use. To change it:: - - >>> from astropy.cosmology import default_cosmology, WMAP7 - >>> with default_cosmology.set(WMAP7): - ... # WMAP7 cosmology in effect - - Or, you may use a string:: - - >>> with default_cosmology.set('WMAP7'): - ... # WMAP7 cosmology in effect - """ - _value = 'WMAP9' - - @staticmethod - def get_cosmology_from_string(arg): - """ Return a cosmology instance from a string. - """ - if arg == 'no_default': - cosmo = None - else: - try: - cosmo = getattr(sys.modules[__name__], arg) - except AttributeError: - s = "Unknown cosmology '%s'. Valid cosmologies:\n%s" % ( - arg, parameters.available) - raise ValueError(s) - return cosmo - - @classmethod - def validate(cls, value): - if value is None: - value = 'WMAP9' - if isinstance(value, six.string_types): - return cls.get_cosmology_from_string(value) - elif isinstance(value, Cosmology): - return value - else: - raise TypeError("default_cosmology must be a string or Cosmology instance.") - - -DEFAULT_COSMOLOGY = ScienceStateAlias( - '0.4', 'DEFAULT_COSMOLOGY', 'default_cosmology', default_cosmology) diff --git a/astropy/cosmology/data/Planck13.ecsv b/astropy/cosmology/data/Planck13.ecsv new file mode 100644 index 000000000000..8ed0494f631a --- /dev/null +++ b/astropy/cosmology/data/Planck13.ecsv @@ -0,0 +1,39 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: name, datatype: string} +# - {name: H0, unit: km / (Mpc s), datatype: float64, format: '', description: Hubble constant as an `~astropy.units.Quantity` at z=0.} +# - {name: Om0, datatype: float64, format: '', description: Omega matter; matter density/critical density at z=0.} +# - {name: Tcmb0, unit: K, datatype: float64, format: '', description: Temperature of the CMB as `~astropy.units.Quantity` at z=0.} +# - {name: Neff, datatype: float64, format: '', description: Number of effective neutrino species.} +# - {name: m_nu, unit: eV, datatype: string, format: '', description: Mass of neutrino species., subtype: 'float64[3]'} +# - {name: Ob0, datatype: float64, format: '', description: Omega baryon; baryonic matter density/critical density at z=0.} +# meta: !!omap +# - {Oc0: 0.25886} +# - {n: 0.9611} +# - {sigma8: 0.8288} +# - {tau: 0.0952} +# - z_reion: !astropy.units.Quantity +# unit: !astropy.units.Unit {unit: redshift} +# value: 11.52 +# - t0: !astropy.units.Quantity +# unit: !astropy.units.Unit {unit: Gyr} +# value: 13.7965 +# - {reference: 'Planck Collaboration 2014, A&A, 571, A16 (Paper XVI), Table 5 (Planck + WP + highL + BAO)'} +# - {cosmology: FlatLambdaCDM} +# - __serialized_columns__: +# H0: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: km / (Mpc s)} +# value: !astropy.table.SerializedColumn {name: H0} +# Tcmb0: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: K} +# value: !astropy.table.SerializedColumn {name: Tcmb0} +# m_nu: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: eV} +# value: !astropy.table.SerializedColumn {name: m_nu} +# schema: astropy-2.0 +name H0 Om0 Tcmb0 Neff m_nu Ob0 +Planck13 67.77 0.30712 2.7255 3.046 [0.0,0.0,0.06] 0.048252 diff --git a/astropy/cosmology/data/Planck15.ecsv b/astropy/cosmology/data/Planck15.ecsv new file mode 100644 index 000000000000..cdb20da91423 --- /dev/null +++ b/astropy/cosmology/data/Planck15.ecsv @@ -0,0 +1,39 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: name, datatype: string} +# - {name: H0, unit: km / (Mpc s), datatype: float64, format: '', description: Hubble constant as an `~astropy.units.Quantity` at z=0.} +# - {name: Om0, datatype: float64, format: '', description: Omega matter; matter density/critical density at z=0.} +# - {name: Tcmb0, unit: K, datatype: float64, format: '', description: Temperature of the CMB as `~astropy.units.Quantity` at z=0.} +# - {name: Neff, datatype: float64, format: '', description: Number of effective neutrino species.} +# - {name: m_nu, unit: eV, datatype: string, format: '', description: Mass of neutrino species., subtype: 'float64[3]'} +# - {name: Ob0, datatype: float64, format: '', description: Omega baryon; baryonic matter density/critical density at z=0.} +# meta: !!omap +# - {Oc0: 0.2589} +# - {n: 0.9667} +# - {sigma8: 0.8159} +# - {tau: 0.066} +# - z_reion: !astropy.units.Quantity +# unit: !astropy.units.Unit {unit: redshift} +# value: 8.8 +# - t0: !astropy.units.Quantity +# unit: !astropy.units.Unit {unit: Gyr} +# value: 13.799 +# - {reference: 'Planck Collaboration 2016, A&A, 594, A13 (Paper XIII), Table 4 (TT, TE, EE + lowP + lensing + ext)'} +# - {cosmology: FlatLambdaCDM} +# - __serialized_columns__: +# H0: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: km / (Mpc s)} +# value: !astropy.table.SerializedColumn {name: H0} +# Tcmb0: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: K} +# value: !astropy.table.SerializedColumn {name: Tcmb0} +# m_nu: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: eV} +# value: !astropy.table.SerializedColumn {name: m_nu} +# schema: astropy-2.0 +name H0 Om0 Tcmb0 Neff m_nu Ob0 +Planck15 67.74 0.3075 2.7255 3.046 [0.0,0.0,0.06] 0.0486 diff --git a/astropy/cosmology/data/Planck18.ecsv b/astropy/cosmology/data/Planck18.ecsv new file mode 100644 index 000000000000..c278799578b4 --- /dev/null +++ b/astropy/cosmology/data/Planck18.ecsv @@ -0,0 +1,39 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: name, datatype: string} +# - {name: H0, unit: km / (Mpc s), datatype: float64, format: '', description: Hubble constant as an `~astropy.units.Quantity` at z=0.} +# - {name: Om0, datatype: float64, format: '', description: Omega matter; matter density/critical density at z=0.} +# - {name: Tcmb0, unit: K, datatype: float64, format: '', description: Temperature of the CMB as `~astropy.units.Quantity` at z=0.} +# - {name: Neff, datatype: float64, format: '', description: Number of effective neutrino species.} +# - {name: m_nu, unit: eV, datatype: string, format: '', description: Mass of neutrino species., subtype: 'float64[3]'} +# - {name: Ob0, datatype: float64, format: '', description: Omega baryon; baryonic matter density/critical density at z=0.} +# meta: !!omap +# - {Oc0: 0.2607} +# - {n: 0.9665} +# - {sigma8: 0.8102} +# - {tau: 0.0561} +# - z_reion: !astropy.units.Quantity +# unit: !astropy.units.Unit {unit: redshift} +# value: 7.82 +# - t0: !astropy.units.Quantity +# unit: !astropy.units.Unit {unit: Gyr} +# value: 13.787 +# - {reference: 'Planck Collaboration 2018, 2020, A&A, 641, A6 (Paper VI), Table 2 (TT, TE, EE + lowE + lensing + BAO)'} +# - {cosmology: FlatLambdaCDM} +# - __serialized_columns__: +# H0: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: km / (Mpc s)} +# value: !astropy.table.SerializedColumn {name: H0} +# Tcmb0: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: K} +# value: !astropy.table.SerializedColumn {name: Tcmb0} +# m_nu: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: eV} +# value: !astropy.table.SerializedColumn {name: m_nu} +# schema: astropy-2.0 +name H0 Om0 Tcmb0 Neff m_nu Ob0 +Planck18 67.66 0.30966 2.7255 3.046 [0.0,0.0,0.06] 0.04897 diff --git a/astropy/cosmology/data/WMAP1.ecsv b/astropy/cosmology/data/WMAP1.ecsv new file mode 100644 index 000000000000..9e5fbb770519 --- /dev/null +++ b/astropy/cosmology/data/WMAP1.ecsv @@ -0,0 +1,40 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: name, datatype: string} +# - {name: H0, unit: km / (Mpc s), datatype: float64, format: '', description: Hubble constant as an `~astropy.units.Quantity` at z=0.} +# - {name: Om0, datatype: float64, format: '', description: Omega matter; matter density/critical density at z=0.} +# - {name: Tcmb0, unit: K, datatype: float64, format: '', description: Temperature of the CMB as `~astropy.units.Quantity` at z=0.} +# - {name: Neff, datatype: float64, format: '', description: Number of effective neutrino species.} +# - {name: m_nu, unit: eV, datatype: string, format: '', description: Mass of neutrino species., subtype: 'float64[3]'} +# - {name: Ob0, datatype: float64, format: '', description: Omega baryon; baryonic matter density/critical density at z=0.} +# meta: !!omap +# - {Oc0: 0.213} +# - {n: 0.96} +# - {sigma8: 0.75} +# - {tau: 0.117} +# - z_reion: !astropy.units.Quantity +# unit: !astropy.units.Unit {unit: redshift} +# value: 17.0 +# - t0: !astropy.units.Quantity +# unit: !astropy.units.Unit {unit: Gyr} +# value: 13.4 +# - {reference: 'Spergel et al. 2003, ApJS, 148, 175, doi: 10.1086/377226. Table 7 (WMAP + CBI + ACBAR + 2dFGRS + Lya).\nPending WMAP +# team approval and subject to change.'} +# - {cosmology: FlatLambdaCDM} +# - __serialized_columns__: +# H0: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: km / (Mpc s)} +# value: !astropy.table.SerializedColumn {name: H0} +# Tcmb0: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: K} +# value: !astropy.table.SerializedColumn {name: Tcmb0} +# m_nu: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: eV} +# value: !astropy.table.SerializedColumn {name: m_nu} +# schema: astropy-2.0 +name H0 Om0 Tcmb0 Neff m_nu Ob0 +WMAP1 72.0 0.257 2.725 3.04 [0.0,0.0,0.0] 0.0436 diff --git a/astropy/cosmology/data/WMAP3.ecsv b/astropy/cosmology/data/WMAP3.ecsv new file mode 100644 index 000000000000..0bed115874ce --- /dev/null +++ b/astropy/cosmology/data/WMAP3.ecsv @@ -0,0 +1,40 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: name, datatype: string} +# - {name: H0, unit: km / (Mpc s), datatype: float64, format: '', description: Hubble constant as an `~astropy.units.Quantity` at z=0.} +# - {name: Om0, datatype: float64, format: '', description: Omega matter; matter density/critical density at z=0.} +# - {name: Tcmb0, unit: K, datatype: float64, format: '', description: Temperature of the CMB as `~astropy.units.Quantity` at z=0.} +# - {name: Neff, datatype: float64, format: '', description: Number of effective neutrino species.} +# - {name: m_nu, unit: eV, datatype: string, format: '', description: Mass of neutrino species., subtype: 'float64[3]'} +# - {name: Ob0, datatype: float64, format: '', description: Omega baryon; baryonic matter density/critical density at z=0.} +# meta: !!omap +# - {Oc0: 0.23} +# - {n: 0.946} +# - {sigma8: 0.784} +# - {tau: 0.079} +# - z_reion: !astropy.units.Quantity +# unit: !astropy.units.Unit {unit: redshift} +# value: 10.3 +# - t0: !astropy.units.Quantity +# unit: !astropy.units.Unit {unit: Gyr} +# value: 13.78 +# - {reference: 'Spergel et al. 2007, ApJS, 170, 377, doi: 10.1086/513700. Table 6 (WMAP + SNGold) obtained from: https://lambda.gsfc.nasa.gov/product/map/dr2/params/lcdm_wmap_sngold.cfm \nPending +# WMAP team approval and subject to change.'} +# - {cosmology: FlatLambdaCDM} +# - __serialized_columns__: +# H0: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: km / (Mpc s)} +# value: !astropy.table.SerializedColumn {name: H0} +# Tcmb0: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: K} +# value: !astropy.table.SerializedColumn {name: Tcmb0} +# m_nu: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: eV} +# value: !astropy.table.SerializedColumn {name: m_nu} +# schema: astropy-2.0 +name H0 Om0 Tcmb0 Neff m_nu Ob0 +WMAP3 70.1 0.276 2.725 3.04 [0.0,0.0,0.0] 0.0454 diff --git a/astropy/cosmology/data/WMAP5.ecsv b/astropy/cosmology/data/WMAP5.ecsv new file mode 100644 index 000000000000..1a6b7f1e707d --- /dev/null +++ b/astropy/cosmology/data/WMAP5.ecsv @@ -0,0 +1,39 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: name, datatype: string} +# - {name: H0, unit: km / (Mpc s), datatype: float64, format: '', description: Hubble constant as an `~astropy.units.Quantity` at z=0.} +# - {name: Om0, datatype: float64, format: '', description: Omega matter; matter density/critical density at z=0.} +# - {name: Tcmb0, unit: K, datatype: float64, format: '', description: Temperature of the CMB as `~astropy.units.Quantity` at z=0.} +# - {name: Neff, datatype: float64, format: '', description: Number of effective neutrino species.} +# - {name: m_nu, unit: eV, datatype: string, format: '', description: Mass of neutrino species., subtype: 'float64[3]'} +# - {name: Ob0, datatype: float64, format: '', description: Omega baryon; baryonic matter density/critical density at z=0.} +# meta: !!omap +# - {Oc0: 0.231} +# - {n: 0.962} +# - {sigma8: 0.817} +# - {tau: 0.088} +# - z_reion: !astropy.units.Quantity +# unit: !astropy.units.Unit {unit: redshift} +# value: 11.3 +# - t0: !astropy.units.Quantity +# unit: !astropy.units.Unit {unit: Gyr} +# value: 13.72 +# - {reference: 'Komatsu et al. 2009, ApJS, 180, 330, doi: 10.1088/0067-0049/180/2/330. Table 1 (WMAP + BAO + SN ML).'} +# - {cosmology: FlatLambdaCDM} +# - __serialized_columns__: +# H0: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: km / (Mpc s)} +# value: !astropy.table.SerializedColumn {name: H0} +# Tcmb0: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: K} +# value: !astropy.table.SerializedColumn {name: Tcmb0} +# m_nu: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: eV} +# value: !astropy.table.SerializedColumn {name: m_nu} +# schema: astropy-2.0 +name H0 Om0 Tcmb0 Neff m_nu Ob0 +WMAP5 70.2 0.277 2.725 3.04 [0.0,0.0,0.0] 0.0459 diff --git a/astropy/cosmology/data/WMAP7.ecsv b/astropy/cosmology/data/WMAP7.ecsv new file mode 100644 index 000000000000..e5a20f8f0e5e --- /dev/null +++ b/astropy/cosmology/data/WMAP7.ecsv @@ -0,0 +1,39 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: name, datatype: string} +# - {name: H0, unit: km / (Mpc s), datatype: float64, format: '', description: Hubble constant as an `~astropy.units.Quantity` at z=0.} +# - {name: Om0, datatype: float64, format: '', description: Omega matter; matter density/critical density at z=0.} +# - {name: Tcmb0, unit: K, datatype: float64, format: '', description: Temperature of the CMB as `~astropy.units.Quantity` at z=0.} +# - {name: Neff, datatype: float64, format: '', description: Number of effective neutrino species.} +# - {name: m_nu, unit: eV, datatype: string, format: '', description: Mass of neutrino species., subtype: 'float64[3]'} +# - {name: Ob0, datatype: float64, format: '', description: Omega baryon; baryonic matter density/critical density at z=0.} +# meta: !!omap +# - {Oc0: 0.226} +# - {n: 0.967} +# - {sigma8: 0.81} +# - {tau: 0.085} +# - z_reion: !astropy.units.Quantity +# unit: !astropy.units.Unit {unit: redshift} +# value: 10.3 +# - t0: !astropy.units.Quantity +# unit: !astropy.units.Unit {unit: Gyr} +# value: 13.76 +# - {reference: 'Komatsu et al. 2011, ApJS, 192, 18, doi: 10.1088/0067-0049/192/2/18. Table 1 (WMAP + BAO + H0 ML).'} +# - {cosmology: FlatLambdaCDM} +# - __serialized_columns__: +# H0: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: km / (Mpc s)} +# value: !astropy.table.SerializedColumn {name: H0} +# Tcmb0: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: K} +# value: !astropy.table.SerializedColumn {name: Tcmb0} +# m_nu: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: eV} +# value: !astropy.table.SerializedColumn {name: m_nu} +# schema: astropy-2.0 +name H0 Om0 Tcmb0 Neff m_nu Ob0 +WMAP7 70.4 0.272 2.725 3.04 [0.0,0.0,0.0] 0.0455 diff --git a/astropy/cosmology/data/WMAP9.ecsv b/astropy/cosmology/data/WMAP9.ecsv new file mode 100644 index 000000000000..1a13d6a67423 --- /dev/null +++ b/astropy/cosmology/data/WMAP9.ecsv @@ -0,0 +1,39 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: name, datatype: string} +# - {name: H0, unit: km / (Mpc s), datatype: float64, format: '', description: Hubble constant as an `~astropy.units.Quantity` at z=0.} +# - {name: Om0, datatype: float64, format: '', description: Omega matter; matter density/critical density at z=0.} +# - {name: Tcmb0, unit: K, datatype: float64, format: '', description: Temperature of the CMB as `~astropy.units.Quantity` at z=0.} +# - {name: Neff, datatype: float64, format: '', description: Number of effective neutrino species.} +# - {name: m_nu, unit: eV, datatype: string, format: '', description: Mass of neutrino species., subtype: 'float64[3]'} +# - {name: Ob0, datatype: float64, format: '', description: Omega baryon; baryonic matter density/critical density at z=0.} +# meta: !!omap +# - {Oc0: 0.2402} +# - {n: 0.9608} +# - {sigma8: 0.82} +# - {tau: 0.081} +# - z_reion: !astropy.units.Quantity +# unit: !astropy.units.Unit {unit: redshift} +# value: 10.1 +# - t0: !astropy.units.Quantity +# unit: !astropy.units.Unit {unit: Gyr} +# value: 13.772 +# - {reference: 'Hinshaw et al. 2013, ApJS, 208, 19, doi: 10.1088/0067-0049/208/2/19. Table 4 (WMAP9 + eCMB + BAO + H0, last column)'} +# - {cosmology: FlatLambdaCDM} +# - __serialized_columns__: +# H0: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: km / (Mpc s)} +# value: !astropy.table.SerializedColumn {name: H0} +# Tcmb0: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: K} +# value: !astropy.table.SerializedColumn {name: Tcmb0} +# m_nu: +# __class__: astropy.units.quantity.Quantity +# unit: !astropy.units.Unit {unit: eV} +# value: !astropy.table.SerializedColumn {name: m_nu} +# schema: astropy-2.0 +name H0 Om0 Tcmb0 Neff m_nu Ob0 +WMAP9 69.32 0.2865 2.725 3.04 [0.0,0.0,0.0] 0.04628 diff --git a/astropy/cosmology/funcs.py b/astropy/cosmology/funcs.py deleted file mode 100644 index f534fb42a8f0..000000000000 --- a/astropy/cosmology/funcs.py +++ /dev/null @@ -1,149 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -""" -Convenience functions for `astropy.cosmology`. -""" -from __future__ import (absolute_import, division, print_function, - unicode_literals) - -import warnings -import numpy as np - -from .core import default_cosmology as _default_cosmology -from .core import CosmologyError -from ..units import Quantity -from ..utils import deprecated - -__all__ = ['z_at_value'] - -__doctest_requires__ = {'*': ['scipy.integrate']} - - -def z_at_value(func, fval, zmin=1e-8, zmax=1000, ztol=1e-8, maxfun=500): - """ Find the redshift ``z`` at which ``func(z) = fval``. - - This finds the redshift at which one of the cosmology functions or - methods (for example Planck13.distmod) is equal to a known value. - - .. warning:: - Make sure you understand the behaviour of the function that you - are trying to invert! Depending on the cosmology, there may not - be a unique solution. For example, in the standard Lambda CDM - cosmology, there are two redshifts which give an angular - diameter distance of 1500 Mpc, z ~ 0.7 and z ~ 3.8. To force - ``z_at_value`` to find the solution you are interested in, use the - ``zmin`` and ``zmax`` keywords to limit the search range (see the - example below). - - Parameters - ---------- - func : function or method - A function that takes a redshift as input. - fval : astropy.Quantity instance - The value of ``func(z)``. - zmin : float, optional - The lower search limit for ``z``. Beware of divergences - in some cosmological functions, such as distance moduli, - at z=0 (default 1e-8). - zmax : float, optional - The upper search limit for ``z`` (default 1000). - ztol : float, optional - The relative error in ``z`` acceptable for convergence. - maxfun : int, optional - The maximum number of function evaluations allowed in the - optimization routine (default 500). - - Returns - ------- - z : float - The redshift ``z`` satisfying ``zmin < z < zmax`` and ``func(z) = - fval`` within ``ztol``. - - Notes - ----- - This works for any arbitrary input cosmology, but is inefficient - if you want to invert a large number of values for the same - cosmology. In this case, it is faster to instead generate an array - of values at many closely-spaced redshifts that cover the relevant - redshift range, and then use interpolation to find the redshift at - each value you're interested in. For example, to efficiently find - the redshifts corresponding to 10^6 values of the distance modulus - in a Planck13 cosmology, you could do the following: - - >>> import astropy.units as u - >>> from astropy.cosmology import Planck13, z_at_value - - Generate 10^6 distance moduli between 24 and 43 for which we - want to find the corresponding redshifts: - - >>> Dvals = (24 + np.random.rand(1e6) * 20) * u.mag - - Make a grid of distance moduli covering the redshift range we - need using 50 equally log-spaced values between zmin and - zmax. We use log spacing to adequately sample the steep part of - the curve at low distance moduli: - - >>> zmin = z_at_value(Planck13.distmod, Dvals.min()) - >>> zmax = z_at_value(Planck13.distmod, Dvals.max()) - >>> zgrid = np.logspace(np.log10(zmin), np.log10(zmax), 50) - >>> Dgrid = Planck13.distmod(zgrid) - - Finally interpolate to find the redshift at each distance modulus: - - >>> zvals = np.interp(Dvals.value, zgrid, Dgrid.value) - - Examples - -------- - >>> import astropy.units as u - >>> from astropy.cosmology import Planck13, z_at_value - - The age and lookback time are monotonic with redshift, and so a - unique solution can be found: - - >>> z_at_value(Planck13.age, 2 * u.Gyr) - 3.19812268... - - The angular diameter is not monotonic however, and there are two - redshifts that give a value of 1500 Mpc. Use the zmin and zmax keywords - to find the one you're interested in: - - >>> z_at_value(Planck13.angular_diameter_distance, 1500 * u.Mpc, zmax=1.5) - 0.6812769577... - >>> z_at_value(Planck13.angular_diameter_distance, 1500 * u.Mpc, zmin=2.5) - 3.7914913242... - - Also note that the luminosity distance and distance modulus (two - other commonly inverted quantities) are monotonic in flat and open - universes, but not in closed universes. - """ - from scipy.optimize import fminbound - - fval_zmin = func(zmin) - fval_zmax = func(zmax) - if np.sign(fval - fval_zmin) != np.sign(fval_zmax - fval): - warnings.warn("""\ -fval is not bracketed by func(zmin) and func(zmax). This means either -there is no solution, or that there is more than one solution between -zmin and zmax satisfying fval = func(z).""") - - if isinstance(fval_zmin, Quantity): - unit = fval_zmin.unit - val = fval.to(unit).value - f = lambda z: abs(func(z).value - val) - else: - f = lambda z: abs(func(z) - fval) - - zbest, resval, ierr, ncall = fminbound(f, zmin, zmax, maxfun=maxfun, - full_output=1, xtol=ztol) - - if ierr != 0: - warnings.warn('Maximum number of function calls ({}) reached'.format( - ncall)) - - if np.allclose(zbest, zmax): - raise CosmologyError("Best guess z is very close the upper z limit.\n" - "Try re-running with a different zmax.") - elif np.allclose(zbest, zmin): - raise CosmologyError("Best guess z is very close the lower z limit.\n" - "Try re-running with a different zmin.") - - return zbest diff --git a/astropy/cosmology/io.py b/astropy/cosmology/io.py new file mode 100644 index 000000000000..1a1153f26161 --- /dev/null +++ b/astropy/cosmology/io.py @@ -0,0 +1,33 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""I/O subpackage for cosmology. + +This subpackage contains classes and functions for reading, writing +and converting cosmology objects to and from various formats. + +User access to the I/O functionality is provided through the methods +on the `~astropy.cosmology.Cosmology` class (and its subclasses) and its instances: + +- |Cosmology.read| for reading from a file, +- |Cosmology.write| for writing to a file, +- |Cosmology.from_format| to construct a Cosmology from an object +- |Cosmology.to_format| to convert a Cosmology to an object +""" + +__all__ = ( + "CosmologyFromFormat", + "CosmologyRead", + "CosmologyToFormat", + "CosmologyWrite", + "convert_registry", + "readwrite_registry", +) + +# Importing the I/O subpackage registers the I/O methods. +from ._src.io.connect import ( + CosmologyFromFormat, + CosmologyRead, + CosmologyToFormat, + CosmologyWrite, + convert_registry, + readwrite_registry, +) diff --git a/astropy/cosmology/parameters.py b/astropy/cosmology/parameters.py index 3ab137df8f2a..9170a5f2e3d3 100644 --- a/astropy/cosmology/parameters.py +++ b/astropy/cosmology/parameters.py @@ -1,153 +1,42 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -""" This module contains dictionaries with sets of parameters for a -given cosmology. - -Each cosmology has the following parameters defined: - - ========== ===================================== - Oc0 Omega cold dark matter at z=0 - Ob0 Omega baryon at z=0 - Om0 Omega matter at z=0 - flat Is this assumed flat? If not, Ode0 must be specified - Ode0 Omega dark energy at z=0 if flat is False - H0 Hubble parameter at z=0 in km/s/Mpc - n Density perturbation spectral index - Tcmb0 Current temperature of the CMB - Neff Effective number of neutrino species - sigma8 Density perturbation amplitude - tau Ionisation optical depth - z_reion Redshift of hydrogen reionisation - t0 Age of the universe in Gyr - reference Reference for the parameters - ========== ===================================== - -The list of cosmologies available are given by the tuple -`available`. Current cosmologies available: - -Planck 15 parameters from Planck Collaboration 2015, arXiv: 1502.01589 - (Paper XIII), Table 4 (TT, TE, EE + lowP + lensing + ext) - -Planck13 parameters from Planck Collaboration 2013, arXiv:1303.5076 - (Paper XVI), Table 5 (Planck + WP + highL + BAO) - -WMAP 9 year parameters from Hinshaw et al. 2013, ApJS, 208, 19, -doi: 10.1088/0067-0049/208/2/19. Table 4 (WMAP9 + eCMB + BAO + H0) - -WMAP 7 year parameters from Komatsu et al. 2011, ApJS, 192, 18, -doi: 10.1088/0067-0049/192/2/18. Table 1 (WMAP + BAO + H0 ML). - -WMAP 5 year parameters from Komatsu et al. 2009, ApJS, 180, 330, -doi: 10.1088/0067-0049/180/2/330. Table 1 (WMAP + BAO + SN ML). +"""This module contains dictionaries with sets of parameters for a given cosmology. +The list of cosmologies available are given by the tuple `available`. """ -from __future__ import (absolute_import, division, print_function, - unicode_literals) -# delete these things from the namespace so we can automatically find -# all of the parameter dictionaries below. -del absolute_import -del division -del print_function -del unicode_literals - -# Note: if you add a new cosmology, please also update the table -# in the 'Built-in Cosmologies' section of astropy/docs/cosmology/index.rst -# in addition to the list above. You also need to add them to -# __all__ in core.py -# Planck 2015 paper XII Table 4 final column (best fit) -Planck15 = dict( - Oc0=0.2589, - Ob0=0.04860, - Om0=0.3075, - H0=67.74, - n=0.9667, - sigma8=0.8159, - tau=0.066, - z_reion=8.8, - t0=13.799, - Tcmb0=2.7255, - Neff=3.046, - flat=True, - m_nu=[0., 0., 0.06], - reference=("Planck Collaboration 2015, Paper XII, arXiv:1502.01589" - " Table 4 (TT, TE, EE + lowP + lensing + ext)") +import sys +from types import MappingProxyType + +from .realizations import available + +__all__ = ( # noqa: F822, RUF022 + "available", + # ---- + "WMAP1", + "WMAP3", + "WMAP5", + "WMAP7", + "WMAP9", + "Planck13", + "Planck15", + "Planck18", ) -# Planck 2013 paper XVI Table 5 penultimate column (best fit) -Planck13 = dict( - Oc0=0.25886, - Ob0=0.048252, - Om0=0.30712, - H0=67.77, - n=0.9611, - sigma8=0.8288, - tau=0.0952, - z_reion=11.52, - t0=13.7965, - Tcmb0=2.7255, - Neff=3.046, - flat=True, - m_nu=[0., 0., 0.06], - reference=("Planck Collaboration 2013, Paper XVI, arXiv:1303.5076" - " Table 5 (Planck + WP + highL + BAO)") -) +def __getattr__(name): + """Get parameters of cosmology representations with lazy import from ``PEP 562``.""" + from astropy.cosmology import realizations -WMAP9 = dict( - Oc0=0.2402, - Ob0=0.04628, - Om0=0.2865, - H0=69.32, - n=0.9608, - sigma8=0.820, - tau=0.081, - z_reion=10.1, - t0=13.772, - Tcmb0=2.725, - Neff=3.04, - m_nu=0.0, - flat=True, - reference=("Hinshaw et al. 2013, ApJS, 208, 19, " - "doi: 10.1088/0067-0049/208/2/19. " - "Table 4 (WMAP9 + eCMB + BAO + H0, last column)") -) + cosmo = getattr(realizations, name) + m = cosmo.to_format("mapping", cosmology_as_str=True, move_from_meta=True) + proxy = MappingProxyType(m) -WMAP7 = dict( - Oc0=0.226, - Ob0=0.0455, - Om0=0.272, - H0=70.4, - n=0.967, - sigma8=0.810, - tau=0.085, - z_reion=10.3, - t0=13.76, - Tcmb0=2.725, - Neff=3.04, - m_nu=0.0, - flat=True, - reference=("Komatsu et al. 2011, ApJS, 192, 18, " - "doi: 10.1088/0067-0049/192/2/18. " - "Table 1 (WMAP + BAO + H0 ML).") -) + # Cache in this module so `__getattr__` is only called once per `name`. + setattr(sys.modules[__name__], name, proxy) + + return proxy -WMAP5 = dict( - Oc0=0.231, - Ob0=0.0459, - Om0=0.277, - H0=70.2, - n=0.962, - sigma8=0.817, - tau=0.088, - z_reion=11.3, - t0=13.72, - Tcmb0=2.725, - Neff=3.04, - m_nu=0.0, - flat=True, - reference=("Komatsu et al. 2009, ApJS, 180, 330, " - "doi: 10.1088/0067-0049/180/2/330. " - "Table 1 (WMAP + BAO + SN ML).") -) -available = tuple(k for k in locals() if not k.startswith('_')) +def __dir__(): + """Directory, including lazily-imported objects.""" + return __all__ diff --git a/astropy/cosmology/realizations.py b/astropy/cosmology/realizations.py new file mode 100644 index 000000000000..df71e7e30625 --- /dev/null +++ b/astropy/cosmology/realizations.py @@ -0,0 +1,75 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""Built-in cosmologies. + +See :attr:`~astropy.cosmology.realizations.available` for a full list. +""" + +__all__ = [ # noqa: F822, RUF022 + "available", + "default_cosmology", + # ---- + "WMAP1", + "WMAP3", + "WMAP5", + "WMAP7", + "WMAP9", + "Planck13", + "Planck15", + "Planck18", +] + +import pathlib +import sys + +from astropy.utils.data import get_pkg_data_path + +from ._src.core import Cosmology +from ._src.default import default_cosmology + +__doctest_requires__ = {"*": ["scipy"]} + +_COSMOLOGY_DATA_DIR = pathlib.Path( + get_pkg_data_path("cosmology", "data", package="astropy") +) +available = ( + "WMAP1", + "WMAP3", + "WMAP5", + "WMAP7", + "WMAP9", + "Planck13", + "Planck15", + "Planck18", +) + + +def __getattr__(name: str) -> Cosmology: + """Make specific realizations from data files with lazy import from ``PEP 562``. + + Raises + ------ + AttributeError + If "name" is not in :mod:`astropy.cosmology.realizations` + """ + if name not in available: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}.") + + cosmo = Cosmology.read( + str(_COSMOLOGY_DATA_DIR / name) + ".ecsv", format="ascii.ecsv" + ) + object.__setattr__( + cosmo, + "__doc__", + f"{name} instance of {cosmo.__class__.__qualname__} " + f"cosmology\n(from {cosmo.meta['reference']})", + ) + + # Cache in this module so `__getattr__` is only called once per `name`. + setattr(sys.modules[__name__], name, cosmo) + + return cosmo + + +def __dir__() -> list[str]: + """Directory, including lazily-imported objects.""" + return __all__ diff --git a/astropy/cosmology/setup_package.py b/astropy/cosmology/setup_package.py deleted file mode 100644 index 3cd9f7c3d928..000000000000 --- a/astropy/cosmology/setup_package.py +++ /dev/null @@ -1,5 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst - - -def requires_2to3(): - return False diff --git a/astropy/cosmology/tests/test_cosmology.py b/astropy/cosmology/tests/test_cosmology.py deleted file mode 100644 index ebbfb00b50a2..000000000000 --- a/astropy/cosmology/tests/test_cosmology.py +++ /dev/null @@ -1,1446 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -from __future__ import (absolute_import, division, print_function, - unicode_literals) - -from io import StringIO - -import numpy as np - -from .. import core, funcs -from ...tests.helper import pytest, quantity_allclose as allclose -from ... import units as u - -try: - import scipy # pylint: disable=W0611 -except ImportError: - HAS_SCIPY = False -else: - HAS_SCIPY = True - - -def test_init(): - """ Tests to make sure the code refuses inputs it is supposed to""" - with pytest.raises(ValueError): - cosmo = core.FlatLambdaCDM(H0=70, Om0=-0.27) - with pytest.raises(ValueError): - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.27, Neff=-1) - with pytest.raises(ValueError): - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.27, - Tcmb0=u.Quantity([0.0, 2], u.K)) - with pytest.raises(ValueError): - h0bad = u.Quantity([70, 100], u.km / u.s / u.Mpc) - cosmo = core.FlatLambdaCDM(H0=h0bad, Om0=0.27) - with pytest.raises(ValueError): - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.2, m_nu=0.5) - with pytest.raises(ValueError): - bad_mnu = u.Quantity([-0.3, 0.2, 0.1], u.eV) - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.2, m_nu=bad_mnu) - with pytest.raises(ValueError): - bad_mnu = u.Quantity([0.15, 0.2, 0.1], u.eV) - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.2, Neff=2, m_nu=bad_mnu) - with pytest.raises(ValueError): - bad_mnu = u.Quantity([-0.3, 0.2], u.eV) # 2, expecting 3 - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.2, m_nu=bad_mnu) - with pytest.raises(ValueError): - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.27, Ob0=-0.04) - with pytest.raises(ValueError): - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.27, Ob0=0.4) - with pytest.raises(ValueError): - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.27) - cosmo.Ob(1) - with pytest.raises(ValueError): - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.27) - cosmo.Odm(1) - with pytest.raises(TypeError): - core.default_cosmology.validate(4) - -def test_basic(): - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.27, Tcmb0=2.0, Neff=3.04, Ob0=0.05) - assert allclose(cosmo.Om0, 0.27) - assert allclose(cosmo.Ode0, 0.729975, rtol=1e-4) - assert allclose(cosmo.Ob0, 0.05) - assert allclose(cosmo.Odm0, 0.27 - 0.05) - # This next test will fail if astropy.const starts returning non-mks - # units by default; see the comment at the top of core.py - assert allclose(cosmo.Ogamma0, 1.463285e-5, rtol=1e-4) - assert allclose(cosmo.Onu0, 1.01026e-5, rtol=1e-4) - assert allclose(cosmo.Ok0, 0.0) - assert allclose(cosmo.Om0 + cosmo.Ode0 + cosmo.Ogamma0 + cosmo.Onu0, - 1.0, rtol=1e-6) - assert allclose(cosmo.Om(1) + cosmo.Ode(1) + cosmo.Ogamma(1) + - cosmo.Onu(1), 1.0, rtol=1e-6) - assert allclose(cosmo.Tcmb0, 2.0 * u.K) - assert allclose(cosmo.Tnu0, 1.4275317 * u.K, rtol=1e-5) - assert allclose(cosmo.Neff, 3.04) - assert allclose(cosmo.h, 0.7) - assert allclose(cosmo.H0, 70.0 * u.km / u.s / u.Mpc) - - # Make sure setting them as quantities gives the same results - H0 = u.Quantity(70, u.km / (u.s * u.Mpc)) - T = u.Quantity(2.0, u.K) - cosmo = core.FlatLambdaCDM(H0=H0, Om0=0.27, Tcmb0=T, Neff=3.04, Ob0=0.05) - assert allclose(cosmo.Om0, 0.27) - assert allclose(cosmo.Ode0, 0.729975, rtol=1e-4) - assert allclose(cosmo.Ob0, 0.05) - assert allclose(cosmo.Odm0, 0.27 - 0.05) - assert allclose(cosmo.Ogamma0, 1.463285e-5, rtol=1e-4) - assert allclose(cosmo.Onu0, 1.01026e-5, rtol=1e-4) - assert allclose(cosmo.Ok0, 0.0) - assert allclose(cosmo.Om0 + cosmo.Ode0 + cosmo.Ogamma0 + cosmo.Onu0, - 1.0, rtol=1e-6) - assert allclose(cosmo.Om(1) + cosmo.Ode(1) + cosmo.Ogamma(1) + - cosmo.Onu(1), 1.0, rtol=1e-6) - assert allclose(cosmo.Tcmb0, 2.0 * u.K) - assert allclose(cosmo.Tnu0, 1.4275317 * u.K, rtol=1e-5) - assert allclose(cosmo.Neff, 3.04) - assert allclose(cosmo.h, 0.7) - assert allclose(cosmo.H0, 70.0 * u.km / u.s / u.Mpc) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_units(): - """ Test if the right units are being returned""" - - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.27, Tcmb0=2.0) - assert cosmo.comoving_distance(1.0).unit == u.Mpc - assert cosmo.angular_diameter_distance(1.0).unit == u.Mpc - assert cosmo.angular_diameter_distance_z1z2(1.0, 2.0).unit == u.Mpc - assert cosmo.comoving_distance(1.0).unit == u.Mpc - assert cosmo.luminosity_distance(1.0).unit == u.Mpc - assert cosmo.lookback_time(1.0).unit == u.Gyr - assert cosmo.lookback_distance(1.0).unit == u.Mpc - assert cosmo.H0.unit == u.km / u.Mpc / u.s - assert cosmo.H(1.0).unit == u.km / u.Mpc / u.s - assert cosmo.Tcmb0.unit == u.K - assert cosmo.Tcmb(1.0).unit == u.K - assert cosmo.Tcmb([0.0, 1.0]).unit == u.K - assert cosmo.Tnu0.unit == u.K - assert cosmo.Tnu(1.0).unit == u.K - assert cosmo.Tnu([0.0, 1.0]).unit == u.K - assert cosmo.arcsec_per_kpc_comoving(1.0).unit == u.arcsec / u.kpc - assert cosmo.arcsec_per_kpc_proper(1.0).unit == u.arcsec / u.kpc - assert cosmo.kpc_comoving_per_arcmin(1.0).unit == u.kpc / u.arcmin - assert cosmo.kpc_proper_per_arcmin(1.0).unit == u.kpc / u.arcmin - assert cosmo.critical_density(1.0).unit == u.g / u.cm ** 3 - assert cosmo.comoving_volume(1.0).unit == u.Mpc ** 3 - assert cosmo.age(1.0).unit == u.Gyr - assert cosmo.distmod(1.0).unit == u.mag - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_distance_broadcast(): - """ Test array shape broadcasting for functions with single - redshift inputs""" - - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.27, - m_nu=u.Quantity([0.0, 0.1, 0.011], u.eV)) - z = np.linspace(0.1, 1, 6) - z_reshape2d = z.reshape(2, 3) - z_reshape3d = z.reshape(3, 2, 1) - # Things with units - methods = ['comoving_distance', 'luminosity_distance', - 'comoving_transverse_distance', 'angular_diameter_distance', - 'distmod', 'lookback_time', 'age', 'comoving_volume', - 'differential_comoving_volume', 'kpc_comoving_per_arcmin'] - for method in methods: - g = getattr(cosmo, method) - value_flat = g(z) - assert value_flat.shape == z.shape - value_2d = g(z_reshape2d) - assert value_2d.shape == z_reshape2d.shape - value_3d = g(z_reshape3d) - assert value_3d.shape == z_reshape3d.shape - assert value_flat.unit == value_2d.unit - assert value_flat.unit == value_3d.unit - assert allclose(value_flat, value_2d.flatten()) - assert allclose(value_flat, value_3d.flatten()) - - # Also test unitless ones - methods = ['absorption_distance', 'Om', 'Ode', 'Ok', 'H', - 'w', 'de_density_scale', 'Onu', 'Ogamma', - 'nu_relative_density'] - for method in methods: - g = getattr(cosmo, method) - value_flat = g(z) - assert value_flat.shape == z.shape - value_2d = g(z_reshape2d) - assert value_2d.shape == z_reshape2d.shape - value_3d = g(z_reshape3d) - assert value_3d.shape == z_reshape3d.shape - assert allclose(value_flat, value_2d.flatten()) - assert allclose(value_flat, value_3d.flatten()) - - # Test some dark energy models - methods = ['Om', 'Ode', 'w', 'de_density_scale'] - for tcosmo in [core.LambdaCDM(H0=70, Om0=0.27, Ode0=0.5), - core.wCDM(H0=70, Om0=0.27, Ode0=0.5, w0=-1.2), - core.w0waCDM(H0=70, Om0=0.27, Ode0=0.5, w0=-1.2, wa=-0.2), - core.wpwaCDM(H0=70, Om0=0.27, Ode0=0.5, - wp=-1.2, wa=-0.2, zp=0.9), - core.w0wzCDM(H0=70, Om0=0.27, Ode0=0.5, w0=-1.2, wz=0.1)]: - for method in methods: - g = getattr(cosmo, method) - value_flat = g(z) - assert value_flat.shape == z.shape - value_2d = g(z_reshape2d) - assert value_2d.shape == z_reshape2d.shape - value_3d = g(z_reshape3d) - assert value_3d.shape == z_reshape3d.shape - assert allclose(value_flat, value_2d.flatten()) - assert allclose(value_flat, value_3d.flatten()) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_clone(): - """ Test clone operation""" - - cosmo = core.FlatLambdaCDM(H0=70 * u.km / u.s / u.Mpc, Om0=0.27, - Tcmb0=3.0 * u.K) - z = np.linspace(0.1, 3, 15) - - # First, test with no changes, which should return same object - newclone = cosmo.clone() - assert newclone is cosmo - - # Now change H0 - # Note that H0 affects Ode0 because it changes Ogamma0 - newclone = cosmo.clone(H0=60 * u.km / u.s / u.Mpc) - assert newclone is not cosmo - assert newclone.__class__ == cosmo.__class__ - assert newclone.name == cosmo.name - assert not np.allclose(newclone.H0.value, cosmo.H0.value) - assert allclose(newclone.H0, 60.0 * u.km / u.s / u.Mpc) - assert allclose(newclone.Om0, cosmo.Om0) - assert allclose(newclone.Ok0, cosmo.Ok0) - assert not np.allclose(newclone.Ogamma0, cosmo.Ogamma0) - assert not np.allclose(newclone.Onu0, cosmo.Onu0) - assert allclose(newclone.Tcmb0, cosmo.Tcmb0) - assert allclose(newclone.m_nu, cosmo.m_nu) - assert allclose(newclone.Neff, cosmo.Neff) - - # Compare modified version with directly instantiated one - cmp = core.FlatLambdaCDM(H0=60 * u.km / u.s / u.Mpc, Om0=0.27, - Tcmb0=3.0 * u.K) - assert newclone.__class__ == cmp.__class__ - assert newclone.name == cmp.name - assert allclose(newclone.H0, cmp.H0) - assert allclose(newclone.Om0, cmp.Om0) - assert allclose(newclone.Ode0, cmp.Ode0) - assert allclose(newclone.Ok0, cmp.Ok0) - assert allclose(newclone.Ogamma0, cmp.Ogamma0) - assert allclose(newclone.Onu0, cmp.Onu0) - assert allclose(newclone.Tcmb0, cmp.Tcmb0) - assert allclose(newclone.m_nu, cmp.m_nu) - assert allclose(newclone.Neff, cmp.Neff) - assert allclose(newclone.Om(z), cmp.Om(z)) - assert allclose(newclone.H(z), cmp.H(z)) - assert allclose(newclone.luminosity_distance(z), - cmp.luminosity_distance(z)) - - # Now try changing multiple things - newclone = cosmo.clone(name="New name", H0=65 * u.km / u.s / u.Mpc, - Tcmb0=2.8 * u.K) - assert newclone.__class__ == cosmo.__class__ - assert not newclone.name == cosmo.name - assert not np.allclose(newclone.H0.value, cosmo.H0.value) - assert allclose(newclone.H0, 65.0 * u.km / u.s / u.Mpc) - assert allclose(newclone.Om0, cosmo.Om0) - assert allclose(newclone.Ok0, cosmo.Ok0) - assert not np.allclose(newclone.Ogamma0, cosmo.Ogamma0) - assert not np.allclose(newclone.Onu0, cosmo.Onu0) - assert not np.allclose(newclone.Tcmb0.value, cosmo.Tcmb0.value) - assert allclose(newclone.Tcmb0, 2.8 * u.K) - assert allclose(newclone.m_nu, cosmo.m_nu) - assert allclose(newclone.Neff, cosmo.Neff) - - # And direct comparison - cmp = core.FlatLambdaCDM(name="New name", H0=65 * u.km / u.s / u.Mpc, - Om0=0.27, Tcmb0=2.8 * u.K) - assert newclone.__class__ == cmp.__class__ - assert newclone.name == cmp.name - assert allclose(newclone.H0, cmp.H0) - assert allclose(newclone.Om0, cmp.Om0) - assert allclose(newclone.Ode0, cmp.Ode0) - assert allclose(newclone.Ok0, cmp.Ok0) - assert allclose(newclone.Ogamma0, cmp.Ogamma0) - assert allclose(newclone.Onu0, cmp.Onu0) - assert allclose(newclone.Tcmb0, cmp.Tcmb0) - assert allclose(newclone.m_nu, cmp.m_nu) - assert allclose(newclone.Neff, cmp.Neff) - assert allclose(newclone.Om(z), cmp.Om(z)) - assert allclose(newclone.H(z), cmp.H(z)) - assert allclose(newclone.luminosity_distance(z), - cmp.luminosity_distance(z)) - - # Try a dark energy class, make sure it can handle w params - cosmo = core.w0waCDM(name="test w0wa", H0=70 * u.km / u.s / u.Mpc, - Om0=0.27, Ode0=0.5, wa=0.1, Tcmb0=4.0 * u.K) - newclone = cosmo.clone(w0=-1.1, wa=0.2) - assert newclone.__class__ == cosmo.__class__ - assert newclone.name == cosmo.name - assert allclose(newclone.H0, cosmo.H0) - assert allclose(newclone.Om0, cosmo.Om0) - assert allclose(newclone.Ode0, cosmo.Ode0) - assert allclose(newclone.Ok0, cosmo.Ok0) - assert not np.allclose(newclone.w0, cosmo.w0) - assert allclose(newclone.w0, -1.1) - assert not np.allclose(newclone.wa, cosmo.wa) - assert allclose(newclone.wa, 0.2) - - # Now test exception if user passes non-parameter - with pytest.raises(AttributeError): - newclone = cosmo.clone(not_an_arg=4) - - -def test_xtfuncs(): - """ Test of absorption and lookback integrand""" - cosmo = core.LambdaCDM(70, 0.3, 0.5) - z = np.array([2.0, 3.2]) - assert allclose(cosmo.lookback_time_integrand(3), 0.052218976654969378, - rtol=1e-4) - assert allclose(cosmo.lookback_time_integrand(z), - [0.10333179, 0.04644541], rtol=1e-4) - assert allclose(cosmo.abs_distance_integrand(3), 3.3420145059180402, - rtol=1e-4) - assert allclose(cosmo.abs_distance_integrand(z), - [2.7899584, 3.44104758], rtol=1e-4) - - -def test_repr(): - """ Test string representation of built in classes""" - cosmo = core.LambdaCDM(70, 0.3, 0.5) - expected = 'LambdaCDM(H0=70 km / (Mpc s), Om0=0.3, '\ - 'Ode0=0.5, Tcmb0=2.725 K, Neff=3.04, m_nu=[ 0. 0. 0.] eV, '\ - 'Ob0=None)' - assert str(cosmo) == expected - - cosmo = core.LambdaCDM(70, 0.3, 0.5, m_nu=u.Quantity(0.01, u.eV)) - expected = 'LambdaCDM(H0=70 km / (Mpc s), Om0=0.3, Ode0=0.5, '\ - 'Tcmb0=2.725 K, Neff=3.04, m_nu=[ 0.01 0.01 0.01] eV, '\ - 'Ob0=None)' - assert str(cosmo) == expected - - cosmo = core.FlatLambdaCDM(50.0, 0.27, Ob0=0.05) - expected = 'FlatLambdaCDM(H0=50 km / (Mpc s), Om0=0.27, '\ - 'Tcmb0=2.725 K, Neff=3.04, m_nu=[ 0. 0. 0.] eV, Ob0=0.05)' - assert str(cosmo) == expected - - cosmo = core.wCDM(60.0, 0.27, 0.6, w0=-0.8, name='test1') - expected = 'wCDM(name="test1", H0=60 km / (Mpc s), Om0=0.27, '\ - 'Ode0=0.6, w0=-0.8, Tcmb0=2.725 K, Neff=3.04, '\ - 'm_nu=[ 0. 0. 0.] eV, Ob0=None)' - assert str(cosmo) == expected - - cosmo = core.FlatwCDM(65.0, 0.27, w0=-0.6, name='test2') - expected = 'FlatwCDM(name="test2", H0=65 km / (Mpc s), Om0=0.27, '\ - 'w0=-0.6, Tcmb0=2.725 K, Neff=3.04, m_nu=[ 0. 0. 0.] eV, '\ - 'Ob0=None)' - assert str(cosmo) == expected - - cosmo = core.w0waCDM(60.0, 0.25, 0.4, w0=-0.6, wa=0.1, name='test3') - expected = 'w0waCDM(name="test3", H0=60 km / (Mpc s), Om0=0.25, '\ - 'Ode0=0.4, w0=-0.6, wa=0.1, Tcmb0=2.725 K, Neff=3.04, '\ - 'm_nu=[ 0. 0. 0.] eV, Ob0=None)' - assert str(cosmo) == expected - - cosmo = core.Flatw0waCDM(55.0, 0.35, w0=-0.9, wa=-0.2, name='test4', - Ob0=0.0456789) - expected = 'Flatw0waCDM(name="test4", H0=55 km / (Mpc s), Om0=0.35, '\ - 'w0=-0.9, Tcmb0=2.725 K, Neff=3.04, m_nu=[ 0. 0. 0.] eV, '\ - 'Ob0=0.0457)' - assert str(cosmo) == expected - - cosmo = core.wpwaCDM(50.0, 0.3, 0.3, wp=-0.9, wa=-0.2, - zp=0.3, name='test5') - expected = 'wpwaCDM(name="test5", H0=50 km / (Mpc s), Om0=0.3, '\ - 'Ode0=0.3, wp=-0.9, wa=-0.2, zp=0.3, Tcmb0=2.725 K, '\ - 'Neff=3.04, m_nu=[ 0. 0. 0.] eV, Ob0=None)' - assert str(cosmo) == expected - - cosmo = core.w0wzCDM(55.0, 0.4, 0.8, w0=-1.05, wz=-0.2, - m_nu=u.Quantity([0.001, 0.01, 0.015], u.eV)) - expected = 'w0wzCDM(H0=55 km / (Mpc s), Om0=0.4, Ode0=0.8, w0=-1.05, '\ - 'wz=-0.2 Tcmb0=2.725 K, Neff=3.04, '\ - 'm_nu=[ 0.001 0.01 0.015] eV, Ob0=None)' - assert str(cosmo) == expected - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_flat_z1(): - """ Test a flat cosmology at z=1 against several other on-line - calculators. - """ - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.27, Tcmb0=0.0) - z = 1 - - # Test values were taken from the following web cosmology - # calculators on 27th Feb 2012: - - # Wright: http://www.astro.ucla.edu/~wright/CosmoCalc.html - # (http://adsabs.harvard.edu/abs/2006PASP..118.1711W) - # Kempner: http://www.kempner.net/cosmic.php - # iCosmos: http://www.icosmos.co.uk/index.html - - # The order of values below is Wright, Kempner, iCosmos' - assert allclose(cosmo.comoving_distance(z), - [3364.5, 3364.8, 3364.7988] * u.Mpc, rtol=1e-4) - assert allclose(cosmo.angular_diameter_distance(z), - [1682.3, 1682.4, 1682.3994] * u.Mpc, rtol=1e-4) - assert allclose(cosmo.luminosity_distance(z), - [6729.2, 6729.6, 6729.5976] * u.Mpc, rtol=1e-4) - assert allclose(cosmo.lookback_time(z), - [7.841, 7.84178, 7.843] * u.Gyr, rtol=1e-3) - assert allclose(cosmo.lookback_distance(z), - [2404.0, 2404.24, 2404.4] * u.Mpc, rtol=1e-3) - - -def test_zeroing(): - """ Tests if setting params to 0s always respects that""" - # Make sure Ode = 0 behaves that way - cosmo = core.LambdaCDM(H0=70, Om0=0.27, Ode0=0.0) - assert allclose(cosmo.Ode([0, 1, 2, 3]), [0, 0, 0, 0]) - assert allclose(cosmo.Ode(1), 0) - # Ogamma0 and Onu - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.27, Tcmb0=0.0) - assert allclose(cosmo.Ogamma(1.5), [0, 0, 0, 0]) - assert allclose(cosmo.Ogamma([0, 1, 2, 3]), [0, 0, 0, 0]) - assert allclose(cosmo.Onu(1.5), [0, 0, 0, 0]) - assert allclose(cosmo.Onu([0, 1, 2, 3]), [0, 0, 0, 0]) - # Obaryon - cosmo = core.LambdaCDM(H0=70, Om0=0.27, Ode0=0.73, Ob0=0.0) - assert allclose(cosmo.Ob([0, 1, 2, 3]), [0, 0, 0, 0]) - - -# This class is to test whether the routines work correctly -# if one only overloads w(z) -class test_cos_sub(core.FLRW): - def __init__(self): - core.FLRW.__init__(self, 70.0, 0.27, 0.73, Tcmb0=0.0, name="test_cos") - self._w0 = -0.9 - - def w(self, z): - return self._w0 * np.ones_like(z) - -# Similar, but with neutrinos -class test_cos_subnu(core.FLRW): - def __init__(self): - core.FLRW.__init__(self, 70.0, 0.27, 0.73, Tcmb0=3.0, - m_nu = 0.1 * u.eV, name="test_cos_nu") - self._w0 = -0.8 - - def w(self, z): - return self._w0 * np.ones_like(z) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_de_subclass(): - # This is the comparison object - z = [0.2, 0.4, 0.6, 0.9] - cosmo = core.wCDM(H0=70, Om0=0.27, Ode0=0.73, w0=-0.9, Tcmb0=0.0) - # Values taken from Ned Wrights advanced cosmo calcluator, Aug 17 2012 - assert allclose(cosmo.luminosity_distance(z), - [975.5, 2158.2, 3507.3, 5773.1] * u.Mpc, - rtol=1e-3) - # Now try the subclass that only gives w(z) - cosmo = test_cos_sub() - assert allclose(cosmo.luminosity_distance(z), - [975.5, 2158.2, 3507.3, 5773.1] * u.Mpc, - rtol=1e-3) - # Test efunc - assert allclose(cosmo.efunc(1.0), 1.7489240754, rtol=1e-5) - assert allclose(cosmo.efunc([0.5, 1.0]), - [1.31744953, 1.7489240754], rtol=1e-5) - assert allclose(cosmo.inv_efunc([0.5, 1.0]), - [0.75904236, 0.57178011], rtol=1e-5) - # Test de_density_scale - assert allclose(cosmo.de_density_scale(1.0), 1.23114444, rtol=1e-4) - assert allclose(cosmo.de_density_scale([0.5, 1.0]), - [1.12934694, 1.23114444], rtol=1e-4) - - # Add neutrinos for efunc, inv_efunc - - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_varyde_lumdist_mathematica(): - """Tests a few varying dark energy EOS models against a mathematica - computation""" - - # w0wa models - z = np.array([0.2, 0.4, 0.9, 1.2]) - cosmo = core.w0waCDM(H0=70, Om0=0.2, Ode0=0.8, w0=-1.1, wa=0.2, Tcmb0=0.0) - assert allclose(cosmo.w0, -1.1) - assert allclose(cosmo.wa, 0.2) - - assert allclose(cosmo.luminosity_distance(z), - [1004.0, 2268.62, 6265.76, 9061.84] * u.Mpc, rtol=1e-4) - assert allclose(cosmo.de_density_scale(0.0), 1.0, rtol=1e-5) - assert allclose(cosmo.de_density_scale([0.0, 0.5, 1.5]), - [1.0, 0.9246310669529021, 0.9184087000251957]) - - cosmo = core.w0waCDM(H0=70, Om0=0.3, Ode0=0.7, w0=-0.9, wa=0.0, Tcmb0=0.0) - assert allclose(cosmo.luminosity_distance(z), - [971.667, 2141.67, 5685.96, 8107.41] * u.Mpc, rtol=1e-4) - cosmo = core.w0waCDM(H0=70, Om0=0.3, Ode0=0.7, w0=-0.9, wa=-0.5, Tcmb0=0.0) - assert allclose(cosmo.luminosity_distance(z), - [974.087, 2157.08, 5783.92, 8274.08] * u.Mpc, rtol=1e-4) - - # wpwa models - cosmo = core.wpwaCDM(H0=70, Om0=0.2, Ode0=0.8, wp=-1.1, wa=0.2, zp=0.5, - Tcmb0=0.0) - assert allclose(cosmo.wp, -1.1) - assert allclose(cosmo.wa, 0.2) - assert allclose(cosmo.zp, 0.5) - assert allclose(cosmo.luminosity_distance(z), - [1010.81, 2294.45, 6369.45, 9218.95] * u.Mpc, rtol=1e-4) - - cosmo = core.wpwaCDM(H0=70, Om0=0.2, Ode0=0.8, wp=-1.1, wa=0.2, zp=0.9, - Tcmb0=0.0) - assert allclose(cosmo.wp, -1.1) - assert allclose(cosmo.wa, 0.2) - assert allclose(cosmo.zp, 0.9) - assert allclose(cosmo.luminosity_distance(z), - [1013.68, 2305.3, 6412.37, 9283.33] * u.Mpc, rtol=1e-4) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_matter(): - # Test non-relativistic matter evolution - tcos = core.FlatLambdaCDM(70.0, 0.3, Ob0=0.045) - assert allclose(tcos.Om0, 0.3) - assert allclose(tcos.H0, 70.0 * u.km / u.s / u.Mpc) - assert allclose(tcos.Om(0), 0.3) - assert allclose(tcos.Ob(0), 0.045) - z = np.array([0.0, 0.5, 1.0, 2.0]) - assert allclose(tcos.Om(z), [0.3, 0.59112134, 0.77387435, 0.91974179], - rtol=1e-4) - assert allclose(tcos.Ob(z), [0.045, 0.08866820, 0.11608115, - 0.13796127], rtol=1e-4) - assert allclose(tcos.Odm(z), [0.255, 0.50245314, 0.6577932, 0.78178052], - rtol=1e-4) - # Consistency of dark and baryonic matter evolution with all - # non-relativistic matter - assert allclose(tcos.Ob(z) + tcos.Odm(z), tcos.Om(z)) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_ocurv(): - # Test Ok evolution - # Flat, boring case - tcos = core.FlatLambdaCDM(70.0, 0.3) - assert allclose(tcos.Ok0, 0.0) - assert allclose(tcos.Ok(0), 0.0) - z = np.array([0.0, 0.5, 1.0, 2.0]) - assert allclose(tcos.Ok(z), [0.0, 0.0, 0.0, 0.0], - rtol=1e-6) - - # Not flat - tcos = core.LambdaCDM(70.0, 0.3, 0.5, Tcmb0=u.Quantity(0.0, u.K)) - assert allclose(tcos.Ok0, 0.2) - assert allclose(tcos.Ok(0), 0.2) - assert allclose(tcos.Ok(z), [0.2, 0.22929936, 0.21621622, 0.17307692], - rtol=1e-4) - - # Test the sum; note that Ogamma/Onu are 0 - assert allclose(tcos.Ok(z) + tcos.Om(z) + tcos.Ode(z), - [1.0, 1.0, 1.0, 1.0], rtol=1e-5) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_ode(): - # Test Ode evolution, turn off neutrinos, cmb - tcos = core.FlatLambdaCDM(70.0, 0.3, Tcmb0=0) - assert allclose(tcos.Ode0, 0.7) - assert allclose(tcos.Ode(0), 0.7) - z = np.array([0.0, 0.5, 1.0, 2.0]) - assert allclose(tcos.Ode(z), [0.7, 0.408759, 0.2258065, 0.07954545], - rtol=1e-5) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_ogamma(): - """Tests the effects of changing the temperature of the CMB""" - - # Tested against Ned Wright's advanced cosmology calculator, - # Sep 7 2012. The accuracy of our comparision is limited by - # how many digits it outputs, which limits our test to about - # 0.2% accuracy. The NWACC does not allow one - # to change the number of nuetrino species, fixing that at 3. - # Also, inspection of the NWACC code shows it uses inaccurate - # constants at the 0.2% level (specifically, a_B), - # so we shouldn't expect to match it that well. The integral is - # also done rather crudely. Therefore, we should not expect - # the NWACC to be accurate to better than about 0.5%, which is - # unfortunate, but reflects a problem with it rather than this code. - # More accurate tests below using Mathematica - z = np.array([1.0, 10.0, 500.0, 1000.0]) - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=0, Neff=3) - assert allclose(cosmo.angular_diameter_distance(z), - [1651.9, 858.2, 26.855, 13.642] * u.Mpc, rtol=5e-4) - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=2.725, Neff=3) - assert allclose(cosmo.angular_diameter_distance(z), - [1651.8, 857.9, 26.767, 13.582] * u.Mpc, rtol=5e-4) - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=4.0, Neff=3) - assert allclose(cosmo.angular_diameter_distance(z), - [1651.4, 856.6, 26.489, 13.405] * u.Mpc, rtol=5e-4) - - # Next compare with doing the integral numerically in Mathematica, - # which allows more precision in the test. It is at least as - # good as 0.01%, possibly better - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=0, Neff=3.04) - assert allclose(cosmo.angular_diameter_distance(z), - [1651.91, 858.205, 26.8586, 13.6469] * u.Mpc, rtol=1e-5) - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=2.725, Neff=3.04) - assert allclose(cosmo.angular_diameter_distance(z), - [1651.76, 857.817, 26.7688, 13.5841] * u.Mpc, rtol=1e-5) - cosmo = core.FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=4.0, Neff=3.04) - assert allclose(cosmo.angular_diameter_distance(z), - [1651.21, 856.411, 26.4845, 13.4028] * u.Mpc, rtol=1e-5) - - # Just to be really sure, we also do a version where the integral - # is analytic, which is a Ode = 0 flat universe. In this case - # Integrate(1/E(x),{x,0,z}) = 2 ( sqrt((1+Or z)/(1+z)) - 1 )/(Or - 1) - # Recall that c/H0 * Integrate(1/E) is FLRW.comoving_distance. - Ogamma0h2 = 4 * 5.670373e-8 / 299792458.0 ** 3 * 2.725 ** 4 / 1.87837e-26 - Onu0h2 = Ogamma0h2 * 7.0 / 8.0 * (4.0 / 11.0) ** (4.0 / 3.0) * 3.04 - Or0 = (Ogamma0h2 + Onu0h2) / 0.7 ** 2 - Om0 = 1.0 - Or0 - hubdis = (299792.458 / 70.0) * u.Mpc - cosmo = core.FlatLambdaCDM(H0=70, Om0=Om0, Tcmb0=2.725, Neff=3.04) - targvals = 2.0 * hubdis * \ - (np.sqrt((1.0 + Or0 * z) / (1.0 + z)) - 1.0) / (Or0 - 1.0) - assert allclose(cosmo.comoving_distance(z), targvals, rtol=1e-5) - - # And integers for z - assert allclose(cosmo.comoving_distance(z.astype(np.int)), - targvals, rtol=1e-5) - - # Try Tcmb0 = 4 - Or0 *= (4.0 / 2.725) ** 4 - Om0 = 1.0 - Or0 - cosmo = core.FlatLambdaCDM(H0=70, Om0=Om0, Tcmb0=4.0, Neff=3.04) - targvals = 2.0 * hubdis * \ - (np.sqrt((1.0 + Or0 * z) / (1.0 + z)) - 1.0) / (Or0 - 1.0) - assert allclose(cosmo.comoving_distance(z), targvals, rtol=1e-5) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_tcmb(): - cosmo = core.FlatLambdaCDM(70.4, 0.272, Tcmb0=2.5) - assert allclose(cosmo.Tcmb0, 2.5 * u.K) - assert allclose(cosmo.Tcmb(2), 7.5 * u.K) - z = [0.0, 1.0, 2.0, 3.0, 9.0] - assert allclose(cosmo.Tcmb(z), - [2.5, 5.0, 7.5, 10.0, 25.0] * u.K, rtol=1e-6) - # Make sure it's the same for integers - z = [0, 1, 2, 3, 9] - assert allclose(cosmo.Tcmb(z), - [2.5, 5.0, 7.5, 10.0, 25.0] * u.K, rtol=1e-6) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_tnu(): - cosmo = core.FlatLambdaCDM(70.4, 0.272, Tcmb0=3.0) - assert allclose(cosmo.Tnu0, 2.1412975665108247 * u.K, rtol=1e-6) - assert allclose(cosmo.Tnu(2), 6.423892699532474 * u.K, rtol=1e-6) - z = [0.0, 1.0, 2.0, 3.0] - expected = [2.14129757, 4.28259513, 6.4238927, 8.56519027] * u.K - assert allclose(cosmo.Tnu(z), expected, rtol=1e-6) - - # Test for integers - z = [0, 1, 2, 3] - assert allclose(cosmo.Tnu(z), expected, rtol=1e-6) - - -def test_efunc_vs_invefunc(): - """ Test that efunc and inv_efunc give inverse values""" - - # Note that all of the subclasses here don't need - # scipy because they don't need to call de_density_scale - # The test following this tests the case where that is needed. - - z0 = 0.5 - z = np.array([0.5, 1.0, 2.0, 5.0]) - - # Below are the 'standard' included cosmologies - # We do the non-standard case in test_efunc_vs_invefunc_flrw, - # since it requires scipy - cosmo = core.LambdaCDM(70, 0.3, 0.5) - assert allclose(cosmo.efunc(z0), 1.0 / cosmo.inv_efunc(z0)) - assert allclose(cosmo.efunc(z), 1.0 / cosmo.inv_efunc(z)) - cosmo = core.LambdaCDM(70, 0.3, 0.5, m_nu=u.Quantity(0.01, u.eV)) - assert allclose(cosmo.efunc(z0), 1.0 / cosmo.inv_efunc(z0)) - assert allclose(cosmo.efunc(z), 1.0 / cosmo.inv_efunc(z)) - cosmo = core.FlatLambdaCDM(50.0, 0.27) - assert allclose(cosmo.efunc(z0), 1.0 / cosmo.inv_efunc(z0)) - assert allclose(cosmo.efunc(z), 1.0 / cosmo.inv_efunc(z)) - cosmo = core.wCDM(60.0, 0.27, 0.6, w0=-0.8) - assert allclose(cosmo.efunc(z0), 1.0 / cosmo.inv_efunc(z0)) - assert allclose(cosmo.efunc(z), 1.0 / cosmo.inv_efunc(z)) - cosmo = core.FlatwCDM(65.0, 0.27, w0=-0.6) - assert allclose(cosmo.efunc(z0), 1.0 / cosmo.inv_efunc(z0)) - assert allclose(cosmo.efunc(z), 1.0 / cosmo.inv_efunc(z)) - cosmo = core.w0waCDM(60.0, 0.25, 0.4, w0=-0.6, wa=0.1) - assert allclose(cosmo.efunc(z0), 1.0 / cosmo.inv_efunc(z0)) - assert allclose(cosmo.efunc(z), 1.0 / cosmo.inv_efunc(z)) - cosmo = core.Flatw0waCDM(55.0, 0.35, w0=-0.9, wa=-0.2) - assert allclose(cosmo.efunc(z0), 1.0 / cosmo.inv_efunc(z0)) - assert allclose(cosmo.efunc(z), 1.0 / cosmo.inv_efunc(z)) - cosmo = core.wpwaCDM(50.0, 0.3, 0.3, wp=-0.9, wa=-0.2, zp=0.3) - assert allclose(cosmo.efunc(z0), 1.0 / cosmo.inv_efunc(z0)) - assert allclose(cosmo.efunc(z), 1.0 / cosmo.inv_efunc(z)) - cosmo = core.w0wzCDM(55.0, 0.4, 0.8, w0=-1.05, wz=-0.2) - assert allclose(cosmo.efunc(z0), 1.0 / cosmo.inv_efunc(z0)) - assert allclose(cosmo.efunc(z), 1.0 / cosmo.inv_efunc(z)) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_efunc_vs_invefunc_flrw(): - """ Test that efunc and inv_efunc give inverse values""" - z0 = 0.5 - z = np.array([0.5, 1.0, 2.0, 5.0]) - - # FLRW is abstract, so requires test_cos_sub defined earlier - # This requires scipy, unlike the built-ins, because it - # calls de_density_scale, which has an integral in it - cosmo = test_cos_sub() - assert allclose(cosmo.efunc(z0), 1.0 / cosmo.inv_efunc(z0)) - assert allclose(cosmo.efunc(z), 1.0 / cosmo.inv_efunc(z)) - # Add neutrinos - cosmo = test_cos_subnu() - assert allclose(cosmo.efunc(z0), 1.0 / cosmo.inv_efunc(z0)) - assert allclose(cosmo.efunc(z), 1.0 / cosmo.inv_efunc(z)) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_kpc_methods(): - cosmo = core.FlatLambdaCDM(70.4, 0.272, Tcmb0=0.0) - assert allclose(cosmo.arcsec_per_kpc_comoving(3), - 0.0317179167 * u.arcsec / u.kpc) - assert allclose(cosmo.arcsec_per_kpc_proper(3), - 0.1268716668 * u.arcsec / u.kpc) - assert allclose(cosmo.kpc_comoving_per_arcmin(3), - 1891.6753126 * u.kpc / u.arcmin) - assert allclose(cosmo.kpc_proper_per_arcmin(3), - 472.918828 * u.kpc / u.arcmin) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_comoving_volume(): - - c_flat = core.LambdaCDM(H0=70, Om0=0.27, Ode0=0.73, Tcmb0=0.0) - c_open = core.LambdaCDM(H0=70, Om0=0.27, Ode0=0.0, Tcmb0=0.0) - c_closed = core.LambdaCDM(H0=70, Om0=2, Ode0=0.0, Tcmb0=0.0) - - # test against ned wright's calculator (cubic Gpc) - redshifts = np.array([0.5, 1, 2, 3, 5, 9]) - wright_flat = np.array([29.123, 159.529, 630.427, 1178.531, 2181.485, - 3654.802]) * u.Gpc**3 - wright_open = np.array([20.501, 99.019, 380.278, 747.049, 1558.363, - 3123.814]) * u.Gpc**3 - wright_closed = np.array([12.619, 44.708, 114.904, 173.709, 258.82, - 358.992]) * u.Gpc**3 - # The wright calculator isn't very accurate, so we use a rather - # modest precision - assert allclose(c_flat.comoving_volume(redshifts), wright_flat, - rtol=1e-2) - assert allclose(c_open.comoving_volume(redshifts), - wright_open, rtol=1e-2) - assert allclose(c_closed.comoving_volume(redshifts), - wright_closed, rtol=1e-2) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_differential_comoving_volume(): - from scipy.integrate import quad - - c_flat = core.LambdaCDM(H0=70, Om0=0.27, Ode0=0.73, Tcmb0=0.0) - c_open = core.LambdaCDM(H0=70, Om0=0.27, Ode0=0.0, Tcmb0=0.0) - c_closed = core.LambdaCDM(H0=70, Om0=2, Ode0=0.0, Tcmb0=0.0) - - # test that integration of differential_comoving_volume() - # yields same as comoving_volume() - redshifts = np.array([0.5, 1, 2, 3, 5, 9]) - wright_flat = np.array([29.123, 159.529, 630.427, 1178.531, 2181.485, - 3654.802]) * u.Gpc**3 - wright_open = np.array([20.501, 99.019, 380.278, 747.049, 1558.363, - 3123.814]) * u.Gpc**3 - wright_closed = np.array([12.619, 44.708, 114.904, 173.709, 258.82, - 358.992]) * u.Gpc**3 - # The wright calculator isn't very accurate, so we use a rather - # modest precision. - ftemp = lambda x: c_flat.differential_comoving_volume(x).value - otemp = lambda x: c_open.differential_comoving_volume(x).value - ctemp = lambda x: c_closed.differential_comoving_volume(x).value - # Multiply by solid_angle (4 * pi) - assert allclose(np.array([4.0 * np.pi * quad(ftemp, 0, redshift)[0] - for redshift in redshifts]) * u.Mpc**3, - wright_flat, rtol=1e-2) - assert allclose(np.array([4.0 * np.pi * quad(otemp, 0, redshift)[0] - for redshift in redshifts]) * u.Mpc**3, - wright_open, rtol=1e-2) - assert allclose(np.array([4.0 * np.pi * quad(ctemp, 0, redshift)[0] - for redshift in redshifts]) * u.Mpc**3, - wright_closed, rtol=1e-2) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_flat_open_closed_icosmo(): - """ Test against the tabulated values generated from icosmo.org - with three example cosmologies (flat, open and closed). - """ - - cosmo_flat = """\ -# from icosmo (icosmo.org) -# Om 0.3 w -1 h 0.7 Ol 0.7 -# z comoving_transvers_dist angular_diameter_dist luminosity_dist - 0.0000000 0.0000000 0.0000000 0.0000000 - 0.16250000 669.77536 576.15085 778.61386 - 0.32500000 1285.5964 970.26143 1703.4152 - 0.50000000 1888.6254 1259.0836 2832.9381 - 0.66250000 2395.5489 1440.9317 3982.6000 - 0.82500000 2855.5732 1564.6976 5211.4210 - 1.0000000 3303.8288 1651.9144 6607.6577 - 1.1625000 3681.1867 1702.2829 7960.5663 - 1.3250000 4025.5229 1731.4077 9359.3408 - 1.5000000 4363.8558 1745.5423 10909.640 - 1.6625000 4651.4830 1747.0359 12384.573 - 1.8250000 4916.5970 1740.3883 13889.387 - 2.0000000 5179.8621 1726.6207 15539.586 - 2.1625000 5406.0204 1709.4136 17096.540 - 2.3250000 5616.5075 1689.1752 18674.888 - 2.5000000 5827.5418 1665.0120 20396.396 - 2.6625000 6010.4886 1641.0890 22013.414 - 2.8250000 6182.1688 1616.2533 23646.796 - 3.0000000 6355.6855 1588.9214 25422.742 - 3.1625000 6507.2491 1563.3031 27086.425 - 3.3250000 6650.4520 1537.6768 28763.205 - 3.5000000 6796.1499 1510.2555 30582.674 - 3.6625000 6924.2096 1485.0852 32284.127 - 3.8250000 7045.8876 1460.2876 33996.408 - 4.0000000 7170.3664 1434.0733 35851.832 - 4.1625000 7280.3423 1410.2358 37584.767 - 4.3250000 7385.3277 1386.9160 39326.870 - 4.5000000 7493.2222 1362.4040 41212.722 - 4.6625000 7588.9589 1340.2135 42972.480 -""" - - cosmo_open = """\ -# from icosmo (icosmo.org) -# Om 0.3 w -1 h 0.7 Ol 0.1 -# z comoving_transvers_dist angular_diameter_dist luminosity_dist - 0.0000000 0.0000000 0.0000000 0.0000000 - 0.16250000 643.08185 553.18868 747.58265 - 0.32500000 1200.9858 906.40441 1591.3062 - 0.50000000 1731.6262 1154.4175 2597.4393 - 0.66250000 2174.3252 1307.8648 3614.8157 - 0.82500000 2578.7616 1413.0201 4706.2399 - 1.0000000 2979.3460 1489.6730 5958.6920 - 1.1625000 3324.2002 1537.2024 7188.5829 - 1.3250000 3646.8432 1568.5347 8478.9104 - 1.5000000 3972.8407 1589.1363 9932.1017 - 1.6625000 4258.1131 1599.2913 11337.226 - 1.8250000 4528.5346 1603.0211 12793.110 - 2.0000000 4804.9314 1601.6438 14414.794 - 2.1625000 5049.2007 1596.5852 15968.097 - 2.3250000 5282.6693 1588.7727 17564.875 - 2.5000000 5523.0914 1578.0261 19330.820 - 2.6625000 5736.9813 1566.4113 21011.694 - 2.8250000 5942.5803 1553.6158 22730.370 - 3.0000000 6155.4289 1538.8572 24621.716 - 3.1625000 6345.6997 1524.4924 26413.975 - 3.3250000 6529.3655 1509.6799 28239.506 - 3.5000000 6720.2676 1493.3928 30241.204 - 3.6625000 6891.5474 1478.0799 32131.840 - 3.8250000 7057.4213 1462.6780 34052.058 - 4.0000000 7230.3723 1446.0745 36151.862 - 4.1625000 7385.9998 1430.7021 38130.224 - 4.3250000 7537.1112 1415.4199 40135.117 - 4.5000000 7695.0718 1399.1040 42322.895 - 4.6625000 7837.5510 1384.1150 44380.133 -""" - - cosmo_closed = """\ -# from icosmo (icosmo.org) -# Om 2 w -1 h 0.7 Ol 0.1 -# z comoving_transvers_dist angular_diameter_dist luminosity_dist - 0.0000000 0.0000000 0.0000000 0.0000000 - 0.16250000 601.80160 517.67879 699.59436 - 0.32500000 1057.9502 798.45297 1401.7840 - 0.50000000 1438.2161 958.81076 2157.3242 - 0.66250000 1718.6778 1033.7912 2857.3019 - 0.82500000 1948.2400 1067.5288 3555.5381 - 1.0000000 2152.7954 1076.3977 4305.5908 - 1.1625000 2312.3427 1069.2914 5000.4410 - 1.3250000 2448.9755 1053.3228 5693.8681 - 1.5000000 2575.6795 1030.2718 6439.1988 - 1.6625000 2677.9671 1005.8092 7130.0873 - 1.8250000 2768.1157 979.86398 7819.9270 - 2.0000000 2853.9222 951.30739 8561.7665 - 2.1625000 2924.8116 924.84161 9249.7167 - 2.3250000 2988.5333 898.80701 9936.8732 - 2.5000000 3050.3065 871.51614 10676.073 - 2.6625000 3102.1909 847.01459 11361.774 - 2.8250000 3149.5043 823.39982 12046.854 - 3.0000000 3195.9966 798.99915 12783.986 - 3.1625000 3235.5334 777.30533 13467.908 - 3.3250000 3271.9832 756.52790 14151.327 - 3.5000000 3308.1758 735.15017 14886.791 - 3.6625000 3339.2521 716.19347 15569.263 - 3.8250000 3368.1489 698.06195 16251.319 - 4.0000000 3397.0803 679.41605 16985.401 - 4.1625000 3422.1142 662.87926 17666.664 - 4.3250000 3445.5542 647.05243 18347.576 - 4.5000000 3469.1805 630.76008 19080.493 - 4.6625000 3489.7534 616.29199 19760.729 -""" - - redshifts, dm, da, dl = np.loadtxt(StringIO(cosmo_flat), unpack=1) - dm = dm * u.Mpc - da = da * u.Mpc - dl = dl * u.Mpc - cosmo = core.LambdaCDM(H0=70, Om0=0.3, Ode0=0.70, Tcmb0=0.0) - assert allclose(cosmo.comoving_transverse_distance(redshifts), dm) - assert allclose(cosmo.angular_diameter_distance(redshifts), da) - assert allclose(cosmo.luminosity_distance(redshifts), dl) - - redshifts, dm, da, dl = np.loadtxt(StringIO(cosmo_open), unpack=1) - dm = dm * u.Mpc - da = da * u.Mpc - dl = dl * u.Mpc - cosmo = core.LambdaCDM(H0=70, Om0=0.3, Ode0=0.1, Tcmb0=0.0) - assert allclose(cosmo.comoving_transverse_distance(redshifts), dm) - assert allclose(cosmo.angular_diameter_distance(redshifts), da) - assert allclose(cosmo.luminosity_distance(redshifts), dl) - - redshifts, dm, da, dl = np.loadtxt(StringIO(cosmo_closed), unpack=1) - dm = dm * u.Mpc - da = da * u.Mpc - dl = dl * u.Mpc - cosmo = core.LambdaCDM(H0=70, Om0=2, Ode0=0.1, Tcmb0=0.0) - assert allclose(cosmo.comoving_transverse_distance(redshifts), dm) - assert allclose(cosmo.angular_diameter_distance(redshifts), da) - assert allclose(cosmo.luminosity_distance(redshifts), dl) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_integral(): - # Test integer vs. floating point inputs - cosmo = core.LambdaCDM(H0=73.2, Om0=0.3, Ode0=0.50) - assert allclose(cosmo.comoving_distance(3), - cosmo.comoving_distance(3.0), rtol=1e-7) - assert allclose(cosmo.comoving_distance([1, 2, 3, 5]), - cosmo.comoving_distance([1.0, 2.0, 3.0, 5.0]), - rtol=1e-7) - assert allclose(cosmo.efunc(6), cosmo.efunc(6.0), rtol=1e-7) - assert allclose(cosmo.efunc([1, 2, 6]), - cosmo.efunc([1.0, 2.0, 6.0]), rtol=1e-7) - assert allclose(cosmo.inv_efunc([1, 2, 6]), - cosmo.inv_efunc([1.0, 2.0, 6.0]), rtol=1e-7) - - -def test_wz(): - cosmo = core.LambdaCDM(H0=70, Om0=0.3, Ode0=0.70) - assert allclose(cosmo.w(1.0), -1.) - assert allclose(cosmo.w([0.1, 0.2, 0.5, 1.5, 2.5, 11.5]), - [-1., -1, -1, -1, -1, -1]) - - cosmo = core.wCDM(H0=70, Om0=0.3, Ode0=0.70, w0=-0.5) - assert allclose(cosmo.w(1.0), -0.5) - assert allclose(cosmo.w([0.1, 0.2, 0.5, 1.5, 2.5, 11.5]), - [-0.5, -0.5, -0.5, -0.5, -0.5, -0.5]) - assert allclose(cosmo.w0, -0.5) - - cosmo = core.w0wzCDM(H0=70, Om0=0.3, Ode0=0.70, w0=-1, wz=0.5) - assert allclose(cosmo.w(1.0), -0.5) - assert allclose(cosmo.w([0.0, 0.5, 1.0, 1.5, 2.3]), - [-1.0, -0.75, -0.5, -0.25, 0.15]) - assert allclose(cosmo.w0, -1.0) - assert allclose(cosmo.wz, 0.5) - - cosmo = core.w0waCDM(H0=70, Om0=0.3, Ode0=0.70, w0=-1, wa=-0.5) - assert allclose(cosmo.w0, -1.0) - assert allclose(cosmo.wa, -0.5) - assert allclose(cosmo.w(1.0), -1.25) - assert allclose(cosmo.w([0.0, 0.5, 1.0, 1.5, 2.3]), - [-1, -1.16666667, -1.25, -1.3, -1.34848485]) - - cosmo = core.wpwaCDM(H0=70, Om0=0.3, Ode0=0.70, wp=-0.9, - wa=0.2, zp=0.5) - assert allclose(cosmo.wp, -0.9) - assert allclose(cosmo.wa, 0.2) - assert allclose(cosmo.zp, 0.5) - assert allclose(cosmo.w(0.5), -0.9) - assert allclose(cosmo.w([0.1, 0.2, 0.5, 1.5, 2.5, 11.5]), - [-0.94848485, -0.93333333, -0.9, -0.84666667, - -0.82380952, -0.78266667]) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_de_densityscale(): - cosmo = core.LambdaCDM(H0=70, Om0=0.3, Ode0=0.70) - z = np.array([0.1, 0.2, 0.5, 1.5, 2.5]) - assert allclose(cosmo.de_density_scale(z), - [1.0, 1.0, 1.0, 1.0, 1.0]) - # Integer check - assert allclose(cosmo.de_density_scale(3), - cosmo.de_density_scale(3.0), rtol=1e-7) - assert allclose(cosmo.de_density_scale([1, 2, 3]), - cosmo.de_density_scale([1., 2., 3.]), rtol=1e-7) - - cosmo = core.wCDM(H0=70, Om0=0.3, Ode0=0.60, w0=-0.5) - assert allclose(cosmo.de_density_scale(z), - [1.15369, 1.31453, 1.83712, 3.95285, 6.5479], - rtol=1e-4) - assert allclose(cosmo.de_density_scale(3), - cosmo.de_density_scale(3.0), rtol=1e-7) - assert allclose(cosmo.de_density_scale([1, 2, 3]), - cosmo.de_density_scale([1., 2., 3.]), rtol=1e-7) - - cosmo = core.w0wzCDM(H0=70, Om0=0.3, Ode0=0.50, w0=-1, wz=0.5) - assert allclose(cosmo.de_density_scale(z), - [0.746048, 0.5635595, 0.25712378, 0.026664129, - 0.0035916468], rtol=1e-4) - assert allclose(cosmo.de_density_scale(3), - cosmo.de_density_scale(3.0), rtol=1e-7) - assert allclose(cosmo.de_density_scale([1, 2, 3]), - cosmo.de_density_scale([1., 2., 3.]), rtol=1e-7) - - cosmo = core.w0waCDM(H0=70, Om0=0.3, Ode0=0.70, w0=-1, wa=-0.5) - assert allclose(cosmo.de_density_scale(z), - [0.9934201, 0.9767912, 0.897450, - 0.622236, 0.4458753], rtol=1e-4) - assert allclose(cosmo.de_density_scale(3), - cosmo.de_density_scale(3.0), rtol=1e-7) - assert allclose(cosmo.de_density_scale([1, 2, 3]), - cosmo.de_density_scale([1., 2., 3.]), rtol=1e-7) - - cosmo = core.wpwaCDM(H0=70, Om0=0.3, Ode0=0.70, wp=-0.9, - wa=0.2, zp=0.5) - assert allclose(cosmo.de_density_scale(z), - [1.012246048, 1.0280102, 1.087439, - 1.324988, 1.565746], rtol=1e-4) - assert allclose(cosmo.de_density_scale(3), - cosmo.de_density_scale(3.0), rtol=1e-7) - assert allclose(cosmo.de_density_scale([1, 2, 3]), - cosmo.de_density_scale([1., 2., 3.]), rtol=1e-7) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_age(): - # WMAP7 but with Omega_relativisitic = 0 - tcos = core.FlatLambdaCDM(70.4, 0.272, Tcmb0=0.0) - assert allclose(tcos.hubble_time, 13.889094057856937 * u.Gyr) - assert allclose(tcos.age(4), 1.5823603508870991 * u.Gyr) - assert allclose(tcos.age([1., 5.]), - [5.97113193, 1.20553129] * u.Gyr) - assert allclose(tcos.age([1, 5]), [5.97113193, 1.20553129] * u.Gyr) - - # Add relativistic species - tcos = core.FlatLambdaCDM(70.4, 0.272, Tcmb0=3.0) - assert allclose(tcos.age(4), 1.5773003779230699 * u.Gyr) - assert allclose(tcos.age([1, 5]), [5.96344942, 1.20093077] * u.Gyr) - - # And massive neutrinos - tcos = core.FlatLambdaCDM(70.4, 0.272, Tcmb0=3.0, - m_nu = 0.1 * u.eV) - assert allclose(tcos.age(4), 1.5546485439853412 * u.Gyr) - assert allclose(tcos.age([1, 5]), [5.88448152, 1.18383759] * u.Gyr) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_distmod(): - # WMAP7 but with Omega_relativisitic = 0 - tcos = core.FlatLambdaCDM(70.4, 0.272, Tcmb0=0.0) - assert allclose(tcos.hubble_distance, 4258.415596590909 * u.Mpc) - assert allclose(tcos.distmod([1, 5]), - [44.124857, 48.40167258] * u.mag) - assert allclose(tcos.distmod([1., 5.]), - [44.124857, 48.40167258] * u.mag) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_neg_distmod(): - # Cosmology with negative luminosity distances (perfectly okay, - # if obscure) - tcos = core.LambdaCDM(70, 0.2, 1.3, Tcmb0=0) - assert allclose(tcos.luminosity_distance([50, 100]), - [16612.44047622, -46890.79092244] * u.Mpc) - assert allclose(tcos.distmod([50, 100]), - [46.102167189, 48.355437790944] * u.mag) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_critical_density(): - # WMAP7 but with Omega_relativistic = 0 - # These tests will fail if astropy.const starts returning non-mks - # units by default; see the comment at the top of core.py - tcos = core.FlatLambdaCDM(70.4, 0.272, Tcmb0=0.0) - assert allclose(tcos.critical_density0, - 9.31000324385361e-30 * u.g / u.cm**3) - assert allclose(tcos.critical_density0, - tcos.critical_density(0)) - assert allclose(tcos.critical_density([1, 5]), - [2.70362491e-29, 5.53758986e-28] * u.g / u.cm**3) - assert allclose(tcos.critical_density([1., 5.]), - [2.70362491e-29, 5.53758986e-28] * u.g / u.cm**3) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_angular_diameter_distance_z1z2(): - - with pytest.raises(core.CosmologyError): # test neg Ok fail - tcos = core.LambdaCDM(H0=70.4, Om0=0.272, Ode0=0.8, Tcmb0=0.0) - tcos.angular_diameter_distance_z1z2(1, 2) - - tcos = core.FlatLambdaCDM(70.4, 0.272, Tcmb0=0.0) - with pytest.raises(ValueError): # test diff size z1, z2 fail - tcos.angular_diameter_distance_z1z2([1, 2], [3, 4, 5]) - with pytest.raises(ValueError): # test z1 > z2 fail - tcos.angular_diameter_distance_z1z2(4, 3) - # Tests that should actually work - assert allclose(tcos.angular_diameter_distance_z1z2(1, 2), - 646.22968662822018 * u.Mpc) - z1 = 0, 0, 1, 0.5, 1 - z2 = 2, 1, 2, 2.5, 1.1 - results = (1760.0628637762106, - 1670.7497657219858, - 646.22968662822018, - 1159.0970895962193, - 115.72768186186921) * u.Mpc - - assert allclose(tcos.angular_diameter_distance_z1z2(z1, z2), - results) - - # Non-flat (positive Ocurv) test - tcos = core.LambdaCDM(H0=70.4, Om0=0.2, Ode0=0.5, Tcmb0=0.0) - assert allclose(tcos.angular_diameter_distance_z1z2(1, 2), - 620.1175337852428 * u.Mpc) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_absorption_distance(): - tcos = core.FlatLambdaCDM(70.4, 0.272, Tcmb0=0.0) - assert allclose(tcos.absorption_distance([1, 3]), - [1.72576635, 7.98685853]) - assert allclose(tcos.absorption_distance([1., 3.]), - [1.72576635, 7.98685853]) - assert allclose(tcos.absorption_distance(3), 7.98685853) - assert allclose(tcos.absorption_distance(3.), 7.98685853) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_massivenu_basic(): - # Test no neutrinos case - tcos = core.FlatLambdaCDM(70.4, 0.272, Neff=4.05, m_nu=u.Quantity(0, u.eV)) - assert allclose(tcos.Neff, 4.05) - assert not tcos.has_massive_nu - mnu = tcos.m_nu - assert len(mnu) == 4 - assert mnu.unit == u.eV - assert allclose(mnu, [0.0, 0.0, 0.0, 0.0] * u.eV) - assert allclose(tcos.nu_relative_density(1.), 0.22710731766 * 4.05, - rtol=1e-6) - assert allclose(tcos.nu_relative_density(1), 0.22710731766 * 4.05, - rtol=1e-6) - - # Alternative no neutrinos case - tcos = core.FlatLambdaCDM(70.4, 0.272, Tcmb0 = 0 * u.K, - m_nu=u.Quantity(0.4, u.eV)) - assert not tcos.has_massive_nu - assert tcos.m_nu is None - - # Test basic setting, retrieval of values - tcos = core.FlatLambdaCDM(70.4, 0.272, - m_nu=u.Quantity([0.0, 0.01, 0.02], u.eV)) - assert tcos.has_massive_nu - mnu = tcos.m_nu - assert len(mnu) == 3 - assert mnu.unit == u.eV - assert allclose(mnu, [0.0, 0.01, 0.02] * u.eV) - - # All massive neutrinos case - tcos = core.FlatLambdaCDM(70.4, 0.272, m_nu=u.Quantity(0.1, u.eV), - Neff=3.1) - assert allclose(tcos.Neff, 3.1) - assert tcos.has_massive_nu - mnu = tcos.m_nu - assert len(mnu) == 3 - assert mnu.unit == u.eV - assert allclose(mnu, [0.1, 0.1, 0.1] * u.eV) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_distances(): - # Test distance calculations for various special case - # scenarios (no relatavistic species, normal, massive neutrinos) - # These do not come from external codes -- they are just internal - # checks to make sure nothing changes if we muck with the distance - # calculators - - z = np.array([1.0, 2.0, 3.0, 4.0]) - - # The pattern here is: no relativistic species, the relativistic - # species with massless neutrinos, then massive neutrinos - cos = core.LambdaCDM(75.0, 0.25, 0.5, Tcmb0=0.0) - assert allclose(cos.comoving_distance(z), - [2953.93001902, 4616.7134253, 5685.07765971, - 6440.80611897] * u.Mpc, rtol=1e-4) - cos = core.LambdaCDM(75.0, 0.25, 0.6, Tcmb0=3.0, Neff=3, - m_nu=u.Quantity(0.0, u.eV)) - assert allclose(cos.comoving_distance(z), - [3037.12620424, 4776.86236327, 5889.55164479, - 6671.85418235] * u.Mpc, rtol=1e-4) - cos = core.LambdaCDM(75.0, 0.3, 0.4, Tcmb0=3.0, Neff=3, - m_nu=u.Quantity(10.0, u.eV)) - assert allclose(cos.comoving_distance(z), - [2471.80626824, 3567.1902565 , 4207.15995626, - 4638.20476018] * u.Mpc, rtol=1e-4) - # Flat - cos = core.FlatLambdaCDM(75.0, 0.25, Tcmb0=0.0) - assert allclose(cos.comoving_distance(z), - [3180.83488552, 5060.82054204, 6253.6721173, - 7083.5374303] * u.Mpc, rtol=1e-4) - cos = core.FlatLambdaCDM(75.0, 0.25, Tcmb0=3.0, Neff=3, - m_nu=u.Quantity(0.0, u.eV)) - assert allclose(cos.comoving_distance(z), - [3180.42662867, 5059.60529655, 6251.62766102, - 7080.71698117] * u.Mpc, rtol=1e-4) - cos = core.FlatLambdaCDM(75.0, 0.25, Tcmb0=3.0, Neff=3, - m_nu=u.Quantity(10.0, u.eV)) - assert allclose(cos.comoving_distance(z), - [2337.54183142, 3371.91131264, 3988.40711188, - 4409.09346922] * u.Mpc, rtol=1e-4) - # Add w - cos = core.FlatwCDM(75.0, 0.25, w0=-1.05, Tcmb0=0.0) - assert allclose(cos.comoving_distance(z), - [3216.8296894 , 5117.2097601 , 6317.05995437, - 7149.68648536] * u.Mpc, rtol=1e-4) - cos = core.FlatwCDM(75.0, 0.25, w0=-0.95, Tcmb0=3.0, Neff=3, - m_nu=u.Quantity(0.0, u.eV)) - assert allclose(cos.comoving_distance(z), - [3143.56537758, 5000.32196494, 6184.11444601, - 7009.80166062] * u.Mpc, rtol=1e-4) - cos = core.FlatwCDM(75.0, 0.25, w0=-0.9, Tcmb0=3.0, Neff=3, - m_nu=u.Quantity(10.0, u.eV)) - assert allclose(cos.comoving_distance(z), - [2337.76035371, 3372.1971387, 3988.71362289, - 4409.40817174] * u.Mpc, rtol=1e-4) - # Non-flat w - cos = core.wCDM(75.0, 0.25, 0.4, w0=-0.9, Tcmb0=0.0) - assert allclose(cos.comoving_distance(z), - [2849.6163356 , 4428.71661565, 5450.97862778, - 6179.37072324] * u.Mpc, rtol=1e-4) - cos = core.wCDM(75.0, 0.25, 0.4, w0=-1.1, Tcmb0=3.0, Neff=3, - m_nu=u.Quantity(0.0, u.eV)) - assert allclose(cos.comoving_distance(z), - [2904.35580229, 4511.11471267, 5543.43643353, - 6275.9206788] * u.Mpc, rtol=1e-4) - cos = core.wCDM(75.0, 0.25, 0.4, w0=-0.9, Tcmb0=3.0, Neff=3, - m_nu=u.Quantity(10.0, u.eV)) - assert allclose(cos.comoving_distance(z), - [2473.32522734, 3581.54519631, 4232.41674426, - 4671.83818117] * u.Mpc, rtol=1e-4) - # w0wa - cos = core.w0waCDM(75.0, 0.3, 0.6, w0=-0.9, wa=0.1, Tcmb0=0.0) - assert allclose(cos.comoving_distance(z), - [2937.7807638, 4572.59950903, 5611.52821924, - 6339.8549956] * u.Mpc, rtol=1e-4) - cos = core.w0waCDM(75.0, 0.25, 0.5, w0=-0.9, wa=0.1, Tcmb0=3.0, Neff=3, - m_nu=u.Quantity(0.0, u.eV)) - assert allclose(cos.comoving_distance(z), - [2907.34722624, 4539.01723198, 5593.51611281, - 6342.3228444] * u.Mpc, rtol=1e-4) - cos = core.w0waCDM(75.0, 0.25, 0.5, w0=-0.9, wa=0.1, Tcmb0=3.0, Neff=3, - m_nu=u.Quantity(10.0, u.eV)) - assert allclose(cos.comoving_distance(z), - [2507.18336722, 3633.33231695, 4292.44746919, - 4736.35404638] * u.Mpc, rtol=1e-4) - # Flatw0wa - cos = core.Flatw0waCDM(75.0, 0.25, w0=-0.95, wa=0.15, Tcmb0=0.0) - assert allclose(cos.comoving_distance(z), - [3123.29892781, 4956.15204302, 6128.15563818, - 6948.26480378] * u.Mpc, rtol=1e-4) - cos = core.Flatw0waCDM(75.0, 0.25, w0=-0.95, wa=0.15, Tcmb0=3.0, Neff=3, - m_nu=u.Quantity(0.0, u.eV)) - assert allclose(cos.comoving_distance(z), - [3122.92671907, 4955.03768936, 6126.25719576, - 6945.61856513] * u.Mpc, rtol=1e-4) - cos = core.Flatw0waCDM(75.0, 0.25, w0=-0.95, wa=0.15, Tcmb0=3.0, Neff=3, - m_nu=u.Quantity(10.0, u.eV)) - assert allclose(cos.comoving_distance(z), - [2337.70072701, 3372.13719963, 3988.6571093, - 4409.35399673] * u.Mpc, rtol=1e-4) - # wpwa - cos = core.wpwaCDM(75.0, 0.3, 0.6, wp=-0.9, zp=0.5, wa=0.1, Tcmb0=0.0) - assert allclose(cos.comoving_distance(z), - [2954.68975298, 4599.83254834, 5643.04013201, - 6373.36147627] * u.Mpc, rtol=1e-4) - cos = core.wpwaCDM(75.0, 0.25, 0.5, wp=-0.9, zp=0.4, wa=0.1, - Tcmb0=3.0, Neff=3, m_nu=u.Quantity(0.0, u.eV)) - assert allclose(cos.comoving_distance(z), - [2919.00656215, 4558.0218123, 5615.73412391, - 6366.10224229] * u.Mpc, rtol=1e-4) - cos = core.wpwaCDM(75.0, 0.25, 0.5, wp=-0.9, zp=1.0, wa=0.1, Tcmb0=3.0, - Neff=4, m_nu=u.Quantity(5.0, u.eV)) - assert allclose(cos.comoving_distance(z), - [2629.48489827, 3874.13392319, 4614.31562397, - 5116.51184842] * u.Mpc, rtol=1e-4) - - # w0wz - cos = core.w0wzCDM(75.0, 0.3, 0.6, w0=-0.9, wz=0.1, Tcmb0=0.0) - assert allclose(cos.comoving_distance(z), - [3051.68786716, 4756.17714818, 5822.38084257, - 6562.70873734] * u.Mpc, rtol=1e-4) - cos = core.w0wzCDM(75.0, 0.25, 0.5, w0=-0.9, wz=0.1, - Tcmb0=3.0, Neff=3, m_nu=u.Quantity(0.0, u.eV)) - assert allclose(cos.comoving_distance(z), - [2997.8115653 , 4686.45599916, 5764.54388557, - 6524.17408738] * u.Mpc, rtol=1e-4) - cos = core.w0wzCDM(75.0, 0.25, 0.5, w0=-0.9, wz=0.1, Tcmb0=3.0, - Neff=4, m_nu=u.Quantity(5.0, u.eV)) - assert allclose(cos.comoving_distance(z), - [2676.73467639, 3940.57967585, 4686.90810278, - 5191.54178243] * u.Mpc, rtol=1e-4) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_massivenu_density(): - # Testing neutrino density calculation - - # Simple test cosmology, where we compare rho_nu and rho_gamma - # against the exact formula (eq 24/25 of Komatsu et al. 2011) - # computed using Mathematica. The approximation we use for f(y) - # is only good to ~ 0.5% (with some redshift dependence), so that's - # what we test to. - ztest = np.array([0.0, 1.0, 2.0, 10.0, 1000.0]) - nuprefac = 7.0 / 8.0 * (4.0 / 11.0) ** (4.0 / 3.0) - # First try 3 massive neutrinos, all 100 eV -- note this is a universe - # seriously dominated by neutrinos! - tcos = core.FlatLambdaCDM(75.0, 0.25, Tcmb0=3.0, Neff=3, - m_nu=u.Quantity(100.0, u.eV)) - assert tcos.has_massive_nu - assert tcos.Neff == 3 - nurel_exp = nuprefac * tcos.Neff * np.array([171969, 85984.5, 57323, - 15633.5, 171.801]) - assert allclose(tcos.nu_relative_density(ztest), nurel_exp, rtol=5e-3) - assert allclose(tcos.efunc([0.0, 1.0]), [1.0, 7.46144727668], rtol=5e-3) - - # Next, slightly less massive - tcos = core.FlatLambdaCDM(75.0, 0.25, Tcmb0=3.0, Neff=3, - m_nu=u.Quantity(0.25, u.eV)) - nurel_exp = nuprefac * tcos.Neff * np.array([429.924, 214.964, 143.312, - 39.1005, 1.11086]) - assert allclose(tcos.nu_relative_density(ztest), nurel_exp, - rtol=5e-3) - - # For this one also test Onu directly - onu_exp = np.array([0.01890217, 0.05244681, 0.0638236, - 0.06999286, 0.1344951]) - assert allclose(tcos.Onu(ztest), onu_exp, rtol=5e-3) - - # And fairly light - tcos = core.FlatLambdaCDM(80.0, 0.30, Tcmb0=3.0, Neff=3, - m_nu=u.Quantity(0.01, u.eV)) - - nurel_exp = nuprefac * tcos.Neff * np.array([17.2347, 8.67345, 5.84348, - 1.90671, 1.00021]) - assert allclose(tcos.nu_relative_density(ztest), nurel_exp, - rtol=5e-3) - onu_exp = np.array([0.00066599, 0.00172677, 0.0020732, - 0.00268404, 0.0978313]) - assert allclose(tcos.Onu(ztest), onu_exp, rtol=5e-3) - assert allclose(tcos.efunc([1.0, 2.0]), [1.76225893, 2.97022048], - rtol=1e-4) - assert allclose(tcos.inv_efunc([1.0, 2.0]), [0.5674535, 0.33667534], - rtol=1e-4) - - # Now a mixture of neutrino masses, with non-integer Neff - tcos = core.FlatLambdaCDM(80.0, 0.30, Tcmb0=3.0, Neff=3.04, - m_nu=u.Quantity([0.0, 0.01, 0.25], u.eV)) - nurel_exp = nuprefac * tcos.Neff * np.array([149.386233, 74.87915, 50.0518, - 14.002403, 1.03702333]) - assert allclose(tcos.nu_relative_density(ztest), nurel_exp, - rtol=5e-3) - onu_exp = np.array([0.00584959, 0.01493142, 0.01772291, - 0.01963451, 0.10227728]) - assert allclose(tcos.Onu(ztest), onu_exp, rtol=5e-3) - - # Integer redshifts - ztest = ztest.astype(np.int) - assert allclose(tcos.nu_relative_density(ztest), nurel_exp, - rtol=5e-3) - assert allclose(tcos.Onu(ztest), onu_exp, rtol=5e-3) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_z_at_value(): - # These are tests of expected values, and hence have less precision - # than the roundtrip tests below (test_z_at_value_roundtrip); - # here we have to worry about the cosmological calculations - # giving slightly different values on different architectures, - # there we are checking internal consistency on the same architecture - # and so can be more demanding - z_at_value = funcs.z_at_value - cosmo = core.Planck13 - d = cosmo.luminosity_distance(3) - assert allclose(z_at_value(cosmo.luminosity_distance, d), 3, - rtol=1e-8) - assert allclose(z_at_value(cosmo.age, 2 * u.Gyr), 3.198122684356, - rtol=1e-6) - assert allclose(z_at_value(cosmo.luminosity_distance, 1e4 * u.Mpc), - 1.3685790653802761, rtol=1e-6) - assert allclose(z_at_value(cosmo.lookback_time, 7 * u.Gyr), - 0.7951983674601507, rtol=1e-6) - assert allclose(z_at_value(cosmo.angular_diameter_distance, 1500*u.Mpc, - zmax=2), 0.68127769625288614, rtol=1e-6) - assert allclose(z_at_value(cosmo.angular_diameter_distance, 1500*u.Mpc, - zmin=2.5), 3.7914908028272083, rtol=1e-6) - assert allclose(z_at_value(cosmo.distmod, 46 * u.mag), - 1.9913891680278133, rtol=1e-6) - - # test behaviour when the solution is outside z limits (should - # raise a CosmologyError) - with pytest.raises(core.CosmologyError): - z_at_value(cosmo.angular_diameter_distance, 1500*u.Mpc, zmax=0.5) - with pytest.raises(core.CosmologyError): - z_at_value(cosmo.angular_diameter_distance, 1500*u.Mpc, zmin=4.) - - -@pytest.mark.skipif('not HAS_SCIPY') -def test_z_at_value_roundtrip(): - """ - Calculate values from a known redshift, and then check that - z_at_value returns the right answer. - """ - z = 0.5 - - # Skip Ok, w, de_density_scale because in the Planck13 cosmolgy - # they are redshift independent and hence uninvertable, - # angular_diameter_distance_z1z2 takes multiple arguments, so requires - # special handling - # clone isn't a redshift-dependent method - skip = ('Ok', 'angular_diameter_distance_z1z2', 'clone', - 'de_density_scale', 'w') - - import inspect - methods = inspect.getmembers(core.Planck13, predicate=inspect.ismethod) - - for name, func in methods: - if name.startswith('_') or name in skip: - continue - print('Round-trip testing {0}'.format(name)) - fval = func(z) - # we need zmax here to pick the right solution for - # angular_diameter_distance and related methods. - # Be slightly more generous with rtol than the default 1e-8 - # used in z_at_value - assert allclose(z, funcs.z_at_value(func, fval, zmax=1.5), - rtol=2e-8) - - # Test angular_diameter_distance_z1z2 - z2 = 2.0 - func = lambda z1: core.Planck13.angular_diameter_distance_z1z2(z1, z2) - fval = func(z) - assert allclose(z, funcs.z_at_value(func, fval, zmax=1.5), - rtol=2e-8) diff --git a/astropy/cosmology/tests/test_pickle.py b/astropy/cosmology/tests/test_pickle.py deleted file mode 100644 index 96e88443239a..000000000000 --- a/astropy/cosmology/tests/test_pickle.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -from __future__ import absolute_import, division, print_function, unicode_literals - -import numpy as np - -from ...tests.helper import pytest, pickle_protocol, check_pickling_recovery -from ...extern.six.moves import cPickle -from ... import cosmology as cosm - -originals = [cosm.FLRW] -xfails = [False] - -@pytest.mark.parametrize("original,xfail", - zip(originals, xfails)) -def test_flrw(pickle_protocol, original, xfail): - if xfail: - pytest.xfail() - check_pickling_recovery(original, pickle_protocol) diff --git a/astropy/cosmology/tests/test_scalar_inv_efuncs.py b/astropy/cosmology/tests/test_scalar_inv_efuncs.py new file mode 100644 index 000000000000..56c35907c92d --- /dev/null +++ b/astropy/cosmology/tests/test_scalar_inv_efuncs.py @@ -0,0 +1,128 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""Tests for the scalar_inv_efuncs Cython extension in astropy.cosmology.""" + +from numpy.testing import assert_allclose + +from astropy.cosmology._src.flrw.scalar_inv_efuncs import ( + flcdm_inv_efunc_nomnu, + flcdm_inv_efunc_norel, + fwcdm_inv_efunc_norel, + lcdm_inv_efunc_nomnu, + lcdm_inv_efunc_norel, + wcdm_inv_efunc_norel, +) + +# --------------------------------------------------------------------------- +# LambdaCDM (no relativistic species) +# At z=0: E(0)=1 => inv_efunc=1 for flat universe (Om0+Ode0=1, Ok0=0) +# --------------------------------------------------------------------------- + + +def test_lcdm_inv_efunc_norel_z0(): + # Flat universe at z=0: Om0+Ode0=1, Ok0=0 => E(0)=1 + assert_allclose(lcdm_inv_efunc_norel(0.0, 0.3, 0.7, 0.0), 1.0) + + +def test_lcdm_inv_efunc_norel_matter_dominated(): + # Matter-only universe (Ode0=0, Ok0=0, Om0=1): E(z)=(1+z)^1.5 + # => inv_efunc(z) = (1+z)^-1.5 + z = 1.0 + expected = (1.0 + z) ** -1.5 + assert_allclose(lcdm_inv_efunc_norel(z, 1.0, 0.0, 0.0), expected) + + +def test_lcdm_inv_efunc_norel_positive(): + """inv_efunc must always be positive for physical parameters.""" + assert lcdm_inv_efunc_norel(0.5, 0.3, 0.7, 0.0) > 0.0 + + +# --------------------------------------------------------------------------- +# LambdaCDM (massless neutrinos) +# --------------------------------------------------------------------------- + + +def test_lcdm_inv_efunc_nomnu_z0(): + # Flat universe at z=0: Om0+Ode0+Or0=1, Ok0=0 => E(0)=1 + assert_allclose(lcdm_inv_efunc_nomnu(0.0, 0.3, 0.6999, 0.0, 0.0001), 1.0) + + +def test_lcdm_inv_efunc_nomnu_positive(): + """inv_efunc must always be positive for physical parameters.""" + assert lcdm_inv_efunc_nomnu(1.0, 0.3, 0.7, 0.0, 0.0) > 0.0 + + +# --------------------------------------------------------------------------- +# FlatLambdaCDM (no relativistic species) +# --------------------------------------------------------------------------- + + +def test_flcdm_inv_efunc_norel_z0(): + # Flat: Om0+Ode0=1 => E(0)=1 + assert_allclose(flcdm_inv_efunc_norel(0.0, 0.3, 0.7), 1.0) + + +def test_flcdm_inv_efunc_norel_matter_dominated(): + # Matter-only flat: Ode0=0, Om0=1 => inv_efunc(z)=(1+z)^-1.5 + z = 2.0 + expected = (1.0 + z) ** -1.5 + assert_allclose(flcdm_inv_efunc_norel(z, 1.0, 0.0), expected) + + +def test_flcdm_inv_efunc_norel_positive(): + """inv_efunc must always be positive for physical parameters.""" + assert flcdm_inv_efunc_norel(0.5, 0.3, 0.7) > 0.0 + + +# --------------------------------------------------------------------------- +# FlatLambdaCDM (massless neutrinos) +# --------------------------------------------------------------------------- + + +def test_flcdm_inv_efunc_nomnu_z0(): + # Flat: Om0+Ode0+Or0=1 => E(0)=1 + assert_allclose(flcdm_inv_efunc_nomnu(0.0, 0.3, 0.6999, 0.0001), 1.0) + + +# --------------------------------------------------------------------------- +# wCDM (no relativistic species) +# --------------------------------------------------------------------------- + + +def test_wcdm_inv_efunc_norel_z0(): + # Flat wCDM at z=0 with Om0+Ode0=1, Ok0=0 => E(0)=1 + assert_allclose(wcdm_inv_efunc_norel(0.0, 0.3, 0.7, 0.0, -1.0), 1.0) + + +def test_wcdm_inv_efunc_norel_lcdm_limit(): + # w0=-1 => wCDM reduces to LambdaCDM + z = 0.5 + Om0, Ode0, Ok0, w0 = 0.3, 0.7, 0.0, -1.0 + assert_allclose( + wcdm_inv_efunc_norel(z, Om0, Ode0, Ok0, w0), + lcdm_inv_efunc_norel(z, Om0, Ode0, Ok0), + ) + + +def test_wcdm_inv_efunc_norel_positive(): + """inv_efunc must always be positive for physical parameters.""" + assert wcdm_inv_efunc_norel(1.0, 0.3, 0.7, 0.0, -0.8) > 0.0 + + +# --------------------------------------------------------------------------- +# FlatwCDM (no relativistic species) +# --------------------------------------------------------------------------- + + +def test_fwcdm_inv_efunc_norel_z0(): + # Flat wCDM at z=0: Om0+Ode0=1 => E(0)=1 + assert_allclose(fwcdm_inv_efunc_norel(0.0, 0.3, 0.7, -1.0), 1.0) + + +def test_fwcdm_inv_efunc_norel_lcdm_limit(): + # w0=-1 => flat wCDM reduces to flat LambdaCDM + z = 1.0 + Om0, Ode0, w0 = 0.3, 0.7, -1.0 + assert_allclose( + fwcdm_inv_efunc_norel(z, Om0, Ode0, w0), + flcdm_inv_efunc_norel(z, Om0, Ode0), + ) diff --git a/astropy/cosmology/traits.py b/astropy/cosmology/traits.py new file mode 100644 index 000000000000..dc4ad40ea2cb --- /dev/null +++ b/astropy/cosmology/traits.py @@ -0,0 +1,32 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""Traits for building ``astropy`` :class:`~astropy.cosmology.Cosmology` classes.""" + +__all__ = ( + "BaryonComponent", + "CriticalDensity", + "CurvatureComponent", + "DarkEnergyComponent", + "DarkMatterComponent", + "HubbleParameter", + "MatterComponent", + "NeutrinoComponent", + "PhotonComponent", + "ScaleFactor", + "TemperatureCMB", + "TotalComponent", +) + +from ._src.traits import ( + BaryonComponent, + CriticalDensity, + CurvatureComponent, + DarkEnergyComponent, + DarkMatterComponent, + HubbleParameter, + MatterComponent, + NeutrinoComponent, + PhotonComponent, + ScaleFactor, + TemperatureCMB, + TotalComponent, +) diff --git a/astropy/cosmology/units.py b/astropy/cosmology/units.py new file mode 100644 index 000000000000..4d2d66cce254 --- /dev/null +++ b/astropy/cosmology/units.py @@ -0,0 +1,44 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""Cosmological units and equivalencies.""" + +__all__ = ( + # redshift equivalencies + "dimensionless_redshift", + "littleh", + "redshift", + "redshift_distance", + "redshift_hubble", + "redshift_temperature", + # other equivalencies + "with_H0", + "with_redshift", +) + +from astropy.units import add_enabled_equivalencies as _add_enabled_equivalencies +from astropy.units.docgen import generate_unit_summary as _generate_unit_summary + +from ._src.units import littleh, redshift +from ._src.units_equivalencies import ( + dimensionless_redshift, + redshift_distance, + redshift_hubble, + redshift_temperature, + with_H0, + with_redshift, +) + +# =================================================================== +# Enable the set of default equivalencies. +# If the cosmology package is imported, this is added to the list astropy-wide. +_add_enabled_equivalencies(dimensionless_redshift()) + + +# ============================================================================= +# DOCSTRING + +# This generates a docstring for this module that describes all of the +# standard units defined here. +if __doc__ is not None: + from ._src.units import _ns + + __doc__ += "\n" + _generate_unit_summary(_ns) diff --git a/astropy/extern/README.rst b/astropy/extern/README.rst index 5ce391a0248f..47880ccd9851 100644 --- a/astropy/extern/README.rst +++ b/astropy/extern/README.rst @@ -7,28 +7,12 @@ party JavaScript libraries used for browser-based features. In particular, this currently includes for Python: -- ConfigObj_ (Python 2 and 3 versions): This provides the core config file - handling for Astropy's configuration system. +- ConfigObj_: This provides the core config file handling for Astropy's + configuration system. - PLY_: This is a parser generator providing lex/yacc-like tools in Python. It is used for Astropy's unit parsing and angle/coordinate string parsing. -- pytest_: This is the test framework used to collect and run Astropy's tests. - It is a fairly large package, so it is bundled as a compressed base64 string - that is unpacked and loaded as a Python module. This packed bundled version - is officially provided by the pytest project. - -- Six_: This is a Python 2/3 compatibility library used to ease development - of a simultaneously Python 2 and 3-compatible code base. - -And for JavaScript: - -- jQuery_: This is used currently for the browsed-based table viewer feature. - -- DataTables_: This is a plug-in for jQuery used also for the browser-based - table viewer. - - Notes for third-party packagers ------------------------------- @@ -36,39 +20,28 @@ Packagers preparing Astropy for inclusion in packaging frameworks have different options for how to handle these third-party extern packages, if they would prefer to use their system packages rather than the bundled versions. -Six -^^^ - -Because use of the ``six`` module is pervasive throughout the Astropy package -(it is crucial for providing Python 2/3 compatibility at all levels) it has -been made easier to replace. The actual ``six`` module is included under -``astropy/extern/bundled``. Importing ``astropy.extern.six`` will first try -to load the ``six`` module from that location, and then will fall back on -trying to import an installed ``six`` module from the standard ``sys.path`` -locations (if it is a new enough version). - -Packagers may wish to prioritize the system version and/or remove the bundled -version altogether. To do this, edit the ``_SIX_SEARCH_PATH`` list in -``astropy.extern.six``. If removing the bundled copy altogether simply delete -``astropy/extern/bundled/six.py`` and update this variable to read:: - - _SIX_SEARCH_PATH = ['six'] +jQuery/DataTables +^^^^^^^^^^^^^^^^^ -No other imports in Astropy need to be updated. Imports from -``astropy.extern.six`` are automatically updated to point to the system version of the module. +It is possible to change the default urls for the remote versions of these +files by using the Astropy +`Configuration system `_. The default +configuration file (``$XDG_CONFIG_HOME/astropy/astropy.cfg``) contains a commented section +``[table.jsviewer]`` with two items for jQuery_ and DataTables_. It is also +possible to display the default value and modify it by importing the +configuration module:: + In [1]: from astropy.table.jsviewer import conf -jQuery/DataTables -^^^^^^^^^^^^^^^^^ + In [2]: conf.jquery_url + Out[2]: u'https://code.jquery.com/jquery-1.11.3.min.js' -Packagers may either use system copies of these JavaScript modules, or require -use of online versions (perhaps via URLs of cloud-hosted versions of these -modules). + In [3]: conf.jquery_url = '...' -To change the default paths for these files edit the ``astropy.table.jsviewer`` -module in Astropy to change the default values for the ``jquery_url`` and/or -``datatables_url`` options. Use a ``file://`` URL for locally-installed -versions of these files. +Third-party packagers can override the defaults for these configuration items +by modifying the configuration objects in ``astropy/table/jsviewer.py``, or +provide astropy config files that include the overrides appropriate for the +packaged version. Other @@ -76,13 +49,10 @@ Other To replace any of the other Python modules included in this package, simply remove them and update any imports in Astropy to import the system versions -rather than the bundled copies. If requested we could add a mechanism similar -to that used by ``six`` for the other modules. +rather than the bundled copies. .. _ConfigObj: https://github.com/DiffSK/configobj .. _PLY: http://www.dabeaz.com/ply/ -.. _pytest: http://pytest.org/latest/ -.. _Six: http://pypi.python.org/pypi/six/ .. _jQuery: http://jquery.com/ .. _DataTables: http://www.datatables.net/ diff --git a/astropy/extern/_strptime.py b/astropy/extern/_strptime.py new file mode 100644 index 000000000000..cbb891454900 --- /dev/null +++ b/astropy/extern/_strptime.py @@ -0,0 +1,529 @@ +"""Strptime-related classes and functions. + +CLASSES: + LocaleTime -- Discovers and stores locale-specific time information + TimeRE -- Creates regexes for pattern matching a string of text containing + time information + +FUNCTIONS: + _getlang -- Figure out what language is being used for the locale + strptime -- Calculates the time struct represented by the passed-in string + +""" +# ----------------------------------------------------------------------------- +# _strptime.py +# +# Licensed under PYTHON SOFTWARE FOUNDATION LICENSE +# See licenses/PYTHON.rst +# +# Copied from https://github.com/python/cpython/blob/3.5/Lib/_strptime.py +# ----------------------------------------------------------------------------- +import time +import locale +import calendar +from re import compile as re_compile +from re import IGNORECASE +from re import escape as re_escape +from datetime import (date as datetime_date, + timedelta as datetime_timedelta, + timezone as datetime_timezone) +try: + from _thread import allocate_lock as _thread_allocate_lock +except ImportError: + from _dummy_thread import allocate_lock as _thread_allocate_lock + +__all__ = [] + +def _getlang(): + # Figure out what the current language is set to. + return locale.getlocale(locale.LC_TIME) + +class LocaleTime(object): + """Stores and handles locale-specific information related to time. + + ATTRIBUTES: + f_weekday -- full weekday names (7-item list) + a_weekday -- abbreviated weekday names (7-item list) + f_month -- full month names (13-item list; dummy value in [0], which + is added by code) + a_month -- abbreviated month names (13-item list, dummy value in + [0], which is added by code) + am_pm -- AM/PM representation (2-item list) + LC_date_time -- format string for date/time representation (string) + LC_date -- format string for date representation (string) + LC_time -- format string for time representation (string) + timezone -- daylight- and non-daylight-savings timezone representation + (2-item list of sets) + lang -- Language used by instance (2-item tuple) + """ + + def __init__(self): + """Set all attributes. + + Order of methods called matters for dependency reasons. + + The locale language is set at the offset and then checked again before + exiting. This is to make sure that the attributes were not set with a + mix of information from more than one locale. This would most likely + happen when using threads where one thread calls a locale-dependent + function while another thread changes the locale while the function in + the other thread is still running. Proper coding would call for + locks to prevent changing the locale while locale-dependent code is + running. The check here is done in case someone does not think about + doing this. + + Only other possible issue is if someone changed the timezone and did + not call tz.tzset . That is an issue for the programmer, though, + since changing the timezone is worthless without that call. + + """ + self.lang = _getlang() + self.__calc_weekday() + self.__calc_month() + self.__calc_am_pm() + self.__calc_timezone() + self.__calc_date_time() + if _getlang() != self.lang: + raise ValueError("locale changed during initialization") + if time.tzname != self.tzname or time.daylight != self.daylight: + raise ValueError("timezone changed during initialization") + + def __pad(self, seq, front): + # Add '' to seq to either the front (is True), else the back. + seq = list(seq) + if front: + seq.insert(0, '') + else: + seq.append('') + return seq + + def __calc_weekday(self): + # Set self.a_weekday and self.f_weekday using the calendar + # module. + a_weekday = [calendar.day_abbr[i].lower() for i in range(7)] + f_weekday = [calendar.day_name[i].lower() for i in range(7)] + self.a_weekday = a_weekday + self.f_weekday = f_weekday + + def __calc_month(self): + # Set self.f_month and self.a_month using the calendar module. + a_month = [calendar.month_abbr[i].lower() for i in range(13)] + f_month = [calendar.month_name[i].lower() for i in range(13)] + self.a_month = a_month + self.f_month = f_month + + def __calc_am_pm(self): + # Set self.am_pm by using time.strftime(). + + # The magic date (1999,3,17,hour,44,55,2,76,0) is not really that + # magical; just happened to have used it everywhere else where a + # static date was needed. + am_pm = [] + for hour in (1, 22): + time_tuple = time.struct_time((1999,3,17,hour,44,55,2,76,0)) + am_pm.append(time.strftime("%p", time_tuple).lower()) + self.am_pm = am_pm + + def __calc_date_time(self): + # Set self.date_time, self.date, & self.time by using + # time.strftime(). + + # Use (1999,3,17,22,44,55,2,76,0) for magic date because the amount of + # overloaded numbers is minimized. The order in which searches for + # values within the format string is very important; it eliminates + # possible ambiguity for what something represents. + time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0)) + date_time = [None, None, None] + date_time[0] = time.strftime("%c", time_tuple).lower() + date_time[1] = time.strftime("%x", time_tuple).lower() + date_time[2] = time.strftime("%X", time_tuple).lower() + replacement_pairs = [('%', '%%'), (self.f_weekday[2], '%A'), + (self.f_month[3], '%B'), (self.a_weekday[2], '%a'), + (self.a_month[3], '%b'), (self.am_pm[1], '%p'), + ('1999', '%Y'), ('99', '%y'), ('22', '%H'), + ('44', '%M'), ('55', '%S'), ('76', '%j'), + ('17', '%d'), ('03', '%m'), ('3', '%m'), + # '3' needed for when no leading zero. + ('2', '%w'), ('10', '%I')] + replacement_pairs.extend([(tz, "%Z") for tz_values in self.timezone + for tz in tz_values]) + for offset,directive in ((0,'%c'), (1,'%x'), (2,'%X')): + current_format = date_time[offset] + for old, new in replacement_pairs: + # Must deal with possible lack of locale info + # manifesting itself as the empty string (e.g., Swedish's + # lack of AM/PM info) or a platform returning a tuple of empty + # strings (e.g., MacOS 9 having timezone as ('','')). + if old: + current_format = current_format.replace(old, new) + # If %W is used, then Sunday, 2005-01-03 will fall on week 0 since + # 2005-01-03 occurs before the first Monday of the year. Otherwise + # %U is used. + time_tuple = time.struct_time((1999,1,3,1,1,1,6,3,0)) + if '00' in time.strftime(directive, time_tuple): + U_W = '%W' + else: + U_W = '%U' + date_time[offset] = current_format.replace('11', U_W) + self.LC_date_time = date_time[0] + self.LC_date = date_time[1] + self.LC_time = date_time[2] + + def __calc_timezone(self): + # Set self.timezone by using time.tzname. + # Do not worry about possibility of time.tzname[0] == time.tzname[1] + # and time.daylight; handle that in strptime. + try: + time.tzset() + except AttributeError: + pass + self.tzname = time.tzname + self.daylight = time.daylight + no_saving = frozenset({"utc", "gmt", self.tzname[0].lower()}) + if self.daylight: + has_saving = frozenset({self.tzname[1].lower()}) + else: + has_saving = frozenset() + self.timezone = (no_saving, has_saving) + + +class TimeRE(dict): + """Handle conversion from format directives to regexes.""" + + def __init__(self, locale_time=None): + """Create keys/values. + + Order of execution is important for dependency reasons. + + """ + if locale_time: + self.locale_time = locale_time + else: + self.locale_time = LocaleTime() + base = super() + base.__init__({ + # The " \d" part of the regex is to make %c from ANSI C work + 'd': r"(?P3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])", + 'f': r"(?P[0-9]{1,6})", + 'H': r"(?P2[0-3]|[0-1]\d|\d)", + 'I': r"(?P1[0-2]|0[1-9]|[1-9])", + 'j': r"(?P36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])", + 'm': r"(?P1[0-2]|0[1-9]|[1-9])", + 'M': r"(?P[0-5]\d|\d)", + 'S': r"(?P6[0-1]|[0-5]\d|\d)", + 'U': r"(?P5[0-3]|[0-4]\d|\d)", + 'w': r"(?P[0-6])", + # W is set below by using 'U' + 'y': r"(?P\d\d)", + #XXX: Does 'Y' need to worry about having less or more than + # 4 digits? + 'Y': r"(?P\d\d\d\d)", + 'z': r"(?P[+-]\d\d[0-5]\d)", + 'A': self.__seqToRE(self.locale_time.f_weekday, 'A'), + 'a': self.__seqToRE(self.locale_time.a_weekday, 'a'), + 'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'), + 'b': self.__seqToRE(self.locale_time.a_month[1:], 'b'), + 'p': self.__seqToRE(self.locale_time.am_pm, 'p'), + 'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone + for tz in tz_names), + 'Z'), + '%': '%'}) + base.__setitem__('W', base.__getitem__('U').replace('U', 'W')) + base.__setitem__('c', self.pattern(self.locale_time.LC_date_time)) + base.__setitem__('x', self.pattern(self.locale_time.LC_date)) + base.__setitem__('X', self.pattern(self.locale_time.LC_time)) + + def __seqToRE(self, to_convert, directive): + """Convert a list to a regex string for matching a directive. + + Want possible matching values to be from longest to shortest. This + prevents the possibility of a match occurring for a value that also + a substring of a larger value that should have matched (e.g., 'abc' + matching when 'abcdef' should have been the match). + + """ + to_convert = sorted(to_convert, key=len, reverse=True) + for value in to_convert: + if value != '': + break + else: + return '' + regex = '|'.join(re_escape(stuff) for stuff in to_convert) + regex = '(?P<%s>%s' % (directive, regex) + return '%s)' % regex + + def pattern(self, format): + """Return regex pattern for the format string. + + Need to make sure that any characters that might be interpreted as + regex syntax are escaped. + + """ + processed_format = '' + # The sub() call escapes all characters that might be misconstrued + # as regex syntax. Cannot use re.escape since we have to deal with + # format directives (%m, etc.). + regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])") + format = regex_chars.sub(r"\\\1", format) + whitespace_replacement = re_compile(r'\s+') + format = whitespace_replacement.sub(r'\\s+', format) + while '%' in format: + directive_index = format.index('%')+1 + processed_format = "%s%s%s" % (processed_format, + format[:directive_index-1], + self[format[directive_index]]) + format = format[directive_index+1:] + return "%s%s" % (processed_format, format) + + def compile(self, format): + """Return a compiled re object for the format string.""" + return re_compile(self.pattern(format), IGNORECASE) + +_cache_lock = _thread_allocate_lock() +# DO NOT modify _TimeRE_cache or _regex_cache without acquiring the cache lock +# first! +_TimeRE_cache = TimeRE() +_CACHE_MAX_SIZE = 5 # Max number of regexes stored in _regex_cache +_regex_cache = {} + +def _calc_julian_from_U_or_W(year, week_of_year, day_of_week, week_starts_Mon): + """Calculate the Julian day based on the year, week of the year, and day of + the week, with week_start_day representing whether the week of the year + assumes the week starts on Sunday or Monday (6 or 0).""" + first_weekday = datetime_date(year, 1, 1).weekday() + # If we are dealing with the %U directive (week starts on Sunday), it's + # easier to just shift the view to Sunday being the first day of the + # week. + if not week_starts_Mon: + first_weekday = (first_weekday + 1) % 7 + day_of_week = (day_of_week + 1) % 7 + # Need to watch out for a week 0 (when the first day of the year is not + # the same as that specified by %U or %W). + week_0_length = (7 - first_weekday) % 7 + if week_of_year == 0: + return 1 + day_of_week - first_weekday + else: + days_to_week = week_0_length + (7 * (week_of_year - 1)) + return 1 + days_to_week + day_of_week + + +def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): + """Return a 2-tuple consisting of a time struct and an int containing + the number of microseconds based on the input string and the + format string.""" + + for index, arg in enumerate([data_string, format]): + if not isinstance(arg, str): + msg = "strptime() argument {} must be str, not {}" + raise TypeError(msg.format(index, type(arg))) + + global _TimeRE_cache, _regex_cache + with _cache_lock: + locale_time = _TimeRE_cache.locale_time + if (_getlang() != locale_time.lang or + time.tzname != locale_time.tzname or + time.daylight != locale_time.daylight): + _TimeRE_cache = TimeRE() + _regex_cache.clear() + locale_time = _TimeRE_cache.locale_time + if len(_regex_cache) > _CACHE_MAX_SIZE: + _regex_cache.clear() + format_regex = _regex_cache.get(format) + if not format_regex: + try: + format_regex = _TimeRE_cache.compile(format) + # KeyError raised when a bad format is found; can be specified as + # \\, in which case it was a stray % but with a space after it + except KeyError as err: + bad_directive = err.args[0] + if bad_directive == "\\": + bad_directive = "%" + del err + raise ValueError("'%s' is a bad directive in format '%s'" % + (bad_directive, format)) from None + # IndexError only occurs when the format string is "%" + except IndexError: + raise ValueError("stray %% in format '%s'" % format) from None + _regex_cache[format] = format_regex + found = format_regex.match(data_string) + if not found: + raise ValueError("time data %r does not match format %r" % + (data_string, format)) + if len(data_string) != found.end(): + raise ValueError("unconverted data remains: %s" % + data_string[found.end():]) + + year = None + month = day = 1 + hour = minute = second = fraction = 0 + tz = -1 + tzoffset = None + # Default to -1 to signify that values not known; not critical to have, + # though + week_of_year = -1 + week_of_year_start = -1 + # weekday and julian defaulted to None so as to signal need to calculate + # values + weekday = julian = None + found_dict = found.groupdict() + for group_key in found_dict.keys(): + # Directives not explicitly handled below: + # c, x, X + # handled by making out of other directives + # U, W + # worthless without day of the week + if group_key == 'y': + year = int(found_dict['y']) + # Open Group specification for strptime() states that a %y + #value in the range of [00, 68] is in the century 2000, while + #[69,99] is in the century 1900 + if year <= 68: + year += 2000 + else: + year += 1900 + elif group_key == 'Y': + year = int(found_dict['Y']) + elif group_key == 'm': + month = int(found_dict['m']) + elif group_key == 'B': + month = locale_time.f_month.index(found_dict['B'].lower()) + elif group_key == 'b': + month = locale_time.a_month.index(found_dict['b'].lower()) + elif group_key == 'd': + day = int(found_dict['d']) + elif group_key == 'H': + hour = int(found_dict['H']) + elif group_key == 'I': + hour = int(found_dict['I']) + ampm = found_dict.get('p', '').lower() + # If there was no AM/PM indicator, we'll treat this like AM + if ampm in ('', locale_time.am_pm[0]): + # We're in AM so the hour is correct unless we're + # looking at 12 midnight. + # 12 midnight == 12 AM == hour 0 + if hour == 12: + hour = 0 + elif ampm == locale_time.am_pm[1]: + # We're in PM so we need to add 12 to the hour unless + # we're looking at 12 noon. + # 12 noon == 12 PM == hour 12 + if hour != 12: + hour += 12 + elif group_key == 'M': + minute = int(found_dict['M']) + elif group_key == 'S': + second = int(found_dict['S']) + elif group_key == 'f': + s = found_dict['f'] + # Pad to always return microseconds. + s += "0" * (6 - len(s)) + fraction = int(s) + elif group_key == 'A': + weekday = locale_time.f_weekday.index(found_dict['A'].lower()) + elif group_key == 'a': + weekday = locale_time.a_weekday.index(found_dict['a'].lower()) + elif group_key == 'w': + weekday = int(found_dict['w']) + if weekday == 0: + weekday = 6 + else: + weekday -= 1 + elif group_key == 'j': + julian = int(found_dict['j']) + elif group_key in ('U', 'W'): + week_of_year = int(found_dict[group_key]) + if group_key == 'U': + # U starts week on Sunday. + week_of_year_start = 6 + else: + # W starts week on Monday. + week_of_year_start = 0 + elif group_key == 'z': + z = found_dict['z'] + tzoffset = int(z[1:3]) * 60 + int(z[3:5]) + if z.startswith("-"): + tzoffset = -tzoffset + elif group_key == 'Z': + # Since -1 is default value only need to worry about setting tz if + # it can be something other than -1. + found_zone = found_dict['Z'].lower() + for value, tz_values in enumerate(locale_time.timezone): + if found_zone in tz_values: + # Deal with bad locale setup where timezone names are the + # same and yet time.daylight is true; too ambiguous to + # be able to tell what timezone has daylight savings + if (time.tzname[0] == time.tzname[1] and + time.daylight and found_zone not in ("utc", "gmt")): + break + else: + tz = value + break + leap_year_fix = False + if year is None and month == 2 and day == 29: + year = 1904 # 1904 is first leap year of 20th century + leap_year_fix = True + elif year is None: + year = 1900 + # If we know the week of the year and what day of that week, we can figure + # out the Julian day of the year. + if julian is None and week_of_year != -1 and weekday is not None: + week_starts_Mon = True if week_of_year_start == 0 else False + julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, + week_starts_Mon) + if julian <= 0: + year -= 1 + yday = 366 if calendar.isleap(year) else 365 + julian += yday + # Cannot pre-calculate datetime_date() since can change in Julian + # calculation and thus could have different value for the day of the week + # calculation. + if julian is None: + # Need to add 1 to result since first day of the year is 1, not 0. + julian = datetime_date(year, month, day).toordinal() - \ + datetime_date(year, 1, 1).toordinal() + 1 + else: # Assume that if they bothered to include Julian day it will + # be accurate. + datetime_result = datetime_date.fromordinal((julian - 1) + datetime_date(year, 1, 1).toordinal()) + year = datetime_result.year + month = datetime_result.month + day = datetime_result.day + if weekday is None: + weekday = datetime_date(year, month, day).weekday() + # Add timezone info + tzname = found_dict.get("Z") + if tzoffset is not None: + gmtoff = tzoffset * 60 + else: + gmtoff = None + + if leap_year_fix: + # the caller didn't supply a year but asked for Feb 29th. We couldn't + # use the default of 1900 for computations. We set it back to ensure + # that February 29th is smaller than March 1st. + year = 1900 + + return (year, month, day, + hour, minute, second, + weekday, julian, tz, tzname, gmtoff), fraction + +def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"): + """Return a time struct based on the input string and the + format string.""" + tt = _strptime(data_string, format)[0] + return time.struct_time(tt[:time._STRUCT_TM_ITEMS]) + +def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"): + """Return a class cls instance based on the input string and the + format string.""" + tt, fraction = _strptime(data_string, format) + tzname, gmtoff = tt[-2:] + args = tt[:6] + (fraction,) + if gmtoff is not None: + tzdelta = datetime_timedelta(seconds=gmtoff) + if tzname: + tz = datetime_timezone(tzdelta, tzname) + else: + tz = datetime_timezone(tzdelta) + args += (tz,) + + return cls(*args) diff --git a/astropy/extern/bundled/six.py b/astropy/extern/bundled/six.py deleted file mode 100644 index 7c285b1155b7..000000000000 --- a/astropy/extern/bundled/six.py +++ /dev/null @@ -1,753 +0,0 @@ -"""Utilities for writing code that runs on Python 2 and 3""" - -# Copyright (c) 2010-2014 Benjamin Peterson -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import functools -import operator -import sys -import types - -__author__ = "Benjamin Peterson " -__version__ = "1.7.3" - - -# Useful for very coarse version differentiation. -PY2 = sys.version_info[0] == 2 -PY3 = sys.version_info[0] == 3 - -if PY3: - string_types = str, - integer_types = int, - class_types = type, - text_type = str - binary_type = bytes - - MAXSIZE = sys.maxsize -else: - string_types = basestring, - integer_types = (int, long) - class_types = (type, types.ClassType) - text_type = unicode - binary_type = str - - if sys.platform.startswith("java"): - # Jython always uses 32 bits. - MAXSIZE = int((1 << 31) - 1) - else: - # It's possible to have sizeof(long) != sizeof(Py_ssize_t). - class X(object): - def __len__(self): - return 1 << 31 - try: - len(X()) - except OverflowError: - # 32-bit - MAXSIZE = int((1 << 31) - 1) - else: - # 64-bit - MAXSIZE = int((1 << 63) - 1) - del X - - -def _add_doc(func, doc): - """Add documentation to a function.""" - func.__doc__ = doc - - -def _import_module(name): - """Import module, returning the module after the last dot.""" - __import__(name) - return sys.modules[name] - - -class _LazyDescr(object): - - def __init__(self, name): - self.name = name - - def __get__(self, obj, tp): - result = self._resolve() - setattr(obj, self.name, result) # Invokes __set__. - # This is a bit ugly, but it avoids running this again. - delattr(obj.__class__, self.name) - return result - - -class MovedModule(_LazyDescr): - - def __init__(self, name, old, new=None): - super(MovedModule, self).__init__(name) - if PY3: - if new is None: - new = name - self.mod = new - else: - self.mod = old - - def _resolve(self): - return _import_module(self.mod) - - def __getattr__(self, attr): - _module = self._resolve() - value = getattr(_module, attr) - setattr(self, attr, value) - return value - - -class _LazyModule(types.ModuleType): - - def __init__(self, name): - super(_LazyModule, self).__init__(name) - self.__doc__ = self.__class__.__doc__ - - def __dir__(self): - attrs = ["__doc__", "__name__"] - attrs += [attr.name for attr in self._moved_attributes] - return attrs - - # Subclasses should override this - _moved_attributes = [] - - -class MovedAttribute(_LazyDescr): - - def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): - super(MovedAttribute, self).__init__(name) - if PY3: - if new_mod is None: - new_mod = name - self.mod = new_mod - if new_attr is None: - if old_attr is None: - new_attr = name - else: - new_attr = old_attr - self.attr = new_attr - else: - self.mod = old_mod - if old_attr is None: - old_attr = name - self.attr = old_attr - - def _resolve(self): - module = _import_module(self.mod) - return getattr(module, self.attr) - - -class _SixMetaPathImporter(object): - """ - A meta path importer to import six.moves and its submodules. - - This class implements a PEP302 finder and loader. It should be compatible - with Python 2.5 and all existing versions of Python3 - """ - def __init__(self, six_module_name): - self.name = six_module_name - self.known_modules = {} - - def _add_module(self, mod, *fullnames): - for fullname in fullnames: - self.known_modules[self.name + "." + fullname] = mod - - def _get_module(self, fullname): - return self.known_modules[self.name + "." + fullname] - - def find_module(self, fullname, path=None): - if fullname in self.known_modules: - return self - return None - - def __get_module(self, fullname): - try: - return self.known_modules[fullname] - except KeyError: - raise ImportError("This loader does not know module " + fullname) - - def load_module(self, fullname): - try: - # in case of a reload - return sys.modules[fullname] - except KeyError: - pass - mod = self.__get_module(fullname) - if isinstance(mod, MovedModule): - mod = mod._resolve() - else: - mod.__loader__ = self - sys.modules[fullname] = mod - return mod - - def is_package(self, fullname): - """ - Return true, if the named module is a package. - - We need this method to get correct spec objects with - Python 3.4 (see PEP451) - """ - return hasattr(self.__get_module(fullname), "__path__") - - def get_code(self, fullname): - """Return None - - Required, if is_package is implemented""" - self.__get_module(fullname) # eventually raises ImportError - return None - get_source = get_code # same as get_code - -_importer = _SixMetaPathImporter(__name__) - - -class _MovedItems(_LazyModule): - """Lazy loading of moved objects""" - __path__ = [] # mark as package - - -_moved_attributes = [ - MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), - MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), - MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), - MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), - MovedAttribute("map", "itertools", "builtins", "imap", "map"), - MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute("reload_module", "__builtin__", "imp", "reload"), - MovedAttribute("reduce", "__builtin__", "functools"), - MovedAttribute("StringIO", "StringIO", "io"), - MovedAttribute("UserDict", "UserDict", "collections"), - MovedAttribute("UserList", "UserList", "collections"), - MovedAttribute("UserString", "UserString", "collections"), - MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), - MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), - - MovedModule("builtins", "__builtin__"), - MovedModule("configparser", "ConfigParser"), - MovedModule("copyreg", "copy_reg"), - MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), - MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), - MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), - MovedModule("http_cookies", "Cookie", "http.cookies"), - MovedModule("html_entities", "htmlentitydefs", "html.entities"), - MovedModule("html_parser", "HTMLParser", "html.parser"), - MovedModule("http_client", "httplib", "http.client"), - MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), - MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), - MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), - MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), - MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), - MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), - MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), - MovedModule("cPickle", "cPickle", "pickle"), - MovedModule("queue", "Queue"), - MovedModule("reprlib", "repr"), - MovedModule("socketserver", "SocketServer"), - MovedModule("_thread", "thread", "_thread"), - MovedModule("tkinter", "Tkinter"), - MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), - MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), - MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), - MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), - MovedModule("tkinter_tix", "Tix", "tkinter.tix"), - MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), - MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), - MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), - MovedModule("tkinter_colorchooser", "tkColorChooser", - "tkinter.colorchooser"), - MovedModule("tkinter_commondialog", "tkCommonDialog", - "tkinter.commondialog"), - MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), - MovedModule("tkinter_font", "tkFont", "tkinter.font"), - MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), - MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", - "tkinter.simpledialog"), - MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), - MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), - MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), - MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), - MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), - MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), - MovedModule("winreg", "_winreg"), -] -for attr in _moved_attributes: - setattr(_MovedItems, attr.name, attr) - if isinstance(attr, MovedModule): - _importer._add_module(attr, "moves." + attr.name) -del attr - -_MovedItems._moved_attributes = _moved_attributes - -moves = _MovedItems(__name__ + ".moves") -_importer._add_module(moves, "moves") - - -class Module_six_moves_urllib_parse(_LazyModule): - """Lazy loading of moved objects in six.moves.urllib_parse""" - - -_urllib_parse_moved_attributes = [ - MovedAttribute("ParseResult", "urlparse", "urllib.parse"), - MovedAttribute("SplitResult", "urlparse", "urllib.parse"), - MovedAttribute("parse_qs", "urlparse", "urllib.parse"), - MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), - MovedAttribute("urldefrag", "urlparse", "urllib.parse"), - MovedAttribute("urljoin", "urlparse", "urllib.parse"), - MovedAttribute("urlparse", "urlparse", "urllib.parse"), - MovedAttribute("urlsplit", "urlparse", "urllib.parse"), - MovedAttribute("urlunparse", "urlparse", "urllib.parse"), - MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), - MovedAttribute("quote", "urllib", "urllib.parse"), - MovedAttribute("quote_plus", "urllib", "urllib.parse"), - MovedAttribute("unquote", "urllib", "urllib.parse"), - MovedAttribute("unquote_plus", "urllib", "urllib.parse"), - MovedAttribute("urlencode", "urllib", "urllib.parse"), - MovedAttribute("splitquery", "urllib", "urllib.parse"), - MovedAttribute("splittag", "urllib", "urllib.parse"), - MovedAttribute("splituser", "urllib", "urllib.parse"), -] -for attr in _urllib_parse_moved_attributes: - setattr(Module_six_moves_urllib_parse, attr.name, attr) -del attr - -Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes - -_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), - "moves.urllib_parse", "moves.urllib.parse") - - -class Module_six_moves_urllib_error(_LazyModule): - """Lazy loading of moved objects in six.moves.urllib_error""" - - -_urllib_error_moved_attributes = [ - MovedAttribute("URLError", "urllib2", "urllib.error"), - MovedAttribute("HTTPError", "urllib2", "urllib.error"), - MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), -] -for attr in _urllib_error_moved_attributes: - setattr(Module_six_moves_urllib_error, attr.name, attr) -del attr - -Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes - -_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), - "moves.urllib_error", "moves.urllib.error") - - -class Module_six_moves_urllib_request(_LazyModule): - """Lazy loading of moved objects in six.moves.urllib_request""" - - -_urllib_request_moved_attributes = [ - MovedAttribute("urlopen", "urllib2", "urllib.request"), - MovedAttribute("install_opener", "urllib2", "urllib.request"), - MovedAttribute("build_opener", "urllib2", "urllib.request"), - MovedAttribute("pathname2url", "urllib", "urllib.request"), - MovedAttribute("url2pathname", "urllib", "urllib.request"), - MovedAttribute("getproxies", "urllib", "urllib.request"), - MovedAttribute("Request", "urllib2", "urllib.request"), - MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), - MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), - MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), - MovedAttribute("BaseHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), - MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), - MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), - MovedAttribute("FileHandler", "urllib2", "urllib.request"), - MovedAttribute("FTPHandler", "urllib2", "urllib.request"), - MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), - MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), - MovedAttribute("urlretrieve", "urllib", "urllib.request"), - MovedAttribute("urlcleanup", "urllib", "urllib.request"), - MovedAttribute("URLopener", "urllib", "urllib.request"), - MovedAttribute("FancyURLopener", "urllib", "urllib.request"), - MovedAttribute("proxy_bypass", "urllib", "urllib.request"), -] -for attr in _urllib_request_moved_attributes: - setattr(Module_six_moves_urllib_request, attr.name, attr) -del attr - -Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes - -_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), - "moves.urllib_request", "moves.urllib.request") - - -class Module_six_moves_urllib_response(_LazyModule): - """Lazy loading of moved objects in six.moves.urllib_response""" - - -_urllib_response_moved_attributes = [ - MovedAttribute("addbase", "urllib", "urllib.response"), - MovedAttribute("addclosehook", "urllib", "urllib.response"), - MovedAttribute("addinfo", "urllib", "urllib.response"), - MovedAttribute("addinfourl", "urllib", "urllib.response"), -] -for attr in _urllib_response_moved_attributes: - setattr(Module_six_moves_urllib_response, attr.name, attr) -del attr - -Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes - -_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), - "moves.urllib_response", "moves.urllib.response") - - -class Module_six_moves_urllib_robotparser(_LazyModule): - """Lazy loading of moved objects in six.moves.urllib_robotparser""" - - -_urllib_robotparser_moved_attributes = [ - MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), -] -for attr in _urllib_robotparser_moved_attributes: - setattr(Module_six_moves_urllib_robotparser, attr.name, attr) -del attr - -Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes - -_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), - "moves.urllib_robotparser", "moves.urllib.robotparser") - - -class Module_six_moves_urllib(types.ModuleType): - """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" - __path__ = [] # mark as package - parse = _importer._get_module("moves.urllib_parse") - error = _importer._get_module("moves.urllib_error") - request = _importer._get_module("moves.urllib_request") - response = _importer._get_module("moves.urllib_response") - robotparser = _importer._get_module("moves.urllib_robotparser") - - def __dir__(self): - return ['parse', 'error', 'request', 'response', 'robotparser'] - -_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), - "moves.urllib") - - -def add_move(move): - """Add an item to six.moves.""" - setattr(_MovedItems, move.name, move) - - -def remove_move(name): - """Remove item from six.moves.""" - try: - delattr(_MovedItems, name) - except AttributeError: - try: - del moves.__dict__[name] - except KeyError: - raise AttributeError("no such move, %r" % (name,)) - - -if PY3: - _meth_func = "__func__" - _meth_self = "__self__" - - _func_closure = "__closure__" - _func_code = "__code__" - _func_defaults = "__defaults__" - _func_globals = "__globals__" -else: - _meth_func = "im_func" - _meth_self = "im_self" - - _func_closure = "func_closure" - _func_code = "func_code" - _func_defaults = "func_defaults" - _func_globals = "func_globals" - - -try: - advance_iterator = next -except NameError: - def advance_iterator(it): - return it.next() -next = advance_iterator - - -try: - callable = callable -except NameError: - def callable(obj): - return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) - - -if PY3: - def get_unbound_function(unbound): - return unbound - - create_bound_method = types.MethodType - - Iterator = object -else: - def get_unbound_function(unbound): - return unbound.im_func - - def create_bound_method(func, obj): - return types.MethodType(func, obj, obj.__class__) - - class Iterator(object): - - def next(self): - return type(self).__next__(self) - - callable = callable -_add_doc(get_unbound_function, - """Get the function out of a possibly unbound function""") - - -get_method_function = operator.attrgetter(_meth_func) -get_method_self = operator.attrgetter(_meth_self) -get_function_closure = operator.attrgetter(_func_closure) -get_function_code = operator.attrgetter(_func_code) -get_function_defaults = operator.attrgetter(_func_defaults) -get_function_globals = operator.attrgetter(_func_globals) - - -if PY3: - def iterkeys(d, **kw): - return iter(d.keys(**kw)) - - def itervalues(d, **kw): - return iter(d.values(**kw)) - - def iteritems(d, **kw): - return iter(d.items(**kw)) - - def iterlists(d, **kw): - return iter(d.lists(**kw)) -else: - def iterkeys(d, **kw): - return iter(d.iterkeys(**kw)) - - def itervalues(d, **kw): - return iter(d.itervalues(**kw)) - - def iteritems(d, **kw): - return iter(d.iteritems(**kw)) - - def iterlists(d, **kw): - return iter(d.iterlists(**kw)) - -_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") -_add_doc(itervalues, "Return an iterator over the values of a dictionary.") -_add_doc(iteritems, - "Return an iterator over the (key, value) pairs of a dictionary.") -_add_doc(iterlists, - "Return an iterator over the (key, [values]) pairs of a dictionary.") - - -if PY3: - def b(s): - return s.encode("latin-1") - def u(s): - return s - unichr = chr - if sys.version_info[1] <= 1: - def int2byte(i): - return bytes((i,)) - else: - # This is about 2x faster than the implementation above on 3.2+ - int2byte = operator.methodcaller("to_bytes", 1, "big") - byte2int = operator.itemgetter(0) - indexbytes = operator.getitem - iterbytes = iter - import io - StringIO = io.StringIO - BytesIO = io.BytesIO -else: - def b(s): - return s - # Workaround for standalone backslash - def u(s): - return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") - unichr = unichr - int2byte = chr - def byte2int(bs): - return ord(bs[0]) - def indexbytes(buf, i): - return ord(buf[i]) - def iterbytes(buf): - return (ord(byte) for byte in buf) - import StringIO - StringIO = BytesIO = StringIO.StringIO -_add_doc(b, """Byte literal""") -_add_doc(u, """Text literal""") - - -if PY3: - exec_ = getattr(moves.builtins, "exec") - - - def reraise(tp, value, tb=None): - if value is None: - value = tp() - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value - -else: - def exec_(_code_, _globs_=None, _locs_=None): - """Execute code in a namespace.""" - if _globs_ is None: - frame = sys._getframe(1) - _globs_ = frame.f_globals - if _locs_ is None: - _locs_ = frame.f_locals - del frame - elif _locs_ is None: - _locs_ = _globs_ - exec("""exec _code_ in _globs_, _locs_""") - - - exec_("""def reraise(tp, value, tb=None): - raise tp, value, tb -""") - - -print_ = getattr(moves.builtins, "print", None) -if print_ is None: - def print_(*args, **kwargs): - """The new-style print function for Python 2.4 and 2.5.""" - fp = kwargs.pop("file", sys.stdout) - if fp is None: - return - def write(data): - if not isinstance(data, basestring): - data = str(data) - # If the file has an encoding, encode unicode with it. - if (isinstance(fp, file) and - isinstance(data, unicode) and - fp.encoding is not None): - errors = getattr(fp, "errors", None) - if errors is None: - errors = "strict" - data = data.encode(fp.encoding, errors) - fp.write(data) - want_unicode = False - sep = kwargs.pop("sep", None) - if sep is not None: - if isinstance(sep, unicode): - want_unicode = True - elif not isinstance(sep, str): - raise TypeError("sep must be None or a string") - end = kwargs.pop("end", None) - if end is not None: - if isinstance(end, unicode): - want_unicode = True - elif not isinstance(end, str): - raise TypeError("end must be None or a string") - if kwargs: - raise TypeError("invalid keyword arguments to print()") - if not want_unicode: - for arg in args: - if isinstance(arg, unicode): - want_unicode = True - break - if want_unicode: - newline = unicode("\n") - space = unicode(" ") - else: - newline = "\n" - space = " " - if sep is None: - sep = space - if end is None: - end = newline - for i, arg in enumerate(args): - if i: - write(sep) - write(arg) - write(end) - -_add_doc(reraise, """Reraise an exception.""") - -if sys.version_info[0:2] < (3, 4): - def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, - updated=functools.WRAPPER_UPDATES): - def wrapper(f): - f = functools.wraps(wrapped)(f) - f.__wrapped__ = wrapped - return f - return wrapper -else: - wraps = functools.wraps - -def with_metaclass(meta, *bases): - """Create a base class with a metaclass.""" - # This requires a bit of explanation: the basic idea is to make a dummy - # metaclass for one level of class instantiation that replaces itself with - # the actual metaclass. - class metaclass(meta): - def __new__(cls, name, this_bases, d): - return meta(name, bases, d) - return type.__new__(metaclass, 'temporary_class', (), {}) - - -def add_metaclass(metaclass): - """Class decorator for creating a class with a metaclass.""" - def wrapper(cls): - orig_vars = cls.__dict__.copy() - orig_vars.pop('__dict__', None) - orig_vars.pop('__weakref__', None) - slots = orig_vars.get('__slots__') - if slots is not None: - if isinstance(slots, str): - slots = [slots] - for slots_var in slots: - orig_vars.pop(slots_var) - return metaclass(cls.__name__, cls.__bases__, orig_vars) - return wrapper - -# Complete the moves implementation. -# This code is at the end of this module to speed up module loading. -# Turn this module into a package. -__path__ = [] # required for PEP 302 and PEP 451 -__package__ = __name__ # see PEP 366 @ReservedAssignment -if globals().get("__spec__") is not None: - __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable -# Remove other six meta path importers, since they cause problems. This can -# happen if six is removed from sys.modules and then reloaded. (Setuptools does -# this for some reason.) -if sys.meta_path: - for i, importer in enumerate(sys.meta_path): - # Here's some real nastiness: Another "instance" of the six module might - # be floating around. Therefore, we can't use isinstance() to check for - # the six meta path importer, since the other six instance will have - # inserted an importer with different class. - if (type(importer).__name__ == "_SixMetaPathImporter" and - importer.name == __name__): - del sys.meta_path[i] - break - del i, importer -# Finally, add the importer to the meta path import hook. -sys.meta_path.append(_importer) diff --git a/astropy/extern/configobj.py b/astropy/extern/configobj.py deleted file mode 100644 index 8d1bd8cd89c0..000000000000 --- a/astropy/extern/configobj.py +++ /dev/null @@ -1,11 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst - -""" -This module just pulls in the appropriate `configobj` package, depending on the -currently installed version of python. - -Also, this should actually never actually show up as a docstring, because -it should get overwritten by the appropriate configobj docstring. -""" - -from .configobj import configobj, validate, __doc__ diff --git a/astropy/extern/configobj/configobj.py b/astropy/extern/configobj/configobj.py index cceaeb4cab53..167608ab37c0 100755 --- a/astropy/extern/configobj/configobj.py +++ b/astropy/extern/configobj/configobj.py @@ -16,13 +16,10 @@ import os import re import sys -import collections +from collections.abc import Mapping from codecs import BOM_UTF8, BOM_UTF16, BOM_UTF16_BE, BOM_UTF16_LE -from ...extern import six -# from __future__ import __version__ - # imported lazily to avoid startup performance hit if it isn't used compiler = None @@ -140,28 +137,28 @@ class UnknownType(Exception): class Builder(object): - + def build(self, o): if m is None: raise UnknownType(o.__class__.__name__) return m(o) - + def build_List(self, o): return list(map(self.build, o.getChildren())) - + def build_Const(self, o): return o.value - + def build_Dict(self, o): d = {} i = iter(map(self.build, o.getChildren())) for el in i: d[el] = next(i) return d - + def build_Tuple(self, o): return tuple(self.build_List(o)) - + def build_Name(self, o): if o.name == 'None': return None @@ -169,10 +166,10 @@ def build_Name(self, o): return True if o.name == 'False': return False - + # An undefined Name raise UnknownType('Undefined Name') - + def build_Add(self, o): real, imag = list(map(self.build_Const, o.getChildren())) try: @@ -182,14 +179,14 @@ def build_Add(self, o): if not isinstance(imag, complex) or imag.real != 0.0: raise UnknownType('Add') return real+imag - + def build_Getattr(self, o): parent = self.build(o.expr) return getattr(parent, o.attrname) - + def build_UnarySub(self, o): return -self.build_Const(o.getChildren()[0]) - + def build_UnaryAdd(self, o): return self.build_Const(o.getChildren()[0]) @@ -200,7 +197,7 @@ def build_UnaryAdd(self, o): def unrepr(s): if not s: return s - + # this is supposed to be safe import ast return ast.literal_eval(s) @@ -305,7 +302,7 @@ def interpolate(self, key, value): # short-cut if not self._cookie in value: return value - + def recursive_interpolate(key, value, section, backtrail): """The function that does the actual work. @@ -405,7 +402,7 @@ def _parse_match(self, match): (e.g., if we interpolated "$$" and returned "$"). """ raise NotImplementedError() - + class ConfigParserInterpolation(InterpolationEngine): @@ -454,27 +451,27 @@ def _parse_match(self, match): def __newobj__(cls, *args): # Hack for pickle - return cls.__new__(cls, *args) + return cls.__new__(cls, *args) class Section(dict): """ A dictionary-like object that represents a section in a config file. - + It does string interpolation if the 'interpolation' attribute of the 'main' object is set to True. - + Interpolation is tried first from this object, then from the 'DEFAULT' section of this object, next from the parent and its 'DEFAULT' section, and so on until the main object is reached. - + A Section will behave like an ordered dictionary - following the order of the ``scalars`` and ``sections`` attributes. You can use this to change the order of members. - + Iteration follows the order: scalars, then sections. """ - + def __setstate__(self, state): dict.update(self, state[0]) self.__dict__.update(state[1]) @@ -482,8 +479,8 @@ def __setstate__(self, state): def __reduce__(self): state = (dict(self), self.__dict__) return (__newobj__, (self.__class__,), state) - - + + def __init__(self, parent, depth, main, indict=None, name=None): """ * parent is the section above @@ -508,8 +505,8 @@ def __init__(self, parent, depth, main, indict=None, name=None): # (rather than just passing to ``dict.__init__``) for entry, value in indict.items(): self[entry] = value - - + + def _initialise(self): # the sequence of scalar values in this Section self.scalars = [] @@ -553,12 +550,12 @@ def _interpolate(self, key, value): def __getitem__(self, key): """Fetch the item and do string interpolation.""" val = dict.__getitem__(self, key) - if self.main.interpolation: - if isinstance(val, six.string_types): + if self.main.interpolation: + if isinstance(val, str): return self._interpolate(key, val) if isinstance(val, list): def _check(entry): - if isinstance(entry, six.string_types): + if isinstance(entry, str): return self._interpolate(key, entry) return entry new = [_check(entry) for entry in val] @@ -570,20 +567,20 @@ def _check(entry): def __setitem__(self, key, value, unrepr=False): """ Correctly set a value. - + Making dictionary values Section instances. (We have to special case 'Section' instances - which are also dicts) - + Keys must be strings. Values need only be strings (or lists of strings) if ``main.stringify`` is set. - + ``unrepr`` must be set when setting a value to a dictionary, without creating a new sub-section. """ - if not isinstance(key, six.string_types): + if not isinstance(key, str): raise ValueError('The key "%s" is not a string.' % key) - + # add the comment if key not in self.comments: self.comments[key] = [] @@ -596,7 +593,7 @@ def __setitem__(self, key, value, unrepr=False): if key not in self: self.sections.append(key) dict.__setitem__(self, key, value) - elif isinstance(value, collections.Mapping) and not unrepr: + elif isinstance(value, Mapping) and not unrepr: # First create the new depth level, # then create the section if key not in self: @@ -615,11 +612,11 @@ def __setitem__(self, key, value, unrepr=False): if key not in self: self.scalars.append(key) if not self.main.stringify: - if isinstance(value, six.string_types): + if isinstance(value, str): pass elif isinstance(value, (list, tuple)): for entry in value: - if not isinstance(entry, six.string_types): + if not isinstance(entry, str): raise TypeError('Value is not a string "%s".' % entry) else: raise TypeError('Value is not a string "%s".' % value) @@ -684,7 +681,7 @@ def clear(self): """ A version of clear that also affects scalars/sections Also clears comments and configspec. - + Leaves other attributes alone : depth/main/parent are not affected """ @@ -758,10 +755,10 @@ def _getval(key): def dict(self): """ Return a deepcopy of self as a dictionary. - + All members that are ``Section`` instances are recursively turned to ordinary dictionaries - by calling their ``dict`` method. - + >>> n = a.dict() >>> n == a 1 @@ -786,7 +783,7 @@ def dict(self): def merge(self, indict): """ A recursive update - useful for merging config files. - + >>> a = '''[section1] ... option1 = True ... [[subsection]] @@ -803,20 +800,20 @@ def merge(self, indict): ConfigObj({'section1': {'option1': 'False', 'subsection': {'more_options': 'False'}}}) """ for key, val in list(indict.items()): - if (key in self and isinstance(self[key], collections.Mapping) and - isinstance(val, collections.Mapping)): + if (key in self and isinstance(self[key], Mapping) and + isinstance(val, Mapping)): self[key].merge(val) - else: + else: self[key] = val def rename(self, oldkey, newkey): """ Change a keyname to another, without changing position in sequence. - + Implemented so that transformations can be made on keys, as well as on values. (used by encode and decode) - + Also renames comments. """ if oldkey in self.scalars: @@ -844,30 +841,30 @@ def walk(self, function, raise_errors=True, call_on_sections=False, **keywargs): """ Walk every member and call a function on the keyword and value. - + Return a dictionary of the return values - + If the function raises an exception, raise the errror unless ``raise_errors=False``, in which case set the return value to ``False``. - - Any unrecognised keyword arguments you pass to walk, will be pased on + + Any unrecognized keyword arguments you pass to walk, will be pased on to the function you pass in. - + Note: if ``call_on_sections`` is ``True`` then - on encountering a subsection, *first* the function is called for the *whole* subsection, and then recurses into it's members. This means your function must be able to handle strings, dictionaries and lists. This allows you to change the key of subsections as well as for ordinary members. The return value when called on the whole subsection has to be discarded. - + See the encode and decode methods for examples, including functions. - + .. admonition:: caution - + You can use ``walk`` to transform the names of members of a section but you mustn't add or delete members. - + >>> config = '''[XXXXsection] ... XXXXkey = XXXXvalue'''.splitlines() >>> cfg = ConfigObj(config) @@ -930,17 +927,17 @@ def as_bool(self, key): Accepts a key as input. The corresponding value must be a string or the objects (``True`` or 1) or (``False`` or 0). We allow 0 and 1 to retain compatibility with Python 2.2. - - If the string is one of ``True``, ``On``, ``Yes``, or ``1`` it returns + + If the string is one of ``True``, ``On``, ``Yes``, or ``1`` it returns ``True``. - - If the string is one of ``False``, ``Off``, ``No``, or ``0`` it returns + + If the string is one of ``False``, ``Off``, ``No``, or ``0`` it returns ``False``. - + ``as_bool`` is not case sensitive. - + Any other input will raise a ``ValueError``. - + >>> a = ConfigObj() >>> a['a'] = 'fish' >>> a.as_bool('a') @@ -960,7 +957,7 @@ def as_bool(self, key): return False else: try: - if not isinstance(val, six.string_types): + if not isinstance(val, str): # TODO: Why do we raise a KeyError here? raise KeyError() else: @@ -972,10 +969,10 @@ def as_bool(self, key): def as_int(self, key): """ A convenience method which coerces the specified value to an integer. - + If the value is an invalid literal for ``int``, a ``ValueError`` will be raised. - + >>> a = ConfigObj() >>> a['a'] = 'fish' >>> a.as_int('a') @@ -995,10 +992,10 @@ def as_int(self, key): def as_float(self, key): """ A convenience method which coerces the specified value to a float. - + If the value is an invalid literal for ``float``, a ``ValueError`` will be raised. - + >>> a = ConfigObj() >>> a['a'] = 'fish' >>> a.as_float('a') #doctest: +IGNORE_EXCEPTION_DETAIL @@ -1012,13 +1009,13 @@ def as_float(self, key): 3.2... """ return float(self[key]) - - + + def as_list(self, key): """ A convenience method which fetches the specified value, guaranteeing that it is a list. - + >>> a = ConfigObj() >>> a['a'] = 1 >>> a.as_list('a') @@ -1034,15 +1031,15 @@ def as_list(self, key): if isinstance(result, (tuple, list)): return list(result) return [result] - + def restore_default(self, key): """ Restore (and return) default value for the specified key. - + This method will only work for a ConfigObj that was created with a configspec and has been validated. - + If there is no default value for this key, ``KeyError`` is raised. """ default = self.default_values[key] @@ -1051,20 +1048,20 @@ def restore_default(self, key): self.defaults.append(key) return default - + def restore_defaults(self): """ Recursively restore default values to all members that have them. - + This method will only work for a ConfigObj that was created with a configspec and has been validated. - + It doesn't delete or modify entries without default values. """ for key in self.default_values: self.restore_default(key) - + for section in self.sections: self[section].restore_defaults() @@ -1179,7 +1176,7 @@ def __init__(self, infile=None, options=None, configspec=None, encoding=None, write_empty_values=False, _inspec=False): """ Parse a config file or create a config file object. - + ``ConfigObj(infile=None, configspec=None, encoding=None, interpolation=True, raise_errors=False, list_values=True, create_empty=False, file_error=False, stringify=True, @@ -1189,9 +1186,9 @@ def __init__(self, infile=None, options=None, configspec=None, encoding=None, self._inspec = _inspec # init the superclass Section.__init__(self, self, 0, self) - + infile = infile or [] - + _options = {'configspec': configspec, 'encoding': encoding, 'interpolation': interpolation, 'raise_errors': raise_errors, 'list_values': list_values, @@ -1207,31 +1204,31 @@ def __init__(self, infile=None, options=None, configspec=None, encoding=None, warnings.warn('Passing in an options dictionary to ConfigObj() is ' 'deprecated. Use **options instead.', DeprecationWarning) - + # TODO: check the values too. for entry in options: if entry not in OPTION_DEFAULTS: - raise TypeError('Unrecognised option "%s".' % entry) + raise TypeError('Unrecognized option "%s".' % entry) for entry, value in list(OPTION_DEFAULTS.items()): if entry not in options: options[entry] = value keyword_value = _options[entry] if value != keyword_value: options[entry] = keyword_value - + # XXXX this ignores an explicit list_values = True in combination # with _inspec. The user should *never* do that anyway, but still... if _inspec: options['list_values'] = False - + self._initialise(options) configspec = options['configspec'] self._original_configspec = configspec self._load(infile, configspec) - - + + def _load(self, infile, configspec): - if isinstance(infile, six.string_types): + if isinstance(infile, str): self.filename = infile if os.path.isfile(infile): with open(infile, 'rb') as h: @@ -1247,10 +1244,10 @@ def _load(self, infile, configspec): with open(infile, 'w') as h: h.write('') content = [] - + elif isinstance(infile, (list, tuple)): content = list(infile) - + elif isinstance(infile, dict): # initialise self # the Section class handles creating subsections @@ -1263,18 +1260,18 @@ def set_section(in_section, this_section): this_section[section] = {} set_section(in_section[section], this_section[section]) set_section(infile, self) - + else: for entry in infile: self[entry] = infile[entry] del self._errors - + if configspec is not None: self._handle_configspec(configspec) else: self.configspec = None return - + elif getattr(infile, 'read', MISSING) is not MISSING: # This supports file like objects content = infile.read() or [] @@ -1299,9 +1296,9 @@ def set_section(in_section, this_section): break break - assert all(isinstance(line, six.string_types) for line in content), repr(content) + assert all(isinstance(line, str) for line in content), repr(content) content = [line.rstrip('\r\n') for line in content] - + self._parse(content) # if we had any errors, now is the time to raise them if self._errors: @@ -1319,17 +1316,17 @@ def set_section(in_section, this_section): raise error # delete private attributes del self._errors - + if configspec is None: self.configspec = None else: self._handle_configspec(configspec) - - + + def _initialise(self, options=None): if options is None: options = OPTION_DEFAULTS - + # initialise a few variables self.filename = None self._errors = [] @@ -1346,18 +1343,18 @@ def _initialise(self, options=None): self.newlines = None self.write_empty_values = options['write_empty_values'] self.unrepr = options['unrepr'] - + self.initial_comment = [] self.final_comment = [] self.configspec = None - + if self._inspec: self.list_values = False - + # Clear section attributes as well Section._initialise(self) - - + + def __repr__(self): def _getval(key): try: @@ -1365,29 +1362,29 @@ def _getval(key): except MissingInterpolationOption: return dict.__getitem__(self, key) return ('%s({%s})' % (self.__class__.__name__, - ', '.join([('%s: %s' % (repr(key), repr(_getval(key)))) + ', '.join([('%s: %s' % (repr(key), repr(_getval(key)))) for key in (self.scalars + self.sections)]))) - - + + def _handle_bom(self, infile): """ Handle any BOM, and decode if necessary. - + If an encoding is specified, that *must* be used - but the BOM should still be removed (and the BOM attribute set). - + (If the encoding is wrongly specified, then a BOM for an alternative encoding won't be discovered or removed.) - + If an encoding is not specified, UTF8 or UTF16 BOM will be detected and removed. The BOM attribute will be set. UTF16 will be decoded to unicode. - + NOTE: This method must not be called with an empty ``infile``. - + Specifying the *wrong* encoding is likely to cause a ``UnicodeDecodeError``. - + ``infile`` must always be returned as a list of lines, but may be passed in as a single string. """ @@ -1398,13 +1395,13 @@ def _handle_bom(self, infile): # the encoding specified doesn't have one # just decode return self._decode(infile, self.encoding) - + if isinstance(infile, (list, tuple)): line = infile[0] else: line = infile - if isinstance(line, six.text_type): + if isinstance(line, str): # it's already decoded and there's no need to do anything # else, just use the _decode utility method to handle # listifying appropriately @@ -1427,18 +1424,18 @@ def _handle_bom(self, infile): ##self.BOM = True # Don't need to remove BOM return self._decode(infile, encoding) - + # If we get this far, will *probably* raise a DecodeError # As it doesn't appear to start with a BOM return self._decode(infile, self.encoding) - + # Must be UTF8 BOM = BOM_SET[enc] if not line.startswith(BOM): return self._decode(infile, self.encoding) - + newline = line[len(BOM):] - + # BOM removed if isinstance(infile, (list, tuple)): infile[0] = newline @@ -1446,10 +1443,10 @@ def _handle_bom(self, infile): infile = newline self.BOM = True return self._decode(infile, self.encoding) - + # No encoding specified - so we need to check for UTF8/UTF16 for BOM, (encoding, final_encoding) in list(BOMS.items()): - if not isinstance(line, six.binary_type) or not line.startswith(BOM): + if not isinstance(line, bytes) or not line.startswith(BOM): # didn't specify a BOM, or it's not a bytestring continue else: @@ -1465,22 +1462,17 @@ def _handle_bom(self, infile): else: infile = newline # UTF-8 - if isinstance(infile, six.text_type): + if isinstance(infile, str): return infile.splitlines(True) - elif isinstance(infile, six.binary_type): + elif isinstance(infile, bytes): return infile.decode('utf-8').splitlines(True) else: return self._decode(infile, 'utf-8') # UTF16 - have to decode return self._decode(infile, encoding) - - if six.PY2 and isinstance(line, str): - # don't actually do any decoding, since we're on python 2 and - # returning a bytestring is fine - return self._decode(infile, None) # No BOM discovered and no encoding specified, default to UTF-8 - if isinstance(infile, six.binary_type): + if isinstance(infile, bytes): return infile.decode('utf-8').splitlines(True) else: return self._decode(infile, 'utf-8') @@ -1488,7 +1480,7 @@ def _handle_bom(self, infile): def _a_to_u(self, aString): """Decode ASCII strings to unicode if a self.encoding is specified.""" - if isinstance(aString, six.binary_type) and self.encoding: + if isinstance(aString, bytes) and self.encoding: return aString.decode(self.encoding) else: return aString @@ -1497,12 +1489,12 @@ def _a_to_u(self, aString): def _decode(self, infile, encoding): """ Decode infile to unicode. Using the specified encoding. - + if is a string, it also needs converting to a list. """ - if isinstance(infile, six.string_types): + if isinstance(infile, str): return infile.splitlines(True) - if isinstance(infile, six.binary_type): + if isinstance(infile, bytes): # NOTE: Could raise a ``UnicodeDecodeError`` if encoding: return infile.decode(encoding).splitlines(True) @@ -1511,7 +1503,7 @@ def _decode(self, infile, encoding): if encoding: for i, line in enumerate(infile): - if isinstance(line, six.binary_type): + if isinstance(line, bytes): # NOTE: The isinstance test here handles mixed lists of unicode/string # NOTE: But the decode will break on any non-string values # NOTE: Or could raise a ``UnicodeDecodeError`` @@ -1521,7 +1513,7 @@ def _decode(self, infile, encoding): def _decode_element(self, line): """Decode element to unicode if necessary.""" - if isinstance(line, six.binary_type) and self.default_encoding: + if isinstance(line, bytes) and self.default_encoding: return line.decode(self.default_encoding) else: return line @@ -1533,9 +1525,7 @@ def _str(self, value): Used by ``stringify`` within validate, to turn non-string values into strings. """ - if not isinstance(value, six.string_types): - # intentially 'str' because it's just whatever the "normal" - # string type is for the python version we're dealing with + if not isinstance(value, str): return str(value) else: return value @@ -1546,14 +1536,14 @@ def _parse(self, infile): temp_list_values = self.list_values if self.unrepr: self.list_values = False - + comment_list = [] done_start = False this_section = self maxline = len(infile) - 1 cur_index = -1 reset_comment = False - + while cur_index < maxline: if reset_comment: comment_list = [] @@ -1565,13 +1555,13 @@ def _parse(self, infile): reset_comment = False comment_list.append(line) continue - + if not done_start: # preserve initial comment self.initial_comment = comment_list comment_list = [] done_start = True - + reset_comment = True # first we check if it's a section marker mat = self._sectionmarker.match(line) @@ -1585,7 +1575,7 @@ def _parse(self, infile): self._handle_error("Cannot compute the section depth", NestingError, infile, cur_index) continue - + if cur_depth < this_section.depth: # the new section is dropping back to a previous level try: @@ -1605,13 +1595,13 @@ def _parse(self, infile): self._handle_error("Section too nested", NestingError, infile, cur_index) continue - + sect_name = self._unquote(sect_name) if sect_name in parent: self._handle_error('Duplicate section name', DuplicateError, infile, cur_index) continue - + # create the new section this_section = Section( parent, @@ -1712,7 +1702,7 @@ def _match_depth(self, sect, depth): """ Given a section and a depth level, walk back through the sections parents to see if the depth level matches a previous section. - + Return a reference to the right section, or raise a SyntaxError. """ @@ -1730,7 +1720,7 @@ def _match_depth(self, sect, depth): def _handle_error(self, text, ErrorClass, infile, cur_index): """ Handle an error according to the error settings. - + Either raise the error or store it. The error will have occured at ``cur_index`` """ @@ -1759,19 +1749,19 @@ def _unquote(self, value): def _quote(self, value, multiline=True): """ Return a safely quoted version of a value. - + Raise a ConfigObjError if the value cannot be safely quoted. If multiline is ``True`` (default) then use triple quotes if necessary. - + * Don't quote values that don't need it. * Recursively quote members of a list and return a comma joined list. * Multiline is ``False`` for lists. * Obey list syntax for empty and single member lists. - + If ``list_values=False`` then the value is only quoted if it contains a ``\\n`` (is multiline) or '#'. - + If ``write_empty_values`` is set, and the value is an empty string, it won't be quoted. """ @@ -1779,7 +1769,7 @@ def _quote(self, value, multiline=True): # Only if multiline is set, so that it is used for values not # keys, and not values that are part of a list return '' - + if multiline and isinstance(value, (list, tuple)): if not value: return ',' @@ -1787,22 +1777,20 @@ def _quote(self, value, multiline=True): return self._quote(value[0], multiline=False) + ',' return ', '.join([self._quote(val, multiline=False) for val in value]) - if not isinstance(value, six.string_types): + if not isinstance(value, str): if self.stringify: - # intentially 'str' because it's just whatever the "normal" - # string type is for the python version we're dealing with value = str(value) else: raise TypeError('Value "%s" is not a string.' % value) if not value: return '""' - + no_lists_no_quotes = not self.list_values and '\n' not in value and '#' not in value need_triple = multiline and ((("'" in value) and ('"' in value)) or ('\n' in value )) hash_triple_quote = multiline and not need_triple and ("'" in value) and ('"' in value) and ('#' in value) check_for_single = (no_lists_no_quotes or not need_triple) and not hash_triple_quote - + if check_for_single: if not self.list_values: # we don't quote if ``list_values=False`` @@ -1820,13 +1808,13 @@ def _quote(self, value, multiline=True): else: # if value has '\n' or "'" *and* '"', it will need triple quotes quot = self._get_triple_quote(value) - + if quot == noquot and '#' in value and self.list_values: quot = self._get_single_quote(value) - + return quot % value - - + + def _get_single_quote(self, value): if ("'" in value) and ('"' in value): raise ConfigObjError('Value "%s" cannot be safely quoted.' % value) @@ -1835,15 +1823,15 @@ def _get_single_quote(self, value): else: quot = dquot return quot - - + + def _get_triple_quote(self, value): if (value.find('"""') != -1) and (value.find("'''") != -1): raise ConfigObjError('Value "%s" cannot be safely quoted.' % value) if value.find('"""') == -1: quot = tdquot else: - quot = tsquot + quot = tsquot return quot @@ -1933,7 +1921,7 @@ def _multiline(self, value, infile, cur_index, maxline): def _handle_configspec(self, configspec): """Parse the configspec.""" - # FIXME: Should we check that the configspec was created with the + # FIXME: Should we check that the configspec was created with the # correct settings ? (i.e. ``list_values=False``) if not isinstance(configspec, ConfigObj): try: @@ -1947,11 +1935,11 @@ def _handle_configspec(self, configspec): raise ConfigspecError('Parsing configspec failed: %s' % e) except IOError as e: raise IOError('Reading configspec failed: %s' % e) - + self.configspec = configspec - - + + def _set_configspec(self, section, copy): """ Called by validate. Handles setting the configspec on subsections @@ -1963,7 +1951,7 @@ def _set_configspec(self, section, copy): for entry in section.sections: if entry not in configspec: section[entry].configspec = many - + for entry in configspec.sections: if entry == '__many__': continue @@ -1974,11 +1962,11 @@ def _set_configspec(self, section, copy): # copy comments section.comments[entry] = configspec.comments.get(entry, []) section.inline_comments[entry] = configspec.inline_comments.get(entry, '') - + # Could be a scalar when we expect a section if isinstance(section[entry], Section): section[entry].configspec = configspec[entry] - + def _write_line(self, indent_string, entry, this_entry, comment): """Write an individual line, for the write method""" @@ -2018,9 +2006,9 @@ def _handle_comment(self, comment): def write(self, outfile=None, section=None): """ Write the current ConfigObj as a file - + tekNico: FIXME: use StringIO instead of real files - + >>> filename = a.filename >>> a.filename = 'test.ini' >>> a.write() @@ -2033,7 +2021,7 @@ def write(self, outfile=None, section=None): if self.indent_type is None: # this can be true if initialised from a dictionary self.indent_type = DEFAULT_INDENT_TYPE - + out = [] cs = self._a_to_u('#') csp = self._a_to_u('# ') @@ -2047,7 +2035,7 @@ def write(self, outfile=None, section=None): if stripped_line and not stripped_line.startswith(cs): line = csp + line out.append(line) - + indent_string = self.indent_type * section.depth for entry in (section.scalars + section.sections): if entry in section.defaults: @@ -2060,7 +2048,7 @@ def write(self, outfile=None, section=None): out.append(indent_string + comment_line) this_entry = section[entry] comment = self._handle_comment(section.inline_comments[entry]) - + if isinstance(this_entry, Section): # a section out.append(self._write_marker( @@ -2075,7 +2063,7 @@ def write(self, outfile=None, section=None): entry, this_entry, comment)) - + if section is self: for line in self.final_comment: line = self._decode_element(line) @@ -2084,10 +2072,10 @@ def write(self, outfile=None, section=None): line = csp + line out.append(line) self.interpolation = int_val - + if section is not self: return out - + if (self.filename is None) and (outfile is None): # output a list of lines # might need to encode @@ -2101,7 +2089,7 @@ def write(self, outfile=None, section=None): out.append('') out[0] = BOM_UTF8 + out[0] return out - + # Turn the list to a string, joined with correct newlines newline = self.newlines or os.linesep if (getattr(outfile, 'mode', None) is not None and outfile.mode == 'w' @@ -2112,7 +2100,7 @@ def write(self, outfile=None, section=None): if not output.endswith(newline): output += newline - if isinstance(output, six.binary_type): + if isinstance(output, bytes): output_bytes = output else: output_bytes = output.encode(self.encoding or @@ -2133,34 +2121,34 @@ def validate(self, validator, preserve_errors=False, copy=False, section=None): """ Test the ConfigObj against a configspec. - + It uses the ``validator`` object from *validate.py*. - + To run ``validate`` on the current ConfigObj, call: :: - + test = config.validate(validator) - + (Normally having previously passed in the configspec when the ConfigObj was created - you can dynamically assign a dictionary of checks to the ``configspec`` attribute of a section though). - + It returns ``True`` if everything passes, or a dictionary of pass/fails (True/False). If every member of a subsection passes, it will just have the value ``True``. (It also returns ``False`` if all members fail). - + In addition, it converts the values from strings to their native types if their checks pass (and ``stringify`` is set). - + If ``preserve_errors`` is ``True`` (``False`` is default) then instead of a marking a fail with a ``False``, it will preserve the actual exception object. This can contain info about the reason for failure. For example the ``VdtValueTooSmallError`` indicates that the value supplied was too small. If a value (or section) is missing it will still be marked as ``False``. - + You must have the validate module to use ``preserve_errors=True``. - + You can then use the ``flatten_errors`` function to turn your nested results dictionary into a flattened list of failures - useful for displaying meaningful error messages. @@ -2171,9 +2159,9 @@ def validate(self, validator, preserve_errors=False, copy=False, if preserve_errors: # We do this once to remove a top level dependency on the validate module # Which makes importing configobj faster - from validate import VdtMissingValue + from .validate import VdtMissingValue self._vdtMissingValue = VdtMissingValue - + section = self if copy: @@ -2183,23 +2171,23 @@ def validate(self, validator, preserve_errors=False, copy=False, section.BOM = section.configspec.BOM section.newlines = section.configspec.newlines section.indent_type = section.configspec.indent_type - + # # section.default_values.clear() #?? configspec = section.configspec self._set_configspec(section, copy) - + def validate_entry(entry, spec, val, missing, ret_true, ret_false): section.default_values.pop(entry, None) - + try: section.default_values[entry] = validator.get_default_value(configspec[entry]) except (KeyError, AttributeError, validator.baseErrorClass): # No default, bad default or validator has no 'get_default_value' # (e.g. SimpleVal) pass - + try: check = validator.check(spec, val, @@ -2233,16 +2221,16 @@ def validate_entry(entry, spec, val, missing, ret_true, ret_false): if not copy and missing and entry not in section.defaults: section.defaults.append(entry) return ret_true, ret_false - + # out = {} ret_true = True ret_false = True - + unvalidated = [k for k in section.scalars if k not in configspec] - incorrect_sections = [k for k in configspec.sections if k in section.scalars] + incorrect_sections = [k for k in configspec.sections if k in section.scalars] incorrect_scalars = [k for k in configspec.scalars if k in section.sections] - + for entry in configspec.scalars: if entry in ('__many__', '___many___'): # reserved names @@ -2262,16 +2250,16 @@ def validate_entry(entry, spec, val, missing, ret_true, ret_false): else: missing = False val = section[entry] - - ret_true, ret_false = validate_entry(entry, configspec[entry], val, + + ret_true, ret_false = validate_entry(entry, configspec[entry], val, missing, ret_true, ret_false) - + many = None if '__many__' in configspec.scalars: many = configspec['__many__'] elif '___many___' in configspec.scalars: many = configspec['___many___'] - + if many is not None: for entry in unvalidated: val = section[entry] @@ -2295,7 +2283,7 @@ def validate_entry(entry, spec, val, missing, ret_true, ret_false): ret_false = False msg = 'Section %r was provided as a single value' % entry out[entry] = validator.baseErrorClass(msg) - + # Missing sections will have been created as empty ones when the # configspec was read. for entry in section.sections: @@ -2316,7 +2304,7 @@ def validate_entry(entry, spec, val, missing, ret_true, ret_false): ret_false = False else: ret_true = False - + section.extra_values = unvalidated if preserve_errors and not section._created: # If the section wasn't created (i.e. it wasn't missing) @@ -2345,16 +2333,16 @@ def reset(self): self.configspec = None # Just to be sure ;-) self._original_configspec = None - - + + def reload(self): """ Reload a ConfigObj from file. - + This method raises a ``ReloadError`` if the ConfigObj doesn't have a filename attribute pointing to a file. """ - if not isinstance(self.filename, six.string_types): + if not isinstance(self.filename, str): raise ReloadError() filename = self.filename @@ -2363,31 +2351,31 @@ def reload(self): if entry == 'configspec': continue current_options[entry] = getattr(self, entry) - + configspec = self._original_configspec current_options['configspec'] = configspec - + self.clear() self._initialise(current_options) self._load(filename, configspec) - + class SimpleVal(object): """ A simple validator. Can be used to check that all members expected are present. - + To use it, provide a configspec with all your members in (the value given will be ignored). Pass an instance of ``SimpleVal`` to the ``validate`` method of your ``ConfigObj``. ``validate`` will return ``True`` if all members are present, or a dictionary with True/False meaning present/missing. (Whole missing sections will be replaced with ``False``) """ - + def __init__(self): self.baseErrorClass = ConfigObjError - + def check(self, check, member, missing=False): """A dummy check method, always returns the value unchanged.""" if missing: @@ -2399,32 +2387,32 @@ def flatten_errors(cfg, res, levels=None, results=None): """ An example function that will turn a nested dictionary of results (as returned by ``ConfigObj.validate``) into a flat list. - + ``cfg`` is the ConfigObj instance being checked, ``res`` is the results dictionary returned by ``validate``. - + (This is a recursive function, so you shouldn't use the ``levels`` or ``results`` arguments - they are used by the function.) - + Returns a list of keys that failed. Each member of the list is a tuple:: - + ([list of sections...], key, result) - + If ``validate`` was called with ``preserve_errors=False`` (the default) then ``result`` will always be ``False``. *list of sections* is a flattened list of sections that the key was found in. - + If the section was missing (or a section was expected and a scalar provided - or vice-versa) then key will be ``None``. - + If the value (or section) was missing then ``result`` will be ``False``. - + If ``validate`` was called with ``preserve_errors=True`` and a value was present, but failed the check, then ``result`` will be the exception object returned. You can use this as a string that describes the failure. - + For example *The value "3" is of the wrong type*. """ if levels is None: @@ -2441,7 +2429,7 @@ def flatten_errors(cfg, res, levels=None, results=None): for (key, val) in list(res.items()): if val == True: continue - if isinstance(cfg.get(key), collections.Mapping): + if isinstance(cfg.get(key), Mapping): # Go down one level levels.append(key) flatten_errors(cfg[key], val, levels, results) @@ -2459,21 +2447,21 @@ def get_extra_values(conf, _prepend=()): """ Find all the values and sections not in the configspec from a validated ConfigObj. - + ``get_extra_values`` returns a list of tuples where each tuple represents either an extra section, or an extra value. - - The tuples contain two values, a tuple representing the section the value + + The tuples contain two values, a tuple representing the section the value is in and the name of the extra values. For extra values in the top level section the first member will be an empty tuple. For values in the 'foo' section the first member will be ``('foo',)``. For members in the 'bar' subsection of the 'foo' section the first member will be ``('foo', 'bar')``. - + NOTE: If you call ``get_extra_values`` on a ConfigObj instance that hasn't been validated it will return an empty list. """ out = [] - + out.extend([(_prepend, name) for name in conf.extra_values]) for name in conf.sections: if name not in conf.extra_values: diff --git a/astropy/extern/configobj/validate.py b/astropy/extern/configobj/validate.py index b7a964c47660..a4cd7ed473d0 100755 --- a/astropy/extern/configobj/validate.py +++ b/astropy/extern/configobj/validate.py @@ -15,114 +15,114 @@ # https://github.com/DiffSK/configobj """ - The Validator object is used to check that supplied values + The Validator object is used to check that supplied values conform to a specification. - + The value can be supplied as a string - e.g. from a config file. In this case the check will also *convert* the value to the required type. This allows you to add validation as a transparent layer to access data stored as strings. The validation checks that the data is correct *and* converts it to the expected type. - + Some standard checks are provided for basic data types. Additional checks are easy to write. They can be provided when the ``Validator`` is instantiated or added afterwards. - + The standard functions work with the following basic data types : - + * integers * floats * booleans * strings * ip_addr - + plus lists of these datatypes - + Adding additional checks is done through coding simple functions. - - The full set of standard checks are : - + + The full set of standard checks are : + * 'integer': matches integer values (including negative) Takes optional 'min' and 'max' arguments : :: - + integer() integer(3, 9) # any value from 3 to 9 integer(min=0) # any positive value integer(max=9) - + * 'float': matches float values Has the same parameters as the integer check. - + * 'boolean': matches boolean values - ``True`` or ``False`` Acceptable string values for True are : true, on, yes, 1 Acceptable string values for False are : false, off, no, 0 - + Any other value raises an error. - + * 'ip_addr': matches an Internet Protocol address, v.4, represented by a dotted-quad string, i.e. '1.2.3.4'. - + * 'string': matches any string. Takes optional keyword args 'min' and 'max' to specify min and max lengths of the string. - + * 'list': matches any list. Takes optional keyword args 'min', and 'max' to specify min and max sizes of the list. (Always returns a list.) - + * 'tuple': matches any tuple. Takes optional keyword args 'min', and 'max' to specify min and max sizes of the tuple. (Always returns a tuple.) - + * 'int_list': Matches a list of integers. Takes the same arguments as list. - + * 'float_list': Matches a list of floats. Takes the same arguments as list. - + * 'bool_list': Matches a list of boolean values. Takes the same arguments as list. - + * 'ip_addr_list': Matches a list of IP addresses. Takes the same arguments as list. - + * 'string_list': Matches a list of strings. Takes the same arguments as list. - - * 'mixed_list': Matches a list with different types in + + * 'mixed_list': Matches a list with different types in specific positions. List size must match the number of arguments. - + Each position can be one of : 'integer', 'float', 'ip_addr', 'string', 'boolean' - + So to specify a list with two strings followed by two integers, you write the check as : :: - + mixed_list('string', 'string', 'integer', 'integer') - + * 'pass': This check matches everything ! It never fails and the value is unchanged. - + It is also the default if no check is specified. - + * 'option': This check matches any from a list of options. You specify this check with : :: - + option('option 1', 'option 2', 'option 3') - + You can supply a default value (returned if no value is supplied) using the default keyword argument. - + You specify a list argument for default using a list constructor syntax in the check : :: - + checkname(arg1, arg2, default=list('val 1', 'val 2', 'val 3')) - + A badly formatted set of arguments will raise a ``VdtParamError``. """ @@ -274,7 +274,7 @@ def bool(val): def dottedQuadToNum(ip): """ Convert decimal dotted quad string to long integer - + >>> int(dottedQuadToNum('1 ')) 1 >>> int(dottedQuadToNum(' 1.2')) @@ -289,10 +289,10 @@ def dottedQuadToNum(ip): Traceback (most recent call last): ValueError: Not a good dotted-quad IP: 255.255.255.256 """ - + # import here to avoid it when ip_addr values are not used import socket, struct - + try: return struct.unpack('!L', socket.inet_aton(ip.strip()))[0] @@ -304,7 +304,7 @@ def dottedQuadToNum(ip): def numToDottedQuad(num): """ Convert int or long int to dotted quad string - + >>> numToDottedQuad(long(-1)) Traceback (most recent call last): ValueError: Not a good numeric IP: -1 @@ -339,10 +339,10 @@ def numToDottedQuad(num): ValueError: Not a good numeric IP: 4294967296 """ - + # import here to avoid it when ip_addr values are not used import socket, struct - + # no need to intercept here, 4294967295L is fine if num > long(4294967295) or num < 0: raise ValueError('Not a good numeric IP: %s' % num) @@ -357,10 +357,10 @@ class ValidateError(Exception): """ This error indicates that the check failed. It can be the base class for more specific errors. - + Any check function that fails ought to raise this error. (or a subclass) - + >>> raise ValidateError Traceback (most recent call last): ValidateError @@ -409,7 +409,7 @@ def __init__(self, value): class VdtValueError(ValidateError): """The value supplied was of the correct type, but was not an allowed value.""" - + def __init__(self, value): """ >>> raise VdtValueError('jedi') @@ -473,18 +473,18 @@ class Validator(object): """ Validator is an object that allows you to register a set of 'checks'. These checks take input and test that it conforms to the check. - + This can also involve converting the value from a string into the correct datatype. - + The ``check`` method takes an input string which configures which check is to be used and applies that check to a supplied value. - + An example input string would be: 'int_range(param1, param2)' - + You would then provide something like: - + >>> def int_range_check(value, min, max): ... # turn min and max from strings to integers ... min = int(min) @@ -507,7 +507,7 @@ class Validator(object): ... if not value <= max: ... raise VdtValueTooBigError(value) ... return value - + >>> fdict = {'int_range': int_range_check} >>> vtr1 = Validator(fdict) >>> vtr1.check('int_range(20, 40)', '30') @@ -515,25 +515,25 @@ class Validator(object): >>> vtr1.check('int_range(20, 40)', '60') Traceback (most recent call last): VdtValueTooBigError: the value "60" is too big. - + New functions can be added with : :: - - >>> vtr2 = Validator() + + >>> vtr2 = Validator() >>> vtr2.functions['int_range'] = int_range_check - - Or by passing in a dictionary of functions when Validator + + Or by passing in a dictionary of functions when Validator is instantiated. - + Your functions *can* use keyword arguments, but the first argument should always be 'value'. - + If the function doesn't take additional arguments, the parentheses are optional in the check. It can be written with either of : :: - + keyword = function_name keyword = function_name() - + The first program to utilise Validator() was Michael Foord's ConfigObj, an alternative to ConfigParser which supports lists and can validate a config file using a config schema. @@ -542,7 +542,7 @@ class Validator(object): """ # this regex does the initial parsing of the checks - _func_re = re.compile(r'(.+?)\((.*)\)', re.DOTALL) + _func_re = re.compile(r'([^\(\)]+?)\((.*)\)', re.DOTALL) # this regex takes apart keyword arguments _key_arg = re.compile(r'^([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*(.*)$', re.DOTALL) @@ -593,35 +593,35 @@ def __init__(self, functions=None): def check(self, check, value, missing=False): """ Usage: check(check, value) - + Arguments: check: string representing check to apply (including arguments) value: object to be checked Returns value, converted to correct type if necessary - + If the check fails, raises a ``ValidateError`` subclass. - + >>> vtor.check('yoda', '') Traceback (most recent call last): VdtUnknownCheckError: the check "yoda" is unknown. >>> vtor.check('yoda()', '') Traceback (most recent call last): VdtUnknownCheckError: the check "yoda" is unknown. - + >>> vtor.check('string(default="")', '', missing=True) '' """ fun_name, fun_args, fun_kwargs, default = self._parse_with_caching(check) - + if missing: if default is None: # no information needed here - to be handled by caller raise VdtMissingValue() value = self._handle_none(default) - + if value is None: return None - + return self._check_value(value, fun_name, fun_args, fun_kwargs) @@ -646,8 +646,8 @@ def _parse_with_caching(self, check): fun_kwargs = dict([(str(key), value) for (key, value) in list(fun_kwargs.items())]) self._cache[check] = fun_name, list(fun_args), dict(fun_kwargs), default return fun_name, fun_args, fun_kwargs, default - - + + def _check_value(self, value, fun_name, fun_args, fun_kwargs): try: fun = self.functions[fun_name] @@ -685,7 +685,7 @@ def _parse_check(self, check): val = self._unquote(val) fun_kwargs[keymatch.group(1)] = val continue - + fun_args.append(self._unquote(arg)) else: # allows for function names without (args) @@ -717,20 +717,20 @@ def _list_handle(self, listmatch): def _pass(self, value): """ Dummy check that always passes - + >>> vtor.check('', 0) 0 >>> vtor.check('', '0') '0' """ return value - - + + def get_default_value(self, check): """ Given a check, return the default value for the check (converted to the right type). - + If the check doesn't specify a default value then a ``KeyError`` will be raised. """ @@ -746,11 +746,11 @@ def get_default_value(self, check): def _is_num_param(names, values, to_float=False): """ Return numbers from inputs or raise VdtParamError. - + Lets ``None`` pass through. Pass in keyword argument ``to_float=True`` to use float for the conversion rather than int. - + >>> _is_num_param(('', ''), (0, 1.0)) [0, 1] >>> _is_num_param(('', ''), (0, 1.0), to_float=True) @@ -785,10 +785,10 @@ def is_integer(value, min=None, max=None): A check that tests that a given value is an integer (int, or long) and optionally, between bounds. A negative value is accepted, while a float will fail. - + If the value is a string, then the conversion is done - if possible. Otherwise a VdtError is raised. - + >>> vtor.check('integer', '-1') -1 >>> vtor.check('integer', '0') @@ -840,17 +840,17 @@ def is_float(value, min=None, max=None): """ A check that tests that a given value is a float (an integer will be accepted), and optionally - that it is between bounds. - + If the value is a string, then the conversion is done - if possible. Otherwise a VdtError is raised. - + This can accept negative values. - + >>> vtor.check('float', '2') 2.0 - + From now on we multiply the value to avoid comparing decimals - + >>> vtor.check('float', '-6.8') * 10 -68.0 >>> vtor.check('float', '12.2') * 10 @@ -889,7 +889,7 @@ def is_float(value, min=None, max=None): bool_dict = { - True: True, 'on': True, '1': True, 'true': True, 'yes': True, + True: True, 'on': True, '1': True, 'true': True, 'yes': True, False: False, 'off': False, '0': False, 'false': False, 'no': False, } @@ -897,7 +897,7 @@ def is_float(value, min=None, max=None): def is_boolean(value): """ Check if the value represents a boolean. - + >>> vtor.check('boolean', 0) 0 >>> vtor.check('boolean', False) @@ -936,7 +936,7 @@ def is_boolean(value): >>> vtor.check('boolean', 'up') Traceback (most recent call last): VdtTypeError: the value "up" is of the wrong type. - + """ if isinstance(value, string_type): try: @@ -958,7 +958,7 @@ def is_ip_addr(value): """ Check that the supplied value is an Internet Protocol address, v.4, represented by a dotted-quad string, i.e. '1.2.3.4'. - + >>> vtor.check('ip_addr', '1 ') '1' >>> vtor.check('ip_addr', ' 1.2') @@ -994,11 +994,11 @@ def is_ip_addr(value): def is_list(value, min=None, max=None): """ Check that the value is a list of values. - + You can optionally specify the minimum and maximum number of members. - + It does no check on list members. - + >>> vtor.check('list', ()) [] >>> vtor.check('list', []) @@ -1039,11 +1039,11 @@ def is_list(value, min=None, max=None): def is_tuple(value, min=None, max=None): """ Check that the value is a tuple of values. - + You can optionally specify the minimum and maximum number of members. - + It does no check on members. - + >>> vtor.check('tuple', ()) () >>> vtor.check('tuple', []) @@ -1073,9 +1073,9 @@ def is_tuple(value, min=None, max=None): def is_string(value, min=None, max=None): """ Check that the supplied value is a string. - + You can optionally specify the minimum and maximum number of members. - + >>> vtor.check('string', '0') '0' >>> vtor.check('string', 0) @@ -1109,11 +1109,11 @@ def is_string(value, min=None, max=None): def is_int_list(value, min=None, max=None): """ Check that the value is a list of integers. - + You can optionally specify the minimum and maximum number of members. - + Each list member is checked that it is an integer. - + >>> vtor.check('int_list', ()) [] >>> vtor.check('int_list', []) @@ -1132,11 +1132,11 @@ def is_int_list(value, min=None, max=None): def is_bool_list(value, min=None, max=None): """ Check that the value is a list of booleans. - + You can optionally specify the minimum and maximum number of members. - + Each list member is checked that it is a boolean. - + >>> vtor.check('bool_list', ()) [] >>> vtor.check('bool_list', []) @@ -1157,11 +1157,11 @@ def is_bool_list(value, min=None, max=None): def is_float_list(value, min=None, max=None): """ Check that the value is a list of floats. - + You can optionally specify the minimum and maximum number of members. - + Each list member is checked that it is a float. - + >>> vtor.check('float_list', ()) [] >>> vtor.check('float_list', []) @@ -1180,11 +1180,11 @@ def is_float_list(value, min=None, max=None): def is_string_list(value, min=None, max=None): """ Check that the value is a list of strings. - + You can optionally specify the minimum and maximum number of members. - + Each list member is checked that it is a string. - + >>> vtor.check('string_list', ()) [] >>> vtor.check('string_list', []) @@ -1206,11 +1206,11 @@ def is_string_list(value, min=None, max=None): def is_ip_addr_list(value, min=None, max=None): """ Check that the value is a list of IP addresses. - + You can optionally specify the minimum and maximum number of members. - + Each list member is checked that it is an IP address. - + >>> vtor.check('ip_addr_list', ()) [] >>> vtor.check('ip_addr_list', []) @@ -1229,11 +1229,11 @@ def force_list(value, min=None, max=None): Check that a value is a list, coercing strings into a list with one member. Useful where users forget the trailing comma that turns a single value into a list. - + You can optionally specify the minimum and maximum number of members. A minumum of greater than one will fail if the user only supplies a string. - + >>> vtor.check('force_list', ()) [] >>> vtor.check('force_list', []) @@ -1244,8 +1244,8 @@ def force_list(value, min=None, max=None): if not isinstance(value, (list, tuple)): value = [value] return is_list(value, min, max) - - + + fun_dict = { 'integer': is_integer, @@ -1261,20 +1261,20 @@ def is_mixed_list(value, *args): Check that the value is a list. Allow specifying the type of each member. Work on lists of specific lengths. - + You specify each member as a positional argument specifying type - + Each type should be one of the following strings : 'integer', 'float', 'ip_addr', 'string', 'boolean' - + So you can specify a list of two strings, followed by two integers as : - + mixed_list('string', 'string', 'integer', 'integer') - + The length of the list must match the number of positional arguments you supply. - + >>> mix_str = "mixed_list('integer', 'float', 'ip_addr', 'string', 'boolean')" >>> check_res = vtor.check(mix_str, (1, 2.0, '1.2.3.4', 'a', True)) >>> check_res == [1, 2.0, '1.2.3.4', 'a', True] @@ -1316,7 +1316,7 @@ def is_mixed_list(value, *args): def is_option(value, *options): """ This check matches the value to any of a set of options. - + >>> vtor.check('option("yoda", "jedi")', 'yoda') 'yoda' >>> vtor.check('option("yoda", "jedi")', 'jed') @@ -1336,7 +1336,7 @@ def is_option(value, *options): def _test(value, *args, **keywargs): """ A function that exists for test purposes. - + >>> checks = [ ... '3, 6, min=1, max=3, test=list(a, b, c)', ... '3', @@ -1370,7 +1370,7 @@ def _test(value, *args, **keywargs): (3, ('3',), {'max': '3', 'test': ['a', 'b', 'c']}) (3, ('3',), {'max': '3', 'test': ["'a'", 'b', 'x=(c)']}) (3, (), {'test': 'x=fish(3)'}) - + >>> v = Validator() >>> v.check('integer(default=6)', '3') 3 @@ -1388,7 +1388,7 @@ def _test(value, *args, **keywargs): KeyError: 'Check "pass" has no default value.' >>> v.get_default_value('pass(default=list(1, 2, 3, 4))') ['1', '2', '3', '4'] - + >>> v = Validator() >>> v.check("pass(default=None)", None, True) >>> v.check("pass(default='None')", None, True) @@ -1397,18 +1397,18 @@ def _test(value, *args, **keywargs): 'None' >>> v.check('pass(default=list(1, 2, 3, 4))', None, True) ['1', '2', '3', '4'] - + Bug test for unicode arguments >>> v = Validator() >>> v.check(unicode('string(min=4)'), unicode('test')) == unicode('test') True - + >>> v = Validator() >>> v.get_default_value(unicode('string(min=4, default="1234")')) == unicode('1234') True >>> v.check(unicode('string(min=4, default="1234")'), unicode('test')) == unicode('test') True - + >>> v = Validator() >>> default = v.get_default_value('string(default=None)') >>> default == None @@ -1419,7 +1419,7 @@ def _test(value, *args, **keywargs): def _test2(): """ - >>> + >>> >>> v = Validator() >>> v.get_default_value('string(default="#ff00dd")') '#ff00dd' @@ -1454,8 +1454,8 @@ def _test3(): >>> vtor.check("string_list(default=list('\n'))", '', missing=True) ['\n'] """ - - + + if __name__ == '__main__': # run the code tests in doctest format import sys diff --git a/astropy/extern/js/jquery-1.11.0.js b/astropy/extern/js/jquery-1.11.0.js deleted file mode 100644 index 3c88fa8b7fd3..000000000000 --- a/astropy/extern/js/jquery-1.11.0.js +++ /dev/null @@ -1,10337 +0,0 @@ -/*! - * jQuery JavaScript Library v1.11.0 - * http://jquery.com/ - * - * Includes Sizzle.js - * http://sizzlejs.com/ - * - * Copyright 2005, 2014 jQuery Foundation, Inc. and other contributors - * Released under the MIT license - * http://jquery.org/license - * - * Date: 2014-01-23T21:02Z - */ - -(function( global, factory ) { - - if ( typeof module === "object" && typeof module.exports === "object" ) { - // For CommonJS and CommonJS-like environments where a proper window is present, - // execute the factory and get jQuery - // For environments that do not inherently posses a window with a document - // (such as Node.js), expose a jQuery-making factory as module.exports - // This accentuates the need for the creation of a real window - // e.g. var jQuery = require("jquery")(window); - // See ticket #14549 for more info - module.exports = global.document ? - factory( global, true ) : - function( w ) { - if ( !w.document ) { - throw new Error( "jQuery requires a window with a document" ); - } - return factory( w ); - }; - } else { - factory( global ); - } - -// Pass this if window is not defined yet -}(typeof window !== "undefined" ? window : this, function( window, noGlobal ) { - -// Can't do this because several apps including ASP.NET trace -// the stack via arguments.caller.callee and Firefox dies if -// you try to trace through "use strict" call chains. (#13335) -// Support: Firefox 18+ -// - -var deletedIds = []; - -var slice = deletedIds.slice; - -var concat = deletedIds.concat; - -var push = deletedIds.push; - -var indexOf = deletedIds.indexOf; - -var class2type = {}; - -var toString = class2type.toString; - -var hasOwn = class2type.hasOwnProperty; - -var trim = "".trim; - -var support = {}; - - - -var - version = "1.11.0", - - // Define a local copy of jQuery - jQuery = function( selector, context ) { - // The jQuery object is actually just the init constructor 'enhanced' - // Need init if jQuery is called (just allow error to be thrown if not included) - return new jQuery.fn.init( selector, context ); - }, - - // Make sure we trim BOM and NBSP (here's looking at you, Safari 5.0 and IE) - rtrim = /^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g, - - // Matches dashed string for camelizing - rmsPrefix = /^-ms-/, - rdashAlpha = /-([\da-z])/gi, - - // Used by jQuery.camelCase as callback to replace() - fcamelCase = function( all, letter ) { - return letter.toUpperCase(); - }; - -jQuery.fn = jQuery.prototype = { - // The current version of jQuery being used - jquery: version, - - constructor: jQuery, - - // Start with an empty selector - selector: "", - - // The default length of a jQuery object is 0 - length: 0, - - toArray: function() { - return slice.call( this ); - }, - - // Get the Nth element in the matched element set OR - // Get the whole matched element set as a clean array - get: function( num ) { - return num != null ? - - // Return a 'clean' array - ( num < 0 ? this[ num + this.length ] : this[ num ] ) : - - // Return just the object - slice.call( this ); - }, - - // Take an array of elements and push it onto the stack - // (returning the new matched element set) - pushStack: function( elems ) { - - // Build a new jQuery matched element set - var ret = jQuery.merge( this.constructor(), elems ); - - // Add the old object onto the stack (as a reference) - ret.prevObject = this; - ret.context = this.context; - - // Return the newly-formed element set - return ret; - }, - - // Execute a callback for every element in the matched set. - // (You can seed the arguments with an array of args, but this is - // only used internally.) - each: function( callback, args ) { - return jQuery.each( this, callback, args ); - }, - - map: function( callback ) { - return this.pushStack( jQuery.map(this, function( elem, i ) { - return callback.call( elem, i, elem ); - })); - }, - - slice: function() { - return this.pushStack( slice.apply( this, arguments ) ); - }, - - first: function() { - return this.eq( 0 ); - }, - - last: function() { - return this.eq( -1 ); - }, - - eq: function( i ) { - var len = this.length, - j = +i + ( i < 0 ? len : 0 ); - return this.pushStack( j >= 0 && j < len ? [ this[j] ] : [] ); - }, - - end: function() { - return this.prevObject || this.constructor(null); - }, - - // For internal use only. - // Behaves like an Array's method, not like a jQuery method. - push: push, - sort: deletedIds.sort, - splice: deletedIds.splice -}; - -jQuery.extend = jQuery.fn.extend = function() { - var src, copyIsArray, copy, name, options, clone, - target = arguments[0] || {}, - i = 1, - length = arguments.length, - deep = false; - - // Handle a deep copy situation - if ( typeof target === "boolean" ) { - deep = target; - - // skip the boolean and the target - target = arguments[ i ] || {}; - i++; - } - - // Handle case when target is a string or something (possible in deep copy) - if ( typeof target !== "object" && !jQuery.isFunction(target) ) { - target = {}; - } - - // extend jQuery itself if only one argument is passed - if ( i === length ) { - target = this; - i--; - } - - for ( ; i < length; i++ ) { - // Only deal with non-null/undefined values - if ( (options = arguments[ i ]) != null ) { - // Extend the base object - for ( name in options ) { - src = target[ name ]; - copy = options[ name ]; - - // Prevent never-ending loop - if ( target === copy ) { - continue; - } - - // Recurse if we're merging plain objects or arrays - if ( deep && copy && ( jQuery.isPlainObject(copy) || (copyIsArray = jQuery.isArray(copy)) ) ) { - if ( copyIsArray ) { - copyIsArray = false; - clone = src && jQuery.isArray(src) ? src : []; - - } else { - clone = src && jQuery.isPlainObject(src) ? src : {}; - } - - // Never move original objects, clone them - target[ name ] = jQuery.extend( deep, clone, copy ); - - // Don't bring in undefined values - } else if ( copy !== undefined ) { - target[ name ] = copy; - } - } - } - } - - // Return the modified object - return target; -}; - -jQuery.extend({ - // Unique for each copy of jQuery on the page - expando: "jQuery" + ( version + Math.random() ).replace( /\D/g, "" ), - - // Assume jQuery is ready without the ready module - isReady: true, - - error: function( msg ) { - throw new Error( msg ); - }, - - noop: function() {}, - - // See test/unit/core.js for details concerning isFunction. - // Since version 1.3, DOM methods and functions like alert - // aren't supported. They return false on IE (#2968). - isFunction: function( obj ) { - return jQuery.type(obj) === "function"; - }, - - isArray: Array.isArray || function( obj ) { - return jQuery.type(obj) === "array"; - }, - - isWindow: function( obj ) { - /* jshint eqeqeq: false */ - return obj != null && obj == obj.window; - }, - - isNumeric: function( obj ) { - // parseFloat NaNs numeric-cast false positives (null|true|false|"") - // ...but misinterprets leading-number strings, particularly hex literals ("0x...") - // subtraction forces infinities to NaN - return obj - parseFloat( obj ) >= 0; - }, - - isEmptyObject: function( obj ) { - var name; - for ( name in obj ) { - return false; - } - return true; - }, - - isPlainObject: function( obj ) { - var key; - - // Must be an Object. - // Because of IE, we also have to check the presence of the constructor property. - // Make sure that DOM nodes and window objects don't pass through, as well - if ( !obj || jQuery.type(obj) !== "object" || obj.nodeType || jQuery.isWindow( obj ) ) { - return false; - } - - try { - // Not own constructor property must be Object - if ( obj.constructor && - !hasOwn.call(obj, "constructor") && - !hasOwn.call(obj.constructor.prototype, "isPrototypeOf") ) { - return false; - } - } catch ( e ) { - // IE8,9 Will throw exceptions on certain host objects #9897 - return false; - } - - // Support: IE<9 - // Handle iteration over inherited properties before own properties. - if ( support.ownLast ) { - for ( key in obj ) { - return hasOwn.call( obj, key ); - } - } - - // Own properties are enumerated firstly, so to speed up, - // if last one is own, then all properties are own. - for ( key in obj ) {} - - return key === undefined || hasOwn.call( obj, key ); - }, - - type: function( obj ) { - if ( obj == null ) { - return obj + ""; - } - return typeof obj === "object" || typeof obj === "function" ? - class2type[ toString.call(obj) ] || "object" : - typeof obj; - }, - - // Evaluates a script in a global context - // Workarounds based on findings by Jim Driscoll - // http://weblogs.java.net/blog/driscoll/archive/2009/09/08/eval-javascript-global-context - globalEval: function( data ) { - if ( data && jQuery.trim( data ) ) { - // We use execScript on Internet Explorer - // We use an anonymous function so that context is window - // rather than jQuery in Firefox - ( window.execScript || function( data ) { - window[ "eval" ].call( window, data ); - } )( data ); - } - }, - - // Convert dashed to camelCase; used by the css and data modules - // Microsoft forgot to hump their vendor prefix (#9572) - camelCase: function( string ) { - return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase ); - }, - - nodeName: function( elem, name ) { - return elem.nodeName && elem.nodeName.toLowerCase() === name.toLowerCase(); - }, - - // args is for internal usage only - each: function( obj, callback, args ) { - var value, - i = 0, - length = obj.length, - isArray = isArraylike( obj ); - - if ( args ) { - if ( isArray ) { - for ( ; i < length; i++ ) { - value = callback.apply( obj[ i ], args ); - - if ( value === false ) { - break; - } - } - } else { - for ( i in obj ) { - value = callback.apply( obj[ i ], args ); - - if ( value === false ) { - break; - } - } - } - - // A special, fast, case for the most common use of each - } else { - if ( isArray ) { - for ( ; i < length; i++ ) { - value = callback.call( obj[ i ], i, obj[ i ] ); - - if ( value === false ) { - break; - } - } - } else { - for ( i in obj ) { - value = callback.call( obj[ i ], i, obj[ i ] ); - - if ( value === false ) { - break; - } - } - } - } - - return obj; - }, - - // Use native String.trim function wherever possible - trim: trim && !trim.call("\uFEFF\xA0") ? - function( text ) { - return text == null ? - "" : - trim.call( text ); - } : - - // Otherwise use our own trimming functionality - function( text ) { - return text == null ? - "" : - ( text + "" ).replace( rtrim, "" ); - }, - - // results is for internal usage only - makeArray: function( arr, results ) { - var ret = results || []; - - if ( arr != null ) { - if ( isArraylike( Object(arr) ) ) { - jQuery.merge( ret, - typeof arr === "string" ? - [ arr ] : arr - ); - } else { - push.call( ret, arr ); - } - } - - return ret; - }, - - inArray: function( elem, arr, i ) { - var len; - - if ( arr ) { - if ( indexOf ) { - return indexOf.call( arr, elem, i ); - } - - len = arr.length; - i = i ? i < 0 ? Math.max( 0, len + i ) : i : 0; - - for ( ; i < len; i++ ) { - // Skip accessing in sparse arrays - if ( i in arr && arr[ i ] === elem ) { - return i; - } - } - } - - return -1; - }, - - merge: function( first, second ) { - var len = +second.length, - j = 0, - i = first.length; - - while ( j < len ) { - first[ i++ ] = second[ j++ ]; - } - - // Support: IE<9 - // Workaround casting of .length to NaN on otherwise arraylike objects (e.g., NodeLists) - if ( len !== len ) { - while ( second[j] !== undefined ) { - first[ i++ ] = second[ j++ ]; - } - } - - first.length = i; - - return first; - }, - - grep: function( elems, callback, invert ) { - var callbackInverse, - matches = [], - i = 0, - length = elems.length, - callbackExpect = !invert; - - // Go through the array, only saving the items - // that pass the validator function - for ( ; i < length; i++ ) { - callbackInverse = !callback( elems[ i ], i ); - if ( callbackInverse !== callbackExpect ) { - matches.push( elems[ i ] ); - } - } - - return matches; - }, - - // arg is for internal usage only - map: function( elems, callback, arg ) { - var value, - i = 0, - length = elems.length, - isArray = isArraylike( elems ), - ret = []; - - // Go through the array, translating each of the items to their new values - if ( isArray ) { - for ( ; i < length; i++ ) { - value = callback( elems[ i ], i, arg ); - - if ( value != null ) { - ret.push( value ); - } - } - - // Go through every key on the object, - } else { - for ( i in elems ) { - value = callback( elems[ i ], i, arg ); - - if ( value != null ) { - ret.push( value ); - } - } - } - - // Flatten any nested arrays - return concat.apply( [], ret ); - }, - - // A global GUID counter for objects - guid: 1, - - // Bind a function to a context, optionally partially applying any - // arguments. - proxy: function( fn, context ) { - var args, proxy, tmp; - - if ( typeof context === "string" ) { - tmp = fn[ context ]; - context = fn; - fn = tmp; - } - - // Quick check to determine if target is callable, in the spec - // this throws a TypeError, but we will just return undefined. - if ( !jQuery.isFunction( fn ) ) { - return undefined; - } - - // Simulated bind - args = slice.call( arguments, 2 ); - proxy = function() { - return fn.apply( context || this, args.concat( slice.call( arguments ) ) ); - }; - - // Set the guid of unique handler to the same of original handler, so it can be removed - proxy.guid = fn.guid = fn.guid || jQuery.guid++; - - return proxy; - }, - - now: function() { - return +( new Date() ); - }, - - // jQuery.support is not used in Core but other projects attach their - // properties to it so it needs to exist. - support: support -}); - -// Populate the class2type map -jQuery.each("Boolean Number String Function Array Date RegExp Object Error".split(" "), function(i, name) { - class2type[ "[object " + name + "]" ] = name.toLowerCase(); -}); - -function isArraylike( obj ) { - var length = obj.length, - type = jQuery.type( obj ); - - if ( type === "function" || jQuery.isWindow( obj ) ) { - return false; - } - - if ( obj.nodeType === 1 && length ) { - return true; - } - - return type === "array" || length === 0 || - typeof length === "number" && length > 0 && ( length - 1 ) in obj; -} -var Sizzle = -/*! - * Sizzle CSS Selector Engine v1.10.16 - * http://sizzlejs.com/ - * - * Copyright 2013 jQuery Foundation, Inc. and other contributors - * Released under the MIT license - * http://jquery.org/license - * - * Date: 2014-01-13 - */ -(function( window ) { - -var i, - support, - Expr, - getText, - isXML, - compile, - outermostContext, - sortInput, - hasDuplicate, - - // Local document vars - setDocument, - document, - docElem, - documentIsHTML, - rbuggyQSA, - rbuggyMatches, - matches, - contains, - - // Instance-specific data - expando = "sizzle" + -(new Date()), - preferredDoc = window.document, - dirruns = 0, - done = 0, - classCache = createCache(), - tokenCache = createCache(), - compilerCache = createCache(), - sortOrder = function( a, b ) { - if ( a === b ) { - hasDuplicate = true; - } - return 0; - }, - - // General-purpose constants - strundefined = typeof undefined, - MAX_NEGATIVE = 1 << 31, - - // Instance methods - hasOwn = ({}).hasOwnProperty, - arr = [], - pop = arr.pop, - push_native = arr.push, - push = arr.push, - slice = arr.slice, - // Use a stripped-down indexOf if we can't use a native one - indexOf = arr.indexOf || function( elem ) { - var i = 0, - len = this.length; - for ( ; i < len; i++ ) { - if ( this[i] === elem ) { - return i; - } - } - return -1; - }, - - booleans = "checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped", - - // Regular expressions - - // Whitespace characters http://www.w3.org/TR/css3-selectors/#whitespace - whitespace = "[\\x20\\t\\r\\n\\f]", - // http://www.w3.org/TR/css3-syntax/#characters - characterEncoding = "(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+", - - // Loosely modeled on CSS identifier characters - // An unquoted value should be a CSS identifier http://www.w3.org/TR/css3-selectors/#attribute-selectors - // Proper syntax: http://www.w3.org/TR/CSS21/syndata.html#value-def-identifier - identifier = characterEncoding.replace( "w", "w#" ), - - // Acceptable operators http://www.w3.org/TR/selectors/#attribute-selectors - attributes = "\\[" + whitespace + "*(" + characterEncoding + ")" + whitespace + - "*(?:([*^$|!~]?=)" + whitespace + "*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|(" + identifier + ")|)|)" + whitespace + "*\\]", - - // Prefer arguments quoted, - // then not containing pseudos/brackets, - // then attribute selectors/non-parenthetical expressions, - // then anything else - // These preferences are here to reduce the number of selectors - // needing tokenize in the PSEUDO preFilter - pseudos = ":(" + characterEncoding + ")(?:\\(((['\"])((?:\\\\.|[^\\\\])*?)\\3|((?:\\\\.|[^\\\\()[\\]]|" + attributes.replace( 3, 8 ) + ")*)|.*)\\)|)", - - // Leading and non-escaped trailing whitespace, capturing some non-whitespace characters preceding the latter - rtrim = new RegExp( "^" + whitespace + "+|((?:^|[^\\\\])(?:\\\\.)*)" + whitespace + "+$", "g" ), - - rcomma = new RegExp( "^" + whitespace + "*," + whitespace + "*" ), - rcombinators = new RegExp( "^" + whitespace + "*([>+~]|" + whitespace + ")" + whitespace + "*" ), - - rattributeQuotes = new RegExp( "=" + whitespace + "*([^\\]'\"]*?)" + whitespace + "*\\]", "g" ), - - rpseudo = new RegExp( pseudos ), - ridentifier = new RegExp( "^" + identifier + "$" ), - - matchExpr = { - "ID": new RegExp( "^#(" + characterEncoding + ")" ), - "CLASS": new RegExp( "^\\.(" + characterEncoding + ")" ), - "TAG": new RegExp( "^(" + characterEncoding.replace( "w", "w*" ) + ")" ), - "ATTR": new RegExp( "^" + attributes ), - "PSEUDO": new RegExp( "^" + pseudos ), - "CHILD": new RegExp( "^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\(" + whitespace + - "*(even|odd|(([+-]|)(\\d*)n|)" + whitespace + "*(?:([+-]|)" + whitespace + - "*(\\d+)|))" + whitespace + "*\\)|)", "i" ), - "bool": new RegExp( "^(?:" + booleans + ")$", "i" ), - // For use in libraries implementing .is() - // We use this for POS matching in `select` - "needsContext": new RegExp( "^" + whitespace + "*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\(" + - whitespace + "*((?:-\\d)?\\d*)" + whitespace + "*\\)|)(?=[^-]|$)", "i" ) - }, - - rinputs = /^(?:input|select|textarea|button)$/i, - rheader = /^h\d$/i, - - rnative = /^[^{]+\{\s*\[native \w/, - - // Easily-parseable/retrievable ID or TAG or CLASS selectors - rquickExpr = /^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/, - - rsibling = /[+~]/, - rescape = /'|\\/g, - - // CSS escapes http://www.w3.org/TR/CSS21/syndata.html#escaped-characters - runescape = new RegExp( "\\\\([\\da-f]{1,6}" + whitespace + "?|(" + whitespace + ")|.)", "ig" ), - funescape = function( _, escaped, escapedWhitespace ) { - var high = "0x" + escaped - 0x10000; - // NaN means non-codepoint - // Support: Firefox - // Workaround erroneous numeric interpretation of +"0x" - return high !== high || escapedWhitespace ? - escaped : - high < 0 ? - // BMP codepoint - String.fromCharCode( high + 0x10000 ) : - // Supplemental Plane codepoint (surrogate pair) - String.fromCharCode( high >> 10 | 0xD800, high & 0x3FF | 0xDC00 ); - }; - -// Optimize for push.apply( _, NodeList ) -try { - push.apply( - (arr = slice.call( preferredDoc.childNodes )), - preferredDoc.childNodes - ); - // Support: Android<4.0 - // Detect silently failing push.apply - arr[ preferredDoc.childNodes.length ].nodeType; -} catch ( e ) { - push = { apply: arr.length ? - - // Leverage slice if possible - function( target, els ) { - push_native.apply( target, slice.call(els) ); - } : - - // Support: IE<9 - // Otherwise append directly - function( target, els ) { - var j = target.length, - i = 0; - // Can't trust NodeList.length - while ( (target[j++] = els[i++]) ) {} - target.length = j - 1; - } - }; -} - -function Sizzle( selector, context, results, seed ) { - var match, elem, m, nodeType, - // QSA vars - i, groups, old, nid, newContext, newSelector; - - if ( ( context ? context.ownerDocument || context : preferredDoc ) !== document ) { - setDocument( context ); - } - - context = context || document; - results = results || []; - - if ( !selector || typeof selector !== "string" ) { - return results; - } - - if ( (nodeType = context.nodeType) !== 1 && nodeType !== 9 ) { - return []; - } - - if ( documentIsHTML && !seed ) { - - // Shortcuts - if ( (match = rquickExpr.exec( selector )) ) { - // Speed-up: Sizzle("#ID") - if ( (m = match[1]) ) { - if ( nodeType === 9 ) { - elem = context.getElementById( m ); - // Check parentNode to catch when Blackberry 4.6 returns - // nodes that are no longer in the document (jQuery #6963) - if ( elem && elem.parentNode ) { - // Handle the case where IE, Opera, and Webkit return items - // by name instead of ID - if ( elem.id === m ) { - results.push( elem ); - return results; - } - } else { - return results; - } - } else { - // Context is not a document - if ( context.ownerDocument && (elem = context.ownerDocument.getElementById( m )) && - contains( context, elem ) && elem.id === m ) { - results.push( elem ); - return results; - } - } - - // Speed-up: Sizzle("TAG") - } else if ( match[2] ) { - push.apply( results, context.getElementsByTagName( selector ) ); - return results; - - // Speed-up: Sizzle(".CLASS") - } else if ( (m = match[3]) && support.getElementsByClassName && context.getElementsByClassName ) { - push.apply( results, context.getElementsByClassName( m ) ); - return results; - } - } - - // QSA path - if ( support.qsa && (!rbuggyQSA || !rbuggyQSA.test( selector )) ) { - nid = old = expando; - newContext = context; - newSelector = nodeType === 9 && selector; - - // qSA works strangely on Element-rooted queries - // We can work around this by specifying an extra ID on the root - // and working up from there (Thanks to Andrew Dupont for the technique) - // IE 8 doesn't work on object elements - if ( nodeType === 1 && context.nodeName.toLowerCase() !== "object" ) { - groups = tokenize( selector ); - - if ( (old = context.getAttribute("id")) ) { - nid = old.replace( rescape, "\\$&" ); - } else { - context.setAttribute( "id", nid ); - } - nid = "[id='" + nid + "'] "; - - i = groups.length; - while ( i-- ) { - groups[i] = nid + toSelector( groups[i] ); - } - newContext = rsibling.test( selector ) && testContext( context.parentNode ) || context; - newSelector = groups.join(","); - } - - if ( newSelector ) { - try { - push.apply( results, - newContext.querySelectorAll( newSelector ) - ); - return results; - } catch(qsaError) { - } finally { - if ( !old ) { - context.removeAttribute("id"); - } - } - } - } - } - - // All others - return select( selector.replace( rtrim, "$1" ), context, results, seed ); -} - -/** - * Create key-value caches of limited size - * @returns {Function(string, Object)} Returns the Object data after storing it on itself with - * property name the (space-suffixed) string and (if the cache is larger than Expr.cacheLength) - * deleting the oldest entry - */ -function createCache() { - var keys = []; - - function cache( key, value ) { - // Use (key + " ") to avoid collision with native prototype properties (see Issue #157) - if ( keys.push( key + " " ) > Expr.cacheLength ) { - // Only keep the most recent entries - delete cache[ keys.shift() ]; - } - return (cache[ key + " " ] = value); - } - return cache; -} - -/** - * Mark a function for special use by Sizzle - * @param {Function} fn The function to mark - */ -function markFunction( fn ) { - fn[ expando ] = true; - return fn; -} - -/** - * Support testing using an element - * @param {Function} fn Passed the created div and expects a boolean result - */ -function assert( fn ) { - var div = document.createElement("div"); - - try { - return !!fn( div ); - } catch (e) { - return false; - } finally { - // Remove from its parent by default - if ( div.parentNode ) { - div.parentNode.removeChild( div ); - } - // release memory in IE - div = null; - } -} - -/** - * Adds the same handler for all of the specified attrs - * @param {String} attrs Pipe-separated list of attributes - * @param {Function} handler The method that will be applied - */ -function addHandle( attrs, handler ) { - var arr = attrs.split("|"), - i = attrs.length; - - while ( i-- ) { - Expr.attrHandle[ arr[i] ] = handler; - } -} - -/** - * Checks document order of two siblings - * @param {Element} a - * @param {Element} b - * @returns {Number} Returns less than 0 if a precedes b, greater than 0 if a follows b - */ -function siblingCheck( a, b ) { - var cur = b && a, - diff = cur && a.nodeType === 1 && b.nodeType === 1 && - ( ~b.sourceIndex || MAX_NEGATIVE ) - - ( ~a.sourceIndex || MAX_NEGATIVE ); - - // Use IE sourceIndex if available on both nodes - if ( diff ) { - return diff; - } - - // Check if b follows a - if ( cur ) { - while ( (cur = cur.nextSibling) ) { - if ( cur === b ) { - return -1; - } - } - } - - return a ? 1 : -1; -} - -/** - * Returns a function to use in pseudos for input types - * @param {String} type - */ -function createInputPseudo( type ) { - return function( elem ) { - var name = elem.nodeName.toLowerCase(); - return name === "input" && elem.type === type; - }; -} - -/** - * Returns a function to use in pseudos for buttons - * @param {String} type - */ -function createButtonPseudo( type ) { - return function( elem ) { - var name = elem.nodeName.toLowerCase(); - return (name === "input" || name === "button") && elem.type === type; - }; -} - -/** - * Returns a function to use in pseudos for positionals - * @param {Function} fn - */ -function createPositionalPseudo( fn ) { - return markFunction(function( argument ) { - argument = +argument; - return markFunction(function( seed, matches ) { - var j, - matchIndexes = fn( [], seed.length, argument ), - i = matchIndexes.length; - - // Match elements found at the specified indexes - while ( i-- ) { - if ( seed[ (j = matchIndexes[i]) ] ) { - seed[j] = !(matches[j] = seed[j]); - } - } - }); - }); -} - -/** - * Checks a node for validity as a Sizzle context - * @param {Element|Object=} context - * @returns {Element|Object|Boolean} The input node if acceptable, otherwise a falsy value - */ -function testContext( context ) { - return context && typeof context.getElementsByTagName !== strundefined && context; -} - -// Expose support vars for convenience -support = Sizzle.support = {}; - -/** - * Detects XML nodes - * @param {Element|Object} elem An element or a document - * @returns {Boolean} True iff elem is a non-HTML XML node - */ -isXML = Sizzle.isXML = function( elem ) { - // documentElement is verified for cases where it doesn't yet exist - // (such as loading iframes in IE - #4833) - var documentElement = elem && (elem.ownerDocument || elem).documentElement; - return documentElement ? documentElement.nodeName !== "HTML" : false; -}; - -/** - * Sets document-related variables once based on the current document - * @param {Element|Object} [doc] An element or document object to use to set the document - * @returns {Object} Returns the current document - */ -setDocument = Sizzle.setDocument = function( node ) { - var hasCompare, - doc = node ? node.ownerDocument || node : preferredDoc, - parent = doc.defaultView; - - // If no document and documentElement is available, return - if ( doc === document || doc.nodeType !== 9 || !doc.documentElement ) { - return document; - } - - // Set our document - document = doc; - docElem = doc.documentElement; - - // Support tests - documentIsHTML = !isXML( doc ); - - // Support: IE>8 - // If iframe document is assigned to "document" variable and if iframe has been reloaded, - // IE will throw "permission denied" error when accessing "document" variable, see jQuery #13936 - // IE6-8 do not support the defaultView property so parent will be undefined - if ( parent && parent !== parent.top ) { - // IE11 does not have attachEvent, so all must suffer - if ( parent.addEventListener ) { - parent.addEventListener( "unload", function() { - setDocument(); - }, false ); - } else if ( parent.attachEvent ) { - parent.attachEvent( "onunload", function() { - setDocument(); - }); - } - } - - /* Attributes - ---------------------------------------------------------------------- */ - - // Support: IE<8 - // Verify that getAttribute really returns attributes and not properties (excepting IE8 booleans) - support.attributes = assert(function( div ) { - div.className = "i"; - return !div.getAttribute("className"); - }); - - /* getElement(s)By* - ---------------------------------------------------------------------- */ - - // Check if getElementsByTagName("*") returns only elements - support.getElementsByTagName = assert(function( div ) { - div.appendChild( doc.createComment("") ); - return !div.getElementsByTagName("*").length; - }); - - // Check if getElementsByClassName can be trusted - support.getElementsByClassName = rnative.test( doc.getElementsByClassName ) && assert(function( div ) { - div.innerHTML = "
"; - - // Support: Safari<4 - // Catch class over-caching - div.firstChild.className = "i"; - // Support: Opera<10 - // Catch gEBCN failure to find non-leading classes - return div.getElementsByClassName("i").length === 2; - }); - - // Support: IE<10 - // Check if getElementById returns elements by name - // The broken getElementById methods don't pick up programatically-set names, - // so use a roundabout getElementsByName test - support.getById = assert(function( div ) { - docElem.appendChild( div ).id = expando; - return !doc.getElementsByName || !doc.getElementsByName( expando ).length; - }); - - // ID find and filter - if ( support.getById ) { - Expr.find["ID"] = function( id, context ) { - if ( typeof context.getElementById !== strundefined && documentIsHTML ) { - var m = context.getElementById( id ); - // Check parentNode to catch when Blackberry 4.6 returns - // nodes that are no longer in the document #6963 - return m && m.parentNode ? [m] : []; - } - }; - Expr.filter["ID"] = function( id ) { - var attrId = id.replace( runescape, funescape ); - return function( elem ) { - return elem.getAttribute("id") === attrId; - }; - }; - } else { - // Support: IE6/7 - // getElementById is not reliable as a find shortcut - delete Expr.find["ID"]; - - Expr.filter["ID"] = function( id ) { - var attrId = id.replace( runescape, funescape ); - return function( elem ) { - var node = typeof elem.getAttributeNode !== strundefined && elem.getAttributeNode("id"); - return node && node.value === attrId; - }; - }; - } - - // Tag - Expr.find["TAG"] = support.getElementsByTagName ? - function( tag, context ) { - if ( typeof context.getElementsByTagName !== strundefined ) { - return context.getElementsByTagName( tag ); - } - } : - function( tag, context ) { - var elem, - tmp = [], - i = 0, - results = context.getElementsByTagName( tag ); - - // Filter out possible comments - if ( tag === "*" ) { - while ( (elem = results[i++]) ) { - if ( elem.nodeType === 1 ) { - tmp.push( elem ); - } - } - - return tmp; - } - return results; - }; - - // Class - Expr.find["CLASS"] = support.getElementsByClassName && function( className, context ) { - if ( typeof context.getElementsByClassName !== strundefined && documentIsHTML ) { - return context.getElementsByClassName( className ); - } - }; - - /* QSA/matchesSelector - ---------------------------------------------------------------------- */ - - // QSA and matchesSelector support - - // matchesSelector(:active) reports false when true (IE9/Opera 11.5) - rbuggyMatches = []; - - // qSa(:focus) reports false when true (Chrome 21) - // We allow this because of a bug in IE8/9 that throws an error - // whenever `document.activeElement` is accessed on an iframe - // So, we allow :focus to pass through QSA all the time to avoid the IE error - // See http://bugs.jquery.com/ticket/13378 - rbuggyQSA = []; - - if ( (support.qsa = rnative.test( doc.querySelectorAll )) ) { - // Build QSA regex - // Regex strategy adopted from Diego Perini - assert(function( div ) { - // Select is set to empty string on purpose - // This is to test IE's treatment of not explicitly - // setting a boolean content attribute, - // since its presence should be enough - // http://bugs.jquery.com/ticket/12359 - div.innerHTML = ""; - - // Support: IE8, Opera 10-12 - // Nothing should be selected when empty strings follow ^= or $= or *= - if ( div.querySelectorAll("[t^='']").length ) { - rbuggyQSA.push( "[*^$]=" + whitespace + "*(?:''|\"\")" ); - } - - // Support: IE8 - // Boolean attributes and "value" are not treated correctly - if ( !div.querySelectorAll("[selected]").length ) { - rbuggyQSA.push( "\\[" + whitespace + "*(?:value|" + booleans + ")" ); - } - - // Webkit/Opera - :checked should return selected option elements - // http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked - // IE8 throws error here and will not see later tests - if ( !div.querySelectorAll(":checked").length ) { - rbuggyQSA.push(":checked"); - } - }); - - assert(function( div ) { - // Support: Windows 8 Native Apps - // The type and name attributes are restricted during .innerHTML assignment - var input = doc.createElement("input"); - input.setAttribute( "type", "hidden" ); - div.appendChild( input ).setAttribute( "name", "D" ); - - // Support: IE8 - // Enforce case-sensitivity of name attribute - if ( div.querySelectorAll("[name=d]").length ) { - rbuggyQSA.push( "name" + whitespace + "*[*^$|!~]?=" ); - } - - // FF 3.5 - :enabled/:disabled and hidden elements (hidden elements are still enabled) - // IE8 throws error here and will not see later tests - if ( !div.querySelectorAll(":enabled").length ) { - rbuggyQSA.push( ":enabled", ":disabled" ); - } - - // Opera 10-11 does not throw on post-comma invalid pseudos - div.querySelectorAll("*,:x"); - rbuggyQSA.push(",.*:"); - }); - } - - if ( (support.matchesSelector = rnative.test( (matches = docElem.webkitMatchesSelector || - docElem.mozMatchesSelector || - docElem.oMatchesSelector || - docElem.msMatchesSelector) )) ) { - - assert(function( div ) { - // Check to see if it's possible to do matchesSelector - // on a disconnected node (IE 9) - support.disconnectedMatch = matches.call( div, "div" ); - - // This should fail with an exception - // Gecko does not error, returns false instead - matches.call( div, "[s!='']:x" ); - rbuggyMatches.push( "!=", pseudos ); - }); - } - - rbuggyQSA = rbuggyQSA.length && new RegExp( rbuggyQSA.join("|") ); - rbuggyMatches = rbuggyMatches.length && new RegExp( rbuggyMatches.join("|") ); - - /* Contains - ---------------------------------------------------------------------- */ - hasCompare = rnative.test( docElem.compareDocumentPosition ); - - // Element contains another - // Purposefully does not implement inclusive descendent - // As in, an element does not contain itself - contains = hasCompare || rnative.test( docElem.contains ) ? - function( a, b ) { - var adown = a.nodeType === 9 ? a.documentElement : a, - bup = b && b.parentNode; - return a === bup || !!( bup && bup.nodeType === 1 && ( - adown.contains ? - adown.contains( bup ) : - a.compareDocumentPosition && a.compareDocumentPosition( bup ) & 16 - )); - } : - function( a, b ) { - if ( b ) { - while ( (b = b.parentNode) ) { - if ( b === a ) { - return true; - } - } - } - return false; - }; - - /* Sorting - ---------------------------------------------------------------------- */ - - // Document order sorting - sortOrder = hasCompare ? - function( a, b ) { - - // Flag for duplicate removal - if ( a === b ) { - hasDuplicate = true; - return 0; - } - - // Sort on method existence if only one input has compareDocumentPosition - var compare = !a.compareDocumentPosition - !b.compareDocumentPosition; - if ( compare ) { - return compare; - } - - // Calculate position if both inputs belong to the same document - compare = ( a.ownerDocument || a ) === ( b.ownerDocument || b ) ? - a.compareDocumentPosition( b ) : - - // Otherwise we know they are disconnected - 1; - - // Disconnected nodes - if ( compare & 1 || - (!support.sortDetached && b.compareDocumentPosition( a ) === compare) ) { - - // Choose the first element that is related to our preferred document - if ( a === doc || a.ownerDocument === preferredDoc && contains(preferredDoc, a) ) { - return -1; - } - if ( b === doc || b.ownerDocument === preferredDoc && contains(preferredDoc, b) ) { - return 1; - } - - // Maintain original order - return sortInput ? - ( indexOf.call( sortInput, a ) - indexOf.call( sortInput, b ) ) : - 0; - } - - return compare & 4 ? -1 : 1; - } : - function( a, b ) { - // Exit early if the nodes are identical - if ( a === b ) { - hasDuplicate = true; - return 0; - } - - var cur, - i = 0, - aup = a.parentNode, - bup = b.parentNode, - ap = [ a ], - bp = [ b ]; - - // Parentless nodes are either documents or disconnected - if ( !aup || !bup ) { - return a === doc ? -1 : - b === doc ? 1 : - aup ? -1 : - bup ? 1 : - sortInput ? - ( indexOf.call( sortInput, a ) - indexOf.call( sortInput, b ) ) : - 0; - - // If the nodes are siblings, we can do a quick check - } else if ( aup === bup ) { - return siblingCheck( a, b ); - } - - // Otherwise we need full lists of their ancestors for comparison - cur = a; - while ( (cur = cur.parentNode) ) { - ap.unshift( cur ); - } - cur = b; - while ( (cur = cur.parentNode) ) { - bp.unshift( cur ); - } - - // Walk down the tree looking for a discrepancy - while ( ap[i] === bp[i] ) { - i++; - } - - return i ? - // Do a sibling check if the nodes have a common ancestor - siblingCheck( ap[i], bp[i] ) : - - // Otherwise nodes in our document sort first - ap[i] === preferredDoc ? -1 : - bp[i] === preferredDoc ? 1 : - 0; - }; - - return doc; -}; - -Sizzle.matches = function( expr, elements ) { - return Sizzle( expr, null, null, elements ); -}; - -Sizzle.matchesSelector = function( elem, expr ) { - // Set document vars if needed - if ( ( elem.ownerDocument || elem ) !== document ) { - setDocument( elem ); - } - - // Make sure that attribute selectors are quoted - expr = expr.replace( rattributeQuotes, "='$1']" ); - - if ( support.matchesSelector && documentIsHTML && - ( !rbuggyMatches || !rbuggyMatches.test( expr ) ) && - ( !rbuggyQSA || !rbuggyQSA.test( expr ) ) ) { - - try { - var ret = matches.call( elem, expr ); - - // IE 9's matchesSelector returns false on disconnected nodes - if ( ret || support.disconnectedMatch || - // As well, disconnected nodes are said to be in a document - // fragment in IE 9 - elem.document && elem.document.nodeType !== 11 ) { - return ret; - } - } catch(e) {} - } - - return Sizzle( expr, document, null, [elem] ).length > 0; -}; - -Sizzle.contains = function( context, elem ) { - // Set document vars if needed - if ( ( context.ownerDocument || context ) !== document ) { - setDocument( context ); - } - return contains( context, elem ); -}; - -Sizzle.attr = function( elem, name ) { - // Set document vars if needed - if ( ( elem.ownerDocument || elem ) !== document ) { - setDocument( elem ); - } - - var fn = Expr.attrHandle[ name.toLowerCase() ], - // Don't get fooled by Object.prototype properties (jQuery #13807) - val = fn && hasOwn.call( Expr.attrHandle, name.toLowerCase() ) ? - fn( elem, name, !documentIsHTML ) : - undefined; - - return val !== undefined ? - val : - support.attributes || !documentIsHTML ? - elem.getAttribute( name ) : - (val = elem.getAttributeNode(name)) && val.specified ? - val.value : - null; -}; - -Sizzle.error = function( msg ) { - throw new Error( "Syntax error, unrecognized expression: " + msg ); -}; - -/** - * Document sorting and removing duplicates - * @param {ArrayLike} results - */ -Sizzle.uniqueSort = function( results ) { - var elem, - duplicates = [], - j = 0, - i = 0; - - // Unless we *know* we can detect duplicates, assume their presence - hasDuplicate = !support.detectDuplicates; - sortInput = !support.sortStable && results.slice( 0 ); - results.sort( sortOrder ); - - if ( hasDuplicate ) { - while ( (elem = results[i++]) ) { - if ( elem === results[ i ] ) { - j = duplicates.push( i ); - } - } - while ( j-- ) { - results.splice( duplicates[ j ], 1 ); - } - } - - // Clear input after sorting to release objects - // See https://github.com/jquery/sizzle/pull/225 - sortInput = null; - - return results; -}; - -/** - * Utility function for retrieving the text value of an array of DOM nodes - * @param {Array|Element} elem - */ -getText = Sizzle.getText = function( elem ) { - var node, - ret = "", - i = 0, - nodeType = elem.nodeType; - - if ( !nodeType ) { - // If no nodeType, this is expected to be an array - while ( (node = elem[i++]) ) { - // Do not traverse comment nodes - ret += getText( node ); - } - } else if ( nodeType === 1 || nodeType === 9 || nodeType === 11 ) { - // Use textContent for elements - // innerText usage removed for consistency of new lines (jQuery #11153) - if ( typeof elem.textContent === "string" ) { - return elem.textContent; - } else { - // Traverse its children - for ( elem = elem.firstChild; elem; elem = elem.nextSibling ) { - ret += getText( elem ); - } - } - } else if ( nodeType === 3 || nodeType === 4 ) { - return elem.nodeValue; - } - // Do not include comment or processing instruction nodes - - return ret; -}; - -Expr = Sizzle.selectors = { - - // Can be adjusted by the user - cacheLength: 50, - - createPseudo: markFunction, - - match: matchExpr, - - attrHandle: {}, - - find: {}, - - relative: { - ">": { dir: "parentNode", first: true }, - " ": { dir: "parentNode" }, - "+": { dir: "previousSibling", first: true }, - "~": { dir: "previousSibling" } - }, - - preFilter: { - "ATTR": function( match ) { - match[1] = match[1].replace( runescape, funescape ); - - // Move the given value to match[3] whether quoted or unquoted - match[3] = ( match[4] || match[5] || "" ).replace( runescape, funescape ); - - if ( match[2] === "~=" ) { - match[3] = " " + match[3] + " "; - } - - return match.slice( 0, 4 ); - }, - - "CHILD": function( match ) { - /* matches from matchExpr["CHILD"] - 1 type (only|nth|...) - 2 what (child|of-type) - 3 argument (even|odd|\d*|\d*n([+-]\d+)?|...) - 4 xn-component of xn+y argument ([+-]?\d*n|) - 5 sign of xn-component - 6 x of xn-component - 7 sign of y-component - 8 y of y-component - */ - match[1] = match[1].toLowerCase(); - - if ( match[1].slice( 0, 3 ) === "nth" ) { - // nth-* requires argument - if ( !match[3] ) { - Sizzle.error( match[0] ); - } - - // numeric x and y parameters for Expr.filter.CHILD - // remember that false/true cast respectively to 0/1 - match[4] = +( match[4] ? match[5] + (match[6] || 1) : 2 * ( match[3] === "even" || match[3] === "odd" ) ); - match[5] = +( ( match[7] + match[8] ) || match[3] === "odd" ); - - // other types prohibit arguments - } else if ( match[3] ) { - Sizzle.error( match[0] ); - } - - return match; - }, - - "PSEUDO": function( match ) { - var excess, - unquoted = !match[5] && match[2]; - - if ( matchExpr["CHILD"].test( match[0] ) ) { - return null; - } - - // Accept quoted arguments as-is - if ( match[3] && match[4] !== undefined ) { - match[2] = match[4]; - - // Strip excess characters from unquoted arguments - } else if ( unquoted && rpseudo.test( unquoted ) && - // Get excess from tokenize (recursively) - (excess = tokenize( unquoted, true )) && - // advance to the next closing parenthesis - (excess = unquoted.indexOf( ")", unquoted.length - excess ) - unquoted.length) ) { - - // excess is a negative index - match[0] = match[0].slice( 0, excess ); - match[2] = unquoted.slice( 0, excess ); - } - - // Return only captures needed by the pseudo filter method (type and argument) - return match.slice( 0, 3 ); - } - }, - - filter: { - - "TAG": function( nodeNameSelector ) { - var nodeName = nodeNameSelector.replace( runescape, funescape ).toLowerCase(); - return nodeNameSelector === "*" ? - function() { return true; } : - function( elem ) { - return elem.nodeName && elem.nodeName.toLowerCase() === nodeName; - }; - }, - - "CLASS": function( className ) { - var pattern = classCache[ className + " " ]; - - return pattern || - (pattern = new RegExp( "(^|" + whitespace + ")" + className + "(" + whitespace + "|$)" )) && - classCache( className, function( elem ) { - return pattern.test( typeof elem.className === "string" && elem.className || typeof elem.getAttribute !== strundefined && elem.getAttribute("class") || "" ); - }); - }, - - "ATTR": function( name, operator, check ) { - return function( elem ) { - var result = Sizzle.attr( elem, name ); - - if ( result == null ) { - return operator === "!="; - } - if ( !operator ) { - return true; - } - - result += ""; - - return operator === "=" ? result === check : - operator === "!=" ? result !== check : - operator === "^=" ? check && result.indexOf( check ) === 0 : - operator === "*=" ? check && result.indexOf( check ) > -1 : - operator === "$=" ? check && result.slice( -check.length ) === check : - operator === "~=" ? ( " " + result + " " ).indexOf( check ) > -1 : - operator === "|=" ? result === check || result.slice( 0, check.length + 1 ) === check + "-" : - false; - }; - }, - - "CHILD": function( type, what, argument, first, last ) { - var simple = type.slice( 0, 3 ) !== "nth", - forward = type.slice( -4 ) !== "last", - ofType = what === "of-type"; - - return first === 1 && last === 0 ? - - // Shortcut for :nth-*(n) - function( elem ) { - return !!elem.parentNode; - } : - - function( elem, context, xml ) { - var cache, outerCache, node, diff, nodeIndex, start, - dir = simple !== forward ? "nextSibling" : "previousSibling", - parent = elem.parentNode, - name = ofType && elem.nodeName.toLowerCase(), - useCache = !xml && !ofType; - - if ( parent ) { - - // :(first|last|only)-(child|of-type) - if ( simple ) { - while ( dir ) { - node = elem; - while ( (node = node[ dir ]) ) { - if ( ofType ? node.nodeName.toLowerCase() === name : node.nodeType === 1 ) { - return false; - } - } - // Reverse direction for :only-* (if we haven't yet done so) - start = dir = type === "only" && !start && "nextSibling"; - } - return true; - } - - start = [ forward ? parent.firstChild : parent.lastChild ]; - - // non-xml :nth-child(...) stores cache data on `parent` - if ( forward && useCache ) { - // Seek `elem` from a previously-cached index - outerCache = parent[ expando ] || (parent[ expando ] = {}); - cache = outerCache[ type ] || []; - nodeIndex = cache[0] === dirruns && cache[1]; - diff = cache[0] === dirruns && cache[2]; - node = nodeIndex && parent.childNodes[ nodeIndex ]; - - while ( (node = ++nodeIndex && node && node[ dir ] || - - // Fallback to seeking `elem` from the start - (diff = nodeIndex = 0) || start.pop()) ) { - - // When found, cache indexes on `parent` and break - if ( node.nodeType === 1 && ++diff && node === elem ) { - outerCache[ type ] = [ dirruns, nodeIndex, diff ]; - break; - } - } - - // Use previously-cached element index if available - } else if ( useCache && (cache = (elem[ expando ] || (elem[ expando ] = {}))[ type ]) && cache[0] === dirruns ) { - diff = cache[1]; - - // xml :nth-child(...) or :nth-last-child(...) or :nth(-last)?-of-type(...) - } else { - // Use the same loop as above to seek `elem` from the start - while ( (node = ++nodeIndex && node && node[ dir ] || - (diff = nodeIndex = 0) || start.pop()) ) { - - if ( ( ofType ? node.nodeName.toLowerCase() === name : node.nodeType === 1 ) && ++diff ) { - // Cache the index of each encountered element - if ( useCache ) { - (node[ expando ] || (node[ expando ] = {}))[ type ] = [ dirruns, diff ]; - } - - if ( node === elem ) { - break; - } - } - } - } - - // Incorporate the offset, then check against cycle size - diff -= last; - return diff === first || ( diff % first === 0 && diff / first >= 0 ); - } - }; - }, - - "PSEUDO": function( pseudo, argument ) { - // pseudo-class names are case-insensitive - // http://www.w3.org/TR/selectors/#pseudo-classes - // Prioritize by case sensitivity in case custom pseudos are added with uppercase letters - // Remember that setFilters inherits from pseudos - var args, - fn = Expr.pseudos[ pseudo ] || Expr.setFilters[ pseudo.toLowerCase() ] || - Sizzle.error( "unsupported pseudo: " + pseudo ); - - // The user may use createPseudo to indicate that - // arguments are needed to create the filter function - // just as Sizzle does - if ( fn[ expando ] ) { - return fn( argument ); - } - - // But maintain support for old signatures - if ( fn.length > 1 ) { - args = [ pseudo, pseudo, "", argument ]; - return Expr.setFilters.hasOwnProperty( pseudo.toLowerCase() ) ? - markFunction(function( seed, matches ) { - var idx, - matched = fn( seed, argument ), - i = matched.length; - while ( i-- ) { - idx = indexOf.call( seed, matched[i] ); - seed[ idx ] = !( matches[ idx ] = matched[i] ); - } - }) : - function( elem ) { - return fn( elem, 0, args ); - }; - } - - return fn; - } - }, - - pseudos: { - // Potentially complex pseudos - "not": markFunction(function( selector ) { - // Trim the selector passed to compile - // to avoid treating leading and trailing - // spaces as combinators - var input = [], - results = [], - matcher = compile( selector.replace( rtrim, "$1" ) ); - - return matcher[ expando ] ? - markFunction(function( seed, matches, context, xml ) { - var elem, - unmatched = matcher( seed, null, xml, [] ), - i = seed.length; - - // Match elements unmatched by `matcher` - while ( i-- ) { - if ( (elem = unmatched[i]) ) { - seed[i] = !(matches[i] = elem); - } - } - }) : - function( elem, context, xml ) { - input[0] = elem; - matcher( input, null, xml, results ); - return !results.pop(); - }; - }), - - "has": markFunction(function( selector ) { - return function( elem ) { - return Sizzle( selector, elem ).length > 0; - }; - }), - - "contains": markFunction(function( text ) { - return function( elem ) { - return ( elem.textContent || elem.innerText || getText( elem ) ).indexOf( text ) > -1; - }; - }), - - // "Whether an element is represented by a :lang() selector - // is based solely on the element's language value - // being equal to the identifier C, - // or beginning with the identifier C immediately followed by "-". - // The matching of C against the element's language value is performed case-insensitively. - // The identifier C does not have to be a valid language name." - // http://www.w3.org/TR/selectors/#lang-pseudo - "lang": markFunction( function( lang ) { - // lang value must be a valid identifier - if ( !ridentifier.test(lang || "") ) { - Sizzle.error( "unsupported lang: " + lang ); - } - lang = lang.replace( runescape, funescape ).toLowerCase(); - return function( elem ) { - var elemLang; - do { - if ( (elemLang = documentIsHTML ? - elem.lang : - elem.getAttribute("xml:lang") || elem.getAttribute("lang")) ) { - - elemLang = elemLang.toLowerCase(); - return elemLang === lang || elemLang.indexOf( lang + "-" ) === 0; - } - } while ( (elem = elem.parentNode) && elem.nodeType === 1 ); - return false; - }; - }), - - // Miscellaneous - "target": function( elem ) { - var hash = window.location && window.location.hash; - return hash && hash.slice( 1 ) === elem.id; - }, - - "root": function( elem ) { - return elem === docElem; - }, - - "focus": function( elem ) { - return elem === document.activeElement && (!document.hasFocus || document.hasFocus()) && !!(elem.type || elem.href || ~elem.tabIndex); - }, - - // Boolean properties - "enabled": function( elem ) { - return elem.disabled === false; - }, - - "disabled": function( elem ) { - return elem.disabled === true; - }, - - "checked": function( elem ) { - // In CSS3, :checked should return both checked and selected elements - // http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked - var nodeName = elem.nodeName.toLowerCase(); - return (nodeName === "input" && !!elem.checked) || (nodeName === "option" && !!elem.selected); - }, - - "selected": function( elem ) { - // Accessing this property makes selected-by-default - // options in Safari work properly - if ( elem.parentNode ) { - elem.parentNode.selectedIndex; - } - - return elem.selected === true; - }, - - // Contents - "empty": function( elem ) { - // http://www.w3.org/TR/selectors/#empty-pseudo - // :empty is negated by element (1) or content nodes (text: 3; cdata: 4; entity ref: 5), - // but not by others (comment: 8; processing instruction: 7; etc.) - // nodeType < 6 works because attributes (2) do not appear as children - for ( elem = elem.firstChild; elem; elem = elem.nextSibling ) { - if ( elem.nodeType < 6 ) { - return false; - } - } - return true; - }, - - "parent": function( elem ) { - return !Expr.pseudos["empty"]( elem ); - }, - - // Element/input types - "header": function( elem ) { - return rheader.test( elem.nodeName ); - }, - - "input": function( elem ) { - return rinputs.test( elem.nodeName ); - }, - - "button": function( elem ) { - var name = elem.nodeName.toLowerCase(); - return name === "input" && elem.type === "button" || name === "button"; - }, - - "text": function( elem ) { - var attr; - return elem.nodeName.toLowerCase() === "input" && - elem.type === "text" && - - // Support: IE<8 - // New HTML5 attribute values (e.g., "search") appear with elem.type === "text" - ( (attr = elem.getAttribute("type")) == null || attr.toLowerCase() === "text" ); - }, - - // Position-in-collection - "first": createPositionalPseudo(function() { - return [ 0 ]; - }), - - "last": createPositionalPseudo(function( matchIndexes, length ) { - return [ length - 1 ]; - }), - - "eq": createPositionalPseudo(function( matchIndexes, length, argument ) { - return [ argument < 0 ? argument + length : argument ]; - }), - - "even": createPositionalPseudo(function( matchIndexes, length ) { - var i = 0; - for ( ; i < length; i += 2 ) { - matchIndexes.push( i ); - } - return matchIndexes; - }), - - "odd": createPositionalPseudo(function( matchIndexes, length ) { - var i = 1; - for ( ; i < length; i += 2 ) { - matchIndexes.push( i ); - } - return matchIndexes; - }), - - "lt": createPositionalPseudo(function( matchIndexes, length, argument ) { - var i = argument < 0 ? argument + length : argument; - for ( ; --i >= 0; ) { - matchIndexes.push( i ); - } - return matchIndexes; - }), - - "gt": createPositionalPseudo(function( matchIndexes, length, argument ) { - var i = argument < 0 ? argument + length : argument; - for ( ; ++i < length; ) { - matchIndexes.push( i ); - } - return matchIndexes; - }) - } -}; - -Expr.pseudos["nth"] = Expr.pseudos["eq"]; - -// Add button/input type pseudos -for ( i in { radio: true, checkbox: true, file: true, password: true, image: true } ) { - Expr.pseudos[ i ] = createInputPseudo( i ); -} -for ( i in { submit: true, reset: true } ) { - Expr.pseudos[ i ] = createButtonPseudo( i ); -} - -// Easy API for creating new setFilters -function setFilters() {} -setFilters.prototype = Expr.filters = Expr.pseudos; -Expr.setFilters = new setFilters(); - -function tokenize( selector, parseOnly ) { - var matched, match, tokens, type, - soFar, groups, preFilters, - cached = tokenCache[ selector + " " ]; - - if ( cached ) { - return parseOnly ? 0 : cached.slice( 0 ); - } - - soFar = selector; - groups = []; - preFilters = Expr.preFilter; - - while ( soFar ) { - - // Comma and first run - if ( !matched || (match = rcomma.exec( soFar )) ) { - if ( match ) { - // Don't consume trailing commas as valid - soFar = soFar.slice( match[0].length ) || soFar; - } - groups.push( (tokens = []) ); - } - - matched = false; - - // Combinators - if ( (match = rcombinators.exec( soFar )) ) { - matched = match.shift(); - tokens.push({ - value: matched, - // Cast descendant combinators to space - type: match[0].replace( rtrim, " " ) - }); - soFar = soFar.slice( matched.length ); - } - - // Filters - for ( type in Expr.filter ) { - if ( (match = matchExpr[ type ].exec( soFar )) && (!preFilters[ type ] || - (match = preFilters[ type ]( match ))) ) { - matched = match.shift(); - tokens.push({ - value: matched, - type: type, - matches: match - }); - soFar = soFar.slice( matched.length ); - } - } - - if ( !matched ) { - break; - } - } - - // Return the length of the invalid excess - // if we're just parsing - // Otherwise, throw an error or return tokens - return parseOnly ? - soFar.length : - soFar ? - Sizzle.error( selector ) : - // Cache the tokens - tokenCache( selector, groups ).slice( 0 ); -} - -function toSelector( tokens ) { - var i = 0, - len = tokens.length, - selector = ""; - for ( ; i < len; i++ ) { - selector += tokens[i].value; - } - return selector; -} - -function addCombinator( matcher, combinator, base ) { - var dir = combinator.dir, - checkNonElements = base && dir === "parentNode", - doneName = done++; - - return combinator.first ? - // Check against closest ancestor/preceding element - function( elem, context, xml ) { - while ( (elem = elem[ dir ]) ) { - if ( elem.nodeType === 1 || checkNonElements ) { - return matcher( elem, context, xml ); - } - } - } : - - // Check against all ancestor/preceding elements - function( elem, context, xml ) { - var oldCache, outerCache, - newCache = [ dirruns, doneName ]; - - // We can't set arbitrary data on XML nodes, so they don't benefit from dir caching - if ( xml ) { - while ( (elem = elem[ dir ]) ) { - if ( elem.nodeType === 1 || checkNonElements ) { - if ( matcher( elem, context, xml ) ) { - return true; - } - } - } - } else { - while ( (elem = elem[ dir ]) ) { - if ( elem.nodeType === 1 || checkNonElements ) { - outerCache = elem[ expando ] || (elem[ expando ] = {}); - if ( (oldCache = outerCache[ dir ]) && - oldCache[ 0 ] === dirruns && oldCache[ 1 ] === doneName ) { - - // Assign to newCache so results back-propagate to previous elements - return (newCache[ 2 ] = oldCache[ 2 ]); - } else { - // Reuse newcache so results back-propagate to previous elements - outerCache[ dir ] = newCache; - - // A match means we're done; a fail means we have to keep checking - if ( (newCache[ 2 ] = matcher( elem, context, xml )) ) { - return true; - } - } - } - } - } - }; -} - -function elementMatcher( matchers ) { - return matchers.length > 1 ? - function( elem, context, xml ) { - var i = matchers.length; - while ( i-- ) { - if ( !matchers[i]( elem, context, xml ) ) { - return false; - } - } - return true; - } : - matchers[0]; -} - -function condense( unmatched, map, filter, context, xml ) { - var elem, - newUnmatched = [], - i = 0, - len = unmatched.length, - mapped = map != null; - - for ( ; i < len; i++ ) { - if ( (elem = unmatched[i]) ) { - if ( !filter || filter( elem, context, xml ) ) { - newUnmatched.push( elem ); - if ( mapped ) { - map.push( i ); - } - } - } - } - - return newUnmatched; -} - -function setMatcher( preFilter, selector, matcher, postFilter, postFinder, postSelector ) { - if ( postFilter && !postFilter[ expando ] ) { - postFilter = setMatcher( postFilter ); - } - if ( postFinder && !postFinder[ expando ] ) { - postFinder = setMatcher( postFinder, postSelector ); - } - return markFunction(function( seed, results, context, xml ) { - var temp, i, elem, - preMap = [], - postMap = [], - preexisting = results.length, - - // Get initial elements from seed or context - elems = seed || multipleContexts( selector || "*", context.nodeType ? [ context ] : context, [] ), - - // Prefilter to get matcher input, preserving a map for seed-results synchronization - matcherIn = preFilter && ( seed || !selector ) ? - condense( elems, preMap, preFilter, context, xml ) : - elems, - - matcherOut = matcher ? - // If we have a postFinder, or filtered seed, or non-seed postFilter or preexisting results, - postFinder || ( seed ? preFilter : preexisting || postFilter ) ? - - // ...intermediate processing is necessary - [] : - - // ...otherwise use results directly - results : - matcherIn; - - // Find primary matches - if ( matcher ) { - matcher( matcherIn, matcherOut, context, xml ); - } - - // Apply postFilter - if ( postFilter ) { - temp = condense( matcherOut, postMap ); - postFilter( temp, [], context, xml ); - - // Un-match failing elements by moving them back to matcherIn - i = temp.length; - while ( i-- ) { - if ( (elem = temp[i]) ) { - matcherOut[ postMap[i] ] = !(matcherIn[ postMap[i] ] = elem); - } - } - } - - if ( seed ) { - if ( postFinder || preFilter ) { - if ( postFinder ) { - // Get the final matcherOut by condensing this intermediate into postFinder contexts - temp = []; - i = matcherOut.length; - while ( i-- ) { - if ( (elem = matcherOut[i]) ) { - // Restore matcherIn since elem is not yet a final match - temp.push( (matcherIn[i] = elem) ); - } - } - postFinder( null, (matcherOut = []), temp, xml ); - } - - // Move matched elements from seed to results to keep them synchronized - i = matcherOut.length; - while ( i-- ) { - if ( (elem = matcherOut[i]) && - (temp = postFinder ? indexOf.call( seed, elem ) : preMap[i]) > -1 ) { - - seed[temp] = !(results[temp] = elem); - } - } - } - - // Add elements to results, through postFinder if defined - } else { - matcherOut = condense( - matcherOut === results ? - matcherOut.splice( preexisting, matcherOut.length ) : - matcherOut - ); - if ( postFinder ) { - postFinder( null, results, matcherOut, xml ); - } else { - push.apply( results, matcherOut ); - } - } - }); -} - -function matcherFromTokens( tokens ) { - var checkContext, matcher, j, - len = tokens.length, - leadingRelative = Expr.relative[ tokens[0].type ], - implicitRelative = leadingRelative || Expr.relative[" "], - i = leadingRelative ? 1 : 0, - - // The foundational matcher ensures that elements are reachable from top-level context(s) - matchContext = addCombinator( function( elem ) { - return elem === checkContext; - }, implicitRelative, true ), - matchAnyContext = addCombinator( function( elem ) { - return indexOf.call( checkContext, elem ) > -1; - }, implicitRelative, true ), - matchers = [ function( elem, context, xml ) { - return ( !leadingRelative && ( xml || context !== outermostContext ) ) || ( - (checkContext = context).nodeType ? - matchContext( elem, context, xml ) : - matchAnyContext( elem, context, xml ) ); - } ]; - - for ( ; i < len; i++ ) { - if ( (matcher = Expr.relative[ tokens[i].type ]) ) { - matchers = [ addCombinator(elementMatcher( matchers ), matcher) ]; - } else { - matcher = Expr.filter[ tokens[i].type ].apply( null, tokens[i].matches ); - - // Return special upon seeing a positional matcher - if ( matcher[ expando ] ) { - // Find the next relative operator (if any) for proper handling - j = ++i; - for ( ; j < len; j++ ) { - if ( Expr.relative[ tokens[j].type ] ) { - break; - } - } - return setMatcher( - i > 1 && elementMatcher( matchers ), - i > 1 && toSelector( - // If the preceding token was a descendant combinator, insert an implicit any-element `*` - tokens.slice( 0, i - 1 ).concat({ value: tokens[ i - 2 ].type === " " ? "*" : "" }) - ).replace( rtrim, "$1" ), - matcher, - i < j && matcherFromTokens( tokens.slice( i, j ) ), - j < len && matcherFromTokens( (tokens = tokens.slice( j )) ), - j < len && toSelector( tokens ) - ); - } - matchers.push( matcher ); - } - } - - return elementMatcher( matchers ); -} - -function matcherFromGroupMatchers( elementMatchers, setMatchers ) { - var bySet = setMatchers.length > 0, - byElement = elementMatchers.length > 0, - superMatcher = function( seed, context, xml, results, outermost ) { - var elem, j, matcher, - matchedCount = 0, - i = "0", - unmatched = seed && [], - setMatched = [], - contextBackup = outermostContext, - // We must always have either seed elements or outermost context - elems = seed || byElement && Expr.find["TAG"]( "*", outermost ), - // Use integer dirruns iff this is the outermost matcher - dirrunsUnique = (dirruns += contextBackup == null ? 1 : Math.random() || 0.1), - len = elems.length; - - if ( outermost ) { - outermostContext = context !== document && context; - } - - // Add elements passing elementMatchers directly to results - // Keep `i` a string if there are no elements so `matchedCount` will be "00" below - // Support: IE<9, Safari - // Tolerate NodeList properties (IE: "length"; Safari: ) matching elements by id - for ( ; i !== len && (elem = elems[i]) != null; i++ ) { - if ( byElement && elem ) { - j = 0; - while ( (matcher = elementMatchers[j++]) ) { - if ( matcher( elem, context, xml ) ) { - results.push( elem ); - break; - } - } - if ( outermost ) { - dirruns = dirrunsUnique; - } - } - - // Track unmatched elements for set filters - if ( bySet ) { - // They will have gone through all possible matchers - if ( (elem = !matcher && elem) ) { - matchedCount--; - } - - // Lengthen the array for every element, matched or not - if ( seed ) { - unmatched.push( elem ); - } - } - } - - // Apply set filters to unmatched elements - matchedCount += i; - if ( bySet && i !== matchedCount ) { - j = 0; - while ( (matcher = setMatchers[j++]) ) { - matcher( unmatched, setMatched, context, xml ); - } - - if ( seed ) { - // Reintegrate element matches to eliminate the need for sorting - if ( matchedCount > 0 ) { - while ( i-- ) { - if ( !(unmatched[i] || setMatched[i]) ) { - setMatched[i] = pop.call( results ); - } - } - } - - // Discard index placeholder values to get only actual matches - setMatched = condense( setMatched ); - } - - // Add matches to results - push.apply( results, setMatched ); - - // Seedless set matches succeeding multiple successful matchers stipulate sorting - if ( outermost && !seed && setMatched.length > 0 && - ( matchedCount + setMatchers.length ) > 1 ) { - - Sizzle.uniqueSort( results ); - } - } - - // Override manipulation of globals by nested matchers - if ( outermost ) { - dirruns = dirrunsUnique; - outermostContext = contextBackup; - } - - return unmatched; - }; - - return bySet ? - markFunction( superMatcher ) : - superMatcher; -} - -compile = Sizzle.compile = function( selector, group /* Internal Use Only */ ) { - var i, - setMatchers = [], - elementMatchers = [], - cached = compilerCache[ selector + " " ]; - - if ( !cached ) { - // Generate a function of recursive functions that can be used to check each element - if ( !group ) { - group = tokenize( selector ); - } - i = group.length; - while ( i-- ) { - cached = matcherFromTokens( group[i] ); - if ( cached[ expando ] ) { - setMatchers.push( cached ); - } else { - elementMatchers.push( cached ); - } - } - - // Cache the compiled function - cached = compilerCache( selector, matcherFromGroupMatchers( elementMatchers, setMatchers ) ); - } - return cached; -}; - -function multipleContexts( selector, contexts, results ) { - var i = 0, - len = contexts.length; - for ( ; i < len; i++ ) { - Sizzle( selector, contexts[i], results ); - } - return results; -} - -function select( selector, context, results, seed ) { - var i, tokens, token, type, find, - match = tokenize( selector ); - - if ( !seed ) { - // Try to minimize operations if there is only one group - if ( match.length === 1 ) { - - // Take a shortcut and set the context if the root selector is an ID - tokens = match[0] = match[0].slice( 0 ); - if ( tokens.length > 2 && (token = tokens[0]).type === "ID" && - support.getById && context.nodeType === 9 && documentIsHTML && - Expr.relative[ tokens[1].type ] ) { - - context = ( Expr.find["ID"]( token.matches[0].replace(runescape, funescape), context ) || [] )[0]; - if ( !context ) { - return results; - } - selector = selector.slice( tokens.shift().value.length ); - } - - // Fetch a seed set for right-to-left matching - i = matchExpr["needsContext"].test( selector ) ? 0 : tokens.length; - while ( i-- ) { - token = tokens[i]; - - // Abort if we hit a combinator - if ( Expr.relative[ (type = token.type) ] ) { - break; - } - if ( (find = Expr.find[ type ]) ) { - // Search, expanding context for leading sibling combinators - if ( (seed = find( - token.matches[0].replace( runescape, funescape ), - rsibling.test( tokens[0].type ) && testContext( context.parentNode ) || context - )) ) { - - // If seed is empty or no tokens remain, we can return early - tokens.splice( i, 1 ); - selector = seed.length && toSelector( tokens ); - if ( !selector ) { - push.apply( results, seed ); - return results; - } - - break; - } - } - } - } - } - - // Compile and execute a filtering function - // Provide `match` to avoid retokenization if we modified the selector above - compile( selector, match )( - seed, - context, - !documentIsHTML, - results, - rsibling.test( selector ) && testContext( context.parentNode ) || context - ); - return results; -} - -// One-time assignments - -// Sort stability -support.sortStable = expando.split("").sort( sortOrder ).join("") === expando; - -// Support: Chrome<14 -// Always assume duplicates if they aren't passed to the comparison function -support.detectDuplicates = !!hasDuplicate; - -// Initialize against the default document -setDocument(); - -// Support: Webkit<537.32 - Safari 6.0.3/Chrome 25 (fixed in Chrome 27) -// Detached nodes confoundingly follow *each other* -support.sortDetached = assert(function( div1 ) { - // Should return 1, but returns 4 (following) - return div1.compareDocumentPosition( document.createElement("div") ) & 1; -}); - -// Support: IE<8 -// Prevent attribute/property "interpolation" -// http://msdn.microsoft.com/en-us/library/ms536429%28VS.85%29.aspx -if ( !assert(function( div ) { - div.innerHTML = ""; - return div.firstChild.getAttribute("href") === "#" ; -}) ) { - addHandle( "type|href|height|width", function( elem, name, isXML ) { - if ( !isXML ) { - return elem.getAttribute( name, name.toLowerCase() === "type" ? 1 : 2 ); - } - }); -} - -// Support: IE<9 -// Use defaultValue in place of getAttribute("value") -if ( !support.attributes || !assert(function( div ) { - div.innerHTML = ""; - div.firstChild.setAttribute( "value", "" ); - return div.firstChild.getAttribute( "value" ) === ""; -}) ) { - addHandle( "value", function( elem, name, isXML ) { - if ( !isXML && elem.nodeName.toLowerCase() === "input" ) { - return elem.defaultValue; - } - }); -} - -// Support: IE<9 -// Use getAttributeNode to fetch booleans when getAttribute lies -if ( !assert(function( div ) { - return div.getAttribute("disabled") == null; -}) ) { - addHandle( booleans, function( elem, name, isXML ) { - var val; - if ( !isXML ) { - return elem[ name ] === true ? name.toLowerCase() : - (val = elem.getAttributeNode( name )) && val.specified ? - val.value : - null; - } - }); -} - -return Sizzle; - -})( window ); - - - -jQuery.find = Sizzle; -jQuery.expr = Sizzle.selectors; -jQuery.expr[":"] = jQuery.expr.pseudos; -jQuery.unique = Sizzle.uniqueSort; -jQuery.text = Sizzle.getText; -jQuery.isXMLDoc = Sizzle.isXML; -jQuery.contains = Sizzle.contains; - - - -var rneedsContext = jQuery.expr.match.needsContext; - -var rsingleTag = (/^<(\w+)\s*\/?>(?:<\/\1>|)$/); - - - -var risSimple = /^.[^:#\[\.,]*$/; - -// Implement the identical functionality for filter and not -function winnow( elements, qualifier, not ) { - if ( jQuery.isFunction( qualifier ) ) { - return jQuery.grep( elements, function( elem, i ) { - /* jshint -W018 */ - return !!qualifier.call( elem, i, elem ) !== not; - }); - - } - - if ( qualifier.nodeType ) { - return jQuery.grep( elements, function( elem ) { - return ( elem === qualifier ) !== not; - }); - - } - - if ( typeof qualifier === "string" ) { - if ( risSimple.test( qualifier ) ) { - return jQuery.filter( qualifier, elements, not ); - } - - qualifier = jQuery.filter( qualifier, elements ); - } - - return jQuery.grep( elements, function( elem ) { - return ( jQuery.inArray( elem, qualifier ) >= 0 ) !== not; - }); -} - -jQuery.filter = function( expr, elems, not ) { - var elem = elems[ 0 ]; - - if ( not ) { - expr = ":not(" + expr + ")"; - } - - return elems.length === 1 && elem.nodeType === 1 ? - jQuery.find.matchesSelector( elem, expr ) ? [ elem ] : [] : - jQuery.find.matches( expr, jQuery.grep( elems, function( elem ) { - return elem.nodeType === 1; - })); -}; - -jQuery.fn.extend({ - find: function( selector ) { - var i, - ret = [], - self = this, - len = self.length; - - if ( typeof selector !== "string" ) { - return this.pushStack( jQuery( selector ).filter(function() { - for ( i = 0; i < len; i++ ) { - if ( jQuery.contains( self[ i ], this ) ) { - return true; - } - } - }) ); - } - - for ( i = 0; i < len; i++ ) { - jQuery.find( selector, self[ i ], ret ); - } - - // Needed because $( selector, context ) becomes $( context ).find( selector ) - ret = this.pushStack( len > 1 ? jQuery.unique( ret ) : ret ); - ret.selector = this.selector ? this.selector + " " + selector : selector; - return ret; - }, - filter: function( selector ) { - return this.pushStack( winnow(this, selector || [], false) ); - }, - not: function( selector ) { - return this.pushStack( winnow(this, selector || [], true) ); - }, - is: function( selector ) { - return !!winnow( - this, - - // If this is a positional/relative selector, check membership in the returned set - // so $("p:first").is("p:last") won't return true for a doc with two "p". - typeof selector === "string" && rneedsContext.test( selector ) ? - jQuery( selector ) : - selector || [], - false - ).length; - } -}); - - -// Initialize a jQuery object - - -// A central reference to the root jQuery(document) -var rootjQuery, - - // Use the correct document accordingly with window argument (sandbox) - document = window.document, - - // A simple way to check for HTML strings - // Prioritize #id over to avoid XSS via location.hash (#9521) - // Strict HTML recognition (#11290: must start with <) - rquickExpr = /^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/, - - init = jQuery.fn.init = function( selector, context ) { - var match, elem; - - // HANDLE: $(""), $(null), $(undefined), $(false) - if ( !selector ) { - return this; - } - - // Handle HTML strings - if ( typeof selector === "string" ) { - if ( selector.charAt(0) === "<" && selector.charAt( selector.length - 1 ) === ">" && selector.length >= 3 ) { - // Assume that strings that start and end with <> are HTML and skip the regex check - match = [ null, selector, null ]; - - } else { - match = rquickExpr.exec( selector ); - } - - // Match html or make sure no context is specified for #id - if ( match && (match[1] || !context) ) { - - // HANDLE: $(html) -> $(array) - if ( match[1] ) { - context = context instanceof jQuery ? context[0] : context; - - // scripts is true for back-compat - // Intentionally let the error be thrown if parseHTML is not present - jQuery.merge( this, jQuery.parseHTML( - match[1], - context && context.nodeType ? context.ownerDocument || context : document, - true - ) ); - - // HANDLE: $(html, props) - if ( rsingleTag.test( match[1] ) && jQuery.isPlainObject( context ) ) { - for ( match in context ) { - // Properties of context are called as methods if possible - if ( jQuery.isFunction( this[ match ] ) ) { - this[ match ]( context[ match ] ); - - // ...and otherwise set as attributes - } else { - this.attr( match, context[ match ] ); - } - } - } - - return this; - - // HANDLE: $(#id) - } else { - elem = document.getElementById( match[2] ); - - // Check parentNode to catch when Blackberry 4.6 returns - // nodes that are no longer in the document #6963 - if ( elem && elem.parentNode ) { - // Handle the case where IE and Opera return items - // by name instead of ID - if ( elem.id !== match[2] ) { - return rootjQuery.find( selector ); - } - - // Otherwise, we inject the element directly into the jQuery object - this.length = 1; - this[0] = elem; - } - - this.context = document; - this.selector = selector; - return this; - } - - // HANDLE: $(expr, $(...)) - } else if ( !context || context.jquery ) { - return ( context || rootjQuery ).find( selector ); - - // HANDLE: $(expr, context) - // (which is just equivalent to: $(context).find(expr) - } else { - return this.constructor( context ).find( selector ); - } - - // HANDLE: $(DOMElement) - } else if ( selector.nodeType ) { - this.context = this[0] = selector; - this.length = 1; - return this; - - // HANDLE: $(function) - // Shortcut for document ready - } else if ( jQuery.isFunction( selector ) ) { - return typeof rootjQuery.ready !== "undefined" ? - rootjQuery.ready( selector ) : - // Execute immediately if ready is not present - selector( jQuery ); - } - - if ( selector.selector !== undefined ) { - this.selector = selector.selector; - this.context = selector.context; - } - - return jQuery.makeArray( selector, this ); - }; - -// Give the init function the jQuery prototype for later instantiation -init.prototype = jQuery.fn; - -// Initialize central reference -rootjQuery = jQuery( document ); - - -var rparentsprev = /^(?:parents|prev(?:Until|All))/, - // methods guaranteed to produce a unique set when starting from a unique set - guaranteedUnique = { - children: true, - contents: true, - next: true, - prev: true - }; - -jQuery.extend({ - dir: function( elem, dir, until ) { - var matched = [], - cur = elem[ dir ]; - - while ( cur && cur.nodeType !== 9 && (until === undefined || cur.nodeType !== 1 || !jQuery( cur ).is( until )) ) { - if ( cur.nodeType === 1 ) { - matched.push( cur ); - } - cur = cur[dir]; - } - return matched; - }, - - sibling: function( n, elem ) { - var r = []; - - for ( ; n; n = n.nextSibling ) { - if ( n.nodeType === 1 && n !== elem ) { - r.push( n ); - } - } - - return r; - } -}); - -jQuery.fn.extend({ - has: function( target ) { - var i, - targets = jQuery( target, this ), - len = targets.length; - - return this.filter(function() { - for ( i = 0; i < len; i++ ) { - if ( jQuery.contains( this, targets[i] ) ) { - return true; - } - } - }); - }, - - closest: function( selectors, context ) { - var cur, - i = 0, - l = this.length, - matched = [], - pos = rneedsContext.test( selectors ) || typeof selectors !== "string" ? - jQuery( selectors, context || this.context ) : - 0; - - for ( ; i < l; i++ ) { - for ( cur = this[i]; cur && cur !== context; cur = cur.parentNode ) { - // Always skip document fragments - if ( cur.nodeType < 11 && (pos ? - pos.index(cur) > -1 : - - // Don't pass non-elements to Sizzle - cur.nodeType === 1 && - jQuery.find.matchesSelector(cur, selectors)) ) { - - matched.push( cur ); - break; - } - } - } - - return this.pushStack( matched.length > 1 ? jQuery.unique( matched ) : matched ); - }, - - // Determine the position of an element within - // the matched set of elements - index: function( elem ) { - - // No argument, return index in parent - if ( !elem ) { - return ( this[0] && this[0].parentNode ) ? this.first().prevAll().length : -1; - } - - // index in selector - if ( typeof elem === "string" ) { - return jQuery.inArray( this[0], jQuery( elem ) ); - } - - // Locate the position of the desired element - return jQuery.inArray( - // If it receives a jQuery object, the first element is used - elem.jquery ? elem[0] : elem, this ); - }, - - add: function( selector, context ) { - return this.pushStack( - jQuery.unique( - jQuery.merge( this.get(), jQuery( selector, context ) ) - ) - ); - }, - - addBack: function( selector ) { - return this.add( selector == null ? - this.prevObject : this.prevObject.filter(selector) - ); - } -}); - -function sibling( cur, dir ) { - do { - cur = cur[ dir ]; - } while ( cur && cur.nodeType !== 1 ); - - return cur; -} - -jQuery.each({ - parent: function( elem ) { - var parent = elem.parentNode; - return parent && parent.nodeType !== 11 ? parent : null; - }, - parents: function( elem ) { - return jQuery.dir( elem, "parentNode" ); - }, - parentsUntil: function( elem, i, until ) { - return jQuery.dir( elem, "parentNode", until ); - }, - next: function( elem ) { - return sibling( elem, "nextSibling" ); - }, - prev: function( elem ) { - return sibling( elem, "previousSibling" ); - }, - nextAll: function( elem ) { - return jQuery.dir( elem, "nextSibling" ); - }, - prevAll: function( elem ) { - return jQuery.dir( elem, "previousSibling" ); - }, - nextUntil: function( elem, i, until ) { - return jQuery.dir( elem, "nextSibling", until ); - }, - prevUntil: function( elem, i, until ) { - return jQuery.dir( elem, "previousSibling", until ); - }, - siblings: function( elem ) { - return jQuery.sibling( ( elem.parentNode || {} ).firstChild, elem ); - }, - children: function( elem ) { - return jQuery.sibling( elem.firstChild ); - }, - contents: function( elem ) { - return jQuery.nodeName( elem, "iframe" ) ? - elem.contentDocument || elem.contentWindow.document : - jQuery.merge( [], elem.childNodes ); - } -}, function( name, fn ) { - jQuery.fn[ name ] = function( until, selector ) { - var ret = jQuery.map( this, fn, until ); - - if ( name.slice( -5 ) !== "Until" ) { - selector = until; - } - - if ( selector && typeof selector === "string" ) { - ret = jQuery.filter( selector, ret ); - } - - if ( this.length > 1 ) { - // Remove duplicates - if ( !guaranteedUnique[ name ] ) { - ret = jQuery.unique( ret ); - } - - // Reverse order for parents* and prev-derivatives - if ( rparentsprev.test( name ) ) { - ret = ret.reverse(); - } - } - - return this.pushStack( ret ); - }; -}); -var rnotwhite = (/\S+/g); - - - -// String to Object options format cache -var optionsCache = {}; - -// Convert String-formatted options into Object-formatted ones and store in cache -function createOptions( options ) { - var object = optionsCache[ options ] = {}; - jQuery.each( options.match( rnotwhite ) || [], function( _, flag ) { - object[ flag ] = true; - }); - return object; -} - -/* - * Create a callback list using the following parameters: - * - * options: an optional list of space-separated options that will change how - * the callback list behaves or a more traditional option object - * - * By default a callback list will act like an event callback list and can be - * "fired" multiple times. - * - * Possible options: - * - * once: will ensure the callback list can only be fired once (like a Deferred) - * - * memory: will keep track of previous values and will call any callback added - * after the list has been fired right away with the latest "memorized" - * values (like a Deferred) - * - * unique: will ensure a callback can only be added once (no duplicate in the list) - * - * stopOnFalse: interrupt callings when a callback returns false - * - */ -jQuery.Callbacks = function( options ) { - - // Convert options from String-formatted to Object-formatted if needed - // (we check in cache first) - options = typeof options === "string" ? - ( optionsCache[ options ] || createOptions( options ) ) : - jQuery.extend( {}, options ); - - var // Flag to know if list is currently firing - firing, - // Last fire value (for non-forgettable lists) - memory, - // Flag to know if list was already fired - fired, - // End of the loop when firing - firingLength, - // Index of currently firing callback (modified by remove if needed) - firingIndex, - // First callback to fire (used internally by add and fireWith) - firingStart, - // Actual callback list - list = [], - // Stack of fire calls for repeatable lists - stack = !options.once && [], - // Fire callbacks - fire = function( data ) { - memory = options.memory && data; - fired = true; - firingIndex = firingStart || 0; - firingStart = 0; - firingLength = list.length; - firing = true; - for ( ; list && firingIndex < firingLength; firingIndex++ ) { - if ( list[ firingIndex ].apply( data[ 0 ], data[ 1 ] ) === false && options.stopOnFalse ) { - memory = false; // To prevent further calls using add - break; - } - } - firing = false; - if ( list ) { - if ( stack ) { - if ( stack.length ) { - fire( stack.shift() ); - } - } else if ( memory ) { - list = []; - } else { - self.disable(); - } - } - }, - // Actual Callbacks object - self = { - // Add a callback or a collection of callbacks to the list - add: function() { - if ( list ) { - // First, we save the current length - var start = list.length; - (function add( args ) { - jQuery.each( args, function( _, arg ) { - var type = jQuery.type( arg ); - if ( type === "function" ) { - if ( !options.unique || !self.has( arg ) ) { - list.push( arg ); - } - } else if ( arg && arg.length && type !== "string" ) { - // Inspect recursively - add( arg ); - } - }); - })( arguments ); - // Do we need to add the callbacks to the - // current firing batch? - if ( firing ) { - firingLength = list.length; - // With memory, if we're not firing then - // we should call right away - } else if ( memory ) { - firingStart = start; - fire( memory ); - } - } - return this; - }, - // Remove a callback from the list - remove: function() { - if ( list ) { - jQuery.each( arguments, function( _, arg ) { - var index; - while ( ( index = jQuery.inArray( arg, list, index ) ) > -1 ) { - list.splice( index, 1 ); - // Handle firing indexes - if ( firing ) { - if ( index <= firingLength ) { - firingLength--; - } - if ( index <= firingIndex ) { - firingIndex--; - } - } - } - }); - } - return this; - }, - // Check if a given callback is in the list. - // If no argument is given, return whether or not list has callbacks attached. - has: function( fn ) { - return fn ? jQuery.inArray( fn, list ) > -1 : !!( list && list.length ); - }, - // Remove all callbacks from the list - empty: function() { - list = []; - firingLength = 0; - return this; - }, - // Have the list do nothing anymore - disable: function() { - list = stack = memory = undefined; - return this; - }, - // Is it disabled? - disabled: function() { - return !list; - }, - // Lock the list in its current state - lock: function() { - stack = undefined; - if ( !memory ) { - self.disable(); - } - return this; - }, - // Is it locked? - locked: function() { - return !stack; - }, - // Call all callbacks with the given context and arguments - fireWith: function( context, args ) { - if ( list && ( !fired || stack ) ) { - args = args || []; - args = [ context, args.slice ? args.slice() : args ]; - if ( firing ) { - stack.push( args ); - } else { - fire( args ); - } - } - return this; - }, - // Call all the callbacks with the given arguments - fire: function() { - self.fireWith( this, arguments ); - return this; - }, - // To know if the callbacks have already been called at least once - fired: function() { - return !!fired; - } - }; - - return self; -}; - - -jQuery.extend({ - - Deferred: function( func ) { - var tuples = [ - // action, add listener, listener list, final state - [ "resolve", "done", jQuery.Callbacks("once memory"), "resolved" ], - [ "reject", "fail", jQuery.Callbacks("once memory"), "rejected" ], - [ "notify", "progress", jQuery.Callbacks("memory") ] - ], - state = "pending", - promise = { - state: function() { - return state; - }, - always: function() { - deferred.done( arguments ).fail( arguments ); - return this; - }, - then: function( /* fnDone, fnFail, fnProgress */ ) { - var fns = arguments; - return jQuery.Deferred(function( newDefer ) { - jQuery.each( tuples, function( i, tuple ) { - var fn = jQuery.isFunction( fns[ i ] ) && fns[ i ]; - // deferred[ done | fail | progress ] for forwarding actions to newDefer - deferred[ tuple[1] ](function() { - var returned = fn && fn.apply( this, arguments ); - if ( returned && jQuery.isFunction( returned.promise ) ) { - returned.promise() - .done( newDefer.resolve ) - .fail( newDefer.reject ) - .progress( newDefer.notify ); - } else { - newDefer[ tuple[ 0 ] + "With" ]( this === promise ? newDefer.promise() : this, fn ? [ returned ] : arguments ); - } - }); - }); - fns = null; - }).promise(); - }, - // Get a promise for this deferred - // If obj is provided, the promise aspect is added to the object - promise: function( obj ) { - return obj != null ? jQuery.extend( obj, promise ) : promise; - } - }, - deferred = {}; - - // Keep pipe for back-compat - promise.pipe = promise.then; - - // Add list-specific methods - jQuery.each( tuples, function( i, tuple ) { - var list = tuple[ 2 ], - stateString = tuple[ 3 ]; - - // promise[ done | fail | progress ] = list.add - promise[ tuple[1] ] = list.add; - - // Handle state - if ( stateString ) { - list.add(function() { - // state = [ resolved | rejected ] - state = stateString; - - // [ reject_list | resolve_list ].disable; progress_list.lock - }, tuples[ i ^ 1 ][ 2 ].disable, tuples[ 2 ][ 2 ].lock ); - } - - // deferred[ resolve | reject | notify ] - deferred[ tuple[0] ] = function() { - deferred[ tuple[0] + "With" ]( this === deferred ? promise : this, arguments ); - return this; - }; - deferred[ tuple[0] + "With" ] = list.fireWith; - }); - - // Make the deferred a promise - promise.promise( deferred ); - - // Call given func if any - if ( func ) { - func.call( deferred, deferred ); - } - - // All done! - return deferred; - }, - - // Deferred helper - when: function( subordinate /* , ..., subordinateN */ ) { - var i = 0, - resolveValues = slice.call( arguments ), - length = resolveValues.length, - - // the count of uncompleted subordinates - remaining = length !== 1 || ( subordinate && jQuery.isFunction( subordinate.promise ) ) ? length : 0, - - // the master Deferred. If resolveValues consist of only a single Deferred, just use that. - deferred = remaining === 1 ? subordinate : jQuery.Deferred(), - - // Update function for both resolve and progress values - updateFunc = function( i, contexts, values ) { - return function( value ) { - contexts[ i ] = this; - values[ i ] = arguments.length > 1 ? slice.call( arguments ) : value; - if ( values === progressValues ) { - deferred.notifyWith( contexts, values ); - - } else if ( !(--remaining) ) { - deferred.resolveWith( contexts, values ); - } - }; - }, - - progressValues, progressContexts, resolveContexts; - - // add listeners to Deferred subordinates; treat others as resolved - if ( length > 1 ) { - progressValues = new Array( length ); - progressContexts = new Array( length ); - resolveContexts = new Array( length ); - for ( ; i < length; i++ ) { - if ( resolveValues[ i ] && jQuery.isFunction( resolveValues[ i ].promise ) ) { - resolveValues[ i ].promise() - .done( updateFunc( i, resolveContexts, resolveValues ) ) - .fail( deferred.reject ) - .progress( updateFunc( i, progressContexts, progressValues ) ); - } else { - --remaining; - } - } - } - - // if we're not waiting on anything, resolve the master - if ( !remaining ) { - deferred.resolveWith( resolveContexts, resolveValues ); - } - - return deferred.promise(); - } -}); - - -// The deferred used on DOM ready -var readyList; - -jQuery.fn.ready = function( fn ) { - // Add the callback - jQuery.ready.promise().done( fn ); - - return this; -}; - -jQuery.extend({ - // Is the DOM ready to be used? Set to true once it occurs. - isReady: false, - - // A counter to track how many items to wait for before - // the ready event fires. See #6781 - readyWait: 1, - - // Hold (or release) the ready event - holdReady: function( hold ) { - if ( hold ) { - jQuery.readyWait++; - } else { - jQuery.ready( true ); - } - }, - - // Handle when the DOM is ready - ready: function( wait ) { - - // Abort if there are pending holds or we're already ready - if ( wait === true ? --jQuery.readyWait : jQuery.isReady ) { - return; - } - - // Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443). - if ( !document.body ) { - return setTimeout( jQuery.ready ); - } - - // Remember that the DOM is ready - jQuery.isReady = true; - - // If a normal DOM Ready event fired, decrement, and wait if need be - if ( wait !== true && --jQuery.readyWait > 0 ) { - return; - } - - // If there are functions bound, to execute - readyList.resolveWith( document, [ jQuery ] ); - - // Trigger any bound ready events - if ( jQuery.fn.trigger ) { - jQuery( document ).trigger("ready").off("ready"); - } - } -}); - -/** - * Clean-up method for dom ready events - */ -function detach() { - if ( document.addEventListener ) { - document.removeEventListener( "DOMContentLoaded", completed, false ); - window.removeEventListener( "load", completed, false ); - - } else { - document.detachEvent( "onreadystatechange", completed ); - window.detachEvent( "onload", completed ); - } -} - -/** - * The ready event handler and self cleanup method - */ -function completed() { - // readyState === "complete" is good enough for us to call the dom ready in oldIE - if ( document.addEventListener || event.type === "load" || document.readyState === "complete" ) { - detach(); - jQuery.ready(); - } -} - -jQuery.ready.promise = function( obj ) { - if ( !readyList ) { - - readyList = jQuery.Deferred(); - - // Catch cases where $(document).ready() is called after the browser event has already occurred. - // we once tried to use readyState "interactive" here, but it caused issues like the one - // discovered by ChrisS here: http://bugs.jquery.com/ticket/12282#comment:15 - if ( document.readyState === "complete" ) { - // Handle it asynchronously to allow scripts the opportunity to delay ready - setTimeout( jQuery.ready ); - - // Standards-based browsers support DOMContentLoaded - } else if ( document.addEventListener ) { - // Use the handy event callback - document.addEventListener( "DOMContentLoaded", completed, false ); - - // A fallback to window.onload, that will always work - window.addEventListener( "load", completed, false ); - - // If IE event model is used - } else { - // Ensure firing before onload, maybe late but safe also for iframes - document.attachEvent( "onreadystatechange", completed ); - - // A fallback to window.onload, that will always work - window.attachEvent( "onload", completed ); - - // If IE and not a frame - // continually check to see if the document is ready - var top = false; - - try { - top = window.frameElement == null && document.documentElement; - } catch(e) {} - - if ( top && top.doScroll ) { - (function doScrollCheck() { - if ( !jQuery.isReady ) { - - try { - // Use the trick by Diego Perini - // http://javascript.nwbox.com/IEContentLoaded/ - top.doScroll("left"); - } catch(e) { - return setTimeout( doScrollCheck, 50 ); - } - - // detach all dom ready events - detach(); - - // and execute any waiting functions - jQuery.ready(); - } - })(); - } - } - } - return readyList.promise( obj ); -}; - - -var strundefined = typeof undefined; - - - -// Support: IE<9 -// Iteration over object's inherited properties before its own -var i; -for ( i in jQuery( support ) ) { - break; -} -support.ownLast = i !== "0"; - -// Note: most support tests are defined in their respective modules. -// false until the test is run -support.inlineBlockNeedsLayout = false; - -jQuery(function() { - // We need to execute this one support test ASAP because we need to know - // if body.style.zoom needs to be set. - - var container, div, - body = document.getElementsByTagName("body")[0]; - - if ( !body ) { - // Return for frameset docs that don't have a body - return; - } - - // Setup - container = document.createElement( "div" ); - container.style.cssText = "border:0;width:0;height:0;position:absolute;top:0;left:-9999px;margin-top:1px"; - - div = document.createElement( "div" ); - body.appendChild( container ).appendChild( div ); - - if ( typeof div.style.zoom !== strundefined ) { - // Support: IE<8 - // Check if natively block-level elements act like inline-block - // elements when setting their display to 'inline' and giving - // them layout - div.style.cssText = "border:0;margin:0;width:1px;padding:1px;display:inline;zoom:1"; - - if ( (support.inlineBlockNeedsLayout = ( div.offsetWidth === 3 )) ) { - // Prevent IE 6 from affecting layout for positioned elements #11048 - // Prevent IE from shrinking the body in IE 7 mode #12869 - // Support: IE<8 - body.style.zoom = 1; - } - } - - body.removeChild( container ); - - // Null elements to avoid leaks in IE - container = div = null; -}); - - - - -(function() { - var div = document.createElement( "div" ); - - // Execute the test only if not already executed in another module. - if (support.deleteExpando == null) { - // Support: IE<9 - support.deleteExpando = true; - try { - delete div.test; - } catch( e ) { - support.deleteExpando = false; - } - } - - // Null elements to avoid leaks in IE. - div = null; -})(); - - -/** - * Determines whether an object can have data - */ -jQuery.acceptData = function( elem ) { - var noData = jQuery.noData[ (elem.nodeName + " ").toLowerCase() ], - nodeType = +elem.nodeType || 1; - - // Do not set data on non-element DOM nodes because it will not be cleared (#8335). - return nodeType !== 1 && nodeType !== 9 ? - false : - - // Nodes accept data unless otherwise specified; rejection can be conditional - !noData || noData !== true && elem.getAttribute("classid") === noData; -}; - - -var rbrace = /^(?:\{[\w\W]*\}|\[[\w\W]*\])$/, - rmultiDash = /([A-Z])/g; - -function dataAttr( elem, key, data ) { - // If nothing was found internally, try to fetch any - // data from the HTML5 data-* attribute - if ( data === undefined && elem.nodeType === 1 ) { - - var name = "data-" + key.replace( rmultiDash, "-$1" ).toLowerCase(); - - data = elem.getAttribute( name ); - - if ( typeof data === "string" ) { - try { - data = data === "true" ? true : - data === "false" ? false : - data === "null" ? null : - // Only convert to a number if it doesn't change the string - +data + "" === data ? +data : - rbrace.test( data ) ? jQuery.parseJSON( data ) : - data; - } catch( e ) {} - - // Make sure we set the data so it isn't changed later - jQuery.data( elem, key, data ); - - } else { - data = undefined; - } - } - - return data; -} - -// checks a cache object for emptiness -function isEmptyDataObject( obj ) { - var name; - for ( name in obj ) { - - // if the public data object is empty, the private is still empty - if ( name === "data" && jQuery.isEmptyObject( obj[name] ) ) { - continue; - } - if ( name !== "toJSON" ) { - return false; - } - } - - return true; -} - -function internalData( elem, name, data, pvt /* Internal Use Only */ ) { - if ( !jQuery.acceptData( elem ) ) { - return; - } - - var ret, thisCache, - internalKey = jQuery.expando, - - // We have to handle DOM nodes and JS objects differently because IE6-7 - // can't GC object references properly across the DOM-JS boundary - isNode = elem.nodeType, - - // Only DOM nodes need the global jQuery cache; JS object data is - // attached directly to the object so GC can occur automatically - cache = isNode ? jQuery.cache : elem, - - // Only defining an ID for JS objects if its cache already exists allows - // the code to shortcut on the same path as a DOM node with no cache - id = isNode ? elem[ internalKey ] : elem[ internalKey ] && internalKey; - - // Avoid doing any more work than we need to when trying to get data on an - // object that has no data at all - if ( (!id || !cache[id] || (!pvt && !cache[id].data)) && data === undefined && typeof name === "string" ) { - return; - } - - if ( !id ) { - // Only DOM nodes need a new unique ID for each element since their data - // ends up in the global cache - if ( isNode ) { - id = elem[ internalKey ] = deletedIds.pop() || jQuery.guid++; - } else { - id = internalKey; - } - } - - if ( !cache[ id ] ) { - // Avoid exposing jQuery metadata on plain JS objects when the object - // is serialized using JSON.stringify - cache[ id ] = isNode ? {} : { toJSON: jQuery.noop }; - } - - // An object can be passed to jQuery.data instead of a key/value pair; this gets - // shallow copied over onto the existing cache - if ( typeof name === "object" || typeof name === "function" ) { - if ( pvt ) { - cache[ id ] = jQuery.extend( cache[ id ], name ); - } else { - cache[ id ].data = jQuery.extend( cache[ id ].data, name ); - } - } - - thisCache = cache[ id ]; - - // jQuery data() is stored in a separate object inside the object's internal data - // cache in order to avoid key collisions between internal data and user-defined - // data. - if ( !pvt ) { - if ( !thisCache.data ) { - thisCache.data = {}; - } - - thisCache = thisCache.data; - } - - if ( data !== undefined ) { - thisCache[ jQuery.camelCase( name ) ] = data; - } - - // Check for both converted-to-camel and non-converted data property names - // If a data property was specified - if ( typeof name === "string" ) { - - // First Try to find as-is property data - ret = thisCache[ name ]; - - // Test for null|undefined property data - if ( ret == null ) { - - // Try to find the camelCased property - ret = thisCache[ jQuery.camelCase( name ) ]; - } - } else { - ret = thisCache; - } - - return ret; -} - -function internalRemoveData( elem, name, pvt ) { - if ( !jQuery.acceptData( elem ) ) { - return; - } - - var thisCache, i, - isNode = elem.nodeType, - - // See jQuery.data for more information - cache = isNode ? jQuery.cache : elem, - id = isNode ? elem[ jQuery.expando ] : jQuery.expando; - - // If there is already no cache entry for this object, there is no - // purpose in continuing - if ( !cache[ id ] ) { - return; - } - - if ( name ) { - - thisCache = pvt ? cache[ id ] : cache[ id ].data; - - if ( thisCache ) { - - // Support array or space separated string names for data keys - if ( !jQuery.isArray( name ) ) { - - // try the string as a key before any manipulation - if ( name in thisCache ) { - name = [ name ]; - } else { - - // split the camel cased version by spaces unless a key with the spaces exists - name = jQuery.camelCase( name ); - if ( name in thisCache ) { - name = [ name ]; - } else { - name = name.split(" "); - } - } - } else { - // If "name" is an array of keys... - // When data is initially created, via ("key", "val") signature, - // keys will be converted to camelCase. - // Since there is no way to tell _how_ a key was added, remove - // both plain key and camelCase key. #12786 - // This will only penalize the array argument path. - name = name.concat( jQuery.map( name, jQuery.camelCase ) ); - } - - i = name.length; - while ( i-- ) { - delete thisCache[ name[i] ]; - } - - // If there is no data left in the cache, we want to continue - // and let the cache object itself get destroyed - if ( pvt ? !isEmptyDataObject(thisCache) : !jQuery.isEmptyObject(thisCache) ) { - return; - } - } - } - - // See jQuery.data for more information - if ( !pvt ) { - delete cache[ id ].data; - - // Don't destroy the parent cache unless the internal data object - // had been the only thing left in it - if ( !isEmptyDataObject( cache[ id ] ) ) { - return; - } - } - - // Destroy the cache - if ( isNode ) { - jQuery.cleanData( [ elem ], true ); - - // Use delete when supported for expandos or `cache` is not a window per isWindow (#10080) - /* jshint eqeqeq: false */ - } else if ( support.deleteExpando || cache != cache.window ) { - /* jshint eqeqeq: true */ - delete cache[ id ]; - - // When all else fails, null - } else { - cache[ id ] = null; - } -} - -jQuery.extend({ - cache: {}, - - // The following elements (space-suffixed to avoid Object.prototype collisions) - // throw uncatchable exceptions if you attempt to set expando properties - noData: { - "applet ": true, - "embed ": true, - // ...but Flash objects (which have this classid) *can* handle expandos - "object ": "clsid:D27CDB6E-AE6D-11cf-96B8-444553540000" - }, - - hasData: function( elem ) { - elem = elem.nodeType ? jQuery.cache[ elem[jQuery.expando] ] : elem[ jQuery.expando ]; - return !!elem && !isEmptyDataObject( elem ); - }, - - data: function( elem, name, data ) { - return internalData( elem, name, data ); - }, - - removeData: function( elem, name ) { - return internalRemoveData( elem, name ); - }, - - // For internal use only. - _data: function( elem, name, data ) { - return internalData( elem, name, data, true ); - }, - - _removeData: function( elem, name ) { - return internalRemoveData( elem, name, true ); - } -}); - -jQuery.fn.extend({ - data: function( key, value ) { - var i, name, data, - elem = this[0], - attrs = elem && elem.attributes; - - // Special expections of .data basically thwart jQuery.access, - // so implement the relevant behavior ourselves - - // Gets all values - if ( key === undefined ) { - if ( this.length ) { - data = jQuery.data( elem ); - - if ( elem.nodeType === 1 && !jQuery._data( elem, "parsedAttrs" ) ) { - i = attrs.length; - while ( i-- ) { - name = attrs[i].name; - - if ( name.indexOf("data-") === 0 ) { - name = jQuery.camelCase( name.slice(5) ); - - dataAttr( elem, name, data[ name ] ); - } - } - jQuery._data( elem, "parsedAttrs", true ); - } - } - - return data; - } - - // Sets multiple values - if ( typeof key === "object" ) { - return this.each(function() { - jQuery.data( this, key ); - }); - } - - return arguments.length > 1 ? - - // Sets one value - this.each(function() { - jQuery.data( this, key, value ); - }) : - - // Gets one value - // Try to fetch any internally stored data first - elem ? dataAttr( elem, key, jQuery.data( elem, key ) ) : undefined; - }, - - removeData: function( key ) { - return this.each(function() { - jQuery.removeData( this, key ); - }); - } -}); - - -jQuery.extend({ - queue: function( elem, type, data ) { - var queue; - - if ( elem ) { - type = ( type || "fx" ) + "queue"; - queue = jQuery._data( elem, type ); - - // Speed up dequeue by getting out quickly if this is just a lookup - if ( data ) { - if ( !queue || jQuery.isArray(data) ) { - queue = jQuery._data( elem, type, jQuery.makeArray(data) ); - } else { - queue.push( data ); - } - } - return queue || []; - } - }, - - dequeue: function( elem, type ) { - type = type || "fx"; - - var queue = jQuery.queue( elem, type ), - startLength = queue.length, - fn = queue.shift(), - hooks = jQuery._queueHooks( elem, type ), - next = function() { - jQuery.dequeue( elem, type ); - }; - - // If the fx queue is dequeued, always remove the progress sentinel - if ( fn === "inprogress" ) { - fn = queue.shift(); - startLength--; - } - - if ( fn ) { - - // Add a progress sentinel to prevent the fx queue from being - // automatically dequeued - if ( type === "fx" ) { - queue.unshift( "inprogress" ); - } - - // clear up the last queue stop function - delete hooks.stop; - fn.call( elem, next, hooks ); - } - - if ( !startLength && hooks ) { - hooks.empty.fire(); - } - }, - - // not intended for public consumption - generates a queueHooks object, or returns the current one - _queueHooks: function( elem, type ) { - var key = type + "queueHooks"; - return jQuery._data( elem, key ) || jQuery._data( elem, key, { - empty: jQuery.Callbacks("once memory").add(function() { - jQuery._removeData( elem, type + "queue" ); - jQuery._removeData( elem, key ); - }) - }); - } -}); - -jQuery.fn.extend({ - queue: function( type, data ) { - var setter = 2; - - if ( typeof type !== "string" ) { - data = type; - type = "fx"; - setter--; - } - - if ( arguments.length < setter ) { - return jQuery.queue( this[0], type ); - } - - return data === undefined ? - this : - this.each(function() { - var queue = jQuery.queue( this, type, data ); - - // ensure a hooks for this queue - jQuery._queueHooks( this, type ); - - if ( type === "fx" && queue[0] !== "inprogress" ) { - jQuery.dequeue( this, type ); - } - }); - }, - dequeue: function( type ) { - return this.each(function() { - jQuery.dequeue( this, type ); - }); - }, - clearQueue: function( type ) { - return this.queue( type || "fx", [] ); - }, - // Get a promise resolved when queues of a certain type - // are emptied (fx is the type by default) - promise: function( type, obj ) { - var tmp, - count = 1, - defer = jQuery.Deferred(), - elements = this, - i = this.length, - resolve = function() { - if ( !( --count ) ) { - defer.resolveWith( elements, [ elements ] ); - } - }; - - if ( typeof type !== "string" ) { - obj = type; - type = undefined; - } - type = type || "fx"; - - while ( i-- ) { - tmp = jQuery._data( elements[ i ], type + "queueHooks" ); - if ( tmp && tmp.empty ) { - count++; - tmp.empty.add( resolve ); - } - } - resolve(); - return defer.promise( obj ); - } -}); -var pnum = (/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/).source; - -var cssExpand = [ "Top", "Right", "Bottom", "Left" ]; - -var isHidden = function( elem, el ) { - // isHidden might be called from jQuery#filter function; - // in that case, element will be second argument - elem = el || elem; - return jQuery.css( elem, "display" ) === "none" || !jQuery.contains( elem.ownerDocument, elem ); - }; - - - -// Multifunctional method to get and set values of a collection -// The value/s can optionally be executed if it's a function -var access = jQuery.access = function( elems, fn, key, value, chainable, emptyGet, raw ) { - var i = 0, - length = elems.length, - bulk = key == null; - - // Sets many values - if ( jQuery.type( key ) === "object" ) { - chainable = true; - for ( i in key ) { - jQuery.access( elems, fn, i, key[i], true, emptyGet, raw ); - } - - // Sets one value - } else if ( value !== undefined ) { - chainable = true; - - if ( !jQuery.isFunction( value ) ) { - raw = true; - } - - if ( bulk ) { - // Bulk operations run against the entire set - if ( raw ) { - fn.call( elems, value ); - fn = null; - - // ...except when executing function values - } else { - bulk = fn; - fn = function( elem, key, value ) { - return bulk.call( jQuery( elem ), value ); - }; - } - } - - if ( fn ) { - for ( ; i < length; i++ ) { - fn( elems[i], key, raw ? value : value.call( elems[i], i, fn( elems[i], key ) ) ); - } - } - } - - return chainable ? - elems : - - // Gets - bulk ? - fn.call( elems ) : - length ? fn( elems[0], key ) : emptyGet; -}; -var rcheckableType = (/^(?:checkbox|radio)$/i); - - - -(function() { - var fragment = document.createDocumentFragment(), - div = document.createElement("div"), - input = document.createElement("input"); - - // Setup - div.setAttribute( "className", "t" ); - div.innerHTML = "
a"; - - // IE strips leading whitespace when .innerHTML is used - support.leadingWhitespace = div.firstChild.nodeType === 3; - - // Make sure that tbody elements aren't automatically inserted - // IE will insert them into empty tables - support.tbody = !div.getElementsByTagName( "tbody" ).length; - - // Make sure that link elements get serialized correctly by innerHTML - // This requires a wrapper element in IE - support.htmlSerialize = !!div.getElementsByTagName( "link" ).length; - - // Makes sure cloning an html5 element does not cause problems - // Where outerHTML is undefined, this still works - support.html5Clone = - document.createElement( "nav" ).cloneNode( true ).outerHTML !== "<:nav>"; - - // Check if a disconnected checkbox will retain its checked - // value of true after appended to the DOM (IE6/7) - input.type = "checkbox"; - input.checked = true; - fragment.appendChild( input ); - support.appendChecked = input.checked; - - // Make sure textarea (and checkbox) defaultValue is properly cloned - // Support: IE6-IE11+ - div.innerHTML = ""; - support.noCloneChecked = !!div.cloneNode( true ).lastChild.defaultValue; - - // #11217 - WebKit loses check when the name is after the checked attribute - fragment.appendChild( div ); - div.innerHTML = ""; - - // Support: Safari 5.1, iOS 5.1, Android 4.x, Android 2.3 - // old WebKit doesn't clone checked state correctly in fragments - support.checkClone = div.cloneNode( true ).cloneNode( true ).lastChild.checked; - - // Support: IE<9 - // Opera does not clone events (and typeof div.attachEvent === undefined). - // IE9-10 clones events bound via attachEvent, but they don't trigger with .click() - support.noCloneEvent = true; - if ( div.attachEvent ) { - div.attachEvent( "onclick", function() { - support.noCloneEvent = false; - }); - - div.cloneNode( true ).click(); - } - - // Execute the test only if not already executed in another module. - if (support.deleteExpando == null) { - // Support: IE<9 - support.deleteExpando = true; - try { - delete div.test; - } catch( e ) { - support.deleteExpando = false; - } - } - - // Null elements to avoid leaks in IE. - fragment = div = input = null; -})(); - - -(function() { - var i, eventName, - div = document.createElement( "div" ); - - // Support: IE<9 (lack submit/change bubble), Firefox 23+ (lack focusin event) - for ( i in { submit: true, change: true, focusin: true }) { - eventName = "on" + i; - - if ( !(support[ i + "Bubbles" ] = eventName in window) ) { - // Beware of CSP restrictions (https://developer.mozilla.org/en/Security/CSP) - div.setAttribute( eventName, "t" ); - support[ i + "Bubbles" ] = div.attributes[ eventName ].expando === false; - } - } - - // Null elements to avoid leaks in IE. - div = null; -})(); - - -var rformElems = /^(?:input|select|textarea)$/i, - rkeyEvent = /^key/, - rmouseEvent = /^(?:mouse|contextmenu)|click/, - rfocusMorph = /^(?:focusinfocus|focusoutblur)$/, - rtypenamespace = /^([^.]*)(?:\.(.+)|)$/; - -function returnTrue() { - return true; -} - -function returnFalse() { - return false; -} - -function safeActiveElement() { - try { - return document.activeElement; - } catch ( err ) { } -} - -/* - * Helper functions for managing events -- not part of the public interface. - * Props to Dean Edwards' addEvent library for many of the ideas. - */ -jQuery.event = { - - global: {}, - - add: function( elem, types, handler, data, selector ) { - var tmp, events, t, handleObjIn, - special, eventHandle, handleObj, - handlers, type, namespaces, origType, - elemData = jQuery._data( elem ); - - // Don't attach events to noData or text/comment nodes (but allow plain objects) - if ( !elemData ) { - return; - } - - // Caller can pass in an object of custom data in lieu of the handler - if ( handler.handler ) { - handleObjIn = handler; - handler = handleObjIn.handler; - selector = handleObjIn.selector; - } - - // Make sure that the handler has a unique ID, used to find/remove it later - if ( !handler.guid ) { - handler.guid = jQuery.guid++; - } - - // Init the element's event structure and main handler, if this is the first - if ( !(events = elemData.events) ) { - events = elemData.events = {}; - } - if ( !(eventHandle = elemData.handle) ) { - eventHandle = elemData.handle = function( e ) { - // Discard the second event of a jQuery.event.trigger() and - // when an event is called after a page has unloaded - return typeof jQuery !== strundefined && (!e || jQuery.event.triggered !== e.type) ? - jQuery.event.dispatch.apply( eventHandle.elem, arguments ) : - undefined; - }; - // Add elem as a property of the handle fn to prevent a memory leak with IE non-native events - eventHandle.elem = elem; - } - - // Handle multiple events separated by a space - types = ( types || "" ).match( rnotwhite ) || [ "" ]; - t = types.length; - while ( t-- ) { - tmp = rtypenamespace.exec( types[t] ) || []; - type = origType = tmp[1]; - namespaces = ( tmp[2] || "" ).split( "." ).sort(); - - // There *must* be a type, no attaching namespace-only handlers - if ( !type ) { - continue; - } - - // If event changes its type, use the special event handlers for the changed type - special = jQuery.event.special[ type ] || {}; - - // If selector defined, determine special event api type, otherwise given type - type = ( selector ? special.delegateType : special.bindType ) || type; - - // Update special based on newly reset type - special = jQuery.event.special[ type ] || {}; - - // handleObj is passed to all event handlers - handleObj = jQuery.extend({ - type: type, - origType: origType, - data: data, - handler: handler, - guid: handler.guid, - selector: selector, - needsContext: selector && jQuery.expr.match.needsContext.test( selector ), - namespace: namespaces.join(".") - }, handleObjIn ); - - // Init the event handler queue if we're the first - if ( !(handlers = events[ type ]) ) { - handlers = events[ type ] = []; - handlers.delegateCount = 0; - - // Only use addEventListener/attachEvent if the special events handler returns false - if ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) { - // Bind the global event handler to the element - if ( elem.addEventListener ) { - elem.addEventListener( type, eventHandle, false ); - - } else if ( elem.attachEvent ) { - elem.attachEvent( "on" + type, eventHandle ); - } - } - } - - if ( special.add ) { - special.add.call( elem, handleObj ); - - if ( !handleObj.handler.guid ) { - handleObj.handler.guid = handler.guid; - } - } - - // Add to the element's handler list, delegates in front - if ( selector ) { - handlers.splice( handlers.delegateCount++, 0, handleObj ); - } else { - handlers.push( handleObj ); - } - - // Keep track of which events have ever been used, for event optimization - jQuery.event.global[ type ] = true; - } - - // Nullify elem to prevent memory leaks in IE - elem = null; - }, - - // Detach an event or set of events from an element - remove: function( elem, types, handler, selector, mappedTypes ) { - var j, handleObj, tmp, - origCount, t, events, - special, handlers, type, - namespaces, origType, - elemData = jQuery.hasData( elem ) && jQuery._data( elem ); - - if ( !elemData || !(events = elemData.events) ) { - return; - } - - // Once for each type.namespace in types; type may be omitted - types = ( types || "" ).match( rnotwhite ) || [ "" ]; - t = types.length; - while ( t-- ) { - tmp = rtypenamespace.exec( types[t] ) || []; - type = origType = tmp[1]; - namespaces = ( tmp[2] || "" ).split( "." ).sort(); - - // Unbind all events (on this namespace, if provided) for the element - if ( !type ) { - for ( type in events ) { - jQuery.event.remove( elem, type + types[ t ], handler, selector, true ); - } - continue; - } - - special = jQuery.event.special[ type ] || {}; - type = ( selector ? special.delegateType : special.bindType ) || type; - handlers = events[ type ] || []; - tmp = tmp[2] && new RegExp( "(^|\\.)" + namespaces.join("\\.(?:.*\\.|)") + "(\\.|$)" ); - - // Remove matching events - origCount = j = handlers.length; - while ( j-- ) { - handleObj = handlers[ j ]; - - if ( ( mappedTypes || origType === handleObj.origType ) && - ( !handler || handler.guid === handleObj.guid ) && - ( !tmp || tmp.test( handleObj.namespace ) ) && - ( !selector || selector === handleObj.selector || selector === "**" && handleObj.selector ) ) { - handlers.splice( j, 1 ); - - if ( handleObj.selector ) { - handlers.delegateCount--; - } - if ( special.remove ) { - special.remove.call( elem, handleObj ); - } - } - } - - // Remove generic event handler if we removed something and no more handlers exist - // (avoids potential for endless recursion during removal of special event handlers) - if ( origCount && !handlers.length ) { - if ( !special.teardown || special.teardown.call( elem, namespaces, elemData.handle ) === false ) { - jQuery.removeEvent( elem, type, elemData.handle ); - } - - delete events[ type ]; - } - } - - // Remove the expando if it's no longer used - if ( jQuery.isEmptyObject( events ) ) { - delete elemData.handle; - - // removeData also checks for emptiness and clears the expando if empty - // so use it instead of delete - jQuery._removeData( elem, "events" ); - } - }, - - trigger: function( event, data, elem, onlyHandlers ) { - var handle, ontype, cur, - bubbleType, special, tmp, i, - eventPath = [ elem || document ], - type = hasOwn.call( event, "type" ) ? event.type : event, - namespaces = hasOwn.call( event, "namespace" ) ? event.namespace.split(".") : []; - - cur = tmp = elem = elem || document; - - // Don't do events on text and comment nodes - if ( elem.nodeType === 3 || elem.nodeType === 8 ) { - return; - } - - // focus/blur morphs to focusin/out; ensure we're not firing them right now - if ( rfocusMorph.test( type + jQuery.event.triggered ) ) { - return; - } - - if ( type.indexOf(".") >= 0 ) { - // Namespaced trigger; create a regexp to match event type in handle() - namespaces = type.split("."); - type = namespaces.shift(); - namespaces.sort(); - } - ontype = type.indexOf(":") < 0 && "on" + type; - - // Caller can pass in a jQuery.Event object, Object, or just an event type string - event = event[ jQuery.expando ] ? - event : - new jQuery.Event( type, typeof event === "object" && event ); - - // Trigger bitmask: & 1 for native handlers; & 2 for jQuery (always true) - event.isTrigger = onlyHandlers ? 2 : 3; - event.namespace = namespaces.join("."); - event.namespace_re = event.namespace ? - new RegExp( "(^|\\.)" + namespaces.join("\\.(?:.*\\.|)") + "(\\.|$)" ) : - null; - - // Clean up the event in case it is being reused - event.result = undefined; - if ( !event.target ) { - event.target = elem; - } - - // Clone any incoming data and prepend the event, creating the handler arg list - data = data == null ? - [ event ] : - jQuery.makeArray( data, [ event ] ); - - // Allow special events to draw outside the lines - special = jQuery.event.special[ type ] || {}; - if ( !onlyHandlers && special.trigger && special.trigger.apply( elem, data ) === false ) { - return; - } - - // Determine event propagation path in advance, per W3C events spec (#9951) - // Bubble up to document, then to window; watch for a global ownerDocument var (#9724) - if ( !onlyHandlers && !special.noBubble && !jQuery.isWindow( elem ) ) { - - bubbleType = special.delegateType || type; - if ( !rfocusMorph.test( bubbleType + type ) ) { - cur = cur.parentNode; - } - for ( ; cur; cur = cur.parentNode ) { - eventPath.push( cur ); - tmp = cur; - } - - // Only add window if we got to document (e.g., not plain obj or detached DOM) - if ( tmp === (elem.ownerDocument || document) ) { - eventPath.push( tmp.defaultView || tmp.parentWindow || window ); - } - } - - // Fire handlers on the event path - i = 0; - while ( (cur = eventPath[i++]) && !event.isPropagationStopped() ) { - - event.type = i > 1 ? - bubbleType : - special.bindType || type; - - // jQuery handler - handle = ( jQuery._data( cur, "events" ) || {} )[ event.type ] && jQuery._data( cur, "handle" ); - if ( handle ) { - handle.apply( cur, data ); - } - - // Native handler - handle = ontype && cur[ ontype ]; - if ( handle && handle.apply && jQuery.acceptData( cur ) ) { - event.result = handle.apply( cur, data ); - if ( event.result === false ) { - event.preventDefault(); - } - } - } - event.type = type; - - // If nobody prevented the default action, do it now - if ( !onlyHandlers && !event.isDefaultPrevented() ) { - - if ( (!special._default || special._default.apply( eventPath.pop(), data ) === false) && - jQuery.acceptData( elem ) ) { - - // Call a native DOM method on the target with the same name name as the event. - // Can't use an .isFunction() check here because IE6/7 fails that test. - // Don't do default actions on window, that's where global variables be (#6170) - if ( ontype && elem[ type ] && !jQuery.isWindow( elem ) ) { - - // Don't re-trigger an onFOO event when we call its FOO() method - tmp = elem[ ontype ]; - - if ( tmp ) { - elem[ ontype ] = null; - } - - // Prevent re-triggering of the same event, since we already bubbled it above - jQuery.event.triggered = type; - try { - elem[ type ](); - } catch ( e ) { - // IE<9 dies on focus/blur to hidden element (#1486,#12518) - // only reproducible on winXP IE8 native, not IE9 in IE8 mode - } - jQuery.event.triggered = undefined; - - if ( tmp ) { - elem[ ontype ] = tmp; - } - } - } - } - - return event.result; - }, - - dispatch: function( event ) { - - // Make a writable jQuery.Event from the native event object - event = jQuery.event.fix( event ); - - var i, ret, handleObj, matched, j, - handlerQueue = [], - args = slice.call( arguments ), - handlers = ( jQuery._data( this, "events" ) || {} )[ event.type ] || [], - special = jQuery.event.special[ event.type ] || {}; - - // Use the fix-ed jQuery.Event rather than the (read-only) native event - args[0] = event; - event.delegateTarget = this; - - // Call the preDispatch hook for the mapped type, and let it bail if desired - if ( special.preDispatch && special.preDispatch.call( this, event ) === false ) { - return; - } - - // Determine handlers - handlerQueue = jQuery.event.handlers.call( this, event, handlers ); - - // Run delegates first; they may want to stop propagation beneath us - i = 0; - while ( (matched = handlerQueue[ i++ ]) && !event.isPropagationStopped() ) { - event.currentTarget = matched.elem; - - j = 0; - while ( (handleObj = matched.handlers[ j++ ]) && !event.isImmediatePropagationStopped() ) { - - // Triggered event must either 1) have no namespace, or - // 2) have namespace(s) a subset or equal to those in the bound event (both can have no namespace). - if ( !event.namespace_re || event.namespace_re.test( handleObj.namespace ) ) { - - event.handleObj = handleObj; - event.data = handleObj.data; - - ret = ( (jQuery.event.special[ handleObj.origType ] || {}).handle || handleObj.handler ) - .apply( matched.elem, args ); - - if ( ret !== undefined ) { - if ( (event.result = ret) === false ) { - event.preventDefault(); - event.stopPropagation(); - } - } - } - } - } - - // Call the postDispatch hook for the mapped type - if ( special.postDispatch ) { - special.postDispatch.call( this, event ); - } - - return event.result; - }, - - handlers: function( event, handlers ) { - var sel, handleObj, matches, i, - handlerQueue = [], - delegateCount = handlers.delegateCount, - cur = event.target; - - // Find delegate handlers - // Black-hole SVG instance trees (#13180) - // Avoid non-left-click bubbling in Firefox (#3861) - if ( delegateCount && cur.nodeType && (!event.button || event.type !== "click") ) { - - /* jshint eqeqeq: false */ - for ( ; cur != this; cur = cur.parentNode || this ) { - /* jshint eqeqeq: true */ - - // Don't check non-elements (#13208) - // Don't process clicks on disabled elements (#6911, #8165, #11382, #11764) - if ( cur.nodeType === 1 && (cur.disabled !== true || event.type !== "click") ) { - matches = []; - for ( i = 0; i < delegateCount; i++ ) { - handleObj = handlers[ i ]; - - // Don't conflict with Object.prototype properties (#13203) - sel = handleObj.selector + " "; - - if ( matches[ sel ] === undefined ) { - matches[ sel ] = handleObj.needsContext ? - jQuery( sel, this ).index( cur ) >= 0 : - jQuery.find( sel, this, null, [ cur ] ).length; - } - if ( matches[ sel ] ) { - matches.push( handleObj ); - } - } - if ( matches.length ) { - handlerQueue.push({ elem: cur, handlers: matches }); - } - } - } - } - - // Add the remaining (directly-bound) handlers - if ( delegateCount < handlers.length ) { - handlerQueue.push({ elem: this, handlers: handlers.slice( delegateCount ) }); - } - - return handlerQueue; - }, - - fix: function( event ) { - if ( event[ jQuery.expando ] ) { - return event; - } - - // Create a writable copy of the event object and normalize some properties - var i, prop, copy, - type = event.type, - originalEvent = event, - fixHook = this.fixHooks[ type ]; - - if ( !fixHook ) { - this.fixHooks[ type ] = fixHook = - rmouseEvent.test( type ) ? this.mouseHooks : - rkeyEvent.test( type ) ? this.keyHooks : - {}; - } - copy = fixHook.props ? this.props.concat( fixHook.props ) : this.props; - - event = new jQuery.Event( originalEvent ); - - i = copy.length; - while ( i-- ) { - prop = copy[ i ]; - event[ prop ] = originalEvent[ prop ]; - } - - // Support: IE<9 - // Fix target property (#1925) - if ( !event.target ) { - event.target = originalEvent.srcElement || document; - } - - // Support: Chrome 23+, Safari? - // Target should not be a text node (#504, #13143) - if ( event.target.nodeType === 3 ) { - event.target = event.target.parentNode; - } - - // Support: IE<9 - // For mouse/key events, metaKey==false if it's undefined (#3368, #11328) - event.metaKey = !!event.metaKey; - - return fixHook.filter ? fixHook.filter( event, originalEvent ) : event; - }, - - // Includes some event props shared by KeyEvent and MouseEvent - props: "altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which".split(" "), - - fixHooks: {}, - - keyHooks: { - props: "char charCode key keyCode".split(" "), - filter: function( event, original ) { - - // Add which for key events - if ( event.which == null ) { - event.which = original.charCode != null ? original.charCode : original.keyCode; - } - - return event; - } - }, - - mouseHooks: { - props: "button buttons clientX clientY fromElement offsetX offsetY pageX pageY screenX screenY toElement".split(" "), - filter: function( event, original ) { - var body, eventDoc, doc, - button = original.button, - fromElement = original.fromElement; - - // Calculate pageX/Y if missing and clientX/Y available - if ( event.pageX == null && original.clientX != null ) { - eventDoc = event.target.ownerDocument || document; - doc = eventDoc.documentElement; - body = eventDoc.body; - - event.pageX = original.clientX + ( doc && doc.scrollLeft || body && body.scrollLeft || 0 ) - ( doc && doc.clientLeft || body && body.clientLeft || 0 ); - event.pageY = original.clientY + ( doc && doc.scrollTop || body && body.scrollTop || 0 ) - ( doc && doc.clientTop || body && body.clientTop || 0 ); - } - - // Add relatedTarget, if necessary - if ( !event.relatedTarget && fromElement ) { - event.relatedTarget = fromElement === event.target ? original.toElement : fromElement; - } - - // Add which for click: 1 === left; 2 === middle; 3 === right - // Note: button is not normalized, so don't use it - if ( !event.which && button !== undefined ) { - event.which = ( button & 1 ? 1 : ( button & 2 ? 3 : ( button & 4 ? 2 : 0 ) ) ); - } - - return event; - } - }, - - special: { - load: { - // Prevent triggered image.load events from bubbling to window.load - noBubble: true - }, - focus: { - // Fire native event if possible so blur/focus sequence is correct - trigger: function() { - if ( this !== safeActiveElement() && this.focus ) { - try { - this.focus(); - return false; - } catch ( e ) { - // Support: IE<9 - // If we error on focus to hidden element (#1486, #12518), - // let .trigger() run the handlers - } - } - }, - delegateType: "focusin" - }, - blur: { - trigger: function() { - if ( this === safeActiveElement() && this.blur ) { - this.blur(); - return false; - } - }, - delegateType: "focusout" - }, - click: { - // For checkbox, fire native event so checked state will be right - trigger: function() { - if ( jQuery.nodeName( this, "input" ) && this.type === "checkbox" && this.click ) { - this.click(); - return false; - } - }, - - // For cross-browser consistency, don't fire native .click() on links - _default: function( event ) { - return jQuery.nodeName( event.target, "a" ); - } - }, - - beforeunload: { - postDispatch: function( event ) { - - // Even when returnValue equals to undefined Firefox will still show alert - if ( event.result !== undefined ) { - event.originalEvent.returnValue = event.result; - } - } - } - }, - - simulate: function( type, elem, event, bubble ) { - // Piggyback on a donor event to simulate a different one. - // Fake originalEvent to avoid donor's stopPropagation, but if the - // simulated event prevents default then we do the same on the donor. - var e = jQuery.extend( - new jQuery.Event(), - event, - { - type: type, - isSimulated: true, - originalEvent: {} - } - ); - if ( bubble ) { - jQuery.event.trigger( e, null, elem ); - } else { - jQuery.event.dispatch.call( elem, e ); - } - if ( e.isDefaultPrevented() ) { - event.preventDefault(); - } - } -}; - -jQuery.removeEvent = document.removeEventListener ? - function( elem, type, handle ) { - if ( elem.removeEventListener ) { - elem.removeEventListener( type, handle, false ); - } - } : - function( elem, type, handle ) { - var name = "on" + type; - - if ( elem.detachEvent ) { - - // #8545, #7054, preventing memory leaks for custom events in IE6-8 - // detachEvent needed property on element, by name of that event, to properly expose it to GC - if ( typeof elem[ name ] === strundefined ) { - elem[ name ] = null; - } - - elem.detachEvent( name, handle ); - } - }; - -jQuery.Event = function( src, props ) { - // Allow instantiation without the 'new' keyword - if ( !(this instanceof jQuery.Event) ) { - return new jQuery.Event( src, props ); - } - - // Event object - if ( src && src.type ) { - this.originalEvent = src; - this.type = src.type; - - // Events bubbling up the document may have been marked as prevented - // by a handler lower down the tree; reflect the correct value. - this.isDefaultPrevented = src.defaultPrevented || - src.defaultPrevented === undefined && ( - // Support: IE < 9 - src.returnValue === false || - // Support: Android < 4.0 - src.getPreventDefault && src.getPreventDefault() ) ? - returnTrue : - returnFalse; - - // Event type - } else { - this.type = src; - } - - // Put explicitly provided properties onto the event object - if ( props ) { - jQuery.extend( this, props ); - } - - // Create a timestamp if incoming event doesn't have one - this.timeStamp = src && src.timeStamp || jQuery.now(); - - // Mark it as fixed - this[ jQuery.expando ] = true; -}; - -// jQuery.Event is based on DOM3 Events as specified by the ECMAScript Language Binding -// http://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html -jQuery.Event.prototype = { - isDefaultPrevented: returnFalse, - isPropagationStopped: returnFalse, - isImmediatePropagationStopped: returnFalse, - - preventDefault: function() { - var e = this.originalEvent; - - this.isDefaultPrevented = returnTrue; - if ( !e ) { - return; - } - - // If preventDefault exists, run it on the original event - if ( e.preventDefault ) { - e.preventDefault(); - - // Support: IE - // Otherwise set the returnValue property of the original event to false - } else { - e.returnValue = false; - } - }, - stopPropagation: function() { - var e = this.originalEvent; - - this.isPropagationStopped = returnTrue; - if ( !e ) { - return; - } - // If stopPropagation exists, run it on the original event - if ( e.stopPropagation ) { - e.stopPropagation(); - } - - // Support: IE - // Set the cancelBubble property of the original event to true - e.cancelBubble = true; - }, - stopImmediatePropagation: function() { - this.isImmediatePropagationStopped = returnTrue; - this.stopPropagation(); - } -}; - -// Create mouseenter/leave events using mouseover/out and event-time checks -jQuery.each({ - mouseenter: "mouseover", - mouseleave: "mouseout" -}, function( orig, fix ) { - jQuery.event.special[ orig ] = { - delegateType: fix, - bindType: fix, - - handle: function( event ) { - var ret, - target = this, - related = event.relatedTarget, - handleObj = event.handleObj; - - // For mousenter/leave call the handler if related is outside the target. - // NB: No relatedTarget if the mouse left/entered the browser window - if ( !related || (related !== target && !jQuery.contains( target, related )) ) { - event.type = handleObj.origType; - ret = handleObj.handler.apply( this, arguments ); - event.type = fix; - } - return ret; - } - }; -}); - -// IE submit delegation -if ( !support.submitBubbles ) { - - jQuery.event.special.submit = { - setup: function() { - // Only need this for delegated form submit events - if ( jQuery.nodeName( this, "form" ) ) { - return false; - } - - // Lazy-add a submit handler when a descendant form may potentially be submitted - jQuery.event.add( this, "click._submit keypress._submit", function( e ) { - // Node name check avoids a VML-related crash in IE (#9807) - var elem = e.target, - form = jQuery.nodeName( elem, "input" ) || jQuery.nodeName( elem, "button" ) ? elem.form : undefined; - if ( form && !jQuery._data( form, "submitBubbles" ) ) { - jQuery.event.add( form, "submit._submit", function( event ) { - event._submit_bubble = true; - }); - jQuery._data( form, "submitBubbles", true ); - } - }); - // return undefined since we don't need an event listener - }, - - postDispatch: function( event ) { - // If form was submitted by the user, bubble the event up the tree - if ( event._submit_bubble ) { - delete event._submit_bubble; - if ( this.parentNode && !event.isTrigger ) { - jQuery.event.simulate( "submit", this.parentNode, event, true ); - } - } - }, - - teardown: function() { - // Only need this for delegated form submit events - if ( jQuery.nodeName( this, "form" ) ) { - return false; - } - - // Remove delegated handlers; cleanData eventually reaps submit handlers attached above - jQuery.event.remove( this, "._submit" ); - } - }; -} - -// IE change delegation and checkbox/radio fix -if ( !support.changeBubbles ) { - - jQuery.event.special.change = { - - setup: function() { - - if ( rformElems.test( this.nodeName ) ) { - // IE doesn't fire change on a check/radio until blur; trigger it on click - // after a propertychange. Eat the blur-change in special.change.handle. - // This still fires onchange a second time for check/radio after blur. - if ( this.type === "checkbox" || this.type === "radio" ) { - jQuery.event.add( this, "propertychange._change", function( event ) { - if ( event.originalEvent.propertyName === "checked" ) { - this._just_changed = true; - } - }); - jQuery.event.add( this, "click._change", function( event ) { - if ( this._just_changed && !event.isTrigger ) { - this._just_changed = false; - } - // Allow triggered, simulated change events (#11500) - jQuery.event.simulate( "change", this, event, true ); - }); - } - return false; - } - // Delegated event; lazy-add a change handler on descendant inputs - jQuery.event.add( this, "beforeactivate._change", function( e ) { - var elem = e.target; - - if ( rformElems.test( elem.nodeName ) && !jQuery._data( elem, "changeBubbles" ) ) { - jQuery.event.add( elem, "change._change", function( event ) { - if ( this.parentNode && !event.isSimulated && !event.isTrigger ) { - jQuery.event.simulate( "change", this.parentNode, event, true ); - } - }); - jQuery._data( elem, "changeBubbles", true ); - } - }); - }, - - handle: function( event ) { - var elem = event.target; - - // Swallow native change events from checkbox/radio, we already triggered them above - if ( this !== elem || event.isSimulated || event.isTrigger || (elem.type !== "radio" && elem.type !== "checkbox") ) { - return event.handleObj.handler.apply( this, arguments ); - } - }, - - teardown: function() { - jQuery.event.remove( this, "._change" ); - - return !rformElems.test( this.nodeName ); - } - }; -} - -// Create "bubbling" focus and blur events -if ( !support.focusinBubbles ) { - jQuery.each({ focus: "focusin", blur: "focusout" }, function( orig, fix ) { - - // Attach a single capturing handler on the document while someone wants focusin/focusout - var handler = function( event ) { - jQuery.event.simulate( fix, event.target, jQuery.event.fix( event ), true ); - }; - - jQuery.event.special[ fix ] = { - setup: function() { - var doc = this.ownerDocument || this, - attaches = jQuery._data( doc, fix ); - - if ( !attaches ) { - doc.addEventListener( orig, handler, true ); - } - jQuery._data( doc, fix, ( attaches || 0 ) + 1 ); - }, - teardown: function() { - var doc = this.ownerDocument || this, - attaches = jQuery._data( doc, fix ) - 1; - - if ( !attaches ) { - doc.removeEventListener( orig, handler, true ); - jQuery._removeData( doc, fix ); - } else { - jQuery._data( doc, fix, attaches ); - } - } - }; - }); -} - -jQuery.fn.extend({ - - on: function( types, selector, data, fn, /*INTERNAL*/ one ) { - var type, origFn; - - // Types can be a map of types/handlers - if ( typeof types === "object" ) { - // ( types-Object, selector, data ) - if ( typeof selector !== "string" ) { - // ( types-Object, data ) - data = data || selector; - selector = undefined; - } - for ( type in types ) { - this.on( type, selector, data, types[ type ], one ); - } - return this; - } - - if ( data == null && fn == null ) { - // ( types, fn ) - fn = selector; - data = selector = undefined; - } else if ( fn == null ) { - if ( typeof selector === "string" ) { - // ( types, selector, fn ) - fn = data; - data = undefined; - } else { - // ( types, data, fn ) - fn = data; - data = selector; - selector = undefined; - } - } - if ( fn === false ) { - fn = returnFalse; - } else if ( !fn ) { - return this; - } - - if ( one === 1 ) { - origFn = fn; - fn = function( event ) { - // Can use an empty set, since event contains the info - jQuery().off( event ); - return origFn.apply( this, arguments ); - }; - // Use same guid so caller can remove using origFn - fn.guid = origFn.guid || ( origFn.guid = jQuery.guid++ ); - } - return this.each( function() { - jQuery.event.add( this, types, fn, data, selector ); - }); - }, - one: function( types, selector, data, fn ) { - return this.on( types, selector, data, fn, 1 ); - }, - off: function( types, selector, fn ) { - var handleObj, type; - if ( types && types.preventDefault && types.handleObj ) { - // ( event ) dispatched jQuery.Event - handleObj = types.handleObj; - jQuery( types.delegateTarget ).off( - handleObj.namespace ? handleObj.origType + "." + handleObj.namespace : handleObj.origType, - handleObj.selector, - handleObj.handler - ); - return this; - } - if ( typeof types === "object" ) { - // ( types-object [, selector] ) - for ( type in types ) { - this.off( type, selector, types[ type ] ); - } - return this; - } - if ( selector === false || typeof selector === "function" ) { - // ( types [, fn] ) - fn = selector; - selector = undefined; - } - if ( fn === false ) { - fn = returnFalse; - } - return this.each(function() { - jQuery.event.remove( this, types, fn, selector ); - }); - }, - - trigger: function( type, data ) { - return this.each(function() { - jQuery.event.trigger( type, data, this ); - }); - }, - triggerHandler: function( type, data ) { - var elem = this[0]; - if ( elem ) { - return jQuery.event.trigger( type, data, elem, true ); - } - } -}); - - -function createSafeFragment( document ) { - var list = nodeNames.split( "|" ), - safeFrag = document.createDocumentFragment(); - - if ( safeFrag.createElement ) { - while ( list.length ) { - safeFrag.createElement( - list.pop() - ); - } - } - return safeFrag; -} - -var nodeNames = "abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|" + - "header|hgroup|mark|meter|nav|output|progress|section|summary|time|video", - rinlinejQuery = / jQuery\d+="(?:null|\d+)"/g, - rnoshimcache = new RegExp("<(?:" + nodeNames + ")[\\s/>]", "i"), - rleadingWhitespace = /^\s+/, - rxhtmlTag = /<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi, - rtagName = /<([\w:]+)/, - rtbody = /\s*$/g, - - // We have to close these tags to support XHTML (#13200) - wrapMap = { - option: [ 1, "" ], - legend: [ 1, "
", "
" ], - area: [ 1, "", "" ], - param: [ 1, "", "" ], - thead: [ 1, "", "
" ], - tr: [ 2, "", "
" ], - col: [ 2, "", "
" ], - td: [ 3, "", "
" ], - - // IE6-8 can't serialize link, script, style, or any html5 (NoScope) tags, - // unless wrapped in a div with non-breaking characters in front of it. - _default: support.htmlSerialize ? [ 0, "", "" ] : [ 1, "X
", "
" ] - }, - safeFragment = createSafeFragment( document ), - fragmentDiv = safeFragment.appendChild( document.createElement("div") ); - -wrapMap.optgroup = wrapMap.option; -wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead; -wrapMap.th = wrapMap.td; - -function getAll( context, tag ) { - var elems, elem, - i = 0, - found = typeof context.getElementsByTagName !== strundefined ? context.getElementsByTagName( tag || "*" ) : - typeof context.querySelectorAll !== strundefined ? context.querySelectorAll( tag || "*" ) : - undefined; - - if ( !found ) { - for ( found = [], elems = context.childNodes || context; (elem = elems[i]) != null; i++ ) { - if ( !tag || jQuery.nodeName( elem, tag ) ) { - found.push( elem ); - } else { - jQuery.merge( found, getAll( elem, tag ) ); - } - } - } - - return tag === undefined || tag && jQuery.nodeName( context, tag ) ? - jQuery.merge( [ context ], found ) : - found; -} - -// Used in buildFragment, fixes the defaultChecked property -function fixDefaultChecked( elem ) { - if ( rcheckableType.test( elem.type ) ) { - elem.defaultChecked = elem.checked; - } -} - -// Support: IE<8 -// Manipulating tables requires a tbody -function manipulationTarget( elem, content ) { - return jQuery.nodeName( elem, "table" ) && - jQuery.nodeName( content.nodeType !== 11 ? content : content.firstChild, "tr" ) ? - - elem.getElementsByTagName("tbody")[0] || - elem.appendChild( elem.ownerDocument.createElement("tbody") ) : - elem; -} - -// Replace/restore the type attribute of script elements for safe DOM manipulation -function disableScript( elem ) { - elem.type = (jQuery.find.attr( elem, "type" ) !== null) + "/" + elem.type; - return elem; -} -function restoreScript( elem ) { - var match = rscriptTypeMasked.exec( elem.type ); - if ( match ) { - elem.type = match[1]; - } else { - elem.removeAttribute("type"); - } - return elem; -} - -// Mark scripts as having already been evaluated -function setGlobalEval( elems, refElements ) { - var elem, - i = 0; - for ( ; (elem = elems[i]) != null; i++ ) { - jQuery._data( elem, "globalEval", !refElements || jQuery._data( refElements[i], "globalEval" ) ); - } -} - -function cloneCopyEvent( src, dest ) { - - if ( dest.nodeType !== 1 || !jQuery.hasData( src ) ) { - return; - } - - var type, i, l, - oldData = jQuery._data( src ), - curData = jQuery._data( dest, oldData ), - events = oldData.events; - - if ( events ) { - delete curData.handle; - curData.events = {}; - - for ( type in events ) { - for ( i = 0, l = events[ type ].length; i < l; i++ ) { - jQuery.event.add( dest, type, events[ type ][ i ] ); - } - } - } - - // make the cloned public data object a copy from the original - if ( curData.data ) { - curData.data = jQuery.extend( {}, curData.data ); - } -} - -function fixCloneNodeIssues( src, dest ) { - var nodeName, e, data; - - // We do not need to do anything for non-Elements - if ( dest.nodeType !== 1 ) { - return; - } - - nodeName = dest.nodeName.toLowerCase(); - - // IE6-8 copies events bound via attachEvent when using cloneNode. - if ( !support.noCloneEvent && dest[ jQuery.expando ] ) { - data = jQuery._data( dest ); - - for ( e in data.events ) { - jQuery.removeEvent( dest, e, data.handle ); - } - - // Event data gets referenced instead of copied if the expando gets copied too - dest.removeAttribute( jQuery.expando ); - } - - // IE blanks contents when cloning scripts, and tries to evaluate newly-set text - if ( nodeName === "script" && dest.text !== src.text ) { - disableScript( dest ).text = src.text; - restoreScript( dest ); - - // IE6-10 improperly clones children of object elements using classid. - // IE10 throws NoModificationAllowedError if parent is null, #12132. - } else if ( nodeName === "object" ) { - if ( dest.parentNode ) { - dest.outerHTML = src.outerHTML; - } - - // This path appears unavoidable for IE9. When cloning an object - // element in IE9, the outerHTML strategy above is not sufficient. - // If the src has innerHTML and the destination does not, - // copy the src.innerHTML into the dest.innerHTML. #10324 - if ( support.html5Clone && ( src.innerHTML && !jQuery.trim(dest.innerHTML) ) ) { - dest.innerHTML = src.innerHTML; - } - - } else if ( nodeName === "input" && rcheckableType.test( src.type ) ) { - // IE6-8 fails to persist the checked state of a cloned checkbox - // or radio button. Worse, IE6-7 fail to give the cloned element - // a checked appearance if the defaultChecked value isn't also set - - dest.defaultChecked = dest.checked = src.checked; - - // IE6-7 get confused and end up setting the value of a cloned - // checkbox/radio button to an empty string instead of "on" - if ( dest.value !== src.value ) { - dest.value = src.value; - } - - // IE6-8 fails to return the selected option to the default selected - // state when cloning options - } else if ( nodeName === "option" ) { - dest.defaultSelected = dest.selected = src.defaultSelected; - - // IE6-8 fails to set the defaultValue to the correct value when - // cloning other types of input fields - } else if ( nodeName === "input" || nodeName === "textarea" ) { - dest.defaultValue = src.defaultValue; - } -} - -jQuery.extend({ - clone: function( elem, dataAndEvents, deepDataAndEvents ) { - var destElements, node, clone, i, srcElements, - inPage = jQuery.contains( elem.ownerDocument, elem ); - - if ( support.html5Clone || jQuery.isXMLDoc(elem) || !rnoshimcache.test( "<" + elem.nodeName + ">" ) ) { - clone = elem.cloneNode( true ); - - // IE<=8 does not properly clone detached, unknown element nodes - } else { - fragmentDiv.innerHTML = elem.outerHTML; - fragmentDiv.removeChild( clone = fragmentDiv.firstChild ); - } - - if ( (!support.noCloneEvent || !support.noCloneChecked) && - (elem.nodeType === 1 || elem.nodeType === 11) && !jQuery.isXMLDoc(elem) ) { - - // We eschew Sizzle here for performance reasons: http://jsperf.com/getall-vs-sizzle/2 - destElements = getAll( clone ); - srcElements = getAll( elem ); - - // Fix all IE cloning issues - for ( i = 0; (node = srcElements[i]) != null; ++i ) { - // Ensure that the destination node is not null; Fixes #9587 - if ( destElements[i] ) { - fixCloneNodeIssues( node, destElements[i] ); - } - } - } - - // Copy the events from the original to the clone - if ( dataAndEvents ) { - if ( deepDataAndEvents ) { - srcElements = srcElements || getAll( elem ); - destElements = destElements || getAll( clone ); - - for ( i = 0; (node = srcElements[i]) != null; i++ ) { - cloneCopyEvent( node, destElements[i] ); - } - } else { - cloneCopyEvent( elem, clone ); - } - } - - // Preserve script evaluation history - destElements = getAll( clone, "script" ); - if ( destElements.length > 0 ) { - setGlobalEval( destElements, !inPage && getAll( elem, "script" ) ); - } - - destElements = srcElements = node = null; - - // Return the cloned set - return clone; - }, - - buildFragment: function( elems, context, scripts, selection ) { - var j, elem, contains, - tmp, tag, tbody, wrap, - l = elems.length, - - // Ensure a safe fragment - safe = createSafeFragment( context ), - - nodes = [], - i = 0; - - for ( ; i < l; i++ ) { - elem = elems[ i ]; - - if ( elem || elem === 0 ) { - - // Add nodes directly - if ( jQuery.type( elem ) === "object" ) { - jQuery.merge( nodes, elem.nodeType ? [ elem ] : elem ); - - // Convert non-html into a text node - } else if ( !rhtml.test( elem ) ) { - nodes.push( context.createTextNode( elem ) ); - - // Convert html into DOM nodes - } else { - tmp = tmp || safe.appendChild( context.createElement("div") ); - - // Deserialize a standard representation - tag = (rtagName.exec( elem ) || [ "", "" ])[ 1 ].toLowerCase(); - wrap = wrapMap[ tag ] || wrapMap._default; - - tmp.innerHTML = wrap[1] + elem.replace( rxhtmlTag, "<$1>" ) + wrap[2]; - - // Descend through wrappers to the right content - j = wrap[0]; - while ( j-- ) { - tmp = tmp.lastChild; - } - - // Manually add leading whitespace removed by IE - if ( !support.leadingWhitespace && rleadingWhitespace.test( elem ) ) { - nodes.push( context.createTextNode( rleadingWhitespace.exec( elem )[0] ) ); - } - - // Remove IE's autoinserted from table fragments - if ( !support.tbody ) { - - // String was a , *may* have spurious - elem = tag === "table" && !rtbody.test( elem ) ? - tmp.firstChild : - - // String was a bare or - wrap[1] === "
" && !rtbody.test( elem ) ? - tmp : - 0; - - j = elem && elem.childNodes.length; - while ( j-- ) { - if ( jQuery.nodeName( (tbody = elem.childNodes[j]), "tbody" ) && !tbody.childNodes.length ) { - elem.removeChild( tbody ); - } - } - } - - jQuery.merge( nodes, tmp.childNodes ); - - // Fix #12392 for WebKit and IE > 9 - tmp.textContent = ""; - - // Fix #12392 for oldIE - while ( tmp.firstChild ) { - tmp.removeChild( tmp.firstChild ); - } - - // Remember the top-level container for proper cleanup - tmp = safe.lastChild; - } - } - } - - // Fix #11356: Clear elements from fragment - if ( tmp ) { - safe.removeChild( tmp ); - } - - // Reset defaultChecked for any radios and checkboxes - // about to be appended to the DOM in IE 6/7 (#8060) - if ( !support.appendChecked ) { - jQuery.grep( getAll( nodes, "input" ), fixDefaultChecked ); - } - - i = 0; - while ( (elem = nodes[ i++ ]) ) { - - // #4087 - If origin and destination elements are the same, and this is - // that element, do not do anything - if ( selection && jQuery.inArray( elem, selection ) !== -1 ) { - continue; - } - - contains = jQuery.contains( elem.ownerDocument, elem ); - - // Append to fragment - tmp = getAll( safe.appendChild( elem ), "script" ); - - // Preserve script evaluation history - if ( contains ) { - setGlobalEval( tmp ); - } - - // Capture executables - if ( scripts ) { - j = 0; - while ( (elem = tmp[ j++ ]) ) { - if ( rscriptType.test( elem.type || "" ) ) { - scripts.push( elem ); - } - } - } - } - - tmp = null; - - return safe; - }, - - cleanData: function( elems, /* internal */ acceptData ) { - var elem, type, id, data, - i = 0, - internalKey = jQuery.expando, - cache = jQuery.cache, - deleteExpando = support.deleteExpando, - special = jQuery.event.special; - - for ( ; (elem = elems[i]) != null; i++ ) { - if ( acceptData || jQuery.acceptData( elem ) ) { - - id = elem[ internalKey ]; - data = id && cache[ id ]; - - if ( data ) { - if ( data.events ) { - for ( type in data.events ) { - if ( special[ type ] ) { - jQuery.event.remove( elem, type ); - - // This is a shortcut to avoid jQuery.event.remove's overhead - } else { - jQuery.removeEvent( elem, type, data.handle ); - } - } - } - - // Remove cache only if it was not already removed by jQuery.event.remove - if ( cache[ id ] ) { - - delete cache[ id ]; - - // IE does not allow us to delete expando properties from nodes, - // nor does it have a removeAttribute function on Document nodes; - // we must handle all of these cases - if ( deleteExpando ) { - delete elem[ internalKey ]; - - } else if ( typeof elem.removeAttribute !== strundefined ) { - elem.removeAttribute( internalKey ); - - } else { - elem[ internalKey ] = null; - } - - deletedIds.push( id ); - } - } - } - } - } -}); - -jQuery.fn.extend({ - text: function( value ) { - return access( this, function( value ) { - return value === undefined ? - jQuery.text( this ) : - this.empty().append( ( this[0] && this[0].ownerDocument || document ).createTextNode( value ) ); - }, null, value, arguments.length ); - }, - - append: function() { - return this.domManip( arguments, function( elem ) { - if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { - var target = manipulationTarget( this, elem ); - target.appendChild( elem ); - } - }); - }, - - prepend: function() { - return this.domManip( arguments, function( elem ) { - if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { - var target = manipulationTarget( this, elem ); - target.insertBefore( elem, target.firstChild ); - } - }); - }, - - before: function() { - return this.domManip( arguments, function( elem ) { - if ( this.parentNode ) { - this.parentNode.insertBefore( elem, this ); - } - }); - }, - - after: function() { - return this.domManip( arguments, function( elem ) { - if ( this.parentNode ) { - this.parentNode.insertBefore( elem, this.nextSibling ); - } - }); - }, - - remove: function( selector, keepData /* Internal Use Only */ ) { - var elem, - elems = selector ? jQuery.filter( selector, this ) : this, - i = 0; - - for ( ; (elem = elems[i]) != null; i++ ) { - - if ( !keepData && elem.nodeType === 1 ) { - jQuery.cleanData( getAll( elem ) ); - } - - if ( elem.parentNode ) { - if ( keepData && jQuery.contains( elem.ownerDocument, elem ) ) { - setGlobalEval( getAll( elem, "script" ) ); - } - elem.parentNode.removeChild( elem ); - } - } - - return this; - }, - - empty: function() { - var elem, - i = 0; - - for ( ; (elem = this[i]) != null; i++ ) { - // Remove element nodes and prevent memory leaks - if ( elem.nodeType === 1 ) { - jQuery.cleanData( getAll( elem, false ) ); - } - - // Remove any remaining nodes - while ( elem.firstChild ) { - elem.removeChild( elem.firstChild ); - } - - // If this is a select, ensure that it displays empty (#12336) - // Support: IE<9 - if ( elem.options && jQuery.nodeName( elem, "select" ) ) { - elem.options.length = 0; - } - } - - return this; - }, - - clone: function( dataAndEvents, deepDataAndEvents ) { - dataAndEvents = dataAndEvents == null ? false : dataAndEvents; - deepDataAndEvents = deepDataAndEvents == null ? dataAndEvents : deepDataAndEvents; - - return this.map(function() { - return jQuery.clone( this, dataAndEvents, deepDataAndEvents ); - }); - }, - - html: function( value ) { - return access( this, function( value ) { - var elem = this[ 0 ] || {}, - i = 0, - l = this.length; - - if ( value === undefined ) { - return elem.nodeType === 1 ? - elem.innerHTML.replace( rinlinejQuery, "" ) : - undefined; - } - - // See if we can take a shortcut and just use innerHTML - if ( typeof value === "string" && !rnoInnerhtml.test( value ) && - ( support.htmlSerialize || !rnoshimcache.test( value ) ) && - ( support.leadingWhitespace || !rleadingWhitespace.test( value ) ) && - !wrapMap[ (rtagName.exec( value ) || [ "", "" ])[ 1 ].toLowerCase() ] ) { - - value = value.replace( rxhtmlTag, "<$1>" ); - - try { - for (; i < l; i++ ) { - // Remove element nodes and prevent memory leaks - elem = this[i] || {}; - if ( elem.nodeType === 1 ) { - jQuery.cleanData( getAll( elem, false ) ); - elem.innerHTML = value; - } - } - - elem = 0; - - // If using innerHTML throws an exception, use the fallback method - } catch(e) {} - } - - if ( elem ) { - this.empty().append( value ); - } - }, null, value, arguments.length ); - }, - - replaceWith: function() { - var arg = arguments[ 0 ]; - - // Make the changes, replacing each context element with the new content - this.domManip( arguments, function( elem ) { - arg = this.parentNode; - - jQuery.cleanData( getAll( this ) ); - - if ( arg ) { - arg.replaceChild( elem, this ); - } - }); - - // Force removal if there was no new content (e.g., from empty arguments) - return arg && (arg.length || arg.nodeType) ? this : this.remove(); - }, - - detach: function( selector ) { - return this.remove( selector, true ); - }, - - domManip: function( args, callback ) { - - // Flatten any nested arrays - args = concat.apply( [], args ); - - var first, node, hasScripts, - scripts, doc, fragment, - i = 0, - l = this.length, - set = this, - iNoClone = l - 1, - value = args[0], - isFunction = jQuery.isFunction( value ); - - // We can't cloneNode fragments that contain checked, in WebKit - if ( isFunction || - ( l > 1 && typeof value === "string" && - !support.checkClone && rchecked.test( value ) ) ) { - return this.each(function( index ) { - var self = set.eq( index ); - if ( isFunction ) { - args[0] = value.call( this, index, self.html() ); - } - self.domManip( args, callback ); - }); - } - - if ( l ) { - fragment = jQuery.buildFragment( args, this[ 0 ].ownerDocument, false, this ); - first = fragment.firstChild; - - if ( fragment.childNodes.length === 1 ) { - fragment = first; - } - - if ( first ) { - scripts = jQuery.map( getAll( fragment, "script" ), disableScript ); - hasScripts = scripts.length; - - // Use the original fragment for the last item instead of the first because it can end up - // being emptied incorrectly in certain situations (#8070). - for ( ; i < l; i++ ) { - node = fragment; - - if ( i !== iNoClone ) { - node = jQuery.clone( node, true, true ); - - // Keep references to cloned scripts for later restoration - if ( hasScripts ) { - jQuery.merge( scripts, getAll( node, "script" ) ); - } - } - - callback.call( this[i], node, i ); - } - - if ( hasScripts ) { - doc = scripts[ scripts.length - 1 ].ownerDocument; - - // Reenable scripts - jQuery.map( scripts, restoreScript ); - - // Evaluate executable scripts on first document insertion - for ( i = 0; i < hasScripts; i++ ) { - node = scripts[ i ]; - if ( rscriptType.test( node.type || "" ) && - !jQuery._data( node, "globalEval" ) && jQuery.contains( doc, node ) ) { - - if ( node.src ) { - // Optional AJAX dependency, but won't run scripts if not present - if ( jQuery._evalUrl ) { - jQuery._evalUrl( node.src ); - } - } else { - jQuery.globalEval( ( node.text || node.textContent || node.innerHTML || "" ).replace( rcleanScript, "" ) ); - } - } - } - } - - // Fix #11809: Avoid leaking memory - fragment = first = null; - } - } - - return this; - } -}); - -jQuery.each({ - appendTo: "append", - prependTo: "prepend", - insertBefore: "before", - insertAfter: "after", - replaceAll: "replaceWith" -}, function( name, original ) { - jQuery.fn[ name ] = function( selector ) { - var elems, - i = 0, - ret = [], - insert = jQuery( selector ), - last = insert.length - 1; - - for ( ; i <= last; i++ ) { - elems = i === last ? this : this.clone(true); - jQuery( insert[i] )[ original ]( elems ); - - // Modern browsers can apply jQuery collections as arrays, but oldIE needs a .get() - push.apply( ret, elems.get() ); - } - - return this.pushStack( ret ); - }; -}); - - -var iframe, - elemdisplay = {}; - -/** - * Retrieve the actual display of a element - * @param {String} name nodeName of the element - * @param {Object} doc Document object - */ -// Called only from within defaultDisplay -function actualDisplay( name, doc ) { - var elem = jQuery( doc.createElement( name ) ).appendTo( doc.body ), - - // getDefaultComputedStyle might be reliably used only on attached element - display = window.getDefaultComputedStyle ? - - // Use of this method is a temporary fix (more like optmization) until something better comes along, - // since it was removed from specification and supported only in FF - window.getDefaultComputedStyle( elem[ 0 ] ).display : jQuery.css( elem[ 0 ], "display" ); - - // We don't have any data stored on the element, - // so use "detach" method as fast way to get rid of the element - elem.detach(); - - return display; -} - -/** - * Try to determine the default display value of an element - * @param {String} nodeName - */ -function defaultDisplay( nodeName ) { - var doc = document, - display = elemdisplay[ nodeName ]; - - if ( !display ) { - display = actualDisplay( nodeName, doc ); - - // If the simple way fails, read from inside an iframe - if ( display === "none" || !display ) { - - // Use the already-created iframe if possible - iframe = (iframe || jQuery( " diff --git a/docs/development/workflow/this_project.inc b/docs/development/workflow/this_project.inc deleted file mode 100644 index 453ebb46bfbb..000000000000 --- a/docs/development/workflow/this_project.inc +++ /dev/null @@ -1,2 +0,0 @@ -.. _`Astropy GitHub`: http://github.com/astropy/astropy - diff --git a/docs/development/workflow/virtual_pythons.rst b/docs/development/workflow/virtual_pythons.rst deleted file mode 100644 index 26e14d8c9e55..000000000000 --- a/docs/development/workflow/virtual_pythons.rst +++ /dev/null @@ -1,189 +0,0 @@ -:orphan: - -.. include:: links.inc -.. _virtual_envs: - -=========================== -Python virtual environments -=========================== - -If you plan to do regular work on astropy you should do your development in -a python virtual environment. Conceptually a virtual environment is a -duplicate of the python environment you normally work in with as many (or as -few) of the packages from your normal environment included in that virtual -environment. It is sandboxed from your normal python environment in the sense -that packages installed in the virtual environment do not affect your normal -environment in any way. - -.. note:: - "Normal python environment" means whatever python you are using when you - log in. - -There are two options for using virtual environments; the choice of method is -dictated by the python distribution you use: - -* If you use the anaconda python distribution you must use `conda`_ to make - and manage your virtual environments. -* If you use any other distribution you use `virtualenvwrapper`_; you *can not* - use `conda`_. As the name suggests, `virtualenvwrapper`_ is a wrapper around - `virtualenv`_. - -In both cases you will go through the same basic steps; the commands to -accomplish each step are given for both `conda`_ and `virtualenvwrapper`_: - -* :ref:`setup_for_env` -* :ref:`list_env` -* :ref:`create_env` -* :ref:`activate_env` -* :ref:`deactivate_env` -* :ref:`delete_env` - -.. note:: - + You **cannot** use `virtualenvwrapper`_ or `virtualenv`_ within anaconda. - + `virtualenvwrapper`_ works with bash and bash-like shells; see - :ref:`using-virtualenv` for alternatives. - -.. _setup_for_env: - - -Set up for virtual environments -------------------------------- - -* `virtualenvwrapper`_: - - + First, install `virtualenvwrapper`_, which will also install `virtualenv`_, - with ``pip install virtualenvwrapper``. - + From the `documentation for virtualenvwrapper`_, you also need to:: - - export WORKON_HOME=$HOME/.virtualenvs - export PROJECT_HOME=$HOME/ - source /usr/local/bin/virtualenvwrapper.sh - -* `conda`_: No setup is necessary beyond installing the anaconda python - distribution. - -.. _list_env: - -List virtual environments -------------------------- - -You do not need to list the virtual environments you have created before using -them...but sooner or later you will forget what environments you have defined -and this is the easy way to find out. - -* `virtualenvwrapper`_: ``workon`` - + If this displays nothing you have no virtual environments - + If this displays ``workon: command not found`` then you haven't done - the :ref:`setup_for_env`; do that. - + For more detailed information about installed environments use - ``lsvirtualenv``. -* `conda`_: ``conda info -e`` - + you will always have at least one environment, called ``root`` - + your active environment is indicated by a ``*`` - -.. _create_env: - -Create a new virtual environment --------------------------------- - -This needs to be done once for each virtual environment you want. There is one -important choice you need to make when you create a virtual environment: -which, if any, of the packages installed in your normal python environment do -you want in your virtual environment? - -Including them in your virtual environment doesn't take much extra space--they -are linked into the virtual environment instead of being copied. Within the -virtual environment you can install new versions of packages like Numpy or -Astropy that override the versions installed in your normal python environment. - -The easiest way to get started is to include in your virtual environment the -packages installed in your your normal python environment; the instructions -below do that. - -In everything that follows, ``ENV`` represents the name you give your virtual -environment. - -**The name you choose cannot have spaces in it.** - -* `virtualenvwrapper`_: - + Make an environment called ``ENV`` with all of the packages in your normal - python environment:: - - ``mkvirtualenv --system-site-packages ENV`` - - + Omit the option ``--system-site-packages`` to create an environment - without the python packages installed in your normal python environment. - + Environments created with `virtualenvwrapper`_ always include `pip`_ - and `setuptools `_ so that you - can install packages within the virtual environment. - + More details and examples are in the - `virtualenvwrapper command documentation`_. -* `conda`_: - + Make an environment called ``ENV`` with all of the packages in your main - anaconda environment:: - - ``conda create -n ENV anaconda`` - - + More details, and examples that start with none of the packages from - your normal python environment, are in the - `documentation for the conda command`_ and the - `blog post announcing anaconda environments`_. - -.. _activate_env: - -Activate a virtual environment ------------------------------- - -To use a new virtual environment you may need to activate it; -`virtualenvwrapper`_ will try to automatically activate your new environment -when you create it. Activation does two things (either of which you could do -manually, though it would be inconvenient): - -* Put the ``bin`` directory for the virtual environment at the front of your - ``$PATH``. -* Add the name of the virtual environment to your command prompt. If you have - successfully switched to a new environment called ``ENV`` your prompt should - look something like this: ``(ENV)[~] $`` - -The commands below allow you to switch between virtual environments in -addition to activating new ones. - -* `virtualenvwrapper`_: Activate the environment ``ENV`` with:: - - workon ENV - -* ` conda`: Activate the environment ``ENV`` with:: - - source activate ENV - - -.. _deactivate_env: - -Deactivate a virtual environment --------------------------------- - -At some point you may want to go back to your normal python environment. Do -that with: - -* `virtualenvwrapper`_: ``deactivate`` - + Note that in ``virtualenvwrapper 4.1.1`` the output of - ``mkvirtualenv`` says you should use ``source deactivate``; that does - not seem to actually work. -* `conda`_: ``source deactivate`` - -.. _delete_env: - -Delete a virtual environment ----------------------------- - -In both `virtualenvwrapper`_ and `conda`_ you can simply delete the directory in -which the ``ENV`` is located; both also provide commands to make that a bit easier. - -* `virtualenvwrapper`_: ``rmvirtualenv ENV`` -* `conda`_: ``conda remove --all -n ENV`` - -.. _documentation for virtualenvwrapper: http://virtualenvwrapper.readthedocs.org/en/latest/install.html -.. _virtualenvwrapper command documentation: http://virtualenvwrapper.readthedocs.org/en/latest/command_ref.html -.. _documentation for the conda command: http://docs.continuum.io/conda/examples/create.html -.. _blog post announcing anaconda environments: http://www.continuum.io/blog/conda - diff --git a/docs/development/workflow/virtualenv_detail.rst b/docs/development/workflow/virtualenv_detail.rst deleted file mode 100644 index 3884e81e981f..000000000000 --- a/docs/development/workflow/virtualenv_detail.rst +++ /dev/null @@ -1,164 +0,0 @@ -:orphan: - -.. _using-virtualenv: - -Using virtualenv -================ - -`virtualenv`_ is a tool for creating and activating isolated Python -environments that allow installing and experimenting with Python packages -without disrupting your production Python environment. When using commands -such as ``python setup.py develop``, for example, it is strongly recommended to -do so within a virtualenv. This is generally preferable to installing a -development version of Astropy into your system site-packages and having to -keep track of whether or not your environment is in a "known good" -configuration for production/science use. - -Using a virtualenv is also a good way to try out new versions of software that -you're not actively doing development work on without disrupting your normal -production environment. - -We won't provide a full tutorial on using virtualenv here |emdash| the -virtualenv documentation linked to above is a better place to start. But here -is a quick overview on how to set up a virtualenv for Astropy development with -your default Python version: - -#. Install virtualenv:: - - $ pip install virtualenv - - or:: - - $ easy_install virtualenv - - or (on Debian/Ubuntu):: - - $ sudo apt-get install python-virtualenv - - etc. - -#. (Recommended) Create a root directory for all your virtualenvs under a path - you have write access to. For example:: - - $ mkdir ~/.virtualenvs - -#. Create the Astropy virtualenv:: - - $ virtualenv --distribute --system-site-packages ~/.virtualenvs/astropy-dev - - The ``--system-site-packages`` option inherits all packages already - installed in your system site-packages directory; this frees you from having - to reinstall packages like Numpy and Scipy in the virtualenv. However, if - you would like your virtualenv to use a development version of Numpy, for - example, you can still install Numpy into the virtualenv and it will take - precedence over the version installed in site-packages. - -#. Activate the virtualenv:: - - $ source ~/.virtualenvs/astropy-dev/bin/activate - - or if you're using a csh-variant:: - - $ source ~/.virtualenvs/astropy-dev/bin/activate.csh - - virtualenv works on Windows too |emdash| see the documentation for details. - -#. If the virtualenv successfully activated its name should appear in your - shell prompt:: - - (astropy-dev) $ - - The virtualenv can be disabled at any time by entering:: - - (astropy-dev) $ deactivate - -#. Now as long as the virtualenv is activated, packages you install with - ``pip``, ``easy_install``, or by manually running ``python setup.py - install`` will automatically install into your virtualenv instead of the - system site-packages. Consider installing Astropy in develop mode into the - virtualenv as described :ref:`activate_development_astropy`. - -Using virtualenv with IPython ------------------------------ - -.. note:: - - As of IPython 0.13 this functionality is built into IPython and these steps - are not necessary for IPython to recognize that it's running with a - virtualenv enabled. - -Each virtualenv has its own ``bin/``, and as IPython is written in pure Python -one can always install IPython directly into a virtualenv. However, if you -would rather not have to install IPython every time you create a virtualenv, it -also suffices to make IPython virtualenv-aware. - -1. Check to see if you already have an IPython profile in - ``~/.ipython/profile_default/``; if not, create one:: - - $ ipython profile create - -2. Edit ``~/.ipython/profile_default/ipython_config.py`` and add the - following to the end:: - - import os - - execfile(os.path.join(os.environ['HOME'], '.ipython', 'virtualenv.py')) - -3. Finally, create the ``~/.ipython/virtualenv.py`` module:: - - import site - from os import environ - from os.path import join - from sys import version_info - - if 'VIRTUAL_ENV' in environ: - virtual_env = join(environ.get('VIRTUAL_ENV'), - 'lib', - 'python%d.%d' % version_info[:2], - 'site-packages') - site.addsitedir(virtual_env) - print 'VIRTUAL_ENV ->', virtual_env - del virtual_env - del site, environ, join, version_info - -Now IPython will import all packages from your virtualenv where applicable. - -.. note:: - - This is not magic. If you switch to a virtualenv that uses a different - Python version from your main IPython installation this won't help you - |emdash| instead use the appropriate IPython installation for the Python - version in question. - -virtualenvwrapper ------------------ - -`virtualenvwrapper`_ is a set of enhancements to virtualenv mostly -implemented through simple shell scripts and aliases. It automatically -organizes all your virtualenvs under a single directory (as suggested -above). To create a new virtualenv you can just use the ``'mkvirtualenv -'`` command and it will automatically create a new virtualenv of -that name in the default location. - -To activate a virtualenv with virtualenvwrapper you don't need to think -about the environment's location of the filesystem or which activate script -to run. Simply run ``'workon '``. You can also list all -virtualenvs with ``lsvirtualenv``. That just scratches the surface of the -goodies included with virtualenvwrapper. - -The one caveat is that it does not support csh-like shells. For csh-like -shells there exists `virtualenvwrapper-csh`_, which implements most of the -virtualenvwrapper functionality and is otherwise compatible with the original. -There also exists `virtualenvwrapper-win`_, which ports virtualenvwrapper to -Windows batch scripts. - -venv ----- - -virtualenv is so commonly used in the Python development community that its -functionality was finally added to the standard library in Python 3.3 under -the name `venv`_. venv has not gained wide use yet and is not explicitly -supported by tools like virtualenvwrapper, but it is expected to see wider -adoption in the future. - -.. include:: links.inc diff --git a/docs/environment_variables.rst b/docs/environment_variables.rst new file mode 100644 index 000000000000..0f8f5e5ad1f0 --- /dev/null +++ b/docs/environment_variables.rst @@ -0,0 +1,39 @@ +.. currentmodule:: astropy + +.. _environment_variables: + +********************* +Environment variables +********************* + +Cache and configuration locations +================================= + +Since version 8.0.0, astropy follows the XDG specification by default, on every platform +(including non-Linux ones). As a result, the default cache location is +``$XDG_CACHE_HOME/astropy``, where ``XDG_CACHE_HOME`` itself defaults to +``$HOME/.cache``. The same goes for configuration, replacing ``cache`` with ``config``, +and preserving case. + +In addition to these, and since v8.0.0, astropy supports comparable, tool-specific +environment variables for finer control: + +.. glossary:: + + ``ASTROPY_CACHE_DIR`` + takes precedence over ``XDG_CACHE_HOME`` and defines an + entire path (as opposed to ``XDG_CACHE_HOME`` which only defines the *parent* + directory of the one used by astropy). Its value must represent an absolute path, + and must not point to file. Invalid values are ignored with a warning when the + variable is evaluated. + See :ref:`utils-data` for how to programmatically set or get the location of the + corresponding directory at runtime. + + ``ASTROPY_CONFIG_DIR`` + takes precedence over ``XDG_CONFIG_HOME`` and defines an + entire path (as opposed to ``XDG_CONFIG_HOME`` which only defines the *parent* + directory of the one used by astropy). Its value must represent an absolute path, + and must not point to file. Invalid values are ignored with a warning when the + variable is evaluated. + See :ref:`astropy_config` for how to programmatically set or get the location of + the corresponding directory at runtime. diff --git a/docs/getting_started.rst b/docs/getting_started.rst deleted file mode 100644 index 0b212bd38cb2..000000000000 --- a/docs/getting_started.rst +++ /dev/null @@ -1,96 +0,0 @@ -**************************** -Getting Started with Astropy -**************************** - -Importing Astropy -================= - -In order to encourage consistency amongst users in importing and using Astropy -functionality, we have put together the following guidelines. - -Since most of the functionality in Astropy resides in sub-packages, importing -astropy as:: - - >>> import astropy - -is not very useful. Instead, it is best to import the desired sub-package -with the syntax:: - - >>> from astropy import subpackage # doctest: +SKIP - -For example, to access the FITS-related functionality, you can import -`astropy.io.fits` with:: - - >>> from astropy.io import fits - >>> hdulist = fits.open('data.fits') # doctest: +SKIP - -In specific cases, we have recommended shortcuts in the documentation for -specific sub-packages, for example:: - - >>> from astropy import units as u - >>> from astropy import coordinates as coord - >>> coord.SkyCoord(ra=10.68458*u.deg, dec=41.26917*u.deg, frame='icrs') - - -Finally, in some cases, most of the required functionality is contained in a -single class (or a few classes). In those cases, the class can be directly -imported:: - - >>> from astropy.cosmology import WMAP7 - >>> from astropy.table import Table - >>> from astropy.wcs import WCS - -Note that for clarity, and to avoid any issues, we recommend to **never** -import any Astropy functionality using ``*``, for example:: - - >>> from astropy.io.fits import * # NOT recommended - -Some components of Astropy started off as standalone packages (e.g. PyFITS, PyWCS), -so in cases where Astropy needs to be used as a drop-in replacement, the following -syntax is also acceptable:: - - >>> from astropy.io import fits as pyfits - -Getting started with subpackages -================================ - -Because different subpackages have very different functionality, further -suggestions for getting started are in the documentation for the subpackages, -which you can reach by browsing the sections listed in the :ref:`user-docs`. - -Or, if you want to dive right in, you can either look at docstrings for -particular a package or object, or access their documentation using the -`~astropy.utils.misc.find_api_page` function. For example, doing this:: - - >>> from astropy import find_api_page - >>> from astropy.units import Quantity - >>> find_api_page(Quantity) # doctest: +SKIP - -Will bring up the documentation for the `~astropy.units.Quantity` class -in your browser. - -Command-line utilities -====================== - -For convenience, several of Astropy's subpackages install utility programs -on your system which allow common tasks to be performed without having -to open a Python interpreter. These utilities include: - -- `~astropy.io.fits.scripts.fitsheader`: prints the headers of a FITS file. - -- `~astropy.io.fits.scripts.fitscheck`: verifies and optionally re-writes - the CHECKSUM and DATASUM keywords of a FITS file. - -- :ref:`fitsdiff`: compares two FITS files and reports the differences. - -- :ref:`fits2bitmap`: converts FITS images to bitmaps, including scaling and - stretching. - -- :ref:`samp_hub `: starts a :ref:`SAMP ` hub. - -- ``volint``: checks a :ref:`VOTable ` - file for compliance against the standards. - -- :ref:`wcslint `: checks the :ref:`WCS ` keywords in a - FITS file for compliance against the standards. diff --git a/docs/glossary.rst b/docs/glossary.rst new file mode 100644 index 000000000000..3a4fb7080211 --- /dev/null +++ b/docs/glossary.rst @@ -0,0 +1,106 @@ +.. currentmodule:: astropy + +**************** +Astropy Glossary +**************** + +.. glossary:: + + (``n``,) + A parenthesized number followed by a comma denotes a tuple with one + element. The trailing comma distinguishes a one-element tuple from a + parenthesized ``n``. + This is from NumPy; see https://numpy.org/doc/stable/glossary.html#term-n. + + -like + ``-like`` is an instance of the ``Class`` or a valid initializer argument + for ``Class`` as ``Class(value)``. E.g. :class:`~astropy.units.Quantity`-like + includes ``"2 * u.km"`` because ``astropy.units.Quantity("2 * u.km")`` works. + + ['physical type'] + The physical type of a quantity can be annotated in square brackets + following a `~astropy.units.Quantity` (or similar :term:`quantity-like`). + + For example, ``distance : quantity-like ['length']`` + + angle-like + :term:`quantity-like` and a valid initializer for `~astropy.coordinates.Angle`. + The ``unit`` must be an angular. A string input is interpreted as an angle as + described in the `~astropy.coordinates.Angle` documentation. + + buffer-like + Object that implements `Python's buffer protocol + `_. + + coordinate-like + :class:`~astropy.coordinates.BaseCoordinateFrame` subclass instance, or a + :class:`~astropy.coordinates.SkyCoord` (or subclass) instance, or a valid + initializer as described in :ref:`coordinates-initialization-coord`. + + file-like (readable) + :term:`python:file-like object` object that supports reading with a method ``read``. + + For a formal definition see :class:`~astropy.io.typing.ReadableFileLike`. + + file-like (writeable) + :term:`python:file-like object` object that supports writing with a method ``write``. + + For a formal definition see :class:`~astropy.io.typing.WriteableFileLike`. + + frame-like + :class:`~astropy.coordinates.BaseCoordinateFrame` subclass or subclass instance or + a valid Frame name (string). + + length-like + :term:`quantity-like` and a valid initializer for + :class:`~astropy.coordinates.Distance`. The ``unit`` must be a convertible to a + unit of length. + + number + Any scalar numeric type. e.g. `float` or `int` or ``numpy.number``. + + quantity-like + `~astropy.units.Quantity` (or subclass) instance, a number or `array-like + `_ object, or a string + which is a valid initializer for `~astropy.units.Quantity`. + + For a formal definition see :obj:`~astropy.units.typing.QuantityLike`. + + table-like + :class:`~astropy.table.Table` (or subclass) instance or valid initializer for + :class:`~astropy.table.Table` as described in :ref:`construct_table`. Common types + include ``dict[list]``, ``list[dict]``, ``list[list]``, and `~numpy.ndarray` + (structured array). + + time-like + :class:`~astropy.time.Time` (or subclass) instance or a valid initializer for + :class:`~astropy.time.Time`, e.g. `str`, array-like[str], `~datetime.datetime`, or + `~numpy.datetime64`. + + trait type + In short, a trait type is a class with the following properties: + + - It is a class that can be used as a mixin to add functionality to another class. + - It should never be instantiated directly. + - It should not be used as a base class for other classes, but only as a mixin. + - It can define methods, properties, and attributes -- any of which can be abstract. + - It can be generic, i.e. it can have type parameters. + - It can subclass other traits, but should have a linear MRO. + + These are the same set of properties as orthogonal mixin classes, with the added + emphasis that they can serve as compiled types, if so enabled by a compilation system such as `mypyc `_. + + unit-like + :class:`~astropy.units.UnitBase` subclass instance or a valid initializer for + :class:`~astropy.units.Unit`, e.g., `str` or scalar `~astropy.units.Quantity`. + + +Optional Packages' Glossary +*************************** + +.. currentmodule:: matplotlib.pyplot + +.. glossary:: + + color + Any valid Matplotlib color. diff --git a/docs/impact_health.rst b/docs/impact_health.rst new file mode 100644 index 000000000000..02f476744c53 --- /dev/null +++ b/docs/impact_health.rst @@ -0,0 +1,71 @@ +################# +Impact and Health +################# + +The figures on this page give a sense of the level of impact the ``astropy`` +codebase has on the astronomy community and research in the field, as well as +the amount of development effort contributed over time (the codebase's "health"). +Assessments of The Astropy Project's engagement with the diverse astronomy +community are an equally important but separate consideration and are detailed +in: + +- `Astropy community engagement study `_ + +- `Astropy diversity, equity and inclusion study `_ + +Concerning the codebase, a major positive trend is that citations to ``astropy``, +as a proxy for its usage in research, have been consistently growing since The +Astropy Project was created. However, the amount of developer resources has +remained small, with a limited number of volunteer maintainers met with an +increasing amount of development responsibility. Hence, we welcome any help +that you can give! + +First considering the citations, this figure estimates ``astropy``'s usage in +astronomy research by its yearly publication impact, using citations drawn from +the NASA ADS database. The continual growth in citations shows the broad and +increasing usage of ``astropy`` in astronomy and scientific research, as well as +the high impact that contributions to ``astropy`` can have. + +|Citation figure| + +To next visualize the size of the developer community contributing to the +``astropy`` core library, this figure shows the number of people authoring commits +over time. While each year ``astropy`` is cited more in papers, the amount of +developers and especially those contributing multiple times is modest and +largely static. + +|Commits figure| + +The workload to maintain ``astropy`` can be traced through the number of issues and +pull requests open and closed in the ``astropy`` core library over time. The +long-term increase in overall pull requests, along with a comparable amount of +opens and closes on average, shows that the small number of maintainers is +increasingly taxed. The discrepancy between issue opens and closes over time +reinforces this. + +|Issue PR history figure| + +In short, ``astropy`` would greatly benefit from more developers, whose +contributions would reach a significant fraction of research in astronomy. This +figure shows the number of open issues and pull requests for each subpackage in +``astropy``. In addition to indicating which functionalities are used more heavily +by the community at present, it gives a sense of where you could start if +you're interested in contributing to ``astropy``. + +|Open issue PR figure| + +.. |Citation figure| image:: https://github.com/astropy/repo_stats/blob/cache/cache/astropy_citations.png?raw=true + :width: 800 + :alt: Astropy citations + +.. |Commits figure| image:: https://github.com/astropy/repo_stats/blob/cache/cache/astropy_authors.png?raw=true + :width: 800 + :alt: Astropy commit author history + +.. |Issue PR history figure| image:: https://github.com/astropy/repo_stats/blob/cache/cache/astropy_issues_PRs.png?raw=true + :width: 800 + :alt: Astropy issue and pull request history + +.. |Open issue PR figure| image:: https://github.com/astropy/repo_stats/blob/cache/cache/astropy_open_items.png?raw=true + :width: 800 + :alt: Astropy open issues and pull requests diff --git a/docs/importing_astropy.rst b/docs/importing_astropy.rst new file mode 100644 index 000000000000..ce2e7dfbe395 --- /dev/null +++ b/docs/importing_astropy.rst @@ -0,0 +1,69 @@ +************************************** +Importing ``astropy`` and Sub-packages +************************************** + +In order to encourage consistency among users in importing and using Astropy +functionality, we have put together the following guidelines. + +Since most of the functionality in Astropy resides in sub-packages, importing +``astropy`` as:: + + >>> import astropy + +is not very useful. Instead, it's best to import the desired sub-package +with the syntax:: + + >>> from astropy import subpackage # doctest: +SKIP + +For example, to access the FITS-related functionality, you can import +`astropy.io.fits` with:: + + >>> from astropy.io import fits + >>> hdulist = fits.open('data.fits') # doctest: +SKIP + +In specific cases, we have recommended shortcuts in the documentation for +specific sub-packages. For example:: + + >>> from astropy import units as u + >>> from astropy import coordinates as coord + >>> coord.SkyCoord(ra=10.68458*u.deg, dec=41.26917*u.deg, frame='icrs') # doctest: +FLOAT_CMP + + +Finally, in some cases, most of the required functionality is contained in a +single class (or a few classes). In those cases, the class can be directly +imported:: + + >>> from astropy.cosmology import WMAP7 + >>> from astropy.table import Table + >>> from astropy.wcs import WCS + +Note that for clarity, and to avoid any issues, we recommend **never** +importing any Astropy functionality using ``*``, for example:: + + >>> from astropy.io.fits import * # NOT recommended + +Some components of Astropy started off as standalone packages (e.g. PyFITS, +PyWCS), so in cases where Astropy needs to be used as a drop-in replacement, +the following syntax is also acceptable:: + + >>> from astropy.io import fits as pyfits + +********************************* +Getting Started with Sub-packages +********************************* + +Because different sub-packages have very different functionalities, each +sub-package has its own getting started guide. These can be found by browsing +the sections listed in the :ref:`user-docs`. + +You can also look at docstrings for a particular package or object, or access +their documentation using the `~astropy.utils.misc.find_api_page` function. For +example, :: + + >>> from astropy import find_api_page + >>> from astropy.units import Quantity + >>> find_api_page(Quantity) # doctest: +SKIP + +will bring up the documentation for the `~astropy.units.Quantity` class +in your browser. diff --git a/docs/index.rst b/docs/index.rst index 79629c5b0956..ad601e63f836 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,230 +1,131 @@ -.. Astropy documentation master file, created by - sphinx-quickstart on Tue Jul 26 02:59:34 2011. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +.. Hide the left hand sidebar on the home page because it's empty -:tocdepth: 2 - -.. the "raw" directive below is used to hide the title in favor of just the logo being visible .. raw:: html - - -################################## -Astropy Core Package Documentation -################################## - -.. |logo_svg| image:: _static/astropy_banner.svg - -.. |logo_png| image:: _static/astropy_banner_96.png - -.. raw:: html - - - -.. only:: latex - - .. image:: _static/astropy_logo.pdf - -Welcome to the Astropy documentation! Astropy is a community-driven -package intended to contain much of the core functionality and some common -tools needed for performing astronomy and astrophysics with Python. + -**Astronomy computations and utilities** +:tocdepth: 3 -.. toctree:: - :maxdepth: 1 +.. _astropy-docs-index: - convolution/index - visualization/index - cosmology/index - stats/index - vo/index +################################################# +astropy: A Community Python Library for Astronomy +################################################# -**Nuts and bolts of Astropy** +**Version**: |release| - :ref:`whatsnew-8.1` -.. toctree:: - :maxdepth: 1 +**Useful links**: +:ref:`Installation ` | +`Issues & Ideas `__ | +:ref:`astropy-org-help` | +:ref:`astropy-org-contribute` | +:ref:`astropy-org-about` - config/index - io/registry - logging - warnings - utils/index +The ``astropy`` package contains key functionality and common tools needed for +performing astronomy and astrophysics with Python. It is at the core of the +:ref:`Astropy Project `, which aims to enable +the community to develop a robust ecosystem of :ref:`astropy-org-affiliated` +covering a broad range of needs for astronomical research, data +processing, and data analysis. -**Astropy project details** +.. Important:: If you use Astropy for work presented in a publication + or talk please help the project via proper + :ref:`astropy-org-acknowledge`. This also applies to use of + software or :ref:`astropy-org-affiliated` that depend on the astropy core + package. .. toctree:: :maxdepth: 1 + :hidden: - stability - whatsnew/index - known_issues - credits - license + index_getting_started + index_user_docs + index_dev + index_project_details -.. _getting_help: +.. grid:: 2 + :gutter: 2 -************ -Getting help -************ + .. grid-item-card:: Getting Started + :link: index_getting_started + :link-type: doc + :text-align: center -If you want to get help or discuss issues with other Astropy users, you can sign -up for the `astropy mailing list`_. Alternatively, the `astropy-dev mailing -list`_ is where you should go to discuss more technical aspects of Astropy with -the developers. You can also email the astropy developers privately at -`astropy-feedback@googlegroups.com`_...but remember that questions you ask -publicly serve as resources for other users! + :material-outlined:`directions_run;8em;sd-text-secondary` -.. _reporting_issues: + New to Astropy? Check out the getting started guides. They contain an + introduction to astropy's main concepts and links to additional tutorials. -**************** -Reporting Issues -**************** + .. grid-item-card:: User Guide + :link: index_user_docs + :link-type: doc + :text-align: center -If you have found a bug in Astropy please report it. The preferred way is to -create a new issue on the Astropy `GitHub issue page -`_; that requires `creating a free -account `_ on GitHub if you do not have one. + :material-outlined:`menu_book;8em;sd-text-secondary` -If you prefer not to create a GitHub account, please report the issue to either -the `astropy mailing list`_, the `astropy-dev mailing list`_ or sending a -private email to the astropy core developers at -`astropy-feedback@googlegroups.com `_. + The user guide provides in-depth information on the key concepts + of astropy with useful background information and explanation. -Please include an example that demonstrates the issue that will allow the -developers to reproduce and fix the problem. You may be asked to also provide -information about your operating system and a full Python stack trace; the -Astropy developers will walk you through obtaining a stack trace if it is -necessary. + .. grid-item-card:: Learn Astropy + :link: https://learn.astropy.org + :link-type: url + :text-align: center + :material-outlined:`psychology;8em;sd-text-secondary` -For astropy-helpers -------------------- + Learn how to use Python for astronomy through tutorials and guides that cover + Astropy and other packages in the astronomy Python ecosystem. -As of Astropy v0.4, Astropy and many affiliated packages use a package of -utilities called astropy-helpers during building and installation. If you have -any build/installation issue--particularly if you're getting a traceback -mentioning the ``astropy_helpers`` or ``ah_bootstrap`` modules--please send a -report to the `astropy-helpers issue tracker -`_. If you're not sure, -however, it's fine to report via the main Astropy issue tracker or one of the -other avenues described above. + .. grid-item-card:: Astropy Packages + :link: astropy-org-affiliated + :link-type: ref + :text-align: center + :material-outlined:`inventory_2;8em;sd-text-secondary` -************ -Contributing -************ + The Astropy Project ecosystem includes numerous `Coordinated + `_ and `Affiliated + `_ packages. + Coordinated packages are maintained by the Project. -The Astropy project is made both by and for its users, so we highly encourage -contributions at all levels. This spans the gamut from sending an email -mentioning a typo in the documentation or requesting a new feature all the way -to developing a major new package. + .. grid-item-card:: Contributor's Guide + :link: index_dev + :link-type: doc + :text-align: center -The full range of ways to be part of the Astropy project are described at -`Contribute to Astropy `_. To get -started contributing code or documentation (no git or GitHub experience -necessary): - -.. toctree:: - :maxdepth: 1 - - development/workflow/get_devel_version - development/workflow/development_workflow + :material-outlined:`person_add;8em;sd-text-secondary` + Saw a typo in the documentation? Want to improve + existing functionalities? The contributing guidelines will show + you how to improve astropy. -.. _developer-docs: + .. grid-item-card:: Project Details + :link: index_project_details + :link-type: doc + :text-align: center -*********************** -Developer Documentation -*********************** + :material-outlined:`more_horiz;8em;sd-text-secondary` -The developer documentation contains instructions for how to contribute to -Astropy or affiliated packages, as well as coding, documentation, and -testing guidelines. For the guiding vision of this process and the project -as a whole, see :doc:`development/vision`. + What's new in the latest release, changelog, and other project details. -There are additional tools of use for developers in the -`astropy/astropy-tools repository -`__. +.. image:: https://github.com/astropy/repo_stats/blob/cache/cache/astropy_user_stats_light.png?raw=true + :class: only-light + :target: https://docs.astropy.org/en/latest/impact_health.html + :alt: Astropy User Statistics -.. toctree:: - :maxdepth: 1 +.. image:: https://github.com/astropy/repo_stats/blob/cache/cache/astropy_user_stats_dark.png?raw=true + :class: only-dark + :target: https://docs.astropy.org/en/latest/impact_health.html + :alt: Astropy User Statistics - development/workflow/development_workflow - development/codeguide - development/docguide - development/testguide - development/scripts - development/building - development/ccython - development/releasing - development/workflow/maintainer_workflow - development/affiliated-packages - changelog - -****************** -Indices and Tables -****************** - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - -.. _astropy mailing list: http://mail.scipy.org/mailman/listinfo/astropy -.. _astropy-feedback@googlegroups.com: mailto:astropy-feedback@googlegroups.com +.. _feedback@astropy.org: mailto:feedback@astropy.org +.. _affiliated packages: https://www.astropy.org/affiliated/ diff --git a/docs/index_dev.rst b/docs/index_dev.rst new file mode 100644 index 000000000000..0f0a728c4f0f --- /dev/null +++ b/docs/index_dev.rst @@ -0,0 +1,81 @@ +.. _developer-docs: + +************ +Contributing +************ + +The contributor documentation contains instructions for how to contribute to ``astropy`` or +affiliated packages. This includes setting up a development environment, installing and +testing the development version, as well as coding, documentation, and testing +guidelines. + +For newcomers the process may initially seem overwhelming, but with a little patience +and practice you will see that it is not so complex. The key is to follow the steps +outlined here and :ref:`ask for help ` if you get stuck. +The Astropy community is welcoming and friendly and will help you! + +{% if is_development %} + +This is divided into two sections, first a quickstart guide that provides an +introduction to the development workflow, followed by a number of detailed guides that +cover provide a deeper dive and a reference for both developers and maintainers. + +.. Important:: There are useful ways to contribute to Astropy without diving + into the developer workflow which is described here. For an + an overview see the :ref:`astropy-org-contribute` page. + + +Contributing quickstart +----------------------- + +This section provides a contributing quickstart guide for Astropy. With minor changes the +process will apply to contributing updates to coordinated and many affiliated packages. + +.. Important:: All contributions must comply with our + `AI policy `_. + +.. toctree:: + :maxdepth: 2 + + development/quickstart + +Now that you have created your development environment and gotten familiar with the +process, you should now read through the detailed tutorial below to see a real-life +example of a simple bug fix. This includes more explanation of the steps and good +advice for making a code change. + +.. toctree:: + :maxdepth: 1 + + development/git_edit_workflow_examples + +Congratulations, now you are ready to be an Astropy contributor! If you are not sure where to contribute, take a look at the `Good First Issues +`_ +list. These issues are the most accessible ones if you are not familiar with the Astropy +source code. + +Details +------- + +.. toctree:: + :maxdepth: 1 + + development/development_details + development/codeguide + development/testguide + development/docguide + development/style-guide + development/git_resources + development/scripts + development/ccython + development/maintainers/index + +.. Note:: Parts of this guide were adapted from the + :ref:`pandas developer documentation `. Astropy is grateful to the ``pandas`` team for their documentation efforts. + +{%else%} + +To read the developer documentation, you will need to go to the +`latest developer version of the documentation `_. + +{%endif%} diff --git a/docs/index_getting_started.rst b/docs/index_getting_started.rst new file mode 100644 index 000000000000..1899826b6ad2 --- /dev/null +++ b/docs/index_getting_started.rst @@ -0,0 +1,14 @@ +.. _getting-started: + +*************** +Getting Started +*************** + +:ref:`whatsnew-8.1` + +.. toctree:: + :maxdepth: 1 + + install + importing_astropy + Tutorials diff --git a/docs/index_project_details.rst b/docs/index_project_details.rst new file mode 100644 index 000000000000..83bd706a0e70 --- /dev/null +++ b/docs/index_project_details.rst @@ -0,0 +1,16 @@ +.. _project-details: + +*************** +Project Details +*************** + +.. toctree:: + :maxdepth: 1 + + whatsnew/index + changelog + lts_policy + known_issues + credits + impact_health + license diff --git a/docs/index_user_docs.rst b/docs/index_user_docs.rst new file mode 100644 index 000000000000..a4e7c7856ffd --- /dev/null +++ b/docs/index_user_docs.rst @@ -0,0 +1,60 @@ +.. _user-docs: + +********** +User Guide +********** + +.. toctree:: + :caption: Data structures and transformations + :maxdepth: 1 + + constants/index + units/index + nddata/index + table/index + time/index + timeseries/index + coordinates/index + wcs/index + modeling/index + uncertainty/index + +.. toctree:: + :caption: File I/O + :maxdepth: 2 + + io/overview + io/unified + +.. toctree:: + :maxdepth: 1 + + io/fits/index + io/ascii/index + io/votable/index + io/misc + +.. toctree:: + :caption: Computations and utilities + :maxdepth: 1 + + cosmology/index + convolution/index + utils/iers + visualization/index + stats/index + utils/masked/index + samp/index + +.. toctree:: + :caption: Nuts and bolts + :maxdepth: 1 + + config/index + io/registry + io/typing + logging + warnings + utils/index + environment_variables + glossary diff --git a/docs/install.rst b/docs/install.rst index b94e59a094da..1b83f7becad6 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -1,392 +1,309 @@ +.. _installing-astropy: + ************ Installation ************ -Requirements -============ - -Astropy has the following strict requirements: - -- `Python `_ 2.6 (>=2.6.5), 2.7, 3.3, or 3.4 - - - Prior to Astropy v1.0 Python 3.1 and 3.2 are also supported. - -- `Numpy`_ |minimum_numpy_version| or later - -Astropy also depends on other packages for optional features: - -- `h5py `_: To read/write - :class:`~astropy.table.Table` objects from/to HDF5 files - -- `BeautifulSoup `_: To read - :class:`~astropy.table.table.Table` objects from HTML files - -- `PyYAML `_: To read/write - :class:`~astropy.table.Table` objects from/to the Enhanced CSV ASCII table format. - -- `scipy`_: To power a variety of features (currently - mainly cosmology-related functionality) - -- `xmllint `_: To validate VOTABLE XML files. - -- `matplotlib `_: To provide plotting functionality that `astropy.visualization` enhances. - -- `WCSAxes `_: To use `astropy.wcs` to define projections in Matplotlib. +Overview +======== -However, note that these only need to be installed if those particular features -are needed. Astropy will import even if these dependencies are not installed. +The first step to installing ``astropy`` is to ensure that you have a Python +environment which is **isolated** from your system Python installation. This is +important because ``astropy`` has many dependencies, and you do not want to accidentally +break your system by installing incompatible versions of these dependencies. -.. TODO: Link to the planned dependency checker/installer tool. +For this installation guide we use the `conda `_ +package manager provided by `miniforge `_. +This is a popular choice and works well, especially for newcomers. It is easy to install +and use on all platforms and it makes it easy to install the latest Python version. If +you already have a ``miniforge``-based Python environment then you can skip to +:ref:`installing-astropy-with-pip`. -Installing Astropy -================== - -Using pip -------------- - -To install Astropy with `pip `_, simply run:: - - pip install --no-deps astropy - -.. warning:: - - Users of the Anaconda python distribution should follow the instructions - for :ref:`anaconda_install`. +Another option for more experienced users is a virtual environment manager such as the +Python standard library `venv `_ module. +There are numerous resources available to help you set up a virtual environment in this +manner if you choose this option. .. note:: + We **do not recommend** using ``astropy`` with an existing `miniconda + `_ or `Anaconda Python + `_ distribution. The ``astropy`` package provided + by Anaconda Inc. in the ``defaults`` channel can be outdated and these distributions + can require a license for use at a large organisation. Instead, use ``miniforge`` as + described below. - You will need a C compiler (e.g. ``gcc`` or ``clang``) to be installed (see - `Building from source`_ below) for the installation to succeed. +Once you have a Python environment set up, you will install ``astropy`` using |pip| or +|conda|. Here we document using |pip| because it is easier to install the optional +dependencies, but feel free to use |conda| if you prefer. -.. note:: - - The ``--no-deps`` flag is optional, but highly recommended if you already - have Numpy installed, since otherwise pip will sometimes try to "help" you - by upgrading your Numpy installation, which may not always be desired. +Install ``miniforge`` +===================== -.. note:: +You will install Python by first installing `miniforge +`__. This provides the `conda +package manager `_ with the default remote package +repository set to the community-led `conda-forge `_ channel. - If you get a ``PermissionError`` this means that you do not have the - required administrative access to install new packages to your Python - installation. In this case you may consider using the ``--user`` option - to install the package into your home directory. You can read more - about how to do this in the `pip documentation - `_. +In a new terminal (miniforge Prompt on Windows) run ``conda list`` to test that the +install has worked. - Alternatively, if you intend to do development on other software that uses - Astropy, such as an affiliated package, consider installing Astropy into a - :ref:`virtualenv`. +Create Python Environment +========================= - Do **not** install Astropy or other third-party packages using ``sudo`` - unless you are fully aware of the risks. +To create a new Python environment for ``astropy`` and other packages, start by +launching a terminal (under a UNIX-like system) or the miniforge Prompt (under Windows). +Now we will create and activate a new virtual environment to install ``astropy`` into: +.. code-block:: bash -.. _anaconda_install: + $ conda create --channel conda-forge --name astropy python + $ conda activate astropy -Anaconda python distribution ----------------------------- +In this case the environment we have created is named ``astropy`` but you can use any +name you like. -Astropy is installed by default with Anaconda. To update to the latest version -run:: +In the future when you make a new terminal, you will need to run ``conda activate +astropy`` to activate this environment. - conda update astropy +.. _installing-astropy-with-pip: -.. note:: - - There may be a delay of a day or two between when a new version of Astropy - is released and when a package is available for Anaconda. You can check - for the list of available versions with ``conda search astropy``. - -.. note:: +Install ``astropy`` +=================== - Attempting to use ``pip`` to upgrade your installation of Astropy may result - in a corrupted installation. +You can install ``astropy`` and the rest of your dependencies using either |pip| or +|conda|. Both methods are fully supported and will work well. +.. warning:: + Once you have created your base Python environment with |conda|, you should try to + stick with one method for installing new packages in your environment. In particular, + |conda| is not aware of packages installed with |pip| and may overwrite them. -Binary installers ------------------ - -Binary installers are available on Windows for Python 2.6, 2.7, and >= 3.3 -at `PyPI `_. - -.. _testing_installed_astropy: - - -Testing an installed Astropy ----------------------------- - -The easiest way to test your installed version of astropy is running -correctly is to use the :ref:`astropy.test()` function:: - - import astropy - astropy.test() +Using pip +--------- +To install ``astropy`` and your choice of :ref:`dependencies `, run +one of the following commands:: -The tests should run and print out any failures, which you can report at -the `Astropy issue tracker `_. + python -m pip install astropy # Minimum required dependencies + python -m pip install "astropy[recommended]" # Recommended dependencies + python -m pip install "astropy[all]" # All optional dependencies -.. note:: +In most cases, this will install a pre-compiled version of ``astropy`` (called a +*wheel*). However, if you are installing astropy on an uncommon platform, astropy will be +installed from a source file. In this unusual case you will need a C compiler to be +installed (see `Build from source`_ below) for the installation to succeed. - This way of running the tests may not work if you do it in the - astropy source distribution. See :ref:`sourcebuildtest` for how to - run the tests from the source code directory, or :ref:`running-tests` - for more details. +.. warning:: Do **not** install ``astropy`` or other packages using ``sudo`` or any + elevated privilege. -.. note:: +Using conda +----------- +To install ``astropy`` and the minimal set of required dependencies, run:: - Running the tests this way is currently disabled in the IPython REPL due - to conflicts with some common display settings in IPython. Please run the - Astropy tests under the standard Python command-line interpreter. + conda install --channel conda-forge astropy +Install the recommended dependencies with:: + conda install --channel conda-forge scipy matplotlib -Building from source -==================== +Install the optional dependencies with:: -Prerequisites -------------- + conda install --channel conda-forge ipython jupyter dask h5py pyarrow \ + beautifulsoup4 html5lib bleach pandas sortedcontainers pytz jplephem mpmath \ + asdf-astropy bottleneck fsspec s3fs certifi -You will need a compiler suite and the development headers for Python and -Numpy in order to build Astropy. On Linux, using the package manager for your -distribution will usually be the easiest route, while on MacOS X you will -need the XCode command line tools. +Testing +------- +You can test that your newly installed version of ``astropy`` is working via the +`documentation on how to test your installed version of astropy +`_. -The `instructions for building Numpy from source -`_ are also a good -resource for setting up your environment to build Python packages. +.. _astropy-main-req: -You will also need `Cython `_ (v0.15 or later) and -`jinja2 `_ (v2.7 or later) installed -to build from source, unless you are installing a numbered release. (The -releases packages have the necessary C files packaged with them, and hence do -not require Cython.) +Requirements +============ -.. note:: +``astropy`` has the following strict requirements: - If you are using MacOS X, you will need to the XCode command line tools. - One way to get them is to install `XCode - `_. If you are using OS X 10.7 (Lion) - or later, you must also explicitly install the command line tools. You can - do this by opening the XCode application, going to **Preferences**, then - **Downloads**, and then under **Components**, click on the Install button - to the right of **Command Line Tools**. Alternatively, on 10.7 (Lion) or - later, you do not need to install XCode, you can download just the command - line tools from https://developer.apple.com/downloads/index.action - (requires an Apple developer account). +- |Python| |minimum_python_version| or later +- |NumPy| |minimum_numpy_version| or later -Obtaining the source packages ------------------------------ +- |PyERFA| |minimum_pyerfa_version| or later -Source packages -^^^^^^^^^^^^^^^ +- `PyYAML `_ |minimum_pyyaml_version| or later -The latest stable source package for Astropy can be `downloaded here -`_. +- |packaging| |minimum_packaging_version| or later -Development repository -^^^^^^^^^^^^^^^^^^^^^^ +``astropy`` also depends on a number of other packages for optional features. +The following are particularly recommended: -The latest development version of Astropy can be cloned from github -using this command:: +- |SciPy| |minimum_scipy_version| or later: To power a variety of features + in several modules. - git clone git://github.com/astropy/astropy.git +- |Matplotlib| |minimum_matplotlib_version| or later: To provide plotting + functionality that `astropy.visualization` enhances. -.. note:: +The further dependencies provide more specific features: - If you wish to participate in the development of Astropy, see - :ref:`developer-docs`. This document covers only the basics - necessary to install Astropy. +- `h5py `_: To read/write + :class:`~astropy.table.Table` objects from/to HDF5 files. -Building and Installing ------------------------ +- `BeautifulSoup `_: To read + :class:`~astropy.table.table.Table` objects from HTML files. -Astropy uses the Python `distutils framework -`_ for building and -installing and requires the -`distribute `_ extension--the later is -automatically downloaded when running ``python setup.py`` if it is not already -provided by your system. +- `html5lib `_: To read + :class:`~astropy.table.table.Table` objects from HTML files using the + `pandas `_ reader. -If Numpy is not already installed in your Python environment, the -astropy setup process will try to download and install it before -continuing to install astropy. +- `bleach `_: Used to sanitize text when + disabling HTML escaping in the :class:`~astropy.table.Table` HTML writer. -To build Astropy (from the root of the source tree):: +- `ipydatagrid `_: Used in + :meth:`astropy.table.Table.show_in_notebook` to display the Astropy table + in Jupyter notebook for ``backend="ipydatagrid"``. - python setup.py build +- `xmllint `_: To validate VOTABLE XML files. + This is a command line tool installed outside of Python. -To install Astropy (from the root of the source tree):: +- `pandas `_: To convert + :class:`~astropy.table.Table` objects from/to pandas DataFrame objects. - python setup.py install +- `sortedcontainers `_ for faster + ``SCEngine`` indexing engine with ``Table``, although this may still be + slower in some cases than the default indexing engine. +- `pytz `_: To specify and convert between + timezones. -Troubleshooting ---------------- +- `jplephem `_: To retrieve JPL + ephemeris of Solar System objects. -If you get an error mentioning that you do not have the correct permissions to -install Astropy into the default ``site-packages`` directory, you can try -installing with:: +- `setuptools `_: Used for discovery of + entry points which are used to insert fitters into `astropy.modeling.fitting`. - python setup.py install --user +- `mpmath `_: Used for the 'kraft-burrows-nousek' + interval in `~astropy.stats.poisson_conf_interval`. -which will install into a default directory in your home directory. +- `asdf-astropy `_ |minimum_asdf_astropy_version| or later: Enables the + serialization of various Astropy classes into a portable, hierarchical, + human-readable representation. +- `bottleneck `_: Improves the performance + of sigma-clipping and other functionality that may require computing + statistics on arrays with NaN values. -External C libraries -^^^^^^^^^^^^^^^^^^^^ +- `certifi `_: Useful when downloading + files from HTTPS or FTP+TLS sites in case Python is not able to locate + up-to-date root CA certificates on your system; this package is usually + already included in many Python installations (e.g., as a dependency of + the ``requests`` package). -The Astropy source ships with the C source code of a number of -libraries. By default, these internal copies are used to build -Astropy. However, if you wish to use the system-wide installation of -one of those libraries, you can pass one or more of the -``--use-system-X`` flags to the ``setup.py build`` command. +- `pyarrow `_ |minimum_pyarrow_version| or later: + To read/write :class:`~astropy.table.Table` objects from/to Parquet files. -For example, to build Astropy using the system `libexpat -`_, use:: +- |fsspec| |minimum_fsspec_version| or later: Enables access to :ref:`subsets + of remote FITS files ` without having to download the entire file. - python setup.py build --use-system-expat +- |s3fs| |minimum_s3fs_version| or later: Enables access to files hosted in + AWS S3 cloud storage. -To build using all of the system libraries, use:: +However, note that these packages require installation only if those particular +features are needed. ``astropy`` will import even if these dependencies are not +installed. - python setup.py build --use-system-libraries +The following packages can optionally be used when testing: -To see which system libraries Astropy knows how to build against, use:: +- |pytest-astropy|: See :ref:`sourcebuildtest` - python setup.py build --help +- `pytest-xdist `_: Used for + distributed testing. -As with all distutils commandline options, they may also be provided in a -``setup.cfg`` in the same directory as ``setup.py``. For example, to use -the system `libexpat `_, add the following to the -``setup.cfg`` file:: +- `pytest-mpl `_: Used for testing + with Matplotlib figures. - [build] - use_system_expat=1 +- `objgraph `_: Used only in tests to test for reference leaks. +- |IPython| |minimum_ipython_version| or later: + Used for testing the notebook interface of `~astropy.table.Table`. -The required version of setuptools is not available -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- `coverage `_: Used for code coverage + measurements. -If upon running the ``setup.py`` script you get a message like +- `skyfield `_: Used for testing Solar System + coordinates. - The required version of setuptools (>=0.9.8) is not available, - and can't be installed while this script is running. Please - install a more recent version first, using - 'easy_install -U setuptools'. +- `sgp4 `_: Used for testing satellite positions. - (Currently using setuptools 0.6c11 (/path/to/setuptools-0.6c11-py2.7.egg)) +- `tox `_: Used to automate testing + and documentation builds. -this is because you have a very outdated version of the `setuptools -`_ package which is used to install -Python packages. Normally Astropy will bootstrap newer version of -setuptools via the network, but setuptools suggests that you first -*uninstall* the old version (the ``easy_install -U setuptools`` command). +.. _sourcebuildinstructions: -However, in the likely case that your version of setuptools was installed by an -OS system package (on Linux check your package manager like apt or yum for a -package called ``python-setuptools``), trying to uninstall with -``easy_install`` and without using ``sudo`` may not work, or may leave your -system package in an inconsistent state. +Build from Source +================= -As the best course of action at this point depends largely on the individual -system and how it is configured, if you are not sure yourself what do please -ask on the Astropy mailing list. +{% if is_development %} +If you want to build the code from source, follow the instructions for +:ref:`contributing_environment`. Note that instead of cloning from your fork, you can +choose to clone from the main repository:: -The Windows installer can't find Python in the registry -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + git clone https://github.com/astropy/astropy.git + cd astropy -This is a common issue with Windows installers for Python packages that do not -support the new User Access Control (UAC) framework added in Windows Vista and -later. In particular, when a Python is installed "for all users" (as opposed -to for a single user) it adds entries for that Python installation under the -``HKEY_LOCAL_MACHINE`` (HKLM) hierarchy and *not* under the -``HKEY_CURRENT_USER`` (HKCU) hierarchy. However, depending on your UAC -settings, if the Astropy installer is not executed with elevated privileges it -will not be able to check in HKLM for the required information about your -Python installation. +Building the documentation is typically not necessary unless you are +developing code or documentation or do not have internet access, because +the stable, latest, and archived versions of Astropy's documentation are +available at `docs.astropy.org `_ . The process +is described in :ref:`builddocs`. -In short: If you encounter this problem it's because you need the appropriate -entries in the Windows registry for Python. You can download `this script`__ -and execute it with the same Python as the one you want to install Astropy -into. For example to add the missing registry entries to your Python 2.7:: +{%else%} - C:\>C:\Python27\python.exe C:\Path\To\Downloads\win_register_python.py +See the `latest documentation on how to build astropy from source `_. -__ https://gist.github.com/embray/6042780#file-win_register_python-py +{%endif%} -.. _builddocs: +.. _sourcebuildtest: -Building documentation +Test Source Code Build ---------------------- -.. note:: - - Building the documentation is in general not necessary unless you - are writing new documentation or do not have internet access, because - the latest (and archive) versions of astropy's documentation should - be available at `docs.astropy.org `_ . - -Building the documentation requires the Astropy source code and some additional -packages: - - - `Sphinx `_ (and its dependencies) 1.0 or later - - - `Graphviz `_ - - - `Astropy-helpers `_ (Astropy - and most affiliated packages include this as a submodule in the source - repository, so it does not need to be installed separately.) +{% if is_development %} -.. note:: - - Sphinx also requires a reasonably modern LaTeX installation to render - equations. Per the `Sphinx documentation - `_, - for the TexLive distribution the following packages are required to be - installed: - - * latex-recommended - * latex-extra - * fonts-recommended - - For other LaTeX distributions your mileage may vary. To build the PDF - documentation using LaTeX, the ``fonts-extra`` TexLive package or the - ``inconsolata`` CTAN package are also required. +The easiest way to run the tests in a source checkout of ``astropy`` +is to use `tox `_:: -There are two ways to build the Astropy documentation. The most straightforward -way is to execute the command (from the astropy source directory):: + tox -e test-alldeps - python setup.py build_sphinx +There are also alternative methods of :ref:`running-tests` if you +would like more control over the testing process. -The documentation will be built in the ``docs/_build/html`` directory, and can -be read by pointing a web browser to ``docs/_build/html/index.html``. +{%else%} -The LaTeX documentation can be generated by using the command:: +See the `latest documentation on how to run the tests in a source +checkout of astropy `_. - python setup.py build_sphinx -b latex +{%endif%} -The LaTeX file ``Astropy.tex`` will be created in the ``docs/_build/latex`` -directory, and can be compiled using ``pdflatex``. -The above method builds the API documentation from the source code. -Alternatively, you can do:: +.. _install_astropy_nightly: - cd docs - make html - -And the documentation will be generated in the same location, but using the -*installed* version of Astropy. - -.. _sourcebuildtest: +Install Pre-built Development Version +===================================== -Testing a source code build of Astropy --------------------------------------- +Most nights a development snapshot of ``astropy`` will be compiled. +This is useful if you want to test against a development version of astropy but +do not want to have to build it yourselves. You can see the +`available astropy dev snapshots page `_ +to find out what is currently being offered. -The easiest way to test that your Astropy built correctly (without -installing astropy) is to run this from the root of the source tree:: +Installing these "nightlies" of ``astropy`` can be achieved by using ``pip``:: - python setup.py test + python -m pip install --upgrade --extra-index-url https://pypi.anaconda.org/astropy/simple astropy --pre -There are also alternative methods of :ref:`running-tests`. +The extra index URL tells ``pip`` to check the ``pip`` index on +pypi.anaconda.org, where the nightlies are stored, and the ``--pre`` command +tells ``pip`` to install pre-release versions (in this case ``.dev`` releases). -.. include:: development/workflow/known_projects.inc +You can test this installation by running the tests as described in the section +`Running tests on an installed astropy `_. diff --git a/docs/io/ascii/base_classes.rst b/docs/io/ascii/base_classes.rst index 6c3df849c555..bd8fa58ca9d8 100644 --- a/docs/io/ascii/base_classes.rst +++ b/docs/io/ascii/base_classes.rst @@ -2,20 +2,20 @@ .. _base_class_elements: -Base class elements ----------------------------- +Base Class Elements +******************* The key elements in :mod:`astropy.io.ascii` are: -* :class:`~astropy.io.ascii.Column`: Internal storage of column properties and data () -* :class:`Reader `: Base class to handle reading and writing tables. -* :class:`Inputter `: Get the lines from the table input. -* :class:`Splitter `: Split the lines into string column values. -* :class:`Header `: Initialize output columns based on the table header or user input. -* :class:`Data `: Populate column data from the table. -* :class:`Outputter `: Convert column data to the specified output format, e.g. `numpy` structured array. +* :class:`~astropy.io.ascii.Column`: internal storage of column properties and data. +* :class:`Reader `: base class to handle reading and writing tables. +* :class:`Inputter `: gets the lines from the table input. +* :class:`Splitter `: splits the lines into string column values. +* :class:`Header `: initializes output columns based on the table header or user input. +* :class:`Data `: populates column data from the table. +* :class:`Outputter `: converts column data to the specified output format (e.g., ``numpy`` structured array). Each of these elements is an inheritable class with attributes that control the -corresponding functionality. In this way the large number of tweakable -parameters is modularized into manageable groups. Where it makes sense these -attributes are actually functions that make it easy to handle special cases. +corresponding functionality. In this way, the large number of tunable +parameters are modularized into manageable groups. In certain places these +attributes are actually functions for handling special cases. diff --git a/docs/io/ascii/ecsv.rst b/docs/io/ascii/ecsv.rst new file mode 100644 index 000000000000..671386b66d59 --- /dev/null +++ b/docs/io/ascii/ecsv.rst @@ -0,0 +1,511 @@ +.. _ecsv_format: + +ECSV Format +=========== + +The `Enhanced Character-Separated Values (ECSV) format +`_ can be used to +write ``astropy`` `~astropy.table.Table` or `~astropy.table.QTable` datasets to +a text-only human readable data file and then read the table back without loss +of information. The format stores column specifications like unit and data type +along with table metadata by using a YAML header data structure. The +actual tabular data are stored in a standard character separated values (CSV) +format, giving compatibility with a wide variety of non-specialized CSV table +readers. + +.. attention:: + + The ECSV format is the recommended way to store Table data in a + human-readable text file. This includes use cases from informal + use in science research to production pipelines and data systems. + + In addition to Python, ECSV is supported in |TOPCAT| + and in the Java |STIL| library. + +Usage +----- + +Reading +""""""" +When reading an ECSV format table we recommend using ``format="ecsv"`` in the call to +``Table.read()`` or ``QTable.read()``. This option uses a code base that was added in +astropy 7.2. With this option you can select one of three built-in engines to read the +actual CSV data: ``"io.ascii"`` (default), ``"pyarrow"`` (via `pyarrow.read_csv() +`_), or ``"pandas"`` (via +`pandas.read_csv`). The latter two require the corresponding optional package +dependency to be installed. + +For a large gzipped ECSV file, the ``pyarrow`` engine is about 15 times faster than ``io.ascii`` +and the ``pandas`` engine is about 3 times faster. Both of these engines are also more +memory efficient than ``io.ascii``. + +Details on this option are available in `~astropy.io.misc.ecsv.read_ecsv` or by +interactively running ``Table.read.help(format="ecsv")`` in Python. + +You can also use ``format="ascii.ecsv"``, which uses the legacy code base that was the +only astropy ECSV reader prior to astropy 7.2. This option is the default if +you read a table file with an ``.ecsv`` extension without supplying the ``format`` +explicitly, e.g., ``QTable.read("my_data.ecsv")``. This default is planned to change in +a future version of astropy and the legacy reader will be deprecated and later removed. + +Writing +""""""" + +When writing in the ECSV format there are only two choices for the delimiter, +either space or comma, with space being the default. Any other value of +``delimiter`` will give an error. For reading the delimiter is specified within +the file itself. + +Apart from the delimiter, the only other applicable write arguments are +``names``, ``include_names``, and ``exclude_names``. All other arguments will be +either ignored or raise an error. + +Simple Table +------------ +.. + EXAMPLE START + Writing Data Tables as ECSV: Simple Table + +The following writes a table as a simple space-delimited file. The +ECSV format is auto-selected due to ``.ecsv`` suffix:: + + >>> import numpy as np + >>> from astropy.table import Table + >>> data = Table() + >>> data['a'] = np.array([1, 2], dtype=np.int8) + >>> data['b'] = np.array([1, 2], dtype=np.float32) + >>> data['c'] = np.array(['hello', 'world']) + >>> data.write('my_data.ecsv') # doctest: +SKIP + +The contents of ``my_data.ecsv`` are shown below:: + + # %ECSV 1.0 + # --- + # datatype: + # - {name: a, datatype: int8} + # - {name: b, datatype: float32} + # - {name: c, datatype: string} + # schema: astropy-2.0 + a b c + 1 1.0 hello + 2 2.0 world + +The ECSV header is the section prefixed by the ``#`` comment character. An ECSV +file must start with the ``%ECSV `` line. The ``datatype`` element +defines the list of columns and the ``schema`` relates to astropy-specific +extensions that are used for writing `Mixin Columns`_. + +.. + EXAMPLE END + +Masked Data +----------- + +You can write masked (or "missing") data in the ECSV format in two different +ways, either using an empty string to represent missing values or by splitting +the masked columns into separate data and mask columns. + +Empty String +"""""""""""" + +The first (default) way uses an empty string as a marker in place of +masked values. This is a bit more common outside of ``astropy`` and does not +require any astropy-specific extensions. + + >>> from astropy.table import MaskedColumn + >>> t = Table() + >>> t['x'] = MaskedColumn([1.0, 2.0, 3.0], unit='m', dtype='float32') + >>> t['x'][1] = np.ma.masked + >>> t['y'] = MaskedColumn([False, True, False], dtype='bool') + >>> t['y'][0] = np.ma.masked + + >>> t.write('my_data.ecsv', format='ascii.ecsv', overwrite=True) # doctest: +SKIP + +The contents of ``my_data.ecsv`` are shown below:: + + # %ECSV 1.0 + # --- + # datatype: + # - {name: x, unit: m, datatype: float32} + # - {name: y, datatype: bool} + # schema: astropy-2.0 + x y + 1.0 "" + "" True + 3.0 False + +To read this back, you would run the following:: + + >>> Table.read('my_data.ecsv') # doctest: +SKIP +
+ x y + m + float32 bool + ------- ----- + 1.0 -- + -- True + 3.0 False + +Data + Mask +""""""""""" + +The second way is to tell the writer to break any masked column into a data +column and a mask column by supplying the ``serialize_method='data_mask'`` +argument:: + + >>> t.write('my_data.ecsv', serialize_method='data_mask', overwrite=True) # doctest: +SKIP + +There are two main reasons you might want to do this: + +- Storing the data "under the mask" instead of replacing it with an empty string. +- Writing a string column that contains empty strings which are not masked. + +The contents of ``my_data.ecsv`` are shown below. First notice that there are +two new columns ``x.mask`` and ``y.mask`` that have been added, and these explicitly +record the mask values for those columns. Next notice now that the ECSV +header is a bit more complex and includes the astropy-specific extensions that +tell the reader how to interpret the plain CSV columns ``x, x.mask, y, y.mask`` +and reassemble them back into the appropriate masked columns. +:: + + # %ECSV 1.0 + # --- + # datatype: + # - {name: x, unit: m, datatype: float32} + # - {name: x.mask, datatype: bool} + # - {name: y, datatype: bool} + # - {name: y.mask, datatype: bool} + # meta: !!omap + # - __serialized_columns__: + # x: + # __class__: astropy.table.column.MaskedColumn + # data: !astropy.table.SerializedColumn {name: x} + # mask: !astropy.table.SerializedColumn {name: x.mask} + # y: + # __class__: astropy.table.column.MaskedColumn + # data: !astropy.table.SerializedColumn {name: y} + # mask: !astropy.table.SerializedColumn {name: y.mask} + # schema: astropy-2.0 + x x.mask y y.mask + 1.0 False False True + 2.0 True True False + 3.0 False False False + +.. note:: + + For the security minded, the ``__class__`` value must within an allowed list + of astropy classes that are trusted by the reader. You cannot use an + arbitrary class here. + +.. + EXAMPLE START + Using ECSV Format to Write Astropy Tables with Masked or Missing Data + +Per-column control +@@@@@@@@@@@@@@@@@@ + +In rare cases it may be necessary to specify the serialization method for each +column individually. This is shown in the example below:: + + >>> from astropy.table.table_helpers import simple_table + >>> t = simple_table(masked=True) + >>> t['c'][0] = "" # Valid empty string in data + >>> t +
+ a b c + int64 float64 str1 + ----- ------- ---- + -- 1.0 + 2 2.0 -- + 3 -- e + +Now we tell ECSV writer to output separate data and mask columns for the +string column ``'c'``: + +.. doctest-skip:: + + >>> t['c'].info.serialize_method['ecsv'] = 'data_mask' + >>> ascii.write(t, format='ecsv') + # %ECSV 1.0 + # --- + # datatype: + # - {name: a, datatype: int64} + # - {name: b, datatype: float64} + # - {name: c, datatype: string} + # - {name: c.mask, datatype: bool} + # meta: !!omap + # - __serialized_columns__: + # c: + # __class__: astropy.table.column.MaskedColumn + # data: !astropy.table.SerializedColumn {name: c} + # mask: !astropy.table.SerializedColumn {name: c.mask} + # schema: astropy-2.0 + a b c c.mask + "" 1.0 "" False + 2 2.0 d True + 3 "" e False + +When you read this back in, both the empty (zero-length) string and the masked +``'d'`` value in the column ``'c'`` will be preserved. + +.. + EXAMPLE END + +.. _ecsv_format_mixin_columns: + +Mixin Columns +------------- + +It is possible to store not only standard `~astropy.table.Column` and +`~astropy.table.MaskedColumn` objects to ECSV but also the following +:ref:`mixin_columns`: + +- `astropy.time.Time` +- `astropy.time.TimeDelta` +- `astropy.units.Quantity` +- `astropy.coordinates.Latitude` +- `astropy.coordinates.Longitude` +- `astropy.coordinates.Angle` +- `astropy.coordinates.Distance` +- `astropy.coordinates.EarthLocation` +- `astropy.coordinates.SkyCoord` +- `astropy.table.NdarrayMixin` +- Coordinate representation types such as `astropy.coordinates.SphericalRepresentation` + +In general, a mixin column may contain multiple data components as well as +object attributes beyond the standard `~astropy.table.Column` attributes like +``format`` or ``description``. Storing such mixin columns is done by replacing +the mixin column with column(s) representing the underlying data component(s) +and then inserting metadata which informs the reader of how to reconstruct the +original column. For example, a `~astropy.coordinates.SkyCoord` mixin column in +``'spherical'`` representation would have data attributes ``ra``, ``dec``, +``distance``, along with object attributes like ``representation_type`` or +``frame``. + +.. + EXAMPLE START + Writing a Table with a SkyCoord Column in ECSV Format + +This example demonstrates writing a `~astropy.table.QTable` that has `~astropy.time.Time` +and `~astropy.coordinates.SkyCoord` mixin columns:: + + >>> from astropy.coordinates import SkyCoord + >>> import astropy.units as u + >>> from astropy.table import QTable + + >>> sc = SkyCoord(ra=[1, 2] * u.deg, dec=[3, 4] * u.deg) + >>> sc.info.description = 'flying circus' + >>> q = [1, 2] * u.m + >>> q.info.format = '.2f' + >>> t = QTable() + >>> t['c'] = [1, 2] + >>> t['q'] = q + >>> t['sc'] = sc + + >>> t.write('my_data.ecsv') # doctest: +SKIP + +The contents of ``my_data.ecsv`` are below:: + + # %ECSV 1.0 + # --- + # datatype: + # - {name: c, datatype: int64} + # - {name: q, unit: m, datatype: float64, format: .2f} + # - {name: sc.ra, unit: deg, datatype: float64} + # - {name: sc.dec, unit: deg, datatype: float64} + # meta: !!omap + # - __serialized_columns__: + # q: + # __class__: astropy.units.quantity.Quantity + # __info__: {format: .2f} + # unit: !astropy.units.Unit {unit: m} + # value: !astropy.table.SerializedColumn {name: q} + # sc: + # __class__: astropy.coordinates.sky_coordinate.SkyCoord + # __info__: {description: flying circus} + # dec: !astropy.table.SerializedColumn + # __class__: astropy.coordinates.angles.Latitude + # unit: &id001 !astropy.units.Unit {unit: deg} + # value: !astropy.table.SerializedColumn {name: sc.dec} + # frame: icrs + # ra: !astropy.table.SerializedColumn + # __class__: astropy.coordinates.angles.Longitude + # unit: *id001 + # value: !astropy.table.SerializedColumn {name: sc.ra} + # wrap_angle: !astropy.coordinates.Angle + # unit: *id001 + # value: 360.0 + # representation_type: spherical + # schema: astropy-2.0 + c q sc.ra sc.dec + 1 1.0 1.0 3.0 + 2 2.0 2.0 4.0 + +The ``'__class__'`` keyword gives the fully-qualified class name and must be +one of the specifically allowed ``astropy`` classes. There is no option to add +user-specified allowed classes. The ``'__info__'`` keyword contains values for +standard `~astropy.table.Column` attributes like ``description`` or ``format``, +for any mixin columns that are represented by more than one serialized column. + +.. + EXAMPLE END + +.. _ecsv_format_masked_columns: + +Multidimensional Columns +------------------------ + +Using ECSV it is possible to write a table that contains multidimensional +columns (both masked and unmasked). This is done by encoding each element as a +string using JSON. This functionality works for all column types that are +supported by ECSV including :ref:`mixin_columns`. This capability is added in +astropy 4.3 and ECSV version 1.0. + +.. + EXAMPLE START + Using ECSV Format to Write Astropy Tables with Multidimensional Columns + +We start by defining a table with 2 rows where each element in the second column +``'b'`` is itself a 3x2 array:: + + >>> t = Table() + >>> t['a'] = ['x', 'y'] + >>> t['b'] = np.arange(12, dtype=np.float64).reshape(2, 3, 2) + >>> t +
+ a b + str1 float64[3,2] + ---- ------------ + x 0.0 .. 5.0 + y 6.0 .. 11.0 + + >>> t['b'][0] + array([[0., 1.], + [2., 3.], + [4., 5.]]) + +Now we can write this to ECSV and observe how the N-d column ``'b'`` has been +written as a string with ``datatype: string``. Notice also that the column +descriptor for the column includes the new ``subtype: float64[3,2]`` attribute +specifying the type and shape of each item. + +.. doctest-skip:: + + >>> ascii.write(t, format='ecsv') # doctest: +SKIP + # %ECSV 1.0 + # --- + # datatype: + # - {name: a, datatype: string} + # - {name: b, datatype: string, subtype: 'float64[3,2]'} + # schema: astropy-2.0 + a b + x [[0.0,1.0],[2.0,3.0],[4.0,5.0]] + y [[6.0,7.0],[8.0,9.0],[10.0,11.0]] + +When you read this back in, the sequence of JSON-encoded column items are then +decoded using JSON back into the original N-d column. + +.. + EXAMPLE END + +Variable-length arrays +---------------------- + +ECSV supports storing multidimensional columns is when the length of each array +element may vary. This data structure is supported in the `FITS standard +`_. While ``numpy`` does not +natively support variable-length arrays, it is possible to represent such a +structure using an object-type array of typed ``np.ndarray`` objects. This is how +the ``astropy`` FITS reader outputs a variable-length array. + +This capability is added in astropy 4.3 and ECSV version 1.0. + +Most commonly variable-length arrays have a 1-d array in each cell of the +column. You might a column with 1-d ``np.ndarray`` cells having lengths of 2, 5, +and 3 respectively. + +The ECSV standard and ``astropy`` also supports arbitrary N-d arrays in each +cell, where all dimensions except the last one must match. For instance you +could have a column with ``np.ndarray`` cells having shapes of ``(4,4,2)``, +``(4,4,5)``, and ``(4,4,3)`` respectively. + +.. + EXAMPLE START + Using ECSV Format to Write Astropy Tables with Variable-Length Arrays + +The example below shows writing a variable-length 1-d array to ECSV. Notice the +new ECSV column attribute ``subtype: 'int64[null]'``. The ``[null]`` indicates a +variable length for the one dimension. If we had been writing the N-d example +above the subtype would have been ``int64[4,4,null]``. + +.. doctest-skip:: + + >>> t = Table() + >>> t['a'] = np.empty(3, dtype=object) + >>> t['a'] = [np.array([1, 2], dtype=np.int64), + ... np.array([3, 4, 5], dtype=np.int64), + ... np.array([6, 7, 8, 9], dtype=np.int64)] + >>> ascii.write(t, format='ecsv') + # %ECSV 1.0 + # --- + # datatype: + # - {name: a, datatype: string, subtype: 'int64[null]'} + # schema: astropy-2.0 + a + [1,2] + [3,4,5] + [6,7,8,9] + +.. + EXAMPLE END + +Object arrays +------------- + +ECSV can store object-type columns with simple Python objects consisting of +``dict``, ``list``, ``str``, ``int``, ``float``, ``bool`` and ``None`` elements. +More precisely, any object that can be serialized to `JSON +`__ using the standard library `json +`__ package is supported. + +.. + EXAMPLE START + Using ECSV Format to Write Astropy Tables with Object Arrays + +The example below shows writing an object array to ECSV. Because JSON requires +a double-quote around strings, and because ECSV requires ``""`` to represent +a double-quote within a string, one tends to get double-double quotes in this +representation. + +.. doctest-skip:: + + >>> t = Table() + >>> t['a'] = np.array([{'a': 1}, + ... {'b': [2.5, None]}, + ... True], dtype=object) + >>> ascii.write(t, format='ecsv') + # %ECSV 1.0 + # --- + # datatype: + # - {name: a, datatype: string, subtype: json} + # schema: astropy-2.0 + a + "{""a"":1}" + "{""b"":[2.5,null]}" + true + +.. + EXAMPLE END + +Meta data +--------- + +The ECSV format allows for different data structures in the header "meta" element. +Typically, this will be a mapping (what Python calls a dict), but it could also be a list or +nested structure. +Astropy's `~astropy.table.Table` requires a `dict` for table meta data. Thus, +the reader will attempt to convert the ECSV header "meta" to a dict or store the entire structure +in a single entry in the table meta. +On writing, that dict will be written back in astropy's form, so ECSV files that were not +originally written by astropy will preserve the content, but not the formatting of the +header "meta" element. diff --git a/docs/io/ascii/extension_classes.rst b/docs/io/ascii/extension_classes.rst index 808fc4a086a9..31b18303b912 100644 --- a/docs/io/ascii/extension_classes.rst +++ b/docs/io/ascii/extension_classes.rst @@ -2,29 +2,32 @@ .. _extension_reader_classes: -Extension Reader classes ------------------------- +Extension Reader Classes +************************ The following classes extend the base :class:`~astropy.io.ascii.BaseReader` functionality to handle reading and writing -different table formats. Some, such as the :class:`~astropy.io.ascii.Basic` Reader class -are fairly general and include a number of configurable attributes. Others +different table formats. Some, such as the :class:`~astropy.io.ascii.Basic` Reader class +are fairly general and include a number of configurable attributes. Others such as :class:`~astropy.io.ascii.Cds` or :class:`~astropy.io.ascii.Daophot` are specialized to read certain well-defined but idiosyncratic formats. -* :class:`~astropy.io.ascii.AASTex`: AASTeX `deluxetable `_ used for AAS journals -* :class:`~astropy.io.ascii.Basic`: basic table with customizable delimiters and header configurations -* :class:`~astropy.io.ascii.Cds`: `CDS format table `_ (also Vizier and ApJ machine readable tables) -* :class:`~astropy.io.ascii.CommentedHeader`: column names given in a line that begins with the comment character -* :class:`~astropy.io.ascii.Daophot`: table from the IRAF DAOphot package -* :class:`~astropy.io.ascii.FixedWidth`: table with fixed-width columns (see also :ref:`fixed_width_gallery`) -* :class:`~astropy.io.ascii.FixedWidthNoHeader`: table with fixed-width columns and no header -* :class:`~astropy.io.ascii.FixedWidthTwoLine`: table with fixed-width columns and a two-line header -* :class:`~astropy.io.ascii.HTML`: HTML format table contained in a
tag -* :class:`~astropy.io.ascii.Ipac`: `IPAC format table `_ -* :class:`~astropy.io.ascii.Latex`: LaTeX table with datavalue in the ``tabular`` environment -* :class:`~astropy.io.ascii.NoHeader`: basic table with no header where columns are auto-named -* :class:`~astropy.io.ascii.Rdb`: tab-separated values with an extra line after the column definition line -* :class:`~astropy.io.ascii.SExtractor`: `SExtractor format table `_ -* :class:`~astropy.io.ascii.Tab`: tab-separated values -* :class:`~astropy.io.ascii.Csv`: comma-separated values - +* :class:`~astropy.io.ascii.AASTex`: AASTeX `deluxetable `_ used for AAS journals. +* :class:`~astropy.io.ascii.Basic`: basic table with customizable delimiters and header configurations. +* :class:`~astropy.io.ascii.Cds`: `CDS format table `_ (also Vizier and ApJ machine readable tables). +* :class:`~astropy.io.ascii.CommentedHeader`: column names given in a line that begins with the comment character. +* :class:`~astropy.io.ascii.Csv`: comma-separated values. +* :class:`~astropy.io.ascii.Daophot`: table from the IRAF DAOphot package. +* :class:`~astropy.io.ascii.FixedWidth`: table with fixed-width columns (see also :ref:`fixed_width_gallery`). +* :class:`~astropy.io.ascii.FixedWidthNoHeader`: table with fixed-width columns and no header. +* :class:`~astropy.io.ascii.FixedWidthTwoLine`: table with fixed-width columns and a two-line header. +* :class:`~astropy.io.ascii.HTML`: HTML format table contained in a
tag. +* :class:`~astropy.io.ascii.Ipac`: `IPAC format table `_. +* :class:`~astropy.io.ascii.Latex`: LaTeX table with datavalue in the ``tabular`` environment. +* :class:`~astropy.io.ascii.Mesa`: MESA stellar evolution code output format. +* :class:`~astropy.io.ascii.Mrt`: `AAS Machine-Readable Table format `_. +* :class:`~astropy.io.ascii.NoHeader`: basic table with no header where columns are auto-named. +* :class:`~astropy.io.ascii.Rdb`: tab-separated values with an extra line after the column definition line. +* :class:`~astropy.io.ascii.RST`: `reStructuredText simple format table `_. +* :class:`~astropy.io.ascii.SExtractor`: `SExtractor format table `_. +* :class:`~astropy.io.ascii.Tab`: tab-separated values. +* :class:`~astropy.io.ascii.Tdat`: Transportable Database Aggregate Table format diff --git a/docs/io/ascii/fast_ascii_io.rst b/docs/io/ascii/fast_ascii_io.rst index d68c7aeeea2e..121f585b666b 100644 --- a/docs/io/ascii/fast_ascii_io.rst +++ b/docs/io/ascii/fast_ascii_io.rst @@ -3,13 +3,13 @@ .. _fast_ascii_io: Fast ASCII I/O --------------- +************** While :mod:`astropy.io.ascii` was designed with flexibility and extensibility -in mind, there is also a less flexible but significantly faster Cython/C engine for -reading and writing ASCII files. By default, |read| and |write| will attempt to -use this engine when dealing with compatible formats. The following formats -are currently compatible with the fast engine: +in mind, there is also a less flexible but significantly faster Cython/C engine +for reading and writing ASCII files. By default, |read| and |write| will +attempt to use this engine when dealing with compatible formats. The following +formats are currently compatible with the fast engine: * ``basic`` * ``commented_header`` @@ -19,9 +19,23 @@ are currently compatible with the fast engine: * ``tab`` The fast engine can also be enabled through the format parameter by prefixing -a compatible format with "fast" and then an underscore. In this case, |read| +a compatible format with "fast" and then an underscore. In this case, or +when enforcing the fast engine by either setting ``fast_reader='force'`` +or explicitly setting any of the :ref:`fast_conversion_opts`, |read| will not fall back on an ordinary reader if fast reading fails. -For example:: + +.. Note:: The fast engine only supports ASCII-encoded data, so reading or writing + unicode text is not possible with the fast engine. For unicode support with large + files, consider using the :ref:`Pandas Table I/O interface `. + +Examples +-------- + +.. + EXAMPLE START + Read and Write a CSV File Using Fast ASCII + +To open a CSV file and write it back out:: >>> from astropy.table import Table >>> t = ascii.read('file.csv', format='fast_csv') # doctest: +SKIP @@ -35,21 +49,32 @@ To disable the fast engine, specify ``fast_reader=False`` or .. Note:: Guessing and Fast reading - By default |read| will try to guess the format of in the input data by successively - trying different formats until one succeeds ([reference the guessing section]). - For the default ``'ascii'`` format this means that a number of pure Python readers - with no fast implementation will be tried before getting to the fast readers. + By default |read| will try to guess the format of the input data by + successively trying different formats until one succeeds + (see the section on :ref:`guess_formats`). For each supported + format it will first try the fast, then the slow version of that + reader. Without any additional options this means that both some pure + Python readers with no fast implementation and the Python versions + of some readers will be tried before getting to some of the fast + readers. To bypass them entirely, a fast reader should be explicitly + requested as above. - **For optimum performance**, turn off guessing entirely (``guess=False``) or - narrow down the format options as much as possible by specifying the format - (e.g. ``format='csv'``) and/or other options such as the delimiter. + **For optimum performance** however, it is recommended to turn off + guessing entirely (``guess=False``) or narrow down the format options + as much as possible by specifying the format (e.g., ``format='csv'``) + and/or other options such as the delimiter. + +.. + EXAMPLE END Reading -^^^^^^^ +======= + Since the fast engine is not part of the ordinary :mod:`astropy.io.ascii` infrastructure, fast readers raise an error when passed certain -parameters which are not implemented in the fast reader -infrastructure. In this case |read| will fall back on the ordinary reader. +parameters which are not implemented in the fast reader infrastructure. +In this case |read| will fall back on the ordinary reader, unless the +fast reader has been explicitly requested (see above). These parameters are: * Negative ``header_start`` (except for commented-header format) @@ -59,115 +84,117 @@ These parameters are: * ``delimiter`` string not of length 1 * ``quotechar`` string not of length 1 * ``converters`` - * ``Outputter`` - * ``Inputter`` - * ``data_Splitter`` - * ``header_Splitter`` + * ``outputter_cls`` + * ``inputter_cls`` + * ``data_splitter_cls`` + * ``header_splitter_cls`` + +.. _fast_conversion_opts: + +Fast Conversion Options +----------------------- -Parallel and fast conversion options -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In addition to ``True`` and ``False``, the parameter ``fast_reader`` can also -be a dict specifying one or both of two additional parameters, ``parallel`` and -``use_fast_converter``. For example:: +be a ``dict`` specifying any of two additional parameters, +``use_fast_converter`` and ``exponent_style``. + +Example +======= - >>> ascii.read('data.txt', format='basic', fast_reader={'parallel': True, 'use_fast_converter': True}) # doctest: +SKIP +.. + EXAMPLE START + Fast Conversion Options for Faster Table Reading + +To specify additional parameters using ``fast_reader``:: + + >>> ascii.read('data.txt', format='basic', + ... fast_reader={'use_fast_converter': True}) # doctest: +SKIP + +.. + EXAMPLE END These options allow for even faster table reading when enabled, but both are disabled by default because they come with some caveats. -The ``parallel`` parameter can be used to enable multiprocessing via -the ``multiprocessing`` module, and can either be set to a number (the number -of processes to use) or ``True``, in which case the number of processes will be -``multiprocessing.cpu_count()``. Note that this can cause issues within the -IPython Notebook and so enabling multiprocessing in this context is discouraged. - Setting ``use_fast_converter`` to be ``True`` enables a faster but -slightly imprecise conversion method for floating-point values, as described below. +slightly imprecise conversion method for floating-point values, as described +below. -Writing -^^^^^^^ -The fast engine supports the same functionality as the ordinary writing engine -and is generally about 2 to 4 times faster than the ordinary engine. An IPython -notebook testing the relative performance of the fast writer against the -ordinary writing system and the data analysis library `Pandas -`__ is available `here `__. -The speed advantage of the faster engine is greatest for integer data and least -for floating-point data; the fast engine is around 3.6 times faster for a -sample file including a mixture of floating-point, integer, and text data. -Also note that stripping string values slows down the writing process, so -specifying ``strip_whitespace=False`` can improve performance. +The ``exponent_style`` parameter allows to define a different character +from the default ``'e'`` for exponential formats in the input file. +The special setting ``'fortran'`` enables auto-detection of any valid +exponent character under Fortran notation. For details see the section on +:ref:`fortran_style_exponents`. + +Fast Converter +-------------- -Fast converter -^^^^^^^^^^^^^^ Input floating-point values should ideally be converted to the nearest possible floating-point approximation; that is, the conversion should be correct within half of the distance between the two closest representable values, or 0.5 `ULP -`__. The ordinary readers, +`__. The ordinary readers, as well as the default fast reader, are guaranteed to convert floating-point values within 0.5 ULP, but there is also a faster and less accurate conversion method accessible via ``use_fast_converter``. If the input -data has less than about 15 significant figures, or if accuracy is relatively -unimportant, this converter might be the best option in +data has less than about fifteen significant figures, or if accuracy is +relatively unimportant, this converter might be the best option in performance-critical scenarios. -`Here -`__ -is an IPython notebook analyzing the error of the fast converter, both in -decimal values and in ULP. For values with a reasonably small number of +For values with a reasonably small number of significant figures, the fast converter is guaranteed to produce an optimal conversion (within 0.5 ULP). Once the number of significant figures exceeds the precision of 64-bit floating-point values, the fast converter is no longer guaranteed to be within 0.5 ULP, but about 60% of values end up -within 0.5 ULP and about 90% within 1.0 ULP. Another notebook analyzing -the fast converter's behavior with extreme values (such as subnormals -and values out of the range of floats) is available `here -`__. +within 0.5 ULP and about 90% within 1.0 ULP. + +Reading Large Tables +-------------------- + +For reading very large tables using the fast reader, see the section on +:ref:`chunk_reading`. + +Writing +======= + +The fast engine supports the same functionality as the ordinary writing engine +and is generally about two to four times faster than the ordinary engine. +The speed advantage of the faster engine is greatest for integer data and least +for floating-point data; the fast engine is around 3.6 times faster for a +sample file including a mixture of floating-point, integer, and text data. +Also note that stripping string values slows down the writing process, so +specifying ``strip_whitespace=False`` can improve performance. + +Speed Gains +=========== -Speed gains -^^^^^^^^^^^ The fast ASCII engine was designed based on the general parsing strategy -used in the `Pandas `__ data analysis library, so +used in the `Pandas `__ data analysis library, so its performance is generally comparable (although slightly slower by default) to the Pandas ``read_csv`` method. -`Here -`__ -is an IPython notebook comparing the performance of the ordinary -:mod:`astropy.io.ascii` reader, the fast reader, the fast reader with the -fast converter enabled, numpy's ``genfromtxt``, and Pandas' ``read_csv`` -for different kinds of table data in a basic space-delimited file. - -In summary, ``genfromtxt`` and the ordinary :mod:`astropy.io.ascii` reader + +The ``genfromtxt`` and the ordinary :mod:`astropy.io.ascii` reader are very similar in terms of speed, while ``read_csv`` is slightly faster than the fast engine for integer and floating-point data; for pure floating-point data, enabling the fast converter yields a speedup of about 50%. Also note that Pandas uses the exact same method as the fast -converter in AstroPy when converting floating-point data. +converter in Astropy when converting floating-point data. The difference in performance between the fast engine and Pandas for text data depends on the extent to which data values are repeated, as Pandas is almost twice as fast as the fast engine when every value is identical and the reverse is true when values are randomized. This is -because the fast engine uses fixed-size numpy string arrays for +because the fast engine uses fixed-size NumPy string arrays for text data, while Pandas uses variable-size object arrays and uses an underlying set to avoid copying repeated values. -Overall, the fast engine tends to be around 4 or 5 times faster than +Overall, the fast engine tends to be around four or five times faster than the ordinary ASCII engine. If the input data is very large (generally -about 100,000 rows or greater), and particularly if the data doesn't -contain primarily integer data or repeated string values, specifying -``parallel`` as ``True`` can yield further performance gains. Although -IPython doesn't work well with ``multiprocessing``, there is a -`script `__ -available for testing the performance of the fast engine in parallel, -and a sample result may be viewed `here -`__. This profile uses the -fast converter for both the serial and parallel AstroPy -readers. +about 100,000 rows or greater), and particularly if the data does not +contain primarily integer data or repeated string values. Another point worth noting is that the fast engine uses memory mapping if a filename is supplied as input. If you want to avoid this for whatever reason, supply an open file object instead. However, this will generally be less efficient from both a time and a memory perspective, as the entire file input will have to be read at once. - diff --git a/docs/io/ascii/fixed_width_gallery.rst b/docs/io/ascii/fixed_width_gallery.rst index 03143b991d0b..cbf2615f2f31 100644 --- a/docs/io/ascii/fixed_width_gallery.rst +++ b/docs/io/ascii/fixed_width_gallery.rst @@ -2,12 +2,12 @@ .. _fixed_width_gallery: -Fixed-width Gallery -------------------- +Fixed-Width Gallery +******************* Fixed-width tables are those where each column has the same width for every row -in the table. This is commonly used to make tables easy to read for humans or -FORTRAN codes. It also reduces issues with quoting and special characters, +in the table. This is commonly used to make tables easy to read for humans or +Fortran codes. It also reduces issues with quoting and special characters, for example:: Col1 Col2 Col3 Col4 @@ -16,23 +16,27 @@ for example:: 2.4 's worlds 2 2 There are a number of common variations in the formatting of fixed-width tables -which :mod:`astropy.io.ascii` can read and write. The most significant difference is -whether there is no header line (:class:`~astropy.io.ascii.FixedWidthNoHeader`), one +which :mod:`astropy.io.ascii` can read and write. The most significant +difference is whether there is no header line (:class:`~astropy.io.ascii.FixedWidthNoHeader`), one header line (:class:`~astropy.io.ascii.FixedWidth`), or two header lines -(:class:`~astropy.io.ascii.FixedWidthTwoLine`). Next, there are variations in the -delimiter character, whether the delimiter appears on either end ("bookends"), -and padding around the delimiter. +(:class:`~astropy.io.ascii.FixedWidthTwoLine`). Next, there are variations in +the delimiter character, like whether the delimiter appears on either end +("bookends"), or if there is padding around the delimiter. Details are available in the class API documentation, but the easiest way to -understand all the options and their interactions is by example. +understand all of the options and their interactions is by example. Reading -^^^^^^^ +======= -FixedWidth -"""""""""" +.. + EXAMPLE START + Reading Fixed-Width Tables -**Nice, typical fixed format table** +Fixed Width +----------- + +**Nice, typical, fixed-format table:** :: >>> from astropy.io import ascii @@ -43,14 +47,14 @@ FixedWidth ... | 2.4 |'s worlds| ... """ >>> ascii.read(table, format='fixed_width') -
+
Col1 Col2 - float64 string72 + float64 str9 ------- --------- 1.2 "hello" 2.4 's worlds -**Typical fixed format table with col names provided** +**Typical fixed-format table with col names provided:** :: >>> table = """ @@ -60,14 +64,14 @@ FixedWidth ... | 2.4 |'s worlds| ... """ >>> ascii.read(table, format='fixed_width', names=['name1', 'name2']) -
+
name1 name2 - float64 string72 + float64 str9 ------- --------- 1.2 "hello" 2.4 's worlds -**Weird input table with data values chopped by col extent** +**Weird input table with data values chopped by col extent:** :: >>> table = """ @@ -76,14 +80,14 @@ FixedWidth ... 2.4 sdf's worlds ... """ >>> ascii.read(table, format='fixed_width') -
+
Col1 Col2 - float64 string56 - ------- -------- - 1.2 "hel - 2.4 df's wo + float64 str7 + ------- ------- + 1.2 "hel + 2.4 df's wo -**Table with double delimiters** +**Table with double delimiters:** :: >>> table = """ @@ -93,15 +97,15 @@ FixedWidth ... | Bob | 555-4527 | 192.168.1.9X| ... """ >>> ascii.read(table, format='fixed_width') -
- Name Phone TCP - string32 string64 string96 - -------- -------- ------------ - John 555-1234 192.168.1.10 - Mary 555-2134 192.168.1.12 - Bob 555-4527 192.168.1.9 - -**Table with space delimiter** +
+ Name Phone TCP + str4 str8 str12 + ---- -------- ------------ + John 555-1234 192.168.1.10 + Mary 555-2134 192.168.1.12 + Bob 555-4527 192.168.1.9 + +**Table with space delimiter:** :: >>> table = """ @@ -111,17 +115,17 @@ FixedWidth ... Bob 555-4527 192.168.1.9 ... """ >>> ascii.read(table, format='fixed_width', delimiter=' ') -
- Name --Phone- ----TCP----- - string32 string64 string96 - -------- -------- ------------ - John 555-1234 192.168.1.10 - Mary 555-2134 192.168.1.12 - Bob 555-4527 192.168.1.9 +
+ Name --Phone- ----TCP----- + str4 str8 str12 + ---- -------- ------------ + John 555-1234 192.168.1.10 + Mary 555-2134 192.168.1.12 + Bob 555-4527 192.168.1.9 -**Table with no header row and auto-column naming.** +**Table with no header row and auto-column naming:** -Use header_start and data_start keywords to indicate no header line. +Use ``header_start`` and ``data_start`` keywords to indicate no header line. :: >>> table = """ @@ -131,40 +135,40 @@ Use header_start and data_start keywords to indicate no header line. ... """ >>> ascii.read(table, format='fixed_width', ... header_start=None, data_start=0) -
- col1 col2 col3 - string32 string64 string96 - -------- -------- ------------ - John 555-1234 192.168.1.10 - Mary 555-2134 192.168.1.12 - Bob 555-4527 192.168.1.9 - -**Table with no header row and with col names provided.** - -Second and third rows also have hanging spaces after final "|". Use header_start and data_start -keywords to indicate no header line. +
+ col1 col2 col3 + str4 str8 str12 + ---- -------- ------------ + John 555-1234 192.168.1.10 + Mary 555-2134 192.168.1.12 + Bob 555-4527 192.168.1.9 + +**Table with no header row and with col names provided:** + +Second and third rows also have hanging spaces after final "|". Use +header_start and data_start keywords to indicate no header line. :: >>> table = ["| John | 555-1234 |192.168.1.10|", ... "| Mary | 555-2134 |192.168.1.12| ", ... "| Bob | 555-4527 | 192.168.1.9| "] >>> ascii.read(table, format='fixed_width', - ... header_start=None, data_start=0, - ... names=('Name', 'Phone', 'TCP')) -
- Name Phone TCP - string32 string64 string96 - -------- -------- ------------ - John 555-1234 192.168.1.10 - Mary 555-2134 192.168.1.12 - Bob 555-4527 192.168.1.9 - - -FixedWidthNoHeader -"""""""""""""""""" - -**Table with no header row and auto-column naming. Use the FixedWidthNoHeader -convenience class.** + ... header_start=None, data_start=0, + ... names=('Name', 'Phone', 'TCP')) +
+ Name Phone TCP + str4 str8 str12 + ---- -------- ------------ + John 555-1234 192.168.1.10 + Mary 555-2134 192.168.1.12 + Bob 555-4527 192.168.1.9 + + +Fixed Width No Header +--------------------- + +**Table with no header row and auto-column naming. Use the +``fixed_width_no_header`` format for convenience:** :: >>> table = """ @@ -173,19 +177,19 @@ convenience class.** ... | Bob | 555-4527 | 192.168.1.9| ... """ >>> ascii.read(table, format='fixed_width_no_header') -
- col1 col2 col3 - string32 string64 string96 - -------- -------- ------------ - John 555-1234 192.168.1.10 - Mary 555-2134 192.168.1.12 - Bob 555-4527 192.168.1.9 - -**Table with no delimiter with column start and end values specified.** - -This uses the col_starts and col_ends keywords. Note that the -col_ends values are inclusive so a position range of 0 to 5 -will select the first 6 characters. +
+ col1 col2 col3 + str4 str8 str12 + ---- -------- ------------ + John 555-1234 192.168.1.10 + Mary 555-2134 192.168.1.12 + Bob 555-4527 192.168.1.9 + +**Table with no delimiter with column start and end values specified:** + +This uses the col_starts and col_ends keywords. Note that the +col_ends values are inclusive so a position range of zero to five +will select the first six characters. :: >>> table = """ @@ -200,30 +204,30 @@ will select the first 6 characters. ... col_starts=(0, 9, 18), ... col_ends=(5, 17, 28), ... ) -
- Name Phone TCP - string32 string72 string80 - -------- --------- ---------- - John 555- 1234 192.168.1. - Mary 555- 2134 192.168.1. - Bob 555- 4527 192.168.1 - -**Table with no delimiter with only column start or end values specified.** +
+ Name Phone TCP + str4 str9 str10 + ---- --------- ---------- + John 555- 1234 192.168.1. + Mary 555- 2134 192.168.1. + Bob 555- 4527 192.168.1 + +**Table with no delimiter with only column start or end values specified:** If only the col_starts keyword is given, it is assumed that each column ends where the next column starts, and the final column ends at the same position as the longest line of data. -Conversely, if only the col_ends keyword is given, it is assumed that the first column -starts at position 0 and that each successive column starts immediately after -the previous one. +Conversely, if only the col_ends keyword is given, it is assumed that the first +column starts at position zero and that each successive column starts +immediately after the previous one. -The two examples below read the same table and produce the same result +The two examples below read the same table and produce the same result. :: >>> table = """ ... #1 9 19 <== Column start indexes - ... #| | | <== Column start positions + ... #| | | <== Column start positions ... #<------><--------><-------------> <== Inferred column positions ... John 555- 1234 192.168.1.10 ... Mary 555- 2134 192.168.1.123 @@ -235,33 +239,34 @@ The two examples below read the same table and produce the same result ... names=('Name', 'Phone', 'TCP'), ... col_starts=(1, 9, 19), ... ) -
- Name Phone TCP - string32 string72 string120 - -------- --------- --------------- - John 555- 1234 192.168.1.10 - Mary 555- 2134 192.168.1.123 - Bob 555- 4527 192.168.1.9 - Bill 555-9875 192.255.255.255 +
+ Name Phone TCP + str4 str9 str15 + ---- --------- --------------- + John 555- 1234 192.168.1.10 + Mary 555- 2134 192.168.1.123 + Bob 555- 4527 192.168.1.9 + Bill 555-9875 192.255.255.255 + >>> ascii.read(table, ... format='fixed_width_no_header', ... names=('Name', 'Phone', 'TCP'), ... col_ends=(8, 18, 32), ... ) -
- Name Phone TCP - string32 string72 string112 - -------- --------- -------------- - John 555- 1234 192.168.1.10 - Mary 555- 2134 192.168.1.123 - Bob 555- 4527 192.168.1.9 - Bill 555-9875 192.255.255.25 +
+ Name Phone TCP + str4 str9 str14 + ---- --------- -------------- + John 555- 1234 192.168.1.10 + Mary 555- 2134 192.168.1.123 + Bob 555- 4527 192.168.1.9 + Bill 555-9875 192.255.255.25 -FixedWidthTwoLine -""""""""""""""""" +Fixed Width Two Line +-------------------- -**Typical fixed format table with two header lines with some cruft** +**Typical fixed-format table with two header lines with some cruft:** :: >>> table = """ @@ -271,14 +276,21 @@ FixedWidthTwoLine ... 2.4 's worlds ... """ >>> ascii.read(table, format='fixed_width_two_line') -
+
Col1 Col2 - float64 string72 + float64 str9 ------- --------- 1.2 "hello" 2.4 's worlds -**Restructured text table** +.. + EXAMPLE END + +.. + EXAMPLE START + Reading a reStructuredText Table + +**reStructuredText table:** :: >>> table = """ @@ -291,14 +303,17 @@ FixedWidthTwoLine ... """ >>> ascii.read(table, format='fixed_width_two_line', ... header_start=1, position_line=2, data_end=-1) -
+
Col1 Col2 - float64 string72 + float64 str9 ------- --------- 1.2 "hello" 2.4 's worlds -**Text table designed for humans and test having position line before the header line.** +.. + EXAMPLE END + +**Text table designed for humans and test having position line before the header line:** :: >>> table = """ @@ -311,20 +326,24 @@ FixedWidthTwoLine ... """ >>> ascii.read(table, format='fixed_width_two_line', delimiter='+', ... header_start=1, position_line=0, data_start=3, data_end=-1) -
+
Col1 Col2 - float64 string72 + float64 str9 ------- --------- 1.2 "hello" 2.4 's worlds Writing -^^^^^^^ +======= + +.. + EXAMPLE START + Writing Fixed-Width Tables -FixedWidth -"""""""""" +Fixed Width +----------- -**Define input values ``dat`` for all write examples.** +**Define input values ``dat`` for all write examples:** :: >>> table = """ @@ -334,7 +353,7 @@ FixedWidth ... """ >>> dat = ascii.read(table, format='fixed_width') -**Write a table as a normal fixed width table.** +**Write a table as a normal fixed-width table:** :: >>> ascii.write(dat, format='fixed_width') @@ -342,7 +361,7 @@ FixedWidth | 1.2 | "hello" | 1 | a | | 2.4 | 's worlds | 2 | 2 | -**Write a table as a fixed width table with no padding.** +**Write a table as a fixed-width table with no padding:** :: >>> ascii.write(dat, format='fixed_width', delimiter_pad=None) @@ -350,7 +369,7 @@ FixedWidth | 1.2| "hello"| 1| a| | 2.4|'s worlds| 2| 2| -**Write a table as a fixed width table with no bookend.** +**Write a table as a fixed-width table with no bookend:** :: >>> ascii.write(dat, format='fixed_width', bookend=False) @@ -358,7 +377,7 @@ FixedWidth 1.2 | "hello" | 1 | a 2.4 | 's worlds | 2 | 2 -**Write a table as a fixed width table with no delimiter.** +**Write a table as a fixed-width table with no delimiter:** :: >>> ascii.write(dat, format='fixed_width', bookend=False, delimiter=None) @@ -366,7 +385,7 @@ FixedWidth 1.2 "hello" 1 a 2.4 's worlds 2 2 -**Write a table as a fixed width table with no delimiter and formatting.** +**Write a table as a fixed-width table with no delimiter and formatting:** :: >>> ascii.write(dat, format='fixed_width', @@ -375,31 +394,31 @@ FixedWidth | 1.200 | "hello" | 1 | a | | 2.400 | 's worlds | 2 | 2 | -FixedWidthNoHeader -"""""""""""""""""" +Fixed Width No Header +--------------------- -**Write a table as a normal fixed width table.** +**Write a table as a normal fixed-width table:** :: >>> ascii.write(dat, format='fixed_width_no_header') | 1.2 | "hello" | 1 | a | | 2.4 | 's worlds | 2 | 2 | -**Write a table as a fixed width table with no padding.** +**Write a table as a fixed-width table with no padding:** :: >>> ascii.write(dat, format='fixed_width_no_header', delimiter_pad=None) |1.2| "hello"|1|a| |2.4|'s worlds|2|2| -**Write a table as a fixed width table with no bookend.** +**Write a table as a fixed-width table with no bookend:** :: >>> ascii.write(dat, format='fixed_width_no_header', bookend=False) 1.2 | "hello" | 1 | a 2.4 | 's worlds | 2 | 2 -**Write a table as a fixed width table with no delimiter.** +**Write a table as a fixed-width table with no delimiter:** :: >>> ascii.write(dat, format='fixed_width_no_header', bookend=False, @@ -407,10 +426,10 @@ FixedWidthNoHeader 1.2 "hello" 1 a 2.4 's worlds 2 2 -FixedWidthTwoLine -""""""""""""""""" +Fixed Width Two Line +-------------------- -**Write a table as a normal fixed width table.** +**Write a table as a normal fixed-width table:** :: >>> ascii.write(dat, format='fixed_width_two_line') @@ -419,7 +438,7 @@ FixedWidthTwoLine 1.2 "hello" 1 a 2.4 's worlds 2 2 -**Write a table as a fixed width table with space padding and '=' position_char.** +**Write a table as a fixed width table with space padding and '=' position_char:** :: >>> ascii.write(dat, format='fixed_width_two_line', @@ -429,7 +448,7 @@ FixedWidthTwoLine 1.2 "hello" 1 a 2.4 's worlds 2 2 -**Write a table as a fixed width table with no bookend.** +**Write a table as a fixed-width table with no bookend:** :: >>> ascii.write(dat, format='fixed_width_two_line', bookend=True, delimiter='|') @@ -437,3 +456,112 @@ FixedWidthTwoLine |----|---------|----|----| | 1.2| "hello"| 1| a| | 2.4|'s worlds| 2| 2| + +.. + EXAMPLE END + +Custom Header Rows +================== + +The ``fixed_width``, ``fixed_width_two_line``, and ``rst`` formats normally include a +single row with the column names in the header. However, for these formats you can +customize the column attributes which appear as header rows. The available +column attributes are ``name``, ``dtype``, ``format``, ``description`` and +``unit``. This is done by listing the desired the header rows using the +``header_rows`` keyword argument. + +.. + EXAMPLE START + Custom Header Rows with Fixed Width + +:: + >>> from astropy.table.table_helpers import simple_table + >>> dat = simple_table(size=3, cols=4) + >>> dat["a"].info.unit = "m" + >>> dat["d"].info.unit = "m/s" + >>> dat["b"].info.format = ".2f" + >>> dat["c"].info.description = "C column" + >>> ascii.write( + ... dat, + ... format="fixed_width", + ... header_rows=["name", "unit", "format", "description"], + ... ) + | a | b | c | d | + | m | | | m / s | + | | .2f | | | + | | | C column | | + | 1 | 1.00 | c | 4 | + | 2 | 2.00 | d | 5 | + | 3 | 3.00 | e | 6 | + +In this example the 1st row is the ``name``, the 2nd row is the ``unit``, and +so forth. You must supply the ``name`` value in the ``header_rows`` list in +order to get an output with the column name included. + +A table with non-standard header rows can be read back in the same way, using +the same list of ``header_rows``:: + + >>> txt = """\ + ... | int32 | float32 | >> dat = ascii.read( + ... txt, + ... format="fixed_width", + ... header_rows=["dtype", "name", "unit", "format", "description"], + ... ) + >>> dat.info +
+ name dtype unit format description + ---- ------- ----- ------ ----------- + a int32 m + b float32 .2f + c str4 C column + d uint8 m / s + +.. + EXAMPLE END + +.. + EXAMPLE START + Custom Header Rows with Fixed Width Two Line + +The same idea can be used with the ``fixed_width_two_line`` format:: + + >>> txt = """\ + ... a b c d + ... int64 float64 >> dat = ascii.read( + ... txt, + ... format="fixed_width_two_line", + ... header_rows=["name", "dtype", "unit"], + ... ) + >>> dat +
+ a b c d + m m / s + int64 float64 str1 int64 + ----- ------- ---- ----- + 1 1.0 c 4 + 2 2.0 d 5 + 3 3.0 e 6 + +.. + EXAMPLE END + +Note that the ``two_line`` in the ``fixed_width_two_line`` format name refers to +the default situation where the header consists two lines, a row of column names +and a row of separator lines. This is a bit of a misnomer when using +``header_rows``. diff --git a/docs/io/ascii/index.rst b/docs/io/ascii/index.rst index ca293fb1ee9b..7a53459a7acd 100644 --- a/docs/io/ascii/index.rst +++ b/docs/io/ascii/index.rst @@ -3,40 +3,47 @@ .. _io-ascii: ********************************* -ASCII Tables (`astropy.io.ascii`) +Text Tables (`astropy.io.ascii`) ********************************* -Introduction -============ - -`astropy.io.ascii` provides methods for reading and writing a wide range of ASCII data table -formats via built-in :ref:`extension_reader_classes`. The emphasis is on flexibility and ease of use, -although readers can optionally use a less flexible C/Cython engine for reading and writing for +`astropy.io.ascii` provides methods for reading and writing a wide range of +text data table formats via built-in :ref:`extension_reader_classes`. The +emphasis is on flexibility and convenience of use, although readers can +optionally use a less flexible C-based engine for reading and writing for improved performance. -The following shows a few of the ASCII formats that are available, while the section on -`Supported formats`_ contains the full list. +.. note:: + + It is strongly encouraged to use the :ref:`Unified I/O Text Tables + ` interface rather than using :mod:`astropy.io.ascii` directly. + + For reading large CSV files, the astropy :ref:`PyArrow CSV ` + reader is the fastest option, while for writing large data tables to CSV, the + :ref:`Table - Pandas interface ` is an option to consider. + + Additional information is available in the :ref:`Unified I/O ` and + :ref:`Unified I/O Table Data ` pages. + +The following shows a few of the text formats that are available, while the +section on `Supported formats`_ contains the full list. * :class:`~astropy.io.ascii.Basic`: basic table with customizable delimiters and header configurations -* :class:`~astropy.io.ascii.Cds`: `CDS format table `_ (also Vizier and ApJ machine readable tables) +* :class:`~astropy.io.ascii.Cds`: `CDS format table `_ (also Vizier) * :class:`~astropy.io.ascii.Daophot`: table from the IRAF DAOphot package -* :class:`~astropy.io.ascii.Ecsv`: `Enhanced CSV format `_ +* :class:`~astropy.io.ascii.Ecsv`: :ref:`ecsv_format` for lossless round-trip of data tables (**recommended**) * :class:`~astropy.io.ascii.FixedWidth`: table with fixed-width columns (see also :ref:`fixed_width_gallery`) -* :class:`~astropy.io.ascii.Ipac`: `IPAC format table `_ +* :class:`~astropy.io.ascii.Ipac`: `IPAC format table `_ * :class:`~astropy.io.ascii.HTML`: HTML format table contained in a
tag * :class:`~astropy.io.ascii.Latex`: LaTeX table with datavalue in the ``tabular`` environment -* :class:`~astropy.io.ascii.Rdb`: tab-separated values with an extra line after the column definition line -* :class:`~astropy.io.ascii.SExtractor`: `SExtractor format table `_ +* :class:`~astropy.io.ascii.Mesa`: MESA stellar evolution code output format -The :mod:`astropy.io.ascii` package is built on a modular and extensible class -structure with independent :ref:`base_class_elements` so that new formats can -be easily accommodated. +* :class:`~astropy.io.ascii.Mrt`: AAS `Machine-Readable Tables (MRT) `_) +* :class:`~astropy.io.ascii.SExtractor`: `SExtractor format table `_ -.. note:: - - It is also possible to use the functionality from - :mod:`astropy.io.ascii` through a higher-level interface in the - :ref:`Data Tables ` package. See :ref:`table_io` for more details. +The strength of `astropy.io.ascii` is the support for astronomy-specific +formats (often with metadata) and specialized data types such as +:ref:`SkyCoord `, :ref:`Time +`, and :ref:`Quantity `. Getting Started =============== @@ -44,71 +51,107 @@ Getting Started Reading Tables -------------- -The majority of commonly encountered ASCII tables can be easily read with the |read| -function. Assume you have a file named ``sources.dat`` with the following contents:: +The majority of commonly encountered text tables can be read with the +|read| function. Assume you have a file named ``sources.dat`` with the +following contents:: obsid redshift X Y object 3102 0.32 4167 4085 Q1250+568-A 877 0.22 4378 3892 "Source 82" -This table can be read with the following:: +.. testsetup:: + >>> from pathlib import Path + >>> from tempfile import TemporaryDirectory + >>> tempdir = TemporaryDirectory() + >>> datadir = Path(tempdir.name) + >>> (datadir / "sources.dat").write_text( + ... "obsid redshift X Y object\n" + ... "3102 0.32 4167 4085 Q1250+568-A\n" + ... "877 0.22 4378 3892 \"Source 82\"\n" + ... ) + 118 + +This table can be read with the following (assuming that the path to the data directory +is set like this: ``datadir=Path('path/to/my/data')``):: >>> from astropy.io import ascii - >>> data = ascii.read("sources.dat") # doctest: +SKIP - >>> print data # doctest: +SKIP + >>> data = ascii.read(datadir / "sources.dat") + >>> print(data) obsid redshift X Y object ----- -------- ---- ---- ----------- 3102 0.32 4167 4085 Q1250+568-A 877 0.22 4378 3892 Source 82 +.. testcleanup:: + + >>> tempdir.cleanup() + The first argument to the |read| function can be the name of a file, a string -representation of a table, or a list of table lines. The return value +representation of a table, or a list of table lines. The return value (``data`` in this case) is a :ref:`Table ` object. -By default |read| will try to `guess the table format <#guess-table-format>`_ -by trying all the `supported formats`_. If this does not work (for unusually -formatted tables) then one needs give ``astropy.io.ascii`` additional hints about -the format, for example:: - - >>> lines = ['objID & osrcid & xsrcid ', - ... '----------------------- & ----------------- & -------------', - ... ' 277955213 & S000.7044P00.7513 & XS04861B6_005', - ... ' 889974380 & S002.9051P14.7003 & XS03957B7_004'] - >>> data = ascii.read(lines, data_start=2, delimiter='&') - >>> print(data) - objID osrcid xsrcid - --------- ----------------- ------------- - 277955213 S000.7044P00.7513 XS04861B6_005 - 889974380 S002.9051P14.7003 XS03957B7_004 - -If the format of a file is known (e.g. it is a fixed width table or an IPAC table), -then it is more efficient and reliable to provide a value for the ``format`` argument from one -of the values in the `supported formats`_. For example:: - - >>> data = ascii.read(lines, format='fixed_width_two_line', delimiter='&') - -For simpler formats such as CSV, |read| will automatically try reading with the -Cython/C parsing engine, which is significantly faster than the ordinary Python -implementation (described in :ref:`fast_ascii_io`). If the fast engine fails, -|read| will fall back on the Python reader by default. The argument -``fast_reader`` can be specified to control this behavior. For example, to -disable the fast engine:: - - >>> data = ascii.read(lines, format='csv', fast_reader=False) +By default, |read| will try to :ref:`guess the table format ` +by trying most of the `supported formats`_. + +.. Warning:: + + Guessing the file format might be convenient, but has two disadvantages: + + - It is often slow for large files because the reader + tries parsing the file with every allowed format until one succeeds. + - Tables sometimes match multiple formats and the first one that succeeds + might not be the one you expected + (:ref:`example `). + + Thus, it is recommended to disable guessing with ``guess=False`` and + explicitly give the table format (e.g. ``format='csv'``) whenever possible. + +If guessing the format does not work, as in the case for unusually formatted +tables, you may need to give `astropy.io.ascii` additional hints about +the format. + +To specify specific data types for one or more columns, use the ``converters`` +argument (see :ref:`io-ascii-read-converters` for details). For instance if the +``obsid`` is actually a string identifier (instead of an integer) you can read +the table with the code below. This also illustrates using the preferred +:ref:`Table interface ` for reading:: + + >>> from astropy.table import Table + >>> sources = """ + ... target observatory obsid + ... TW_Hya Chandra 22178 + ... MP_Mus XMM 0406030101""" + >>> data = Table.read(sources, format='ascii', converters={'obsid': str}) + >>> data +
+ target observatory obsid + str6 str7 str10 + ------ ----------- ---------- + TW_Hya Chandra 22178 + MP_Mus XMM 0406030101 Writing Tables -------------- -The |write| function provides a way to write a data table as a formatted ASCII -table. For example the following writes a table as a simple space-delimited -file:: +The |write| function provides a way to write a data table as a formatted text +table. Most of the input table :ref:`supported_formats` for reading are also +available for writing. This provides a great deal of flexibility in the format +for writing. + +.. + EXAMPLE START + Writing Data Tables as Formatted Text Tables + +The following shows how to write a formatted text table using the |write| +function:: >>> import numpy as np - >>> from astropy.table import Table, Column - >>> x = np.array([1, 2, 3]) - >>> y = x ** 2 - >>> data = Table([x, y], names=['x', 'y']) - >>> ascii.write(data, 'values.dat') + >>> from astropy.io import ascii + >>> from astropy.table import Table + >>> data = Table() + >>> data['x'] = np.array([1, 2, 3], dtype=np.int32) + >>> data['y'] = data['x'] ** 2 + >>> ascii.write(data, 'values.dat', overwrite=True) The ``values.dat`` file will then contain:: @@ -117,92 +160,94 @@ The ``values.dat`` file will then contain:: 2 4 3 9 -Most of the input Reader formats supported by `astropy.io.ascii` for reading are -also supported for writing. This provides a great deal of flexibility in the -format for writing. The example below writes the data as a LaTeX table, using -the option to send the output to ``sys.stdout`` instead of a file:: - - >>> import sys - >>> ascii.write(data, sys.stdout, format='latex') - \begin{table} - \begin{tabular}{cc} - x & y \\ - 1 & 1 \\ - 2 & 4 \\ - 3 & 9 \\ - \end{tabular} - \end{table} - -There is also a faster Cython engine for writing simple formats, -which is enabled by default for these formats (see :ref:`fast_ascii_io`). -To disable this engine, use the parameter ``fast_writer``:: - - >>> ascii.write(data, 'values.csv', format='csv', fast_writer=False) # doctest: +SKIP - -Finally, one can write data in the `ECSV table format -`_ which allows -preserving table meta-data such as column data types and units. In this way a -data table can be stored and read back as ASCII with no loss of information. - - >>> t = Table() - >>> t['x'] = Column([1.0, 2.0], unit='m', dtype='float32') - >>> t['y'] = Column([False, True], dtype='bool') - - >>> from astropy.extern.six.moves import StringIO - >>> fh = StringIO() - >>> t.write(fh, format='ascii.ecsv') # doctest: +SKIP - >>> table_string = fh.getvalue() # doctest: +SKIP - >>> print(table_string) # doctest: +SKIP - # %ECSV 0.9 +It is also possible and encouraged to use the write functionality from +:mod:`astropy.io.ascii` through a higher level interface in the :ref:`Data +Tables ` package (see :ref:`table_io` for more details). For +example:: + + >>> data.write('values.dat', format='ascii', overwrite=True) + +.. attention:: **ECSV is recommended** + + For a reproducible text version of your table, we recommend using the + :ref:`ecsv_format`. This stores all the table meta-data (in particular the + column types and units) to a comment section at the beginning while + maintaining compatibility with most plain CSV readers. It also allows storing + richer data like `~astropy.coordinates.SkyCoord` or multidimensional or + variable-length columns. ECSV is also supported in Java by |STIL| and + |TOPCAT| (see :ref:`ecsv_format`). + +To write our simple example table to ECSV we use:: + + >>> data.write('values.ecsv', overwrite=True) # doctest: +SKIP + +The ``.ecsv`` extension is recognized and implies using ECSV (equivalent to +``format='ascii.ecsv'``). The ``values.ecsv`` file will then contain:: + + # %ECSV 1.0 # --- - # columns: - # - {name: x, unit: m, type: float32} - # - {name: y, type: bool} + # datatype: + # - {name: x, datatype: int32} + # - {name: y, datatype: int32} + # schema: astropy-2.0 x y - 1.0 False - 2.0 True - - >>> Table.read(table_string, format='ascii') # doctest: +SKIP -
- x y - m - float32 bool - ------- ----- - 1.0 False - 2.0 True + 1 1 + 2 4 + 3 9 + +.. + EXAMPLE END .. _supported_formats: -Supported formats +Supported Formats ================= -A full list of the supported ``format`` values and corresponding format types for ASCII -tables is given below. The ``Write`` column indicates which formats support write -functionality, and the ``Fast`` column indicates which formats are compatible with -the fast Cython/C engine for reading and writing. +A full list of the supported ``format`` values and corresponding format types +for text tables is given below. The ``Write`` column indicates which formats +support write functionality, and the ``Fast`` column indicates which formats +are compatible with the fast Cython/C engine for reading and writing. ========================= ===== ==== ============================================================================================ Format Write Fast Description ========================= ===== ==== ============================================================================================ ``aastex`` Yes :class:`~astropy.io.ascii.AASTex`: AASTeX deluxetable used for AAS journals ``basic`` Yes Yes :class:`~astropy.io.ascii.Basic`: Basic table with custom delimiters -``cds`` :class:`~astropy.io.ascii.Cds`: CDS format table +``cds`` Yes :class:`~astropy.io.ascii.Cds`: CDS format table ``commented_header`` Yes Yes :class:`~astropy.io.ascii.CommentedHeader`: Column names in a commented line ``csv`` Yes Yes :class:`~astropy.io.ascii.Csv`: Basic table with comma-separated values ``daophot`` :class:`~astropy.io.ascii.Daophot`: IRAF DAOphot format table -``ecsv`` Yes :class:`~astropy.io.ascii.Ecsv`: Enhanced CSV format +``ecsv`` Yes :class:`~astropy.io.ascii.Ecsv`: Enhanced CSV format (**recommended**) ``fixed_width`` Yes :class:`~astropy.io.ascii.FixedWidth`: Fixed width -``fixed_width_no_header`` Yes :class:`~astropy.io.ascii.FixedWidthNoHeader`: Fixed width with no header -``fixed_width_two_line`` Yes :class:`~astropy.io.ascii.FixedWidthTwoLine`: Fixed width with second header line +``fixed_width_no_header`` Yes :class:`~astropy.io.ascii.FixedWidthNoHeader`: Fixed-width with no header +``fixed_width_two_line`` Yes :class:`~astropy.io.ascii.FixedWidthTwoLine`: Fixed-width with second header line ``html`` Yes :class:`~astropy.io.ascii.HTML`: HTML format table ``ipac`` Yes :class:`~astropy.io.ascii.Ipac`: IPAC format table ``latex`` Yes :class:`~astropy.io.ascii.Latex`: LaTeX table +``mesa`` No :class:`~astropy.io.ascii.Mesa`: MESA stellar evolution code format +``mrt`` Yes :class:`~astropy.io.ascii.Mrt`: AAS Machine-Readable Table format ``no_header`` Yes Yes :class:`~astropy.io.ascii.NoHeader`: Basic table with no headers +``qdp`` Yes :class:`~astropy.io.ascii.QDP`: Quick and Dandy Plotter files ``rdb`` Yes Yes :class:`~astropy.io.ascii.Rdb`: Tab-separated with a type definition header line +``rst`` Yes :class:`~astropy.io.ascii.RST`: reStructuredText simple format table ``sextractor`` :class:`~astropy.io.ascii.SExtractor`: SExtractor format table ``tab`` Yes Yes :class:`~astropy.io.ascii.Tab`: Basic table with tab-separated values +``tdat`` Yes :class:`~astropy.io.ascii.Tdat`: Transportable Database Aggregate Table format ========================= ===== ==== ============================================================================================ +Getting Help +============ + +Some formats have additional options that can be set to control the behavior of the +reader or writer. For more information on these options, you can either see the +documentation for the specific format class (e.g. :class:`~astropy.io.ascii.HTML`) or +use the ``help`` function of the ``read`` or ``write`` functions. For example: + +.. doctest-skip:: + + >>> ascii.read.help() # Common help for all formats + >>> ascii.read.help("html") # Common help plus "html" format-specific args + >>> ascii.write.help("latex") # Common help plus "latex" format-specific args Using `astropy.io.ascii` ======================== @@ -225,7 +270,15 @@ Writing tables write -Fixed-width Gallery +ECSV Format +----------- + +.. toctree:: + :maxdepth: 2 + + ecsv + +Fixed-Width Gallery -------------------- .. toctree:: @@ -241,7 +294,7 @@ Fast ASCII Engine fast_ascii_io -Base class elements +Base Class Elements ------------------- .. toctree:: @@ -249,7 +302,7 @@ Base class elements base_classes -Extension Reader classes +Extension Reader Classes ------------------------ .. toctree:: @@ -257,8 +310,15 @@ Extension Reader classes extension_classes +.. note that if this section gets too long, it should be moved to a separate + doc page - see the top of performance.inc.rst for the instructions on how to do + that +.. include:: performance.inc.rst Reference/API ============= -.. automodapi:: astropy.io.ascii +.. toctree:: + :maxdepth: 2 + + ref_api diff --git a/docs/io/ascii/performance.inc.rst b/docs/io/ascii/performance.inc.rst new file mode 100644 index 000000000000..5a74658966ab --- /dev/null +++ b/docs/io/ascii/performance.inc.rst @@ -0,0 +1,33 @@ +.. note that if this is changed from the default approach of using an *include* + (in index.rst) to a separate performance page, the header needs to be changed + from === to ***, the filename extension needs to be changed from .inc.rst to + .rst, and a link needs to be added in the subpackage toctree + +.. _astropy-io-ascii-performance: + +Performance Tips +================ + +By default, when trying to read a file the reader will guess the format, which +involves trying to read it with many different readers. For better performance +when dealing with large tables, it is recommended to specify the format and any +options explicitly, and turn off guessing as well. + +Example +------- + +.. + EXAMPLE START + Performance Tips for Reading Large Tables with astropy.io.ascii + +If you are reading a simple CSV file with a one-line header with column names, +the following:: + + read('example.csv', format='basic', delimiter=',', guess=False) # doctest: +SKIP + +can be at least an order of magnitude faster than:: + + read('example.csv') # doctest: +SKIP + +.. + EXAMPLE END diff --git a/docs/io/ascii/read.rst b/docs/io/ascii/read.rst index 9b93c7e81cb0..47fad0b253ba 100644 --- a/docs/io/ascii/read.rst +++ b/docs/io/ascii/read.rst @@ -2,156 +2,247 @@ .. _astropy.io.ascii_read: -Reading tables --------------- +Reading Tables +************** -The majority of commonly encountered ASCII tables can be easily read with the |read| +The majority of commonly encountered text tables can be read with the |read| function:: >>> from astropy.io import ascii >>> data = ascii.read(table) # doctest: +SKIP -where ``table`` is the name of a file, a string representation of a table, or a -list of table lines. The return value (``data`` in this case) is a :ref:`Table +Here ``table`` is the name of a file, a string representation of a table, or a +list of table lines. The return value (``data`` in this case) is a :ref:`Table ` object. -By default |read| will try to `guess the table format <#guess-table-format>`_ -by trying all the supported formats. If this does not work (for unusually -formatted tables) then one needs give `astropy.io.ascii` additional hints about the -format, for example:: +Help on the ``read()`` function arguments is available interactively as shown in +this example: + +.. doctest-skip:: + + >>> ascii.read.help() # Common help for all formats + >>> ascii.read.help("html") # Common help plus "html" format-specific args + +By default, |read| will try to `guess the table format <#guess-table-format>`_ +by trying all of the supported formats. + +.. Warning:: + + Guessing the file format is often slow for large files because the reader + tries parsing the file with every allowed format until one succeeds. + For large files it is recommended to disable guessing with ``guess=False``. + +.. + EXAMPLE START + Reading Text Tables Using astropy.io.ascii + +For unusually formatted tables where guessing does not work, give additional +hints about the format:: + + >>> lines = ['objID & osrcid & xsrcid ', + ... '----------------------- & ----------------- & -------------', + ... ' 277955213 & S000.7044P00.7513 & XS04861B6_005', + ... ' 889974380 & S002.9051P14.7003 & XS03957B7_004'] + >>> data = ascii.read(lines, data_start=2, delimiter='&') + >>> print(data) + objID osrcid xsrcid + --------- ----------------- ------------- + 277955213 S000.7044P00.7513 XS04861B6_005 + 889974380 S002.9051P14.7003 XS03957B7_004 + +Other examples are as follows:: + + >>> data = astropy.io.ascii.read('data/nls1_stackinfo.dbout', data_start=2, delimiter='|') # doctest: +SKIP + >>> data = astropy.io.ascii.read('data/simple.txt', quotechar="'") # doctest: +SKIP + >>> data = astropy.io.ascii.read('data/simple4.txt', format='no_header', delimiter='|') # doctest: +SKIP + >>> data = astropy.io.ascii.read('data/tab_and_space.txt', delimiter=r'\s') # doctest: +SKIP + +If the format of a file is known (e.g., it is a fixed-width table or an IPAC +table), then it is more efficient and reliable to provide a value for the +``format`` argument from one of the values in the :ref:`supported_formats`. For +example:: + + >>> data = ascii.read(lines, format='fixed_width_two_line', delimiter='&') - >>> data = astropy.io.ascii.read('t/nls1_stackinfo.dbout', data_start=2, delimiter='|') # doctest: +SKIP - >>> data = astropy.io.ascii.read('t/simple.txt', quotechar="'") # doctest: +SKIP - >>> data = astropy.io.ascii.read('t/simple4.txt', format='no_header', delimiter='|') # doctest: +SKIP +See the :ref:`guess_formats` section for additional details on format guessing. + +.. + EXAMPLE END + +For simpler formats such as CSV, |read| will automatically try reading with the +Cython/C parsing engine, which is significantly faster than the ordinary Python +implementation (described in :ref:`fast_ascii_io`). If the fast engine fails, +|read| will fall back on the Python reader by default. The argument +``fast_reader`` can be specified to control this behavior. For example, to +disable the fast engine:: + + >>> data = ascii.read(lines, format='csv', fast_reader=False) + +For reading very large tables see the section on :ref:`chunk_reading` or +use `pandas `_ (see Note below). + +.. Note:: + + Reading a table which contains unicode characters is supported with the + pure Python readers by specifying the ``encoding`` parameter. The fast + C-readers do not support unicode. For large data files containing unicode, + we recommend reading the file using `pandas `_ + and converting to a :ref:`Table ` via the :ref:`Table - + Pandas interface `. The |read| function accepts a number of parameters that specify the detailed -table format. Different formats can define different defaults, so the -descriptions below sometimes mention "typical" default values. This refers to +table format. Different formats can define different defaults, so the +descriptions below sometimes mention "typical" default values. This refers to the :class:`~astropy.io.ascii.Basic` format reader and other similar character-separated formats. .. _io_ascii_read_parameters: Parameters for ``read()`` -^^^^^^^^^^^^^^^^^^^^^^^^^ +========================= **table** : input table There are four ways to specify the table to be read: - - Name of a file (string) + - Path to a file (string) - Single string containing all table lines separated by newlines - File-like object with a callable read() method - List of strings where each list element is a table line - The first two options are distinguished by the presence of a newline in the string. - This assumes that valid file names will not normally contain a newline. + The first two options are distinguished by the presence of a newline in the + string. This assumes that valid file names will not normally contain a + newline, and a valid table input will at least contain two rows. + Note that a table read in ``no_header`` format can legitimately consist + of a single row; in this case passing the string as a list with a single + item will ensure that it is not interpreted as a file name. **format** : file format (default='basic') - This specifies the top-level format of the ASCII table, for example + This specifies the top-level format of the text table; for example, if it is a basic character delimited table, fixed format table, or - a CDS-compatible table, etc. The value of this parameter must + a CDS-compatible table, etc. The value of this parameter must be one of the :ref:`supported_formats`. -**guess** : try to guess table format (default=True) - If set to True then |read| will try to guess the table format by cycling +**guess** : try to guess table format (default=None) + If set to True, then |read| will try to guess the table format by cycling through a number of possible table format permutations and attempting to read - the table in each case. See the `Guess table format`_ section for further details. + the table in each case. See the `Guess table format`_ section for further details. **delimiter** : column delimiter string A one-character string used to separate fields which typically defaults to - the space character. Other common values might be "\\s" (whitespace), "," or - "|" or "\\t" (tab). A value of "\\s" allows any combination of the tab and + the space character. Other common values might be "\\s" (whitespace), "," or + "|" or "\\t" (tab). A value of "\\s" allows any combination of the tab and space characters to delimit columns. **comment** : regular expression defining a comment line in table - If the ``comment`` regular expression matches the beginning of a table line then that line - will be discarded from header or data processing. For the ``basic`` format this - defaults to "\\s*#" (any whitespace followed by #). + If the ``comment`` regular expression matches the beginning of a table line + then that line will be discarded from header or data processing. For the + ``basic`` format this defaults to "\\s*#" (any whitespace followed by #). **quotechar** : one-character string to quote fields containing special characters - This specifies the quote character and will typically be either the single or double - quote character. This is can be useful for reading text fields with spaces in a space-delimited - table. The default is typically the double quote. + This specifies the quote character and will typically be either the single or + double quote character. This is can be useful for reading text fields with + spaces in a space-delimited table. The default is typically the double quote. **header_start** : line index for the header line This includes only significant non-comment lines and counting starts at 0. If set to None this indicates that there is no header line and the column names - will be auto-generated. See `Specifying header and data location`_ for more + will be auto-generated. See `Specifying header and data location`_ for more details. **data_start** : line index for the start of data counting - This includes only significant non-comment lines and counting starts at 0. See - `Specifying header and data location`_ for more details. + This includes only significant non-comment lines and counting starts at 0. + See `Specifying header and data location`_ for more details. **data_end** : line index for the end of data - This includes only significant non-comment line and can be negative to count - from end. See `Specifying header and data location`_ for more details. + This includes only significant non-comment lines and can be negative to count + from end. See `Specifying header and data location`_ for more details. + +**encoding**: encoding to read the file (``default=None``) + When `None` use `locale.getpreferredencoding` as an encoding. This matches + the default behavior of the built-in `open` when no ``mode`` argument is + provided. + +**converters** : ``dict`` specifying output data types + See the :ref:`io-ascii-read-converters` section for examples. Each key in the + dictionary is a column name or else a name matching pattern including + wildcards. The value is one of: -**converters** : dict of data type converters - See the `Converters`_ section for more information. + - Python data type or numpy dtype such as ``int`` or ``np.float32`` + - list of such types which is tried in order until conversion is successful + - list of converter tuples (this is not common, but see the + `~astropy.io.ascii.convert_numpy` function for an example). **names** : list of names corresponding to each data column - Define the complete list of names for each data column. This will override - names found in the header (if it exists). If not supplied then + Define the complete list of names for each data column. This will override + names found in the header (if it exists). If not supplied then use names from the header or auto-generated names if there is no header. **include_names** : list of names to include in output From the list of column names found from the header or the ``names`` - parameter, select for output only columns within this list. If not supplied + parameter, select for output only columns within this list. If not supplied, then include all names. **exclude_names** : list of names to exclude from output - Exclude these names from the list of output columns. This is applied *after* - the ``include_names`` filtering. If not specified then no columns are excluded. + Exclude these names from the list of output columns. This is applied *after* + the ``include_names`` filtering. If not specified then no columns are excluded. **fill_values** : list of fill value specifiers Specify input table entries which should be masked in the output table - because they are bad or missing. See the `Bad or missing values`_ section - for more information and examples. The default is that any blank table + because they are bad or missing. See the `Bad or missing values`_ section + for more information and examples. The default is that any blank table values are treated as missing. -**fill_include_names** : list of column names, which are affected by ``fill_values``. - If not supplied, then ``fill_values`` can affect all columns. -**fill_exclude_names** : list of column names, which are not affected by ``fill_values``. - If not supplied, then ``fill_values`` can affect all columns. +**fill_include_names** : list of column names affected by ``fill_values`` + This is a list of column names (found from the header or the ``names`` + parameter) for all columns where values will be filled. `None` (the default) will + apply ``fill_values`` to all columns. + +**fill_exclude_names** : list of column names not affected by ``fill_values`` + This is a list of column names (found from the header or the ``names`` + parameter) for all columns where values will be **not** be filled. + This parameter takes precedence over ``fill_include_names``. A value + of `None` (default) does not exclude any columns. + +**fast_reader** : whether to use the C engine + This can be ``True`` or ``False``, and also be a ``dict`` with options. + (see :ref:`fast_ascii_io`) -**Outputter** : Outputter class +**outputter_cls** : Outputter class This converts the raw data tables value into the - output object that gets returned by |read|. The default is + output object that gets returned by |read|. The default is :class:`~astropy.io.ascii.TableOutputter`, which returns a :class:`~astropy.table.Table` object (see :ref:`Data Tables `). -**Inputter** : Inputter class +**inputter_cls** : Inputter class This is generally not specified. -**data_Splitter** : Splitter class to split data columns +**data_splitter_cls** : Splitter class to split data columns -**header_Splitter** : Splitter class to split header columns +**header_splitter_cls** : Splitter class to split header columns -**fast_reader** : whether to use the C engine - This can be ``True`` or ``False``, and also be a dict with options. - (see :ref:`fast_ascii_io`) +Specifying Header and Data Location +=================================== -**Reader** : Reader class (*deprecated* in favor of ``format``) - This specifies the top-level format of the ASCII table, for example - if it is a basic character delimited table, fixed format table, or - a CDS-compatible table, etc. The value of this parameter must - be a Reader class. For basic usage this means one of the - built-in :ref:`extension_reader_classes`. - -Specifying header and data location -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The three parameters ``header_start``, ``data_start`` and ``data_end`` make it +The three parameters ``header_start``, ``data_start``, and ``data_end`` make it possible to read a table file that has extraneous non-table data included. -This is a case where you need to help out ``io.ascii`` and tell it where to -find the header and data. +This is a case where you need to help out `astropy.io.ascii` and tell it where +to find the header and data. -When processing of a file into a header and data components any blank lines +When a file is processed into a header and data components, any blank lines (which might have whitespace characters) and commented lines (starting with the -comment character, typically ``#``) are stripped out *before* the header and data -parsing code sees the table content. For example imagine you have the file -below. The column on the left is not part of the file but instead shows how -``io.ascii`` is viewing each line and the line count index. :: +comment character, typically ``#``) are stripped out *before* the header and +data parsing code sees the table content. + +Example +------- + +.. + EXAMPLE START + Specifying Header and Data Location for Text Tables + +To use the parameters ``header_start``, ``data_start``, and ``data_end`` +to read a table with non-table data included, take the file below. The column +on the left is not part of the file but instead shows how `astropy.io.ascii` is +viewing each line and the line count index. :: Index Table content ------ ---------------------------------------------------------------- @@ -171,27 +262,31 @@ below. The column on the left is not part of the file but instead shows how 7 | Run completed at 2012:01-01T12:14:01 In this case you would have ``header_start=3``, ``data_start=5``, and -``data_end=7``. The convention for ``data_end`` follows the normal Python +``data_end=7``. The convention for ``data_end`` follows the normal Python slicing convention where to select data rows 5 and 6 you would do -``rows[5:7]``. For ``data_end`` you can also supply a negative index to +``rows[5:7]``. For ``data_end`` you can also supply a negative index to count backward from the end, so ``data_end=-1`` (like ``rows[5:-1]``) would work in this case. -.. note:: - - Prior to astropy v1.1 there was a bug in which a blank line that had one or - more whitespace characters was mistakenly counted for ``header_start`` but - was (correctly) not counted for ``data_start`` and ``data_end``. If you - have code that was depending on the incorrect pre-1.1 behavior then it needs - to be modified. +.. + EXAMPLE END .. _replace_bad_or_missing_values: -Bad or missing values -^^^^^^^^^^^^^^^^^^^^^ +Bad or Missing Values +===================== + +text data tables can contain bad or missing values. A common case is when a +table contains blank entries with no available data. + +Examples +-------- -ASCII data tables can contain bad or missing values. A common case is when a table -contains blank entries with no available data, for example:: +.. + EXAMPLE START + Text Tables with Bad or Missing Values + +Take this example of a table with blank entries:: >>> weather_data = """ ... day,precip,type @@ -200,38 +295,38 @@ contains blank entries with no available data, for example:: ... Wed,1.1,snow ... """ -By default |read| will interpret blank entries as being bad/missing and output a masked -Table with those entries masked out by setting the corresponding mask value set to -``True``:: +By default, |read| will interpret blank entries as being bad/missing and output +a masked Table with those entries masked out by setting the corresponding mask +value set to ``True``:: >>> dat = ascii.read(weather_data) - >>> print dat + >>> print(dat) day precip type ---- ------ ---- Mon 1.5 rain Tues -- -- Wed 1.1 snow -If you want to replace the masked (missing) values with particular values, set the masked -column ``fill_value`` attribute and then get the "filled" version of the table. This -looks like the following:: +If you want to replace the masked (missing) values with particular values, set +the masked column ``fill_value`` attribute and then get the "filled" version of +the table. This looks like the following:: >>> dat['precip'].fill_value = -999 >>> dat['type'].fill_value = 'N/A' - >>> print dat.filled() + >>> print(dat.filled()) day precip type ---- ------ ---- Mon 1.5 rain Tues -999.0 N/A Wed 1.1 snow -ASCII tables may also have other indicators of bad or missing data. For -example a table may contain string values that are not a valid representation -of a number, e.g. ``"..."``, or a table may have special values like ``-999`` -that are chosen to indicate missing data. The |read| function has a flexible +Text tables may have other indicators of bad or missing data as well. For +example, a table may contain string values that are not a valid representation +of a number (e.g., ``"..."``), or a table may have special values like ``-999`` +that are chosen to indicate missing data. The |read| function has a flexible system to accommodate these cases by marking specified character sequences in -the input data as "missing data" during the conversion process. Whenever -missing data is found then the output will be a masked table. +the input data as "missing data" during the conversion process. Whenever +missing data is found the output will be a masked table. This is done with the ``fill_values`` keyword argument, which can be set to a single missing-value specification ```` or a list of ```` tuples:: @@ -239,11 +334,10 @@ single missing-value specification ```` or a list of `` | [, , ...] = (, '0', , , ...) -When reading a table the second element of a ```` should always -be the string ``'0'``, -otherwise you may get unexpected behavior [#f1]_. By default the -```` is applied to all columns unless column name strings are -supplied. An alterate way to limit the columns is via the +When reading a table, the second element of a ```` should always +be the string ``'0'``, otherwise you may get unexpected behavior [#f1]_. By +default, the ```` is applied to all columns unless column name +strings are supplied. An alternate way to limit the columns is via the ``fill_include_names`` and ``fill_exclude_names`` keyword arguments in |read|. In the example below we read back the weather table after filling the missing @@ -254,7 +348,7 @@ values in with typical placeholders:: ... 'Tues -999.0 N/A', ... ' Wed 1.1 snow'] >>> t = ascii.read(table, fill_values=[('-999.0', '0', 'precip'), ('N/A', '0', 'type')]) - >>> print t + >>> print(t) day precip type ---- ------ ---- Mon 1.5 rain @@ -263,11 +357,11 @@ values in with typical placeholders:: .. note:: - The default in |read| is ``fill_values=('','0')``. This marks blank entries as being - missing for any data type (int, float, or string). If ``fill_values`` is explicitly + The default in |read| is ``fill_values=('','0')``. This marks blank entries as being + missing for any data type (int, float, or string). If ``fill_values`` is explicitly set in the call to |read| then the default behavior of marking blank entries as missing - no longer applies. For instance setting ``fill_values=None`` will disable this - auto-masking without setting any other fill values. This can be useful for a string + no longer applies. For instance setting ``fill_values=None`` will disable this + auto-masking without setting any other fill values. This can be useful for a string column where one of values happens to be ``""``. @@ -277,71 +371,122 @@ values in with typical placeholders:: ``fill_value`` used for writing tables. On reading, the second element of the ```` tuple can actually be an arbitrary string value which replaces occurrences of the ```` - string in the input stream prior to type conversion. This ends up + string in the input stream prior to type conversion. This ends up being the value "behind the mask", which should never be directly - accessed. Only the value ``'0'`` is neutral when attempting to detect - the column data type and perform type conversion. For instance if you + accessed. Only the value ``'0'`` is neutral when attempting to detect + the column data type and perform type conversion. For instance if you used ``'nan'`` for the ```` value then integer columns would wind up as float. -Guess table format -^^^^^^^^^^^^^^^^^^ +.. + EXAMPLE END + +Selecting columns for masking +----------------------------- +The |read| function provides the parameters ``fill_include_names`` and ``fill_exclude_names`` +to select which columns will be used in the ``fill_values`` masking process described above. + +.. + EXAMPLE START + Using the ``fill_include_names`` and ``fill_exclude_names`` parameters for Text tables -If the ``guess`` parameter in |read| is set to True (which is the default) then +The use of these parameters is not common but in some cases can considerably simplify +the code required to read a table. The following gives a simple example to illustrate how +``fill_include_names`` and ``fill_exclude_names`` can be used +in the most basic and typical cases:: + + >>> from astropy.io import ascii + >>> lines = ['a,b,c,d', '1.0,2.0,3.0,4.0', ',,,'] + >>> ascii.read(lines) +
+ a b c d + float64 float64 float64 float64 + ------- ------- ------- ------- + 1.0 2.0 3.0 4.0 + -- -- -- -- + + >>> ascii.read(lines, fill_include_names=['a', 'c']) +
+ a b c d + float64 str3 float64 str3 + ------- ---- ------- ---- + 1.0 2.0 3.0 4.0 + -- -- + + >>> ascii.read(lines, fill_exclude_names=['a', 'c']) +
+ a b c d + str3 float64 str3 float64 + ---- ------- ---- ------- + 1.0 2.0 3.0 4.0 + -- -- + +.. + EXAMPLE END + +.. _guess_formats: + +Guess Table Format +================== + +If the ``guess`` parameter in |read| is set to True, then |read| will try to guess the table format by cycling through a number of -possible table format permutations and attempting to read the table in each case. -The first format which succeeds and will be used to read the table. To succeed -the table must be successfully parsed by the Reader and satisfy the following -column requirements: +possible table format permutations and attempting to read the table in each +case. The first format which succeeds will be used to read the table. To +succeed, the table must be successfully parsed by the Reader and satisfy the +following column requirements: - * At least two table columns - * No column names are a float or int number - * No column names begin or end with space, comma, tab, single quote, double quote, or - a vertical bar (|). + * At least two table columns. + * No column names are a float or int number. + * No column names begin or end with space, comma, tab, single quote, double + quote, or a vertical bar (|). These requirements reduce the chance for a false positive where a table is -successfully parsed with the wrong format. A common situation is a table -with numeric columns but no header row, and in this case ``astropy.io.ascii`` will +successfully parsed with the wrong format. A common situation is a table +with numeric columns but no header row, and in this case `astropy.io.ascii` will auto-assign column names because of the restriction on column names that look like a number. -Guess order -""""""""""" -The order of guessing is shown by this Python code, where ``Reader`` is the -class which actually implements reading the different file formats:: +Guess Order +----------- + +The order of guessing is shown by this Python code:: - for Reader in (Ecsv, FixedWidthTwoLine, FastBasic, Basic, - Rdb, FastTab, Tab, Cds, Daophot, SExtractor, - Ipac, Latex, AASTex): - read(Reader=Reader) + for format in ("ecsv", "fixed_width_two_line", "rst", "fast_basic", "basic", + "fast_rdb", "rdb", "fast_tab", "tab", "cds", "daophot", "sextractor", + "ipac", "latex", "aastex"): + read(format=format) - for Reader in (CommentedHeader, FastBasic, Basic, FastNoHeader, NoHeader): + for format in ("commented_header", "fast_basic", "basic", "fast_noheader", "noheader"): for delimiter in ("|", ",", " ", "\\s"): for quotechar in ('"', "'"): - read(Reader=Reader, delimiter=delimiter, quotechar=quotechar) + read(format=format, delimiter=delimiter, quotechar=quotechar) -Note that the :class:`~astropy.io.ascii.FixedWidth` derived-readers are not included -in the default guess sequence (this causes problems), so to read such tables -one must explicitly specify the format with the ``format`` keyword. Also notice -that formats compatible with the fast reading engine attempt to use the fast -engine before the ordinary reading engine. +Note that the :class:`~astropy.io.ascii.FixedWidth` derived-readers are not +included in the default guess sequence (this causes problems), so to read such +tables you must explicitly specify the format with the ``format`` keyword. Also +notice that formats compatible with the fast reading engine attempt to use the +fast engine before the ordinary reading engine. If none of the guesses succeed in reading the table (subject to the column -requirements) a final try is made using just the user-supplied parameters but -without checking the column requirements. In this way a table with only one +requirements), a final try is made using just the user-supplied parameters but +without checking the column requirements. In this way, a table with only one column or column names that look like a number can still be successfully read. -The guessing process respects any values of the Reader, delimiter, and -quotechar parameters that were supplied to the read() function. Any guesses -that would conflict are skipped. For example the call:: +The guessing process respects any values of the format, delimiter, and +quotechar parameters as well as options for the fast reader that were +supplied to the read() function. Any guesses that would conflict are +skipped. For example, the call:: - >>> data = ascii.read(table, Reader=ascii.NoHeader, quotechar="'") + >>> data = ascii.read(table, format="no_header", quotechar="'") would only try the four delimiter possibilities, skipping all the conflicting -Reader and quotechar combinations. +format and quotechar combinations. Similarly, with any setting of +``fast_reader`` that requires use of the fast engine, only the fast +variants in the format list above will be tried. Disabling -""""""""" +--------- Guessing can be disabled in two ways:: @@ -352,18 +497,28 @@ Guessing can be disabled in two ways:: data = astropy.io.ascii.read(table) # guessing disabled Debugging -""""""""" +--------- In order to get more insight into the guessing process and possibly debug if -something isn't working as expected, use the -`~astropy.io.ascii.get_read_trace()` function. This returns a traceback of the +something is not working as expected, use the +`~astropy.io.ascii.get_read_trace()` function. This returns a traceback of the attempted read formats for the last call to `~astropy.io.ascii.read()`. -Comments and metadata -^^^^^^^^^^^^^^^^^^^^^ +Comments and Metadata +===================== Any comment lines detected during reading are inserted into the output table -via the ``comments`` key in the table's ``.meta`` dictionary. For example:: +via the ``comments`` key in the table's ``.meta`` dictionary. + +Example +------- + +.. + EXAMPLE START + Comments and Metadata in Text Tables + +Comment lines detected during reading are inserted into the output table as +such:: >>> table='''# TELESCOPE = 30 inch ... # TARGET = PV Ceph @@ -376,7 +531,7 @@ via the ``comments`` key in the table's ``.meta`` dictionary. For example:: ['TELESCOPE = 30 inch', 'TARGET = PV Ceph', 'BAND = V'] While :mod:`astropy.io.ascii` will not do any post-processing on comment lines, -custom post-processing can be accomplished by re-reading with the metadata line +custom post-processing can be accomplished by rereading with the metadata line comments. Here is one example, where comments are of the form "# KEY = VALUE":: >>> header = ascii.read(dat.meta['comments'], delimiter='=', @@ -388,155 +543,641 @@ comments. Here is one example, where comments are of the form "# KEY = VALUE":: TARGET PV Ceph BAND V +.. + EXAMPLE END -Converters -^^^^^^^^^^ +.. _io-ascii-read-converters: + +Converters for Specifying Dtype +=============================== :mod:`astropy.io.ascii` converts the raw string values from the table into numeric data types by using converter functions such as the Python ``int`` and -``float`` functions. For example ``int("5.0")`` will fail while float("5.0") -will succeed and return 5.0 as a Python float. +``float`` functions or numpy dtype types such as ``np.float64``. The default converters are:: - default_converters = [astropy.io.ascii.convert_numpy(numpy.int), - astropy.io.ascii.convert_numpy(numpy.float), - astropy.io.ascii.convert_numpy(numpy.str)] - -These take advantage of the :func:`~astropy.io.ascii.convert_numpy` -function which returns a 2-element tuple ``(converter_func, converter_type)`` -as described in the previous section. The type provided to -:func:`~astropy.io.ascii.convert_numpy` must be a valid `numpy type -`_, for example -``numpy.int``, ``numpy.uint``, ``numpy.int8``, ``numpy.int64``, -``numpy.float``, ``numpy.float64``, ``numpy.str``. + default_converters = [int, float, str] The default converters for each column can be overridden with the ``converters`` keyword:: >>> import numpy as np - >>> converters = {'col1': [ascii.convert_numpy(np.uint)], - ... 'col2': [ascii.convert_numpy(np.float32)]} + >>> converters = {'col1': np.uint, + ... 'col2': np.float32} >>> ascii.read('file.dat', converters=converters) # doctest: +SKIP -Advanced customization -^^^^^^^^^^^^^^^^^^^^^^ +In addition to single column names you can use wildcards via `fnmatch` to +select multiple columns. For example, we can set the format for all columns +with a name starting with "col" to an unsigned integer while applying default +converters to all other columns in the table:: + + >>> import numpy as np + >>> converters = {'col*': np.uint} + >>> ascii.read('file.dat', converters=converters) # doctest: +SKIP + +.. + EXAMPLE START + Reading True / False values as boolean type + +The value in the converters ``dict`` can also be a list of types, in which case +these will be tried in order. This allows for flexible type conversions. For +example, imagine you get read the following table:: + + >>> txt = """\ + ... a b c + ... --- --- ----- + ... 1 3.5 True + ... 2 4.0 False""" + >>> t = ascii.read(txt, format='fixed_width_two_line') + +By default the ``True`` and ``False`` values will be interpreted as strings. +However, if you want those values to be read as booleans you can do the +following:: + + >>> converters = {'*': [int, float, bool, str]} + >>> t = ascii.read(txt, format='fixed_width_two_line', converters=converters) + >>> print(t['c'].dtype) + bool + +.. + EXAMPLE END + +Advanced usage +-------------- + +Internally type conversion uses the :func:`~astropy.io.ascii.convert_numpy` +function which returns a two-element tuple ``(converter_func, converter_type)``. +This two-element tuple can be used as the value in a ``converters`` dict. +The type provided to +:func:`~astropy.io.ascii.convert_numpy` must be a valid `NumPy type +`_ such as +``numpy.int``, ``numpy.uint``, ``numpy.int8``, ``numpy.int64``, +``numpy.float``, ``numpy.float64``, or ``numpy.str``. + +It is also possible to directly pass an arbitrary conversion function as the +``converter_func`` element of the two-element tuple. + +.. _fortran_style_exponents: + +Fortran-Style Exponents +======================= + +The :ref:`fast converter ` available with the C +input parser provides an ``exponent_style`` option to define a custom +character instead of the standard ``'e'`` for exponential formats in +the input file, to read, for example, Fortran-style double precision +numbers like ``'1.495978707D+13'``: + + >>> ascii.read('double.dat', format='basic', guess=False, + ... fast_reader={'exponent_style': 'D'}) # doctest: +SKIP + +The special setting ``'fortran'`` is provided to allow for the +auto-detection of any valid Fortran exponent character (``'E'``, +``'D'``, ``'Q'``), as well as of triple-digit exponents prefixed with no +character at all (e.g., ``'2.1127123261674622-107'``). +All values and exponent characters in the input data are +case-insensitive; any value other than the default ``'E'`` implies the +automatic setting of ``'use_fast_converter': True``. + +.. _advanced_customization: + +Advanced Customization +====================== Here we provide a few examples that demonstrate how to extend the base -functionality to handle special cases. To go beyond these simple examples the +functionality to handle special cases. To go beyond these examples, the best reference is to read the code for the existing :ref:`extension_reader_classes`. +Examples +-------- + +.. + EXAMPLE START + Advanced Customization to Extend Base Functionality of astropy.io.ascii + +For special cases, these examples demonstrate how to extend the base +functionality of `astropy.io.ascii`. + **Define custom readers by class inheritance** The most useful way to define a new reader class is by inheritance. -This is the way all the build-in readers are defined, so there are plenty +This is the way all of the built-in readers are defined, so there are plenty of examples in the code. In most cases, you will define one class to handle the header, -one class that handles the data and a reader class that ties it all together. -Here is a simple example from the code that defines a reader that is just like +one class that handles the data, and a reader class that ties it all together. +Here is an example from the code that defines a reader that is just like the basic reader, but header and data start in different lines of the file:: - # Note: NoHeader is already included in astropy.io.ascii for convenience. - class NoHeaderHeader(BasicHeader): - '''Reader for table header without a header - - Set the start of header line number to `None`, which tells the basic - reader there is no header line. - ''' - start_line = None - - class NoHeaderData(BasicData): - '''Reader for table data without a header - - Data starts at first uncommented line since there is no header line. - ''' - start_line = 0 - - class NoHeader(Basic): - """Read a table with no header line. Columns are autonamed using - header.auto_format which defaults to "col%d". Otherwise this reader - the same as the :class:`Basic` class from which it is derived. Example:: - - # Table data - 1 2 "hello there" - 3 4 world - """ - _format_name = 'no_header' - _description = 'Basic table with no headers' - header_class = NoHeaderHeader - data_class = NoHeaderData + >>> # Note: NoHeader is already included in astropy.io.ascii for convenience. + >>> from astropy.io.ascii.basic import BasicHeader, BasicData, Basic + >>> + >>> class NoHeaderHeader(BasicHeader): + ... """Reader for table header without a header + ... + ... Set the start of header line number to `None`, which tells the basic + ... reader there is no header line. + ... """ + ... start_line = None + >>> + >>> class NoHeaderData(BasicData): + ... """Reader for table data without a header + ... + ... Data starts at first uncommented line since there is no header line. + ... """ + ... start_line = 0 + >>> + >>> class NoHeader(Basic): + ... """Read a table with no header line. Columns are autonamed using + ... header.auto_format which defaults to "col%d". Otherwise this reader + ... the same as the :class:`Basic` class from which it is derived. Example:: + ... + ... # Table data + ... 1 2 "hello there" + ... 3 4 world + ... """ + ... _format_name = 'custom_no_header' + ... _description = 'Basic table with no headers' + ... header_class = NoHeaderHeader + ... data_class = NoHeaderData In a slightly more involved case, the implementation can also override some of the methods in the base class:: - # Note: CommentedHeader is already included in astropy.io.ascii for convenience. - class CommentedHeaderHeader(BasicHeader): - """Header class for which the column definition line starts with the - comment character. See the :class:`CommentedHeader` class for an example. - """ - def process_lines(self, lines): - """Return only lines that start with the comment regexp. For these - lines strip out the matching characters.""" - re_comment = re.compile(self.comment) - for line in lines: - match = re_comment.match(line) - if match: - yield line[match.end():] - - def write(self, lines): - lines.append(self.write_comment + self.splitter.join(self.colnames)) - - - class CommentedHeader(Basic): - """Read a file where the column names are given in a line that begins with - the header comment character. ``header_start`` can be used to specify the - line index of column names, and it can be a negative index (for example -1 - for the last commented line). The default delimiter is the - character.:: - - # col1 col2 col3 - # Comment line - 1 2 3 - 4 5 6 - """ - _format_name = 'commented_header' - _description = 'Column names in a commented line' - - header_class = CommentedHeaderHeader - data_class = NoHeaderData - + >>> # Note: CommentedHeader is already included in astropy.io.ascii for convenience. + >>> class CommentedHeaderHeader(BasicHeader): + ... """Header class for which the column definition line starts with the + ... comment character. See the :class:`CommentedHeader` class for an example. + ... """ + ... def process_lines(self, lines): + ... """Return only lines that start with the comment regexp. For these + ... lines strip out the matching characters.""" + ... re_comment = re.compile(self.comment) + ... for line in lines: + ... match = re_comment.match(line) + ... if match: + ... yield line[match.end():] + ... + ... def write(self, lines): + ... lines.append(self.write_comment + self.splitter.join(self.colnames)) + >>> + >>> + >>> class CommentedHeader(Basic): + ... """Read a file where the column names are given in a line that begins with + ... the header comment character. ``header_start`` can be used to specify the + ... line index of column names, and it can be a negative index (for example -1 + ... for the last commented line). The default delimiter is the + ... character.:: + ... + ... # col1 col2 col3 + ... # Comment line + ... 1 2 3 + ... 4 5 6 + ... """ + ... _format_name = 'custom_commented_header' + ... _description = 'Column names in a commented line' + ... + ... header_class = CommentedHeaderHeader + ... data_class = NoHeaderData + +**Application: Write a "fixed_width" table with a "commented_header"** + +This module provides formats for tables where the header line is marked with a comment +character and a separate class that writes fixed-width tables, but there is no functionality +to write a fixed-width table with a commented header. Fixed width tables can be easier to read +by eye because the rows are aligned and certain other programs require the header line to be +commented. So, we now want to make a writer that can write this format; for this example we do +not bother to work out how to read this format, but just raise an error on reading: + + >>> from astropy.io.ascii.fixedwidth import FixedWidthData, FixedWidth + >>> + >>> class FixedWidthDataCommentedHeaderData(FixedWidthData): + ... def write(self, lines): + ... lines = super().write(lines) + ... lines[0] = self.write_comment + lines[0] + ... for i in range(1, len(lines)): + ... lines[i] = ' ' * len(self.write_comment) + lines[i] + ... return lines + >>> + >>> class FixedWidthCommentedHeader(FixedWidth): + ... _format_name = "fixed_width_commented_header" + ... _description = "Fixed width with commented header" + ... + ... data_class = FixedWidthDataCommentedHeaderData + ... + ... def read(self, table): + ... raise NotImplementedError + +This new format is automatically added to the list of formats that can be read by +the :ref:`io_registry` (note that our format has no mechanism to write out the units): + + >>> import sys + >>> import astropy.units as u + >>> from astropy.table import Table + >>> tab = Table({'v': [15.4, 223.45] * u.km/u.s, 'type': ['star', 'jet']}) + >>> tab.write(sys.stdout, format='ascii.fixed_width', delimiter=None) + v type + 15.4 star + 223.45 jet + >>> tab.write(sys.stdout, format='ascii.commented_header') + # v type + 15.4 star + 223.45 jet + >>> tab.write(sys.stdout, format='ascii.fixed_width_commented_header', delimiter=None) + # v type + 15.4 star + 223.45 jet + +.. testcleanup:: + + >>> from astropy.io import registry + >>> from astropy.io.ascii.core import FORMAT_CLASSES + >>> for format_name in ['custom_no_header', 'custom_commented_header', 'fixed_width_commented_header']: + ... registry.unregister_reader(f"ascii.{format_name}", Table) + ... registry.unregister_writer(f"ascii.{format_name}", Table) + ... del FORMAT_CLASSES[format_name] **Define a custom reader functionally** + Instead of defining a new class, it is also possible to obtain an instance -of a reader and then to modify the properties of this one reader instance +of a reader, and then to modify the properties of this one reader instance in a function:: - def read_rdb_table(table): - reader = astropy.io.ascii.Basic() - reader.header.splitter.delimiter = '\t' - reader.data.splitter.delimiter = '\t' - reader.header.splitter.process_line = None - reader.data.splitter.process_line = None - reader.data.start_line = 2 - - return reader.read(table) + >>> from astropy.io import ascii + >>> + >>> def read_rdb_table(table): + ... reader = ascii.Basic() + ... reader.header.splitter.delimiter = '\t' + ... reader.data.splitter.delimiter = '\t' + ... reader.header.splitter.process_line = None + ... reader.data.splitter.process_line = None + ... reader.data.start_line = 2 + ... + ... return reader.read(table) **Create a custom splitter.process_val function** :: - # The default process_val() normally just strips whitespace. - # In addition have it replace empty fields with -999. - def process_val(x): - """Custom splitter process_val function: Remove whitespace at the beginning - or end of value and substitute -999 for any blank entries.""" - x = x.strip() - if x == '': - x = '-999' - return x - - # Create an RDB reader and override the splitter.process_val function - rdb_reader = astropy.io.ascii.get_reader(Reader=astropy.io.ascii.Rdb) - rdb_reader.data.splitter.process_val = process_val + >>> # The default process_val() normally just strips whitespace. + >>> # In addition have it replace empty fields with -999. + >>> def process_val(x): + ... """Custom splitter process_val function: Remove whitespace at the beginning + ... or end of value and substitute -999 for any blank entries.""" + ... x = x.strip() + ... if x == '': + ... x = '-999' + ... return x + >>> + >>> # Create an RDB reader and override the splitter.process_val function + >>> rdb_reader = ascii.get_reader(reader_cls=ascii.Rdb) + >>> rdb_reader.data.splitter.process_val = process_val + +.. + EXAMPLE END + +.. _chunk_reading: + +Reading Large Tables in Chunks +============================== + +The default process for reading ASCII tables is not memory efficient and may +temporarily require much more memory than the size of the file (up to a factor +of 5 to 10). In cases where the temporary memory requirement exceeds available +memory this can cause significant slowdown when disk cache gets used. + +In this situation, there is a way to read the table in smaller chunks which are +limited in size. There are two possible ways to do this: + +- Read the table in chunks and aggregate the final table along the way. This + uses only somewhat more memory than the final table requires. +- Use a Python generator function to return a `~astropy.table.Table` object for + each chunk of the input table. This allows for scanning through arbitrarily + large tables since it never returns the final aggregate table. + +The chunk reading functionality is most useful for very large tables, so this is +available only for the :ref:`fast_ascii_io` readers. The following formats are +supported: ``tab``, ``csv``, ``no_header``, ``rdb``, and ``basic``. The +``commented_header`` format is not directly supported, but as a workaround one +can read using the ``no_header`` format and explicitly supply the column names +using the ``names`` argument. + +In order to read a table in chunks you must provide the ``fast_reader`` keyword +argument with a ``dict`` that includes the ``chunk_size`` key with the value +being the approximate size (in bytes) of each chunk of the input table to read. +In addition, if you provide a ``chunk_generator`` key which is set to +``True``, then instead of returning a single table for the whole input it +returns an iterator that provides a table for each chunk of the input. + +Examples +-------- + +.. + EXAMPLE START + Reading Large Tables in Chunks with astropy.io.ascii + +.. testsetup:: + + >>> # For performance we don't actually make a > 100 MB table. + >>> # The code works this way, too. + >>> tab = Table({'Vmag': [7] * 10}) + >>> tab.write('large_table.csv') + +To read an entire table while limiting peak memory usage: +:: + + # Read a large CSV table in 100 Mb chunks. + + tbl = ascii.read('large_table.csv', format='csv', guess=False, + fast_reader={'chunk_size': 100 * 1000000}) + +To read the table in chunks with an iterator, we iterate over a CSV table and +select all rows where the ``Vmag`` column is less than 8.0 (e.g., all stars in +table brighter than 8.0 mag). We collect all of these subtables and then stack +them at the end. +:: + + from astropy.table import vstack + + # tbls is an iterator over the chunks (no actual reading done yet) + tbls = ascii.read('large_table.csv', format='csv', guess=False, + fast_reader={'chunk_size': 100 * 1000000, + 'chunk_generator': True}) + + out_tbls = [] + + # At this point the file is actually read in chunks. + for tbl in tbls: + bright = tbl['Vmag'] < 8.0 + if np.count_nonzero(bright): + out_tbls.append(tbl[bright]) + + out_tbl = vstack(out_tbls) + +.. testcleanup:: + + >>> import pathlib + >>> pathlib.Path.unlink('large_table.csv') + +.. Note:: **Performance** + + Specifying the ``format`` explicitly and using ``guess=False`` is a good idea + for large tables. This prevents unnecessary guessing in the typical case + where the format is already known. + + The ``chunk_size`` should generally be set to the largest value that is + reasonable given available system memory. There is overhead associated + with processing each chunk, so the fewer chunks the better. + + .. + EXAMPLE END + +.. _io_ascii_how_to_examples: + +How to Find and Fix Problems Reading a Table +============================================ + +The purpose of this section is to provide a few examples how we can +deal with tables that fail to read. + +.. _io_ascii_should_specify_format: + +Specify as much detail as possible about the format +--------------------------------------------------- +One of the most common ways to read text tables with astropy is to get the +reader guess the format. This is convenient, but it takes extra time because +the reader tries different formats until one of them looks like it's +working (see :ref:`guess_formats` for details on the guessing process) and +sometimes that's not the format you expect. +Thus, if you know the format of the table, is it safer and faster to specify +as much detail as possible. + +Here is an example: + + >>> tab_text = """ + ... , a, b + ... 0, x, 3 + ... 1, y, d + ... """ + +This could be read either as table with three rows and no header (and a missing +data entry in the first column) or the first row could be the column names. +In either case, the commas could be part of the data of a space-delimited table +or they could be delimiters for a comma-delimited table. +If we explicitly set the format, astropy will read it for any of these cases. + +The "no_header" format will try a space-delimited table first, so all the columns will +come out to be string columns: + + >>> ascii.read(tab_text, format="no_header") +
+ col1 col2 col3 + str2 str2 str1 + ---- ---- ---- + , a, b + 0, x, 3 + 1, y, d + +If we set the delimiter to a comma, then the first column will be an integer: + + >>> ascii.read(tab_text, format="no_header", delimiter=",") +
+ col1 col2 col3 + int64 str1 str1 + ----- ---- ---- + -- a b + 0 x 3 + 1 y d + +We can also read it using a format that expects a header line and a comma delimiter. +In this case, only the name for the first column with an empty name will be auto-assigned: + + >>> ascii.read(tab_text, format='csv') +
+ col0 a b + int64 str1 str1 + ----- ---- ---- + 0 x 3 + 1 y d + +However, if we let astropy guess the format, it cannot know what is intended. When +guessing tries out different format with ``ascii.read(tab_text)`` the first one that +matches in this particular example is the "no header with comma delimiter" format. +While the astropy developers spent a lot of time trying to make the guessing process +return what a human would naturally expect, there is no way to make it work for all cases. +Thus, it is safest to always specify the format if known. + + +Sometimes it is easy to obtain the data in a more structured format that +more clearly defines columns and metadata, e.g. a FITS or VO/XML table, or +a text table that uses a different column separator (e.g. comma instead of +white space) or fixed-width columns. +In that case, the fastest solution can be to simply download or export the +data again in a different format. + +Find the Problem +---------------- +Usually, `astropy.io.ascii.read` tries many different formats until one +succeeds in reading. If it works, that saves you from finding and +setting right options for reading. However, if it fails to find any combination +of format and format options that correctly parses the file, then you will get +a long exception message which shows every format that was tried and ends +with this advice:: + + ************************************************************************ + ** ERROR: Unable to guess table format with the guesses listed above. ** + ** ** + ** To figure out why the table did not read, use guess=False and ** + ** fast_reader=False, along with any appropriate arguments to read(). ** + ** In particular specify the format and any known attributes like the ** + ** delimiter. ** + ************************************************************************ + +To expand on this a bit, you probably know from looking at the file +what format it is in, which must be one of the :ref:`supported_formats`. +For instance maybe it is a basic space-delimited file but has the header +line as a comment like below, which corresponds to the ``commented_header`` +format:: + + >>> table = """# name id + ... Jill 1232 + ... Jack Johnson 456""" + +In order to find the actual problem with the reading this file, you would do:: + + >>> ascii.read(table, format='commented_header', delimiter=' ', guess=False, fast_reader=False) + Traceback (most recent call last): + ... + astropy.io.ascii.core.InconsistentTableError: Number of header columns (2) inconsistent with data columns (3) at data line 1 + Header values: ['name', 'id'] + Data values: ['Jack', 'Johnson', '456'] + +At this point you can see that the problem is that the 2nd data line has 3 columns while the header says there should be only 2. You might be initially confused by the ``data line 1`` since the problem was in the 3rd line of the file. There are two things happening here. First, ``data line 1`` refers to the count of data lines and does not include any header lines, blank lines, or commented out lines. Second, the count starts from zero, so that ``1`` is the 2nd data line. +See the :ref:`guess_formats` section for additional details on format guessing. + + +Make the Table Easier to Read +----------------------------- + +Sometimes, the parameters for `astropy.io.ascii.read` to specify, for example +``format``, ``delimiter``, ``comment``, ``quote_char``, ``header_start``, +``data_start``, ``data_end``, and ``encoding`` are not enough. +To read just a single table that has a format close to, but not identical +with, any of the :ref:`supported_formats`, the fastest solution may be to open +that one table file in a text editor to modify it until it does conform to a +format that can be read. On the other hand, if we need to +read tables of that specific format again and again, it is better to find a way +to read them with `~astropy.io.ascii` without modifying every file by hand. + + +Badly formatted header line +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The following table will fail to parse (raising an +`~astropy.io.ascii.InconsistentTableError`) because the header line looks as +if there were three columns, while in fact, there are only two:: + + Name spectral type + Vega A0 + Altair A7 + +Opening this file in a text editor to fix the format is easy:: + + Name "spectral type" + Vega A0 + Altair A7 + +or:: + + Name spectral_type + Vega A0 + Altair A7 + +With either of the above changes you can read the file with no problem using default settings. + +.. + EXAMPLE START + Make a table easier to read + +To read the table without editing the files, we need to ignore the badly formatted header line and +pass in the names of the column ourselves. +That can be done without any modification of the table file by setting the ``data_start`` parameter:: + + >>> table = """ + ... Star spectral type + ... Vega A0 + ... Altair A7 + ... """ + >>> ascii.read(table, names=["Star", "spectral type"], data_start=1) +
+ Star spectral type + str6 str2 + ------ ------------- + Vega A0 + Altair A7 + +.. + EXAMPLE END + +Badly formatted data line +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Similar principles apply to badly formatted data lines. Here is a +table where the number of columns is not consistent (``alpha Cen`` +should be written as ``"alpha Cen"`` to make clear that the two words +"alpha" and "Cen" are part of the same column):: + + Star SpT + Vega A0 + alpha Cen G2V+K1 + +When we try to read that with ``guess=False``, astropy throws an +`astropy.io.ascii.InconsistentTableError`:: + + >>> from astropy.io import ascii + >>> table = ''' + ... Star SpT + ... Vega A0 + ... alpha Cen G2V+K1 + ... ''' + >>> ascii.read(table, guess=False) + Traceback (most recent call last): + ... + astropy.io.ascii.core.InconsistentTableError: Number of header columns (2) inconsistent with data columns in data line 1 + +This points us to the line with the problem, here line 1 (starting to count +after the header lines and counting the data lines from 0 as usual in Python). In this table with +just two lines the problem is easy to spot, but for longer tables, the line number is +very helpful. We can now fix that line by hand in the file by adding quotes +around ``"alpha Cen"``. Then we can try to read the table again and see +if it works or if there is a another badly formatted data line. + +.. _io-ascii-read-gaia-tables: + +Reading Gaia Data Tables +======================== + +.. note:: + + The recommended way to access Gaia is via its + `astroquery.gaia `_ module. + However, if you need to access its data file separately via + ``astropy``, then read on. + +Gaia data tables are available in `ECSV +`_ format including detailed +metadata for the tables and columns (e.g., column descriptions, units, and data types). +For example the DR3 tables are at http://cdn.gea.esac.esa.int/Gaia/gdr3/gaia_source/. + +The DR3 data files are not strictly compliant with the ECSV standard because they use +the marker ``null`` to indicate a missing value instead of the required ``""``. In order +to read these files correctly with the full metadata, we need to tell the ECSV reader +to treat ``null`` as the missing value:: + + >>> from astropy.table import QTable + >>> dat = QTable.read( + ... "GaiaSource_000000-003111.csv.gz", + ... format="ascii.ecsv", + ... fill_values=("null", "0") + ... ) # doctest: +SKIP diff --git a/docs/io/ascii/ref_api.rst b/docs/io/ascii/ref_api.rst new file mode 100644 index 000000000000..f4fff26884a2 --- /dev/null +++ b/docs/io/ascii/ref_api.rst @@ -0,0 +1,4 @@ +Reference/API +************* + +.. automodapi:: astropy.io.ascii diff --git a/docs/io/ascii/references.txt b/docs/io/ascii/references.txt index 1d2ab2974123..58fefccda0a6 100644 --- a/docs/io/ascii/references.txt +++ b/docs/io/ascii/references.txt @@ -1,4 +1,3 @@ .. |read| replace:: :func:`~astropy.io.ascii.read` .. |write| replace:: :func:`~astropy.io.ascii.write` -.. |Table| replace:: :class:`~astropy.table.Table` -.. _structured array: http://docs.scipy.org/doc/numpy/user/basics.rec.html +.. _structured array: https://numpy.org/doc/stable/user/basics.rec.html diff --git a/docs/io/ascii/write.rst b/docs/io/ascii/write.rst index 93f90d40e67a..cbee040acb11 100644 --- a/docs/io/ascii/write.rst +++ b/docs/io/ascii/write.rst @@ -2,21 +2,41 @@ .. _astropy.io.ascii_write: -Writing tables --------------- +Writing Tables +============== -:mod:`astropy.io.ascii` is able to write ASCII tables out to a file or file-like +:mod:`astropy.io.ascii` is able to write text tables out to a file or file-like object using the same class structure and basic user interface as for reading tables. +Help on the ``write()`` function arguments is available interactively as shown in +this example: + +.. doctest-skip:: + + >>> from astropy.io import ascii + >>> ascii.write.help() # Common help for all formats + >>> ascii.write.help("html") # Common help plus "html" format-specific args + The |write| function provides a way to write a data table as a -formatted ASCII table. For example:: +formatted text table. + +Examples +-------- + +.. + EXAMPLE START + Writing Text Tables Using astropy.io.ascii + +To write a formatted text table using the |write| function:: >>> import numpy as np >>> from astropy.io import ascii - >>> x = np.array([1, 2, 3]) - >>> y = x ** 2 - >>> ascii.write([x, y], 'values.dat', names=['x', 'y']) + >>> from astropy.table import Table + >>> data = Table() + >>> data['x'] = np.array([1, 2, 3], dtype=np.int32) + >>> data['y'] = data['x'] ** 2 + >>> ascii.write(data, 'values.dat', overwrite=True) # doctest: +SKIP The ``values.dat`` file will then contain:: @@ -25,9 +45,39 @@ The ``values.dat`` file will then contain:: 2 4 3 9 +It is also possible and encouraged to use the write functionality from +:mod:`astropy.io.ascii` through a higher level interface in the :ref:`Data +Tables ` package (see :ref:`table_io` for more details). For +example:: + + >>> data.write('values.dat', format='ascii', overwrite=True) # doctest: +SKIP + +For a more reproducible text version of your table, we recommend using the +:ref:`ecsv_format`. This stores all the table meta-data (in particular the +column types and units) to a comment section at the beginning while still +maintaining compatibility with most plain CSV readers. It also allows storing +richer data like `~astropy.coordinates.SkyCoord` or multidimensional or +variable-length columns. For our simple example:: + + >>> data.write('values.ecsv', overwrite=True) # doctest: +SKIP + +The ``.ecsv`` extension is recognized and implies using ECSV (equivalent to +``format='ascii.ecsv'``). The ``values.ecsv`` file will then contain:: + + # %ECSV 1.0 + # --- + # datatype: + # - {name: x, datatype: int32} + # - {name: y, datatype: int32} + # schema: astropy-2.0 + x y + 1 1 + 2 4 + 3 9 + Most of the input table :ref:`supported_formats` for -reading are also available for writing. This provides a great deal of -flexibility in the format for writing. The example below writes the data as a +reading are also available for writing. This provides a great deal of +flexibility in the format for writing. The example below writes the data as a LaTeX table, using the option to send the output to ``sys.stdout`` instead of a file:: @@ -47,138 +97,57 @@ To disable this engine, use the parameter ``fast_writer``:: >>> ascii.write(data, 'values.csv', format='csv', fast_writer=False) # doctest: +SKIP -Input data format -^^^^^^^^^^^^^^^^^ - -The input ``table`` argument to |write| can be any value that is supported for -initializing a |Table| object. This is documented in detail in the -:ref:`construct_table` section and includes creating a table with a list of -columns, a dictionary of columns, or from `numpy` arrays (either structured or -homogeneous). The sections below show a few examples. - -Table or NumPy structured array -""""""""""""""""""""""""""""""" - -An AstroPy |Table| object or a NumPy `structured array`_ (or record array) can -serve as input to the |write| function. - -:: - - >>> from astropy.io import ascii - >>> from astropy.table import Table - - >>> data = Table({'a': [1, 2, 3], - ... 'b': [4.0, 5.0, 6.0]}, - ... names=['a', 'b']) - >>> ascii.write(data) - a b - 1 4.0 - 2 5.0 - 3 6.0 - - >>> data = np.array([(1, 2., 'Hello'), (2, 3., "World")], - ... dtype=('i4,f4,a10')) - >>> ascii.write(data) - f0 f1 f2 - 1 2.0 Hello - 2 3.0 World - -The output of :mod:`astropy.io.ascii.read` is a |Table| or NumPy array data -object that can be an input to the |write| function. - -:: - - >>> data = ascii.read('t/daophot.dat', format='daophot') # doctest: +SKIP - >>> ascii.write(data, 'space_delimited_table.dat') # doctest: +SKIP +.. + EXAMPLE END -List of lists -""""""""""""" - -A list of Python lists (or any iterable object) can be used as input:: - - >>> x = [1, 2, 3] - >>> y = [4, 5.2, 6.1] - >>> z = ['hello', 'world', '!!!'] - >>> data = [x, y, z] - - >>> ascii.write(data) - col0 col1 col2 - 1 4.0 hello - 2 5.2 world - 3 6.1 !!! - -The ``data`` object does not contain information about the column names so -|Table| has chosen them automatically. To specify the names, provide the -``names`` keyword argument. This example also shows excluding one of the columns -from the output:: - - >>> ascii.write(data, names=['x', 'y', 'z'], exclude_names=['y']) - x z - 1 hello - 2 world - 3 !!! - - -Dict of lists -""""""""""""" - -A dictionary containing iterable objects can serve as input to |write|. Each -dict key is taken as the column name while the value must be an iterable object -containing the corresponding column values. - -Since a Python dictionary is not ordered the output column order will be -unpredictable unless the ``names`` argument is provided. - -:: - - >>> data = {'x': [1, 2, 3], - ... 'y': [4, 5.2, 6.1], - ... 'z': ['hello', 'world', '!!!']} - >>> ascii.write(data, names=['x', 'y', 'z']) - x y z - 1 4.0 hello - 2 5.2 world - 3 6.1 !!! +.. Note:: + For most supported formats one can write a masked table and then read it back + without losing information about the masked table entries. This is + accomplished by using a blank string entry to indicate a masked (missing) + value. See the :ref:`replace_bad_or_missing_values` section for more + information. .. _io_ascii_write_parameters: Parameters for ``write()`` -^^^^^^^^^^^^^^^^^^^^^^^^^^ +-------------------------- -The |write| function accepts a number of parameters that specify the detailed output table -format. Each of the :ref:`supported_formats` is handled by a corresponding Writer class that -can define different defaults, so the descriptions below sometimes mention "typical" -default values. This refers to the :class:`~astropy.io.ascii.Basic` writer and other -similar Writer classes. +The |write| function accepts a number of parameters that specify the detailed +output table format. Each of the :ref:`supported_formats` is handled by a +corresponding Writer class that can define different defaults, so the +descriptions below sometimes mention "typical" default values. This refers to +the :class:`~astropy.io.ascii.Basic` writer and other similar Writer classes. -Some output format Writer classes, e.g. :class:`~astropy.io.ascii.Latex` or -:class:`~astropy.io.ascii.AASTex` accept additional keywords, that can +Some output format Writer classes (e.g., :class:`~astropy.io.ascii.Latex` or +:class:`~astropy.io.ascii.AASTex`) accept additional keywords that can customize the output further. See the documentation of these classes for details. -**output** : output specifier +**output**: output specifier There are two ways to specify the output for the write operation: - Name of a file (string) - - File-like object (from open(), StringIO, etc) + - File-like object (from open(), StringIO, etc.) -**table** : input table - Any value that is supported for initializing a |Table| object (see :ref:`construct_table`). +**table**: input table + Any value that is supported for initializing a |Table| object (see + :ref:`construct_table`). This includes a table with a list of columns, a + dictionary of columns, or from `numpy` arrays (either structured or + homogeneous). -**format** : output format (default='basic') - This specifies the format of the ASCII table to be written, for - example if it is a basic character delimited table, fixed format table, or a - CDS-compatible table, etc. The value of this parameter must - be one of the :ref:`supported_formats`. +**format**: output format (default='basic') + This specifies the format of the text table to be written, such as a basic + character delimited table, fixed-format table, or a CDS-compatible table, + etc. The value of this parameter must be one of the :ref:`supported_formats`. -**delimiter** : column delimiter string - A one-character string used to separate fields which typically defaults to the space character. - Other common values might be "," or "|" or "\\t". +**delimiter**: column delimiter string + A one-character string used to separate fields which typically defaults to + the space character. Other common values might be "," or "|" or "\\t". -**comment** : string defining start of a comment line in output table +**comment**: string defining start of a comment line in output table For the :class:`~astropy.io.ascii.Basic` Writer this defaults to "# ". - Which and how comments are written depends on the format chosen. + Which comments are written and how depends on the format chosen. The comments are defined as a list of strings in the input table ``meta['comments']`` element. Comments in the metadata of the given |Table| will normally be written before the header, although @@ -186,36 +155,36 @@ details. commented header. To disable writing comments, set ``comment=False``. **formats**: dict of data type converters - For each key (column name) use the given value to convert the column data to a string. - If the format value is string-like then it is used as a Python format statement, - e.g. '%0.2f' % value. If it is a callable function then that function - is called with a single argument containing the column value to be converted. - Example:: + For each key (column name) use the given value to convert the column data to + a string. If the format value is string-like, then it is used as a Python + format statement (e.g., '%0.2f' % value). If it is a callable function, then + that function is called with a single argument containing the column value to + be converted. Example:: astropy.io.ascii.write(table, sys.stdout, formats={'XCENTER': '%12.1f', 'YCENTER': lambda x: round(x, 1)}, -**names**: list of names corresponding to each data column - Define the complete list of names for each data column. This will override - names determined from the data table (if available). If not supplied then - use names from the data table or auto-generated names. +**names**: list of output column names + Define the complete list of output column names to write for the data table, + overriding the existing column names. **include_names**: list of names to include in output From the list of column names found from the data table or the ``names`` - parameter, select for output only columns within this list. If not supplied + parameter, select for output only columns within this list. If not supplied then include all names. **exclude_names**: list of names to exclude from output - Exclude these names from the list of output columns. This is applied *after* - the ``include_names`` filtering. If not specified then no columns are excluded. + Exclude these names from the list of output columns. This is applied *after* + the ``include_names`` filtering. If not specified then no columns are excluded. -**fill_values**: fill value specifier of lists +**fill_values**: list of fill value specifiers This can be used to fill missing values in the table or replace values with special meaning. - See the :ref:`replace_bad_or_missing_values` section for more information on the syntax. - The syntax is almost the same as when reading a table. - There is a special value ``astropy.io.ascii.masked`` that is used a say "output this string - for all masked values in a masked table (the default is to use a ``'--'``):: + See the :ref:`replace_bad_or_missing_values` section for more information on + the syntax. The syntax is almost the same as when reading a table. + There is a special value ``astropy.io.ascii.masked`` that is used to say + "output this string for all masked values in a masked table" (the default is + to use an empty string ``""``):: >>> import sys >>> from astropy.table import Table, Column, MaskedColumn @@ -224,22 +193,14 @@ details. >>> t['a'].mask = [True, False] >>> ascii.write(t, sys.stdout) a b - -- 3 + "" 3 2 4 >>> ascii.write(t, sys.stdout, fill_values=[(ascii.masked, 'N/A')]) a b N/A 3 2 4 - If no ``fill_values`` is applied for masked values in ``astropy.io.ascii``, the default set - with ``numpy.ma.masked_print_option.set_display`` applies (usually that is also ``'--'``):: - - >>> ascii.write(t, sys.stdout, fill_values=[]) - a b - -- 3 - 2 4 - - Note that when writing a table all values are converted to strings, before + Note that when writing a table, all values are converted to strings before any value is replaced. Because ``fill_values`` only replaces cells that are an exact match to the specification, you need to provide the string representation (stripped of whitespace) for each value. For example, in @@ -253,15 +214,15 @@ details. "no data" 3 2.00 4 - Similarly, if you replace a value in a column that has a fixed length format, - e.g. ``'f4.2'``, then the string you want to replace must have the same - number of characters, in the example above ``fill_values=[(' nan',' N/A')]`` + Similarly, if you replace a value in a column that has a fixed length format + (e.g., ``'f4.2'``), then the string you want to replace must have the same + number of characters. In the example above, ``fill_values=[(' nan',' N/A')]`` would work. -**fill_include_names**: list of column names, which are affected by ``fill_values``. +**fill_include_names**: list of column names, which are affected by ``fill_values`` If not supplied, then ``fill_values`` can affect all columns. -**fill_exclude_names**: list of column names, which are not affected by ``fill_values``. +**fill_exclude_names**: list of column names, which are not affected by ``fill_values`` If not supplied, then ``fill_values`` can affect all columns. **fast_writer**: whether to use the fast Cython writer @@ -269,11 +230,213 @@ details. to use the faster writer (described in :ref:`fast_ascii_io`) if possible. Specifying ``fast_writer=False`` disables this behavior. -**Writer** : Writer class (*deprecated* in favor of ``format``) - This specifies the top-level format of the ASCII table to be written, for - example if it is a basic character delimited table, fixed format table, or a - CDS-compatible table, etc. The value of this parameter must be a Writer - class. For basic usage this means one of the built-in :ref:`extension_reader_classes`. - Note: Reader classes and Writer classes are synonymous, in other - words Reader classes can also write, but for historical reasons they are - often called Reader classes. +.. _cds_mrt_format: + +Machine-Readable Table Format +----------------------------- + +The American Astronomical Society Journals' `Machine-Readable Table (MRT) +`_ format consists of single file with +the table description header and the table data itself. MRT is similar to the +`CDS `_ format standard, but differs +in the table description sections and the lack of a separate ``ReadMe`` file. +Astropy does not support writing in the CDS format. + +The :class:`~astropy.io.ascii.Mrt` writer supports writing tables to MRT format. + +.. note:: + + The metadata of the table, apart from column ``unit``, ``name`` and + ``description``, are not written in the output file. Placeholders for + the title, authors, and table name fields are put into the output file and + can be edited after writing. + +Examples +"""""""" + +.. + EXAMPLE START + Writing MRT Format Tables Using astropy.io.ascii + +The command ``ascii.write(format='mrt')`` writes an ``astropy`` `~astropy.table.Table` +to the MRT format. Section dividers ``---`` and ``===`` are used to divide the table +into different sections, with the last section always been the actual data. + +As the MRT standard requires, +for columns that have a ``unit`` attribute not set to ``None``, +the unit names are tabulated in the Byte-By-Byte +description of the column. When columns do not contain any units, ``---`` is put instead. +A ``?`` is prefixed to the column description in the Byte-By-Byte for ``Masked`` +columns or columns that have null values, indicating them as such. + +The example below initializes a table with columns that have a ``unit`` attribute and +has masked values. + + >>> from astropy.io import ascii + >>> from astropy.table import Table, Column, MaskedColumn + >>> from astropy import units as u + >>> table = Table() + >>> table['Name'] = ['ASASSN-15lh', 'ASASSN-14li'] + >>> # MRT Standard requires all quantities in SI units. + >>> temperature = [0.0334, 0.297] * u.K + >>> table['Temperature'] = temperature.to(u.keV, equivalencies=u.temperature_energy()) + >>> table['nH'] = Column([0.025, 0.0188], unit=u.Unit(10**22)) + >>> table['Flux'] = ([2.044 * 10**-11] * u.erg * u.cm**-2).to(u.Jy * u.Unit(10**12)) + >>> table['Flux'] = MaskedColumn(table['Flux'], mask=[True, False]) + >>> table['magnitude'] = [u.Magnitude(25), u.Magnitude(-9)] + +Note that for columns with `~astropy.time.Time`, `~astropy.time.TimeDelta` and related values, +the writer does not do any internal conversion or modification. These columns should be +converted to regular columns with proper ``unit`` and ``name`` attribute before writing +the table. Thus:: + + >>> from astropy.time import Time, TimeDelta + >>> from astropy.timeseries import TimeSeries + >>> ts = TimeSeries(time_start=Time('2019-01-01'), time_delta=2*u.day, n_samples=1) + >>> table['Obs'] = Column(ts.time.decimalyear, description='Time of Observation') + >>> table['Cadence'] = Column(TimeDelta(100.0, format='sec').datetime.seconds, + ... unit=u.s) + +Columns that are `~astropy.coordinates.SkyCoord` objects or columns with +values that are such objects are recognized as such, and some predefined labels and +description is used for them. Coordinate columns that have `~astropy.coordinates.SphericalRepresentation` +are additionally sub-divided into their coordinate component columns. Representations that have +``ra`` and ``dec`` components are divided into their ``hour``-``min``-``sec`` +and ``deg``-``arcmin``-``arcsec`` components respectively. Whereas columns with +``SkyCoord`` objects in the ``Galactic`` or any of the ``Ecliptic`` frames are divided +into their latitude(``ELAT``/``GLAT``) and longitude components (``ELON``/``GLAT``) only. +The original table remains accessible as such, while the file is written from a modified +copy of the table. The new coordinate component columns are appended to the end of the table. + +It should be noted that the default precision of the latitude, longitude and seconds (of arc) +columns is set at a default number of 12, 10 and 9 digits after the decimal for ``deg``, ``sec`` +and ``arcsec`` values, respectively. This default is set to match a machine precision of 1e-15 +relative to the original ``SkyCoord`` those columns were extracted from. +As all other columns, the format can be expliclty set by passing the ``formats`` keyword to the +``write`` function or by setting the ``format`` attribute of individual columns (the latter +will only work for columns that are not decomposed). +To customize the number of significant digits, presicions should therefore be specified in the +``formats`` dictionary for the *output* column names, such as +``formats={'RAs': '07.4f', 'DEs': '06.3f'}`` or ``formats={'GLAT': '+10.6f', 'GLON': '9.6f'}`` +for milliarcsecond accuracy. Note that the forms with leading zeros for the seconds and +including the sign for latitudes are recommended for better consistency and readability. + +The following code illustrates the above. + + >>> from astropy.coordinates import SkyCoord + >>> table['coord'] = [SkyCoord.from_name('ASASSN-15lh'), + ... SkyCoord.from_name('ASASSN-14li')] # doctest: +REMOTE_DATA + >>> table.write('coord_cols.dat', format='ascii.mrt') # doctest: +SKIP + >>> table['coord'] = table['coord'].geocentrictrueecliptic # doctest: +REMOTE_DATA + >>> table['Temperature'].format = '.5E' # Set default column format. + >>> table.write('ecliptic_cols.dat', format='ascii.mrt') # doctest: +SKIP + +After execution, the contents of ``coords_cols.dat`` will be:: + + Title: + Authors: + Table: + ================================================================================ + Byte-by-byte Description of file: table.dat + -------------------------------------------------------------------------------- + Bytes Format Units Label Explanations + -------------------------------------------------------------------------------- + 1-11 A11 --- Name Description of Name + 13-23 E11.6 keV Temperature [0.0/0.01] Description of Temperature + 25-30 F6.4 10+22 nH [0.01/0.03] Description of nH + 32-36 F5.3 10+12Jy Flux ? Description of Flux + 38-42 E5.1 mag magnitude [0.0/3981.08] Description of magnitude + 44-49 F6.1 --- Obs [2019.0/2019.0] Time of Observation + 51-53 I3 s Cadence [100] Description of Cadence + 55-56 I2 h RAh Right Ascension (hour) + 58-59 I2 min RAm Right Ascension (minute) + 61-73 F13.10 s RAs Right Ascension (second) + 75 A1 --- DE- Sign of Declination + 76-77 I2 deg DEd Declination (degree) + 79-80 I2 arcmin DEm Declination (arcmin) + 82-93 F12.9 arcsec DEs Declination (arcsec) + -------------------------------------------------------------------------------- + Notes: + -------------------------------------------------------------------------------- + ASASSN-15lh 2.87819e-09 0.0250 1e-10 2019.0 100 22 02 15.4500000000 -61 39 34.599996000 + ASASSN-14li 2.55935e-08 0.0188 2.044 4e+03 2019.0 100 12 48 15.2244072000 +17 46 26.496624000 + +And the file ``ecliptic_cols.dat`` will look like:: + + Title: + Authors: + Table: + ================================================================================ + Byte-by-byte Description of file: table.dat + -------------------------------------------------------------------------------- + Bytes Format Units Label Explanations + -------------------------------------------------------------------------------- + 1- 11 A11 --- Name Description of Name + 13- 23 E11.6 keV Temperature [0.0/0.01] Description of Temperature + 25- 30 F6.4 10+22 nH [0.01/0.03] Description of nH + 32- 36 F5.3 10+12Jy Flux ? Description of Flux + 38- 42 E5.1 mag magnitude [0.0/3981.08] Description of magnitude + 44- 49 F6.1 --- Obs [2019.0/2019.0] Time of Observation + 51- 53 I3 s Cadence [100] Description of Cadence + 55- 70 F16.12 deg ELON Ecliptic Longitude (geocentrictrueecliptic) + 72- 87 F16.12 deg ELAT Ecliptic Latitude (geocentrictrueecliptic) + -------------------------------------------------------------------------------- + Notes: + -------------------------------------------------------------------------------- + ASASSN-15lh 2.87819e-09 0.0250 1e-10 2019.0 100 306.224208650096 -45.621789850825 + ASASSN-14li 2.55935e-08 0.0188 2.044 4e+03 2019.0 100 183.754980099243 21.051410763027 + +Finally, MRT has some specific naming conventions for columns +(``_). For example, if a column contains +the mean error for the data in a column named ``label``, then this column should be named ``e_label``. +These kinds of relative column naming cannot be enforced by the MRT writer +because it does not know what the column data means and thus, the relation between the +columns cannot be figured out. Therefore, it is up to the user to use ``Table.rename_columns`` +to appropriately rename any columns before writing the table to MRT format. +The following example shows a similar situation, using the option to send the output to +``sys.stdout`` instead of a file:: + + >>> table['error'] = [1e4, 450] * u.Jy # Error in the Flux values. + >>> outtab = table.copy() # So that changes don't affect the original table. + >>> outtab.rename_column('error', 'e_Flux') + >>> # re-order so that related columns are placed next to each other. + >>> outtab = outtab['Name', 'Obs', 'coord', 'Cadence', 'nH', 'magnitude', + ... 'Temperature', 'Flux', 'e_Flux'] # doctest: +REMOTE_DATA + + >>> ascii.write(outtab, format='mrt') # doctest: +SKIP + Title: + Authors: + Table: + ================================================================================ + Byte-by-byte Description of file: table.dat + -------------------------------------------------------------------------------- + Bytes Format Units Label Explanations + -------------------------------------------------------------------------------- + 1- 11 A11 --- Name Description of Name + 13- 18 F6.1 --- Obs [2019.0/2019.0] Time of Observation + 20- 22 I3 s Cadence [100] Description of Cadence + 24- 29 F6.4 10+22 nH [0.01/0.03] Description of nH + 31- 35 E5.1 mag magnitude [0.0/3981.08] Description of magnitude + 37- 47 E11.6 keV Temperature [0.0/0.01] Description of Temperature + 49- 53 F5.3 10+12Jy Flux ? Description of Flux + 55- 61 F7.1 Jy e_Flux [450.0/10000.0] Description of e_Flux + 63- 78 F16.12 deg ELON Ecliptic Longitude (geocentrictrueecliptic) + 80- 95 F16.12 deg ELAT Ecliptic Latitude (geocentrictrueecliptic) + -------------------------------------------------------------------------------- + Notes: + -------------------------------------------------------------------------------- + ASASSN-15lh 2019.0 100 0.0250 1e-10 2.87819e-09 10000.0 306.224208650096 -45.621789850825 + ASASSN-14li 2019.0 100 0.0188 4e+03 2.55935e-08 2.044 450.0 183.754980099243 21.051410763027 + +.. + EXAMPLE END + +.. attention:: + + The MRT writer currently supports automatic writing of a single coordinate column + in ``Tables``. For tables with more than one coordinate column of a given kind + (e.g. equatorial, galactic or ecliptic), only the first found coordinate column + will be decomposed into its component columns, and the rest of the coordinate + columns of the same type will be converted to string columns. Thus users should take + care that the additional coordinate columns are dealt with (e.g. by converting them + to unique ``float``-valued columns) before using ``SkyCoord`` methods. diff --git a/docs/io/fits/api/cards.rst b/docs/io/fits/api/cards.rst index af22cc3c53f7..ad0ab08c0121 100644 --- a/docs/io/fits/api/cards.rst +++ b/docs/io/fits/api/cards.rst @@ -1,31 +1,13 @@ .. currentmodule:: astropy.io.fits Cards ------ +***** :class:`Card` -^^^^^^^^^^^^^ +============= .. autoclass:: Card :members: :inherited-members: :undoc-members: :show-inheritance: - -Deprecated Interfaces -^^^^^^^^^^^^^^^^^^^^^ - -The following classes and functions are deprecated as of the PyFITS 3.1 header -refactoring, though they are currently still available for backwards-compatibility. - -.. autoclass:: CardList - :members: - :undoc-members: - :show-inheritance: - -.. autofunction:: create_card - -.. autofunction:: create_card_from_string - -.. autofunction:: upper_key - diff --git a/docs/io/fits/api/diff.rst b/docs/io/fits/api/diff.rst index 1e14069b0632..fb6458a42813 100644 --- a/docs/io/fits/api/diff.rst +++ b/docs/io/fits/api/diff.rst @@ -1,46 +1,46 @@ Differs -------- +******* .. automodule:: astropy.io.fits.diff .. currentmodule:: astropy.io.fits :class:`FITSDiff` -^^^^^^^^^^^^^^^^^ +================= .. autoclass:: FITSDiff :members: :inherited-members: :show-inheritance: :class:`HDUDiff` -^^^^^^^^^^^^^^^^ +================ .. autoclass:: HDUDiff :members: :inherited-members: :show-inheritance: :class:`HeaderDiff` -^^^^^^^^^^^^^^^^^^^ +=================== .. autoclass:: HeaderDiff :members: :inherited-members: :show-inheritance: :class:`ImageDataDiff` -^^^^^^^^^^^^^^^^^^^^^^ +====================== .. autoclass:: ImageDataDiff :members: :inherited-members: :show-inheritance: :class:`RawDataDiff` -^^^^^^^^^^^^^^^^^^^^ +==================== .. autoclass:: RawDataDiff :members: :inherited-members: :show-inheritance: :class:`TableDataDiff` -^^^^^^^^^^^^^^^^^^^^^^ +====================== .. autoclass:: TableDataDiff :members: :inherited-members: diff --git a/docs/io/fits/api/files.rst b/docs/io/fits/api/files.rst index 15af19dac9f0..9514524437cc 100644 --- a/docs/io/fits/api/files.rst +++ b/docs/io/fits/api/files.rst @@ -1,44 +1,48 @@ .. currentmodule:: astropy.io.fits File Handling and Convenience Functions ---------------------------------------- +*************************************** :func:`open` -^^^^^^^^^^^^ +============ .. autofunction:: open :func:`writeto` -^^^^^^^^^^^^^^^ +=============== .. autofunction:: writeto :func:`info` -^^^^^^^^^^^^ +============ .. autofunction:: info +:func:`printdiff` +================= +.. autofunction:: printdiff + :func:`append` -^^^^^^^^^^^^^^ +============== .. autofunction:: append :func:`update` -^^^^^^^^^^^^^^ +============== .. autofunction:: update :func:`getdata` -^^^^^^^^^^^^^^^ +=============== .. autofunction:: getdata :func:`getheader` -^^^^^^^^^^^^^^^^^ +================= .. autofunction:: getheader :func:`getval` -^^^^^^^^^^^^^^ +============== .. autofunction:: getval :func:`setval` -^^^^^^^^^^^^^^ +============== .. autofunction:: setval :func:`delval` -^^^^^^^^^^^^^^ +============== .. autofunction:: delval diff --git a/docs/io/fits/api/hdulists.rst b/docs/io/fits/api/hdulists.rst index 6df042360646..da1afdbf410d 100644 --- a/docs/io/fits/api/hdulists.rst +++ b/docs/io/fits/api/hdulists.rst @@ -1,12 +1,12 @@ .. currentmodule:: astropy.io.fits HDU Lists ---------- +********* .. inheritance-diagram:: HDUList :class:`HDUList` -^^^^^^^^^^^^^^^^ +================ .. autoclass:: HDUList :members: diff --git a/docs/io/fits/api/hdus.rst b/docs/io/fits/api/hdus.rst index 47f17ba6062e..5433f6f38156 100644 --- a/docs/io/fits/api/hdus.rst +++ b/docs/io/fits/api/hdus.rst @@ -1,7 +1,11 @@ .. currentmodule:: astropy.io.fits -Header Data Units ------------------ +Header Data Unit +**************** + +Header Data Units are the fundamental container structure of the FITS format +consisting of a ``data`` member and its associated metadata in a ``header``. +They are defined in ``astropy.io.fits.hdu``. The :class:`ImageHDU` and :class:`CompImageHDU` classes are discussed in the section on :ref:`Images`. @@ -10,33 +14,33 @@ The :class:`TableHDU` and :class:`BinTableHDU` classes are discussed in the section on :ref:`Tables`. :class:`PrimaryHDU` -^^^^^^^^^^^^^^^^^^^ +=================== .. autoclass:: PrimaryHDU :members: :inherited-members: :show-inheritance: :class:`GroupsHDU` -^^^^^^^^^^^^^^^^^^ +================== .. autoclass:: GroupsHDU :members: :inherited-members: :show-inheritance: :class:`GroupData` -^^^^^^^^^^^^^^^^^^ +================== .. autoclass:: GroupData :members: :show-inheritance: :class:`Group` -============== +-------------- .. autoclass:: Group :members: :show-inheritance: :class:`StreamingHDU` -^^^^^^^^^^^^^^^^^^^^^ +===================== .. autoclass:: StreamingHDU :members: :inherited-members: diff --git a/docs/io/fits/api/headers.rst b/docs/io/fits/api/headers.rst index d8eb18738472..02a399e7b3f2 100644 --- a/docs/io/fits/api/headers.rst +++ b/docs/io/fits/api/headers.rst @@ -1,10 +1,10 @@ .. currentmodule:: astropy.io.fits Headers -------- +******* :class:`Header` -^^^^^^^^^^^^^^^ +=============== .. autoclass:: Header :members: diff --git a/docs/io/fits/api/images.rst b/docs/io/fits/api/images.rst index 6024e04dbe06..cee287d42c47 100644 --- a/docs/io/fits/api/images.rst +++ b/docs/io/fits/api/images.rst @@ -3,10 +3,10 @@ .. _images: Images ------- +****** `ImageHDU` -^^^^^^^^^^ +========== .. autoclass:: ImageHDU :members: @@ -14,7 +14,7 @@ Images :show-inheritance: `CompImageHDU` -^^^^^^^^^^^^^^ +============== .. autoclass:: CompImageHDU :members: @@ -28,3 +28,11 @@ Images :members: :inherited-members: :show-inheritance: + +`CompImageSection` +================== + +.. autoclass:: CompImageSection + :members: + :inherited-members: + :show-inheritance: diff --git a/docs/io/fits/api/index.rst b/docs/io/fits/api/index.rst new file mode 100644 index 000000000000..e45f5f50a727 --- /dev/null +++ b/docs/io/fits/api/index.rst @@ -0,0 +1,17 @@ +Reference/API +************* + + +.. toctree:: + :maxdepth: 3 + + files.rst + hdulists.rst + hdus.rst + headers.rst + cards.rst + tables.rst + images.rst + diff.rst + verification.rst + tiled_compression.rst diff --git a/docs/io/fits/api/tables.rst b/docs/io/fits/api/tables.rst index 1c9138e9560d..909c072b811f 100644 --- a/docs/io/fits/api/tables.rst +++ b/docs/io/fits/api/tables.rst @@ -3,44 +3,44 @@ .. _tables: Tables ------- +****** :class:`BinTableHDU` -^^^^^^^^^^^^^^^^^^^^ +==================== .. autoclass:: BinTableHDU :members: :inherited-members: :show-inheritance: :class:`TableHDU` -^^^^^^^^^^^^^^^^^ +================= .. autoclass:: TableHDU :members: :inherited-members: :show-inheritance: :class:`Column` -^^^^^^^^^^^^^^^ +=============== .. autoclass:: Column :members: :inherited-members: :show-inheritance: :class:`ColDefs` -^^^^^^^^^^^^^^^^ +================ .. autoclass:: ColDefs :members: :inherited-members: :show-inheritance: :class:`FITS_rec` -^^^^^^^^^^^^^^^^^ +================= .. autoclass:: FITS_rec :members: :show-inheritance: :class:`FITS_record` -^^^^^^^^^^^^^^^^^^^^ +==================== .. autoclass:: FITS_record :members: :inherited-members: @@ -48,16 +48,16 @@ Tables Table Functions -^^^^^^^^^^^^^^^ - -:func:`new_table` -""""""""""""""""" -.. autofunction:: new_table +=============== :func:`tabledump` -""""""""""""""""" +----------------- .. autofunction:: tabledump :func:`tableload` -""""""""""""""""" +----------------- .. autofunction:: tableload + +:func:`table_to_hdu` +-------------------- +.. autofunction:: table_to_hdu diff --git a/docs/io/fits/api/tiled_compression.rst b/docs/io/fits/api/tiled_compression.rst new file mode 100644 index 000000000000..3a033d21e251 --- /dev/null +++ b/docs/io/fits/api/tiled_compression.rst @@ -0,0 +1,27 @@ +***************** +Tiled Compression +***************** + +.. warning:: + This module is in development (so marked as private), anything may change in future releases. + This documentation is provided to aid in further development of this submodule and related functionality. + +This module implements the compression and decompression algorithms, and associated functionality for FITS Tiled Image Compression. +The goal of this submodule is to expose a useful Python API, which different functionality can be built on for reading these files. + +The functionality is roughly split up into the following sections: + +1. Low level compression and decompression functions implemented in cfitsio (for all algorithms other than the GZIP ones, which use the Python stdlib). +2. The quantize and dequantize functions from cfitsio. +3. A Python C-API module which wraps all the compression and quantize cfitsio functions. +4. `numcodecs `__ style ``Codec`` classes for each compression algorithms. +5. `~astropy.io.fits.hdu.compressed._tiled_compression.compress_image_data` and + `~astropy.io.fits.hdu.compressed._tiled_compression.decompress_image_data_section` functions which + are called from `~astropy.io.fits.CompImageHDU`. + + +.. automodapi:: astropy.io.fits.hdu.compressed._tiled_compression + +.. automodapi:: astropy.io.fits.hdu.compressed._codecs + +.. automodapi:: astropy.io.fits.hdu.compressed._quantization diff --git a/docs/io/fits/api/verification.rst b/docs/io/fits/api/verification.rst index efca766bc82e..5415d892e478 100644 --- a/docs/io/fits/api/verification.rst +++ b/docs/io/fits/api/verification.rst @@ -2,25 +2,25 @@ .. _verify: -Verification options --------------------- +Verification Options +******************** -There are 5 options for the ``output_verify`` argument of the following methods -of :class:`HDUList`: :meth:`~HDUList.close`, :meth:`~HDUList.writeto`, and -:meth:`~HDUList.flush`, or the :meth:``~_BaseHDU.writeto`` method on any HDU -object. In these cases, the verification option is passed to a :meth:``verify`` +There are five options for the ``output_verify`` argument of the following +methods of :class:`HDUList`: :meth:`~HDUList.close`, :meth:`~HDUList.writeto`, +and :meth:`~HDUList.flush`, or the ``_BaseHDU.writeto`` method on any HDU +object. In these cases, the verification option is passed to a ``verify`` call within these methods. -exception -^^^^^^^^^ +``'exception'`` +=============== This option will raise an exception if any FITS standard is violated. This is -the default option for output (i.e. when :meth:`~HDUList.writeto`, -:meth:`~HDUList.close`, or :meth:`~HDUList.flush` is called. If a user wants to +the default option for output (i.e., when :meth:`~HDUList.writeto`, +:meth:`~HDUList.close`, or :meth:`~HDUList.flush` is called). If a user wants to overwrite this default on output, the other options listed below can be used. -ignore -^^^^^^ +``'ignore'`` +============ This option will ignore any FITS standard violation. On output, it will write the HDU List content to the output FITS file, whether or not it is conforming @@ -38,14 +38,14 @@ The ``ignore`` option is useful in these situations, for example: No warning message will be printed out. This is like a silent warn (see below) option. -fix -^^^ +``'fix'`` +========= This option will try to fix any FITS standard violations. It is not always possible to fix such violations. In general, there are two kinds of FITS -standard violation: fixable and not fixable. For example, if a keyword has a -floating number with an exponential notation in lower case 'e' (e.g. 1.23e11) -instead of the upper case 'E' as required by the FITS standard, it's a fixable +standard violations: fixable and not fixable. For example, if a keyword has a +floating number with an exponential notation in lower case 'e' (e.g., 1.23e11) +instead of the upper case 'E' as required by the FITS standard, it is a fixable violation. On the other hand, a keyword name like ``P.I.`` is not fixable, since it will not know what to use to replace the disallowed periods. If a violation is fixable, this option will print out a message noting it is fixed. @@ -53,20 +53,20 @@ If it is not fixable, it will throw an exception. The principle behind the fixing is do no harm. For example, it is plausible to 'fix' a :class:`Card` with a keyword name like ``P.I.`` by deleting it, but -Astropy will not take such action to hurt the integrity of the data. +``astropy`` will not take such action to hurt the integrity of the data. -Not all fixes may be the "correct" fix, but at least Astropy will try to make -the fix in such a way that it will not throw off other FITS readers. +Not all fixes may be the "correct" fix, but at least ``astropy`` will try to +make the fix in such a way that it will not throw off other FITS readers. -silentfix -^^^^^^^^^ +``'silentfix'`` +=============== Same as fix, but will not print out informative messages. This may be useful in -a large script where the user does not want excessive harmless messages. If the -violation is not fixable, it will still throw an exception. +a large script where the user does not want excessive harmless messages. If +the violation is not fixable, it will still throw an exception. -warn -^^^^ +``'warn'`` +========== This option is the same as the ignore option but will send warning messages. It will not try to fix any FITS standard violations whether fixable or not. diff --git a/docs/io/fits/appendix/faq.rst b/docs/io/fits/appendix/faq.rst index 8576fb55eedd..3402284e6929 100644 --- a/docs/io/fits/appendix/faq.rst +++ b/docs/io/fits/appendix/faq.rst @@ -1,215 +1,196 @@ -.. doctest-skip-all - .. _io-fits-faq: astropy.io.fits FAQ -------------------- +******************* .. contents:: General Questions -^^^^^^^^^^^^^^^^^ +================= -What is PyFITS and how does it relate to Astropy? -""""""""""""""""""""""""""""""""""""""""""""""""" +What is PyFITS and how does it relate to ``astropy``? +----------------------------------------------------- -PyFITS_ is a library written in, and for use with the Python_ programming -language for reading, writing, and manipulating FITS_ formatted files. It -includes a high-level interface to FITS headers with the ability for high and +PyFITS_ is a library written in, and for use with the |Python| programming +language for reading, writing, and manipulating FITS_ formatted files. It +includes a high-level interface to FITS headers with the ability for high- and low-level manipulation of headers, and it supports reading image and table -data as Numpy_ arrays. It also supports more obscure and non-standard formats +data as |NumPy| arrays. It also supports more obscure and nonstandard formats found in some FITS files. The `astropy.io.fits` module is identical to PyFITS but with the names changed. -When development began on Astropy it was clear that one of the core -requirements would be a FITS reader. Rather than starting from scratch, -PyFITS--being the most flexible FITS reader available for Python--was ported -into Astropy. There are plans to gradually phase out PyFITS as a stand-alone -module and deprecate it in favor of `astropy.io.fits`. See more about that in +When the development of ``astropy`` began, it was clear that one of the core +requirements would be a FITS reader. Rather than starting from scratch, +PyFITS — being the most flexible FITS reader available for Python — was ported +into ``astropy``. There are plans to gradually phase out PyFITS as a stand-alone +module and deprecate it in favor of `astropy.io.fits`. See more about this in the next question. Although PyFITS is written mostly in Python, it includes an optional module -written in C that's required to read/write compressed image data. However, +written in C that is required to read/write compressed image data. However, the rest of PyFITS functions without this extension module. -.. _PyFITS: http://www.stsci.edu/institute/software_hardware/pyfits -.. _Python: http://www.python.org -.. _FITS: http://fits.gsfc.nasa.gov/ -.. _Numpy: http://numpy.scipy.org/ +.. _PyFITS: https://github.com/spacetelescope/pyfits +.. _FITS: https://fits.gsfc.nasa.gov/ What is the development status of PyFITS? -""""""""""""""""""""""""""""""""""""""""" +----------------------------------------- -PyFITS is written and maintained by the Science Software Branch at the `Space +PyFITS was written and maintained by the Science Software Branch at the `Space Telescope Science Institute`_, and is licensed by AURA_ under a `3-clause BSD -license`_ (see `LICENSE.txt`_ in the PyFITS source code). +license`_. -It is now primarily developed as primarily as a component of Astropy -(`astropy.io.fits`) rather than as a stand-alone module. There are a few +It is now exclusively developed as a component of ``astropy`` +(`astropy.io.fits`) rather than as a stand-alone module. There are a few reasons for this: The first is simply to reduce development effort; the overhead of maintaining both PyFITS *and* `astropy.io.fits` in separate code -bases is non-trivial. The second is that there are many features of Astropy +bases is nontrivial. The second is that there are many features of ``astropy`` (units, tables, etc.) from which the `astropy.io.fits` module can benefit -greatly. Since PyFITS is already integrated into Astropy, it makes more sense -to continue development there rather than make Astropy a dependency of PyFITS. - -PyFITS' current primary developer and active maintainer is `Erik Bray`_, though -patch submissions are welcome from anyone. PyFITS is now primarily developed -in a Git repository for ease of merging to and from Astropy. Patches and issue -reports can be posted to the `GitHub project`_ for PyFITS, or for Astropy. -There is also a legacy `Trac site`_ with some older issue reports still open, -but new issues should be submitted via GitHub if possible. An `SVN mirror`_ of -the repository is still maintained as well. - -The current stable release series is 3.3.x. Each 3.3.x release tries to -contain only bug fixes, and to not introduce any significant behavioral or API -changes (though this isn't guaranteed to be perfect). Patch releases for older -release series may be released upon request. Older versions of PyFITS (2.4 and -earlier) are no longer actively supported. - -.. _Space Telescope Science Institute: http://www.stsci.edu/ -.. _AURA: http://www.aura-astronomy.org/ -.. _3-clause BSD license: http://en.wikipedia.org/wiki/BSD_licenses#3-clause_license_.28.22New_BSD_License.22_or_.22Modified_BSD_License.22.29 -.. _LICENSE.txt: https://aeon.stsci.edu/ssb/trac/pyfits/browser/trunk/LICENSE.txt -.. _Erik Bray: mailto:embray@stsci.edu -.. _Trac site: https://aeon.stsci.edu/ssb/trac/pyfits/ -.. _SVN mirror: https://aeon.stsci.edu/ssb/svn/pyfits/ +greatly. Since PyFITS is already integrated into ``astropy``, it makes more +sense to continue development there rather than make ``astropy`` a dependency +of PyFITS. + +PyFITS' past primary developer and active maintainer was Erik Bray. There +is a `GitHub project`_ for PyFITS, but PyFITS is not actively developed anymore +so patches and issue reports should be posted on the Astropy issue tracker. + +The current (and last) stable release is 3.4.0. + +.. _Space Telescope Science Institute: https://www.stsci.edu/ +.. _AURA: https://www.aura-astronomy.org/ +.. _3-clause BSD license: https://en.wikipedia.org/wiki/BSD_licenses#3-clause_license_.28.22New_BSD_License.22_or_.22Modified_BSD_License.22.29 .. _GitHub project: https://github.com/spacetelescope/PyFITS Usage Questions -^^^^^^^^^^^^^^^ +=============== -Something didn't work as I expected. Did I do something wrong? -""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +Something did not work as I expected. Did I do something wrong? +--------------------------------------------------------------- -Possibly. But if you followed the documentation and things still did not work +Possibly. But if you followed the documentation and things still did not work as expected, it is entirely possible that there is a mistake in the -documentation, a bug in the code, or both. So feel free to report it as a bug. +documentation, a bug in the code, or both. So feel free to report it as a bug. There are also many, many corner cases in FITS files, with new ones discovered -almost every week. `astropy.io.fits` is always improving, but does not support -all cases perfectly. There are some features of the FITS format (scaled data, +almost every week. `astropy.io.fits` is always improving, but does not support +all cases perfectly. There are some features of the FITS format (scaled data, for example) that are difficult to support correctly and can sometimes cause unexpected behavior. For the most common cases, however, such as reading and updating FITS headers, -images, and tables, `astropy.io.fits`. is very stable and well-tested. Before -every Astropy/PyFITS release it is ensured that all its tests pass on a variety -of platforms, and those tests cover the majority of use-cases (until new corner +images, and tables, `astropy.io.fits` is very stable and well-tested. Before +every ``astropy`` release it is ensured that all of its tests pass on a variety +of platforms, and those tests cover the majority of use cases (until new corner cases are discovered). -Astropy crashed and output a long string of code. What do I do? -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +``astropy`` crashed and output a long string of code. What do I do? +------------------------------------------------------------------- -This listing of code is what is knows as a `stack trace`_ (or in Python -parlance a "traceback"). When an unhandled exception occurs in the code, +This listing of code is what is known as a `stack trace`_ (or in Python +parlance a "traceback"). When an unhandled exception occurs in the code causing the program to end, this is a way of displaying where the exception occurred and the path through the code that led to it. -As Astropy is meant to be used as a piece in other software projects, some -exceptions raised by Astropy are by design. For example, one of the most -common exceptions is a `~.exceptions.KeyError` when an attempt is made to read -the value of a non-existent keyword in a header:: +As ``astropy`` is meant to be used as a piece in other software projects, some +exceptions raised by ``astropy`` are by design. For example, one of the most +common exceptions is a `KeyError` when an attempt is made to read +the value of a nonexistent keyword in a header:: >>> from astropy.io import fits >>> h = fits.Header() >>> h['NAXIS'] Traceback (most recent call last): - File "", line 1, in - File "/path/to/astropy/io/fits/header.py", line 125, in __getitem__ - return self._cards[self._cardindex(key)].value - File "/path/to/astropy/io/fits/header.py", line 1535, in _cardindex - raise KeyError("Keyword %r not found." % keyword) + ... KeyError: "Keyword 'NAXIS' not found." This indicates that something was looking for a keyword called "NAXIS" that -does not exist. If an error like this occurs in some other software that uses -Astropy, it may indicate a bug in that software, in that it expected to find a -keyword that didn't exist in a file. +does not exist. If an error like this occurs in some other software that uses +``astropy``, it may indicate a bug in that software, in that it expected to +find a keyword that did not exist in a file. Most "expected" exceptions will output a message at the end of the traceback -giving some idea of why the exception occurred and what to do about it. The +giving some idea of why the exception occurred and what to do about it. The more vague and mysterious the error message in an exception appears, the more -likely that it was caused by a bug in Astropy. So if you're getting an -exception and you really don't know why or what to do about it, feel free to +likely that it was caused by a bug in ``astropy``. So if you are getting an +exception and you really do not know why or what to do about it, feel free to report it as a bug. -.. _stack trace: http://en.wikipedia.org/wiki/Stack_trace +.. _stack trace: https://en.wikipedia.org/wiki/Stack_trace -Why does opening a file work in CFITSIO, ds9, etc. but not in Astropy? -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +Why does opening a file work in CFITSIO, ds9, etc., but not in ``astropy``? +--------------------------------------------------------------------------- As mentioned elsewhere in this FAQ, there are many unusual corner cases when -dealing with FITS files. It's possible that a file should work, but isn't -support due to a bug. Sometimes it's even possible for a file to work in an -older version of Astropy or PyFITS, but not a newer version due to a regression -that isn't tested for yet. +dealing with FITS files. It is possible that a file should work, but is not +supported due to a bug. Sometimes it is even possible for a file to work in an +older version of ``astropy``, but not a newer version due to a regression +that has not been tested for yet. Another problem with the FITS format is that, as old as it is, there are many conventions that appear in files from certain sources that do not meet the FITS -standard. And yet they are so common-place that it is necessary to support -them in any FITS readers. CONTINUE cards are one such example. There are -non-standard conventions supported by Astropy/PyFITS that are not supported by -CFITSIO and possibly vice-versa. You may have hit one of those cases. - -If Astropy is having trouble opening a file, a good way to rule out whether not -the problem is with Astropy is to run the file through the `fitsverify`_ -program. For smaller files you can even use the `online FITS verifier`_. +standard. And yet they are so commonplace that it is necessary to support +them in any FITS readers. CONTINUE cards are one such example. There are +nonstandard conventions supported by ``astropy`` that are not supported by +CFITSIO and possibly vice versa. You may have hit one of those cases. + +If ``astropy`` is having trouble opening a file, a good way to rule out whether +not the problem is with ``astropy`` is to run the file through the `fitsverify`_ +program. For smaller files you can even use the `online FITS verifier`_. These use CFITSIO under the hood, and should give a good indication of whether -or not there is something erroneous about the file. If the file is +or not there is something erroneous about the file. If the file is malformatted, fitsverify will output errors and warnings. -If fitsverify confirms no problems with a file, and Astropy is still having -trouble opening it (especially if it produces a traceback) then it's possible -there is a bug in Astropy. +If fitsverify confirms no problems with a file, and ``astropy`` is still having +trouble opening it (especially if it produces a traceback), then it is possible +there is a bug in ``astropy``. -.. _fitsverify: http://heasarc.gsfc.nasa.gov/docs/software/ftools/fitsverify/ -.. _online FITS verifier: http://fits.gsfc.nasa.gov/fits_verify.html +.. _fitsverify: https://heasarc.gsfc.nasa.gov/docs/software/ftools/fitsverify/ +.. _online FITS verifier: https://fits.gsfc.nasa.gov/fits_verify.html -How do I turn off the warning messages Astropy keeps outputting to my console? -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +How do I turn off the warning messages ``astropy`` outputs to my console? +------------------------------------------------------------------------- -Astropy uses Python's built-in `warnings`_ subsystem for informing about +``astropy`` uses Python's built-in `warnings`_ subsystem for informing about exceptional conditions in the code that are recoverable, but that the user may -want to be informed of. One of the most common warnings in `astropy.io.fits` +want to be informed of. One of the most common warnings in `astropy.io.fits` occurs when updating a header value in such a way that the comment must be truncated to preserve space:: Card is too long, comment is truncated. -Any console output generated by Astropy can be assumed to be from the warnings -subsystem. See Astropy's documentation on the :ref:`python-warnings` for more -information on how to control and quiet warnings. +Any console output generated by ``astropy`` can be assumed to be from the +warnings subsystem. See Astropy's documentation on the :ref:`python-warnings` +for more information on how to control and quiet warnings. -.. _warnings: http://docs.python.org/library/warnings.html +.. _warnings: https://docs.python.org/3/library/warnings.html -What convention does Astropy use for indexing, such as of image coordinates? -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +What convention does ``astropy`` use for indexing, such as of image coordinates? +-------------------------------------------------------------------------------- -All arrays and sequences in Astropy use a zero-based indexing scheme. For +All arrays and sequences in ``astropy`` use a zero-based indexing scheme. For example, the first keyword in a header is ``header[0]``, not ``header[1]``. This is in accordance with Python itself, as well as C, on which Python is based. This may come as a surprise to veteran FITS users coming from IRAF, where -1-based indexing is typically used, due to its origins in FORTRAN. +1-based indexing is typically used, due to its origins in Fortran. -Likewise, the top-left pixel in an N x N array is ``data[0,0]``. The indices +Likewise, the top-left pixel in an N x N array is ``data[0,0]``. The indices for 2-dimensional arrays are row-major order, in that the first index is the -row number, and the second index is the column number. Or put in terms of -axes, the first axis is the y-axis, and the second axis is the x-axis. This is -the opposite of column-major order, which is used by FORTRAN and hence FITS. +row number, and the second index is the column number. Or put in terms of +axes, the first axis is the y-axis, and the second axis is the x-axis. This is +the opposite of column-major order, which is used by Fortran and hence FITS. For example, the second index refers to the axis specified by NAXIS1 in the FITS header. In general, for N-dimensional arrays, row-major orders means that the right-most axis is the one that varies the fastest while moving over the -array data linearly. For example, the 3-dimensional array:: +array data linearly. For example, the 3-dimensional array:: [[[1, 2], [3, 4]], @@ -224,184 +205,176 @@ Since 2 immediately follows 1, you can see that the right-most (or inner-most) axis is the one that varies the fastest. The discrepancy in axis-ordering may take some getting used to, but it is a -necessary evil. Since most other Python and C software assumes row-major -ordering, trying to enforce column-major ordering in arrays returned by Astropy -is likely to cause more difficulties than it's worth. +necessary evil. Since most other Python and C software assumes row-major +ordering, trying to enforce column-major ordering in arrays returned by +``astropy`` is likely to cause more difficulties than it is worth. -How do I open a very large image that won't fit in memory? -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +How do I open a very large image that will not fit in memory? +------------------------------------------------------------- -In PyFITS, prior to version 3.1, when the data portion of an HDU is accessed, -the data is read into memory in its entirety. For example:: - - >>> hdul = pyfits.open('myimage.fits') - >>> hdul[0].data - ... - -reads the entire image array from disk into memory. For very large images or -tables this is clearly undesirable, if not impossible given the available -resources. - -However, `astropy.io.fits.open` has an option to access the data portion of an -HDU by memory mapping using `mmap`_. In both Astropy and newer versions of -PyFITS this is used by *default*. +`astropy.io.fits.open` has an option to access the data portion of an +HDU by memory mapping using `mmap`_. In ``astropy`` this is used by default. What this means is that accessing the data as in the example above only reads -portions of the data into memory on demand. For example, if I request just a -slice of the image, such as ``hdul[0].data[100:200]``, then just rows 100-200 -will be read into memory. This happens transparently, as though the entire -image were already in memory. This works the same way for tables. For most +portions of the data into memory on demand. For example, if we request just a +slice of the image, such as ``hdul[0].data[100:200]``, then only rows 100-200 +will be read into memory. This happens transparently, as though the entire +image were already in memory. This works the same way for tables. For most cases this is your best bet for working with large files. -To ensure use of memory mapping, just add the ``memmap=True`` argument to -`fits.open `. Likewise, using ``memmap=False`` will +To ensure use of memory mapping, add the ``memmap=True`` argument to +:func:`fits.open `. Likewise, using ``memmap=False`` will force data to be read entirely into memory. - The default can also be controlled through a configuration option called -``USE_MEMMAP``. Setting this to ``0`` will disable mmap by default. +``USE_MEMMAP``. Setting this to ``0`` will disable mmap by default. Unfortunately, memory mapping does not currently work as well with scaled image data, where BSCALE and BZERO factors need to be applied to the data to -yield physical values. Currently this requires enough memory to hold the +yield physical values. Currently this requires enough memory to hold the entire array, though this is an area that will see improvement in the future. An alternative, which currently only works for image data (that is, non-tables) -is the sections interface. It is largely replaced by the better support for -mmap, but may still be useful on systems with more limited virtual-memory -space, such as on 32-bit systems. Support for scaled image data is flakey with -sections too, though that will be fixed. See the documentation on :ref:`image +is the sections interface. It is largely replaced by the better support for +mmap, but may still be useful on systems with more limited virtual memory +space, such as on 32-bit systems. Support for scaled image data is flaky with +sections too, though that will be fixed. See the documentation on :ref:`image sections ` for more details on using this interface. -.. _mmap: http://en.wikipedia.org/wiki/Mmap +.. _mmap: https://en.wikipedia.org/wiki/Mmap +.. _sphx_glr_generated_examples_io_skip_create-large-fits.py: How can I create a very large FITS file from scratch? -""""""""""""""""""""""""""""""""""""""""""""""""""""" +----------------------------------------------------- -This is a very common issue, but unfortunately Astropy does not come with any -built-in facilities for creating large files (larger than will fit in memory) -from scratch (though it may in the future). +This example demonstrates how to create a large file (larger than will fit in +memory) from scratch using `astropy.io.fits`. -Normally to create a single image FITS file one would do something like:: +Normally to create a single image FITS file one would do something like: - >>> import numpy - >>> from astropy.io import fits - >> data = numpy.zeros((40000, 40000), dtype=numpy.float64) - >> hdu = fits.PrimaryHDU(data=data) - >> hdu.writeto('large.fits') +.. code:: python + + import os + import numpy as np + from astropy.io import fits + + data = np.zeros((40000, 40000), dtype=np.float64) + hdu = fits.PrimaryHDU(data=data) + +Then use the `astropy.io.fits.writeto()` method to write out the new file to disk: + +.. code:: python + + hdu.writeto("large.fits") -However, a 40000 x 40000 array of doubles is nearly twelve gigabytes! Most -systems won't be able to create that in memory just to write out to disk. In +However, a 40000 x 40000 array of doubles is nearly twelve gigabytes! Most +systems won't be able to create that in memory just to write out to disk. In order to create such a large file efficiently requires a little extra work, and a few assumptions. First, it is helpful to anticipate about how large (as in, how many keywords) -the header will have in it. FITS headers must be written in 2880 byte -blocks--large enough for 36 keywords per block (including the END keyword in -the final block). Typical headers have somewhere between 1 and 4 blocks, +the header will have in it. FITS headers must be written in 2880 byte +blocks, large enough for 36 keywords per block (including the END keyword in +the final block). Typical headers have somewhere between 1 and 4 blocks, though sometimes more. Since the first thing we write to a FITS file is the header, we want to write enough header blocks so that there is plenty of padding in which to add new -keywords without having to resize the whole file. Say you want the header to -use 4 blocks by default. Then, excluding the END card which Astropy will add -automatically, create the header and pad it out to 36 * 4 cards like so:: - - >>> data = numpy.zeros((100, 100), dtype=numpy.float64) - # This is a stub array that we'll be using the initialize the HDU; its - # exact size is irrelevant, as long as it has the desired number of - # dimensions - >>> hdu = fits.PrimaryHDU(data=data) - >>> header = hdu.header - >>> while len(header) < (36 * 4 - 1): - ... header.append() # Adds a blank card to the end +keywords without having to resize the whole file. Say you want the header to +use 4 blocks by default. Then, excluding the END card which Astropy will add +automatically, create the header and pad it out to 36 * 4 cards. + +Create a stub array to initialize the HDU; its +exact size is irrelevant, as long as it has the desired number of +dimensions: + +.. code:: python + + data = np.zeros((100, 100), dtype=np.float64) + hdu = fits.PrimaryHDU(data=data) + header = hdu.header + while len(header) < (36 * 4 - 1): + header.append() # Adds a blank card to the end Now adjust the NAXISn keywords to the desired size of the array, and write -*only* the header out to a file. Using the ``hdu.writeto()`` method will -cause Astropy to "helpfully" reset the NAXISn keywords to match the size of the -dummy array. That is because it works hard to ensure that only valid FITS -files are written. Instead, we can write *just* the header to a file using -the `Header.tofile ` method:: - - >>> header['NAXIS1'] = 40000 - >>> header['NAXIS2'] = 40000 - >>> header.tofile('large.fits') - -Finally, we need to grow out the end of the file to match the length of the -data (plus the length of the header). This can be done very efficiently on +only the header out to a file. Using the ``hdu.writeto()`` method will cause +astropy to "helpfully" reset the NAXISn keywords to match the size of the +dummy array. That is because it works hard to ensure that only valid FITS +files are written. Instead, we can write just the header to a file using the +`astropy.io.fits.Header.tofile` method: + +.. code:: python + + header["NAXIS1"] = 40000 + header["NAXIS2"] = 40000 + header.tofile("large.fits") + +Finally, grow out the end of the file to match the length of the +data (plus the length of the header). This can be done very efficiently on most systems by seeking past the end of the file and writing a single byte, -like so:: +like so: - >>> with open('large.fits', 'rb+') as fobj: - ... # Seek past the length of the header, plus the length of the - ... # Data we want to write. - ... # The -1 is to account for the final byte taht we are about to - ... # write: - ... fobj.seek(len(header.tostring()) + (40000 * 40000 * 8) - 1) - ... fobj.write('\0') +.. code:: python + + with open("large.fits", "rb+") as fobj: + # Seek past the length of the header, plus the length of the + # Data we want to write. + # 8 is the number of bytes per value, i.e. abs(header['BITPIX'])/8 + # (this example is assuming a 64-bit float) + file_length = len(header.tostring()) + (40000 * 40000 * 8) + # FITS files must be a multiple of 2880 bytes long; the final -1 + # is to account for the final byte that we are about to write. + file_length = ((file_length + 2880 - 1) // 2880) * 2880 - 1 + fobj.seek(file_length) + fobj.write(b"\0") + +More generally, this can be written: + +.. code:: python + + shape = tuple(header[f"NAXIS{ii}"] for ii in range(1, header["NAXIS"] + 1)) + with open("large.fits", "rb+") as fobj: + file_length = len(header.tostring()) + (np.prod(shape) * np.abs(header["BITPIX"] // 8)) + file_length = ((file_length + 2880 - 1) // 2880) * 2880 - 1 + fobj.seek(file_length) + fobj.write(b"\0") On modern operating systems this will cause the file (past the header) to be -filled with zeros out to the ~12GB needed to hold a 40000 x 40000 image. On +filled with zeros out to the ~12GB needed to hold a 40000 x 40000 image. On filesystems that support sparse file creation (most Linux filesystems, but not the HFS+ filesystem used by most Macs) this is a very fast, efficient -operation. On other systems your mileage may vary. +operation. On other systems your mileage may vary. This isn't the only way to build up a large file, but probably one of the -safest. This method can also be used to create large multi-extension FITS +safest. This method can also be used to create large multi-extension FITS files, with a little care. -For creating very large tables, this method may also be used. Though it can be -difficult to determine ahead of time how many rows a table will need. In +For creating very large tables, this method may also be used, though it can be +difficult to determine ahead of time how many rows a table will need. In general, use of the `astropy.io.fits` module is currently discouraged for the -creation and manipulation of large tables. The FITS format itself is not +creation and manipulation of large tables. The FITS format itself is not designed for efficient on-disk or in-memory manipulation of table structures. For large, heavy-duty table data it might be better too look into using `HDF5`_ -through the `PyTables`_ library. The :ref:`Astropy Table ` +through the `PyTables`_ library. The :ref:`Astropy Table ` interface can provide an abstraction layer between different on-disk table -formats as well (for example for converting a table between FITS and HDF5). +formats as well (for example, for converting a table between FITS and HDF5). -PyTables makes use of Numpy under the hood, and can be used to write binary -table data to disk in the same format required by FITS. It is then possible -to serialize your table to the FITS format for distribution. At some point +PyTables makes use of NumPy under the hood, and can be used to write binary +table data to disk in the same format required by FITS. It is then possible +to serialize your table to the FITS format for distribution. At some point this FAQ might provide an example of how to do this. -.. _HDF5: http://www.hdfgroup.org/HDF5/ -.. _PyTables: http://www.pytables.org/moin - - -How do I create a multi-extension FITS file from scratch? -""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -When you open a FITS file with `astropy.io.fits.open`, an -`~astropy.io.fits.HDUList` object is returned, which holds all the HDUs in the -file. This ``HDUList`` class is a subclass of Python's builtin `list`, and can -be created from scratch and used as such:: - - >>> from astropy.io import fits - >>> new_hdul = fits.HDUList() - >>> new_hdul.append(fits.ImageHDU()) - >>> new_hdul.append(fits.ImageHDU()) - >>> new_hdul.writeto('test.fits') - -Or the HDU instances can be created first (or read from an existing FITS file) -and the HDUList instantiated like so:: - - >>> hdu1 = fits.PrimaryHDU() - >>> hdu2 = fits.ImageHDU() - >>> new_hdul = fits.HDUList([hdu1, hdu2]) - >>> new_hdul.writeto('test.fits') - -That will create a new multi-extension FITS file with two empty IMAGE -extensions (a default PRIMARY HDU is prepended automatically if one was not -provided manually). +.. _HDF5: https://www.hdfgroup.org/HDF5/ +.. _PyTables: http://www.pytables.org/ +.. _fits-scaled-data-faq: Why is an image containing integer data being converted unexpectedly to floats? -""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +------------------------------------------------------------------------------- -If the header for your image contains non-trivial values for the optional +If the header for your image contains nontrivial values for the optional BSCALE and/or BZERO keywords (that is, BSCALE != 1 and/or BZERO != 0), then the raw data in the file must be rescaled to its physical values according to the formula:: @@ -409,79 +382,83 @@ the formula:: physical_value = BZERO + BSCALE * array_value As BZERO and BSCALE are floating point values, the resulting value must be a -float as well. If the original values were 16-bit integers, the resulting -values are single-precision (32-bit) floats. If the original values were -32-bit integers the resulting values are double-precision (64-bit floats). +float as well. If the original values were 16-bit integers, the resulting +values are single-precision (32-bit) floats. If the original values were +32-bit integers, the resulting values are double-precision (64-bit floats). -This automatic scaling can easily catch you of guard if you're not expecting -it, because it doesn't happen until the data portion of the HDU is accessed -(to allow things like updating the header without rescaling the data). For +This automatic scaling can easily catch you off guard if you are not expecting +it, because it does not happen until the data portion of the HDU is accessed +(to allow for things like updating the header without rescaling the data). For example:: - >>> hdul = fits.open('scaled.fits') - >>> image = hdul['SCI', 1] + >>> fits_scaledimage_filename = fits.util.get_testdata_filepath('scale.fits') + + >>> hdul = fits.open(fits_scaledimage_filename) + >>> image = hdul[0] >>> image.header['BITPIX'] - 32 + 16 >>> image.header['BSCALE'] - 2.0 + 0.045777764213996 >>> data = image.data # Read the data into memory - >>> data.dtype - dtype('float64') # Got float64 despite BITPIX = 32 (32-bit int) + >>> data.dtype.name # Got float32 despite BITPIX = 16 (16-bit int) + 'float32' >>> image.header['BITPIX'] # The BITPIX will automatically update too - -64 + -32 >>> 'BSCALE' in image.header # And the BSCALE keyword removed False The reason for this is that once a user accesses the data they may also -manipulate it and perform calculations on it. If the data were forced to -remain as integers, a great deal of precision is lost. So it is best to err +manipulate it and perform calculations on it. If the data were forced to +remain as integers, a great deal of precision is lost. So it is best to err on the side of not losing data, at the cost of causing some confusion at first. -If the data must be returned to integers before saving, use the `ImageHDU.scale -` method:: +If the data must be returned to integers before saving, use the +`~astropy.io.fits.ImageHDU.scale` method:: >>> image.scale('int32') >>> image.header['BITPIX'] 32 + >>> hdul.close() Alternatively, if a file is opened with ``mode='update'`` along with the ``scale_back=True`` argument, the original BSCALE and BZERO scaling will -be automatically re-applied to the data before saving. Usually this is -not desirable, especially when converting from floating point back to -unsigned integer values. But this may be useful in cases where the raw +be automatically reapplied to the data before saving. Usually this is +not desirable, especially when converting from floating point values back to +unsigned integer values. But this may be useful in cases where the raw data needs to be modified corresponding to changes in the physical values. -To prevent rescaling from occurring at all (good for updating headers--even if -you don't intend for the code to access the data, it's good to err on the side -of caution here), use the ``do_not_scale_image_data`` argument when opening -the file:: +To prevent rescaling from occurring at all (which is good for updating headers +— even if you do not intend for the code to access the data, it is good to err +on the side of caution here), use the ``do_not_scale_image_data`` argument when +opening the file:: - >>> hdul = fits.open('scaled.fits', do_not_scale_image_data=True) - >>> image = hdul['SCI', 1] - >>> image.data.dtype - dtype('int32') + >>> hdul = fits.open(fits_scaledimage_filename, do_not_scale_image_data=True) + >>> image = hdul[0] + >>> image.data.dtype.name + 'int16' + >>> hdul.close() Why am I losing precision when I assign floating point values in the header? -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +---------------------------------------------------------------------------- -The FITS standard allows two formats for storing floating-point numbers in a -header value. The "fixed" format requires the ASCII representation of the +The FITS standard allows two formats for storing floating point numbers in a +header value. The "fixed" format requires the ASCII representation of the number to be in bytes 11 through 30 of the header card, and to be -right-justified. This leaves a standard number of characters for any comment +right-justified. This leaves a standard number of characters for any comment string. The fixed format is not wide enough to represent the full range of values that -can be stored in a 64-bit float with full precision. So FITS also supports a +can be stored in a 64-bit float with full precision. So FITS also supports a "free" format in which the ASCII representation can be stored anywhere, using the full 70 bytes of the card (after the keyword). -Currently Astropy/PyFITS only supports writing fixed format (it can read both +Currently ``astropy`` only supports writing fixed format (it can read both formats), so all floating point values assigned to a header are stored in the -fixed format. There are plans to add support for more flexible formatting. +fixed format. There are plans to add support for more flexible formatting. -In the meantime it is possible to add or update cards by manually formatting +In the meantime, it is possible to add or update cards by manually formatting the card image from a string, as it should appear in the FITS file:: >>> c = fits.Card.fromstring('FOO = 1234567890.123456789') @@ -490,211 +467,282 @@ the card image from a string, as it should appear in the FITS file:: >>> h FOO = 1234567890.123456789 -As long as you don't assign new values to 'FOO' via ``h['FOO'] = 123``, will +As long as you do not assign new values to 'FOO' via ``h['FOO'] = 123``, will maintain the header value exactly as you formatted it (as long as it is valid according to the FITS standard). Why is reading rows out of a FITS table so slow? -"""""""""""""""""""""""""""""""""""""""""""""""" +------------------------------------------------ -Underlying every table data array returned by `astropy.io.fits` is a Numpy -`~numpy.recarray` which is a Numpy array type specifically for representing -structured array data (i.e. a table). As with normal image arrays, Astropy +Underlying every table data array returned by `astropy.io.fits` is a ``numpy`` +`~numpy.recarray` which is a ``numpy`` array type specifically for representing +structured array data (i.e., a table). As with normal image arrays, ``astropy`` accesses the underlying binary data from the FITS file via mmap (see the question "`What performance differences are there between astropy.io.fits and -fitsio?`_" for a deeper explanation of this). The underlying mmap is then +fitsio?`_" for a deeper explanation of this). The underlying mmap is then exposed as a `~numpy.recarray` and in general this is a very efficient way to read the data. -However, for many (if not most) FITS tables it isn't all that simple. For +However, for many (if not most) FITS tables it is not all that simple. For many columns there are conversions that have to take place between the actual -data that's "on disk" (in the FITS file) and the data values that are returned -to the user. For example FITS binary tables represent boolean values -differently from how Numpy expects them to be represented, "Logical" columns -need to be converted on the fly to a format Numpy (and hence the user) can -understand. This issue also applies to data that is linearly scaled via the +data that is "on disk" (in the FITS file) and the data values that are returned +to the user. For example, FITS binary tables represent boolean values +differently from how ``numpy`` expects them to be represented, "Logical" columns +need to be converted on the fly to a format ``numpy`` (and hence the user) can +understand. This issue also applies to data that is linearly scaled via the ``TSCALn`` and ``TZEROn`` header keywords. Supporting all of these "FITS-isms" introduces a lot of overhead that might -not be necessary for all tables, but are still common nonetheless. That's -not to say it can't be faster even while supporting the peculiarities of -FITS--CFITSIO for example supports all the same features but is orders of -magnitude faster. Astropy could do much better here too, and there are many -known issues causing slowdown. There are plenty of opportunities for speedups, -and patches are welcome. In the meantime for high-performance applications +not be necessary for all tables, but are still common nonetheless. That is +not to say it cannot be faster even while supporting the peculiarities of +FITS — CFITSIO, for example, supports all of the same features but is orders of +magnitude faster. ``astropy`` could do much better here too, and there are many +known issues causing slowdown. There are plenty of opportunities for speedups, +and patches are welcome. In the meantime, for high-performance applications with FITS tables some users might find the ``fitsio`` library more to their liking. +I am opening many FITS files in a loop and getting OSError: Too many open files +------------------------------------------------------------------------------- + +Say you have some code like: + +.. code:: python + + from astropy.io import fits + + for filename in filenames: + with fits.open(filename) as hdul: + for hdu in hdul: + hdu_data = hdul.data + # Do some stuff with the data + + +The details may differ, but the qualitative point is that the data to many +HDUs and/or FITS files are being accessed in a loop. This may result in +an exception like:: + + Traceback (most recent call last): + File "", line 2, in + OSError: [Errno 24] Too many open files: 'my_data.fits' + +As explained in the :ref:`note on working with large files `, +because ``astropy`` uses mmap by default to read the data in a FITS file, even +if you correctly close a file with :meth:`HDUList.close +` a handle is kept open to that file so +that the memory-mapped data array can still continue to be read transparently. + +The way ``numpy`` supports mmap is such that the file mapping is not closed +until the overlying `~numpy.ndarray` object has no references to it and is freed +memory. However, when looping over a large number of files (or even just HDUs) +rapidly, this may not happen immediately. Or in some cases if the HDU object +persists, the data array attached to it may persist too. The recommended +workaround is to *manually* delete the ``.data`` attribute on the HDU object so +that the `~numpy.ndarray` reference is freed and the mmap can be closed: + +.. code:: python + + from astropy.io import fits + + for filename in filenames: + with fits.open(filename) as hdul: + for hdu in hdul: + hdu_data = hdul.data + # Do some stuff with the data + # ... + # Don't need the data anymore; delete all references to it + # so that it can be garbage collected + del hdu_data + del hdu.data + + +In some extreme cases files are opened and closed fast enough that Python's +garbage collector does not free them (and hence free the file handles) often +enough. To mitigate this, your code can manually force a garbage collection +by calling :func:`gc.collect` at the end of the loop. + +In a future release it will be more convenient to automatically perform this +sort of cleanup when closing FITS files, where needed. + +Using header['NAXIS2'] += 1 does not add another row to my Table +---------------------------------------------------------------- + +``NAXIS`` and similar keywords are FITS *structural* keywords and should not be +modified by the user. They are automatically updated by :mod:`astropy.io.fits` +when checking the validity of the data and headers. See :ref:`structural_keywords` +for more information. + +To add rows to a table, you can modify the actual data. + Comparison with Other FITS Readers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +================================== What is the difference between astropy.io.fits and fitsio? -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +---------------------------------------------------------- The `astropy.io.fits` module (originally PyFITS) is a "pure Python" FITS -reader in that all the code for parsing the FITS file format is in Python, -though Numpy is used to provide access to the FITS data via the -`~numpy.ndarray` interface. `astropy.io.fits` currently also accesses the -`CFITSIO `_ to support the -FITS Tile Compression convention, but this feature is optional. It does not +reader in that all of the code for parsing the FITS file format is in Python, +though ``numpy`` is used to provide access to the FITS data via the +`~numpy.ndarray` interface. `astropy.io.fits` currently also accesses the +`CFITSIO `_ to support the +FITS Tile Compression convention, but this feature is optional. It does not use CFITSIO outside of reading compressed images. `fitsio `_, on the other hand, is a Python -wrapper for the CFITSIO library. All the heavy lifting of reading the FITS -format is handled by CFITSIO, while ``fitsio`` provides an easier to use -object-oriented API including providing a Numpy interface to FITS files read -from CFITSIO. Much of it is written in C (to provide the interface between -Python and CFITSIO), and the rest is in Python. The Python end mostly +wrapper for the CFITSIO library. All of the heavy lifting of reading the FITS +format is handled by CFITSIO, while ``fitsio`` provides a better way to use +object-oriented API, including providing a ``numpy`` interface to FITS files +read from CFITSIO. Much of it is written in C (to provide the interface between +Python and CFITSIO), and the rest is in Python. The Python end mostly provides the documentation and user-level API. Because ``fitsio`` wraps CFITSIO it inherits most of its strengths and -weaknesses, though it has an added strength of providing an easier to use +weaknesses, though it has an added strength of providing a more convenient API than if one were to use CFITSIO directly. Why did Astropy adopt PyFITS as its FITS reader instead of fitsio? -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +------------------------------------------------------------------ -When the Astropy project was first started it was clear from the start that +When the Astropy Project was first started it was clear from the start that one of its core components should be a submodule for reading and writing FITS files, as many other components would be likely to depend on this -functionality. At the time, the ``fitsio`` package was in its infancy (it -goes back to roughly 2011) while PyFITS had already been established going -back to before the year 2000). It was already a mature package with support +functionality. At the time, the ``fitsio`` package was in its infancy (it +goes back to roughly 2011) while PyFITS had already been established (going +back to before the year 2000). It was already a mature package with support for the vast majority of FITS files found in the wild, including outdated formats such as "Random Groups" FITS files still used extensively in the radio astronomy community. Although many aspects of PyFITS' interface have evolved over the years, much of it has also remained the same, and is already familiar to astronomers -working with FITS files in Python. Most of not all existing training -materials were also based around PyFITS. PyFITS was developed at STScI, which +working with FITS files in Python. Most of if not all existing training +materials were also based around PyFITS. PyFITS was developed at STScI, which also put forward significant resources to develop Astropy, with an eye toward -integrating Astropy into STScI's own software stacks. As most of the Python -software at STScI uses PyFITS it was the only practical choice for making that +integrating Astropy into STScI's own software stacks. As most of the Python +software at STScI uses PyFITS, it was the only practical choice for making that transition. Finally, although CFITSIO (and by extension ``fitsio``) can read any FITS files -that conform to the FITS standard, it does not support all of the non-standard -conventions that have been added to FITS files in the wild. It does have some -support for some of these conventions (such as CONTINUE cards and, to a limited -extent, HIERARCH cards), it is not easy to add support for other conventions -to a large and complex C codebase. +that conform to the FITS standard, it does not support all of the nonstandard +conventions that have been added to FITS files in the wild. While it does have +some support for some of these conventions (such as CONTINUE cards and, to a +limited extent, HIERARCH cards), it is not easy to add support for other +conventions to a large and complex C codebase. -PyFITS' object-oriented design makes supporting non-standard conventions +PyFITS' object-oriented design makes supporting nonstandard conventions somewhat easier in most cases, and as such PyFITS can be more flexible in the -types of FITS files it can read and return *useful* data from. This includes +types of FITS files it can read and return *useful* data from. This includes better support for files that fail to meet the FITS standard, but still contain -useful data that should still be readable at least well-enough to correct any -violations of the FITS standard. For example, a common error in non-English- -speaking regions is to insert non-ASCII characters into FITS headers. This -is not a valid FITS file, but still should be readable in some sense. -Supporting structural errors such as this is more difficult in CFITSIO which -assumes a more rigid structure. +useful data that should be readable enough to correct any violations of the +FITS standard. For example, a common error in non-English speaking regions is +to insert non-ASCII characters into FITS headers. This is not a valid FITS +file, but should still be readable in some sense. Supporting structural errors +such as this is more difficult in CFITSIO which assumes a more rigid structure. What performance differences are there between astropy.io.fits and fitsio? -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +-------------------------------------------------------------------------- There are two main performance areas to look at: reading/parsing FITS headers and reading FITS data (image-like arrays as well as tables). -In the area of headers ``fitsio`` is significantly faster in most cases. This +In the area of headers, ``fitsio`` is significantly faster in most cases. This is due in large part to the (almost) pure C implementation (due to the use of CFITSIO), but also due to fact that it is more rigid and does not support as many local conventions and other special cases as `astropy.io.fits` tries to support in its pure Python implementation. -That said the difference is small, and only likely to be a bottleneck either +That said, the difference is small and only likely to be a bottleneck either when opening files containing thousands of HDUs, or reading the headers out of thousands of FITS files in succession (in either case the difference is not even an order of magnitude). Where data is concerned the situation is a little more complicated, and -requires some understanding of how PyFITS is implemented versus CFITSIO and -``fitsio``. First it's important to understand how they differ in terms of -memory management. +requires some understanding of how `astropy.io.fits` is implemented versus +CFITSIO and ``fitsio``. First, it is important to understand how they differ in +terms of memory management. -`astropy.io.fits`/PyFITS uses mmap, by default, to provide access to the raw -binary data in FITS files. Mmap is a system call (or in most cases these days +`astropy.io.fits` uses mmap, by default, to provide access to the raw +binary data in FITS files. Mmap is a system call (or in most cases these days a wrapper in your libc for a lower-level system call) which allows user-space applications to essentially do the same thing your OS is doing when it uses a -pagefile (swap space) for virtual memory: It allows data in a file on disk to +pagefile (swap space) for virtual memory: it allows data in a file on disk to be paged into physical memory one page (or in practice usually several pages) -at a time on an as-needed basis. These cached pages of the file are also +at a time on an as-needed basis. These cached pages of the file are also accessible from all processes on the system, so multiple processes can read -from the same file with little additional overhead. In the case of reading -over all the data in the file the performance difference between using mmap +from the same file with little additional overhead. In the case of reading +over all of the data in the file, the performance difference between using mmap versus reading the entire data into physical memory at once can vary widely between systems, hardware, and depending on what else is happening on the -system at the moment, but mmap almost always going to be better. - -In principle it requires more overhead since accessing each page will result in -a page fault, and the system requires more requests to the disk. But in -practice the OS will optimize this pretty aggressively, especially for the most -common case of sequential access--also in reality reading the entire thing into -memory is still going to result in a whole lot of page faults too. For random -access having all the data in physical memory is always going to be best, -though with mmap it's usually going to be pretty good too (one doesn't normally -access all the data in a file in totally random order--usually a few sections -of it will be accessed most frequently, the OS will keep those pages in -physical memory as best it can). So for the most general case of reading FITS -files (or most large data on disk) this is the best choice, especially for -casual users, and is hence enabled by default. - -CFITSIO/``fitsio``, on the other hand, doesn't assume the existence of -technologies like mmap and page caching. Thus it implements its own LRU cache +system at the moment, but mmap is almost always going to be better. + +In principle, it requires more overhead since accessing each page will result in +a page fault and the system requires more requests to the disk. But in +practice, the OS will optimize this pretty aggressively, especially for the most +common case of sequential access — also in reality, reading the entire thing +into memory is still going to result in a whole lot of page faults too. For +random access, having all of the data in physical memory is always going to be +best, though with mmap it is usually going to be pretty good too. (Most users +do not normally access all of the data in a file in a totally random order — +usually a few sections of it will be accessed most frequently, so the OS will +keep those pages in physical memory as best it can.) For the most general case +of reading FITS files (or most large data on disk) this is therefore the best +choice, especially for casual users, and is hence enabled by default. + +CFITSIO/``fitsio``, on the other hand, does not assume the existence of +technologies like mmap and page caching. Thus it implements its own LRU cache of I/O buffers that store sections of FITS files read from disk in memory in -FITS' famous 2880 byte chunk size. The I/O buffers are used heavily in -particular for keeping the headers in memory. Though for large data reads (for -example reading an entire image from a file) it *does* bypass the cache and +FITS' famous 2880 byte chunk size. The I/O buffers are used heavily in +particular for keeping the headers in memory. Though for large data reads (for +example, reading an entire image from a file), it *does* bypass the cache and instead does a read directly from disk into a user-provided memory buffer. However, even when CFITSIO reads direct from the file, this is still largely -less efficient than using mmap: Normally when your OS reads a file from disk, +less efficient than using mmap. Normally when your OS reads a file from disk, it caches as much of that read as it can in physical memory (in its page cache) so that subsequent access to those same pages does not require a subsequent -expensive disk read. This happens when using mmap too, since the data has to -be copied from disk into RAM at some point. The difference is that when using +expensive disk read. This happens when using mmap too, since the data has to +be copied from disk into RAM at some point. The difference is that when using mmap to access the data, the program is able to read that data *directly* out -of the OS's page cache (so long as it's only being read). On the other hand +of the OS's page cache (as long as it is only being read). On the other hand, when reading data from a file into a local buffer such as with fread(), the data is first read into the page cache (if not already present) and then copied -from the page cache into the local buffer. So every read performs at least one +from the page cache into the local buffer. So every read performs at least one additional memory copy per page read (requiring twice as much physical memory, and possibly lots of paging if the file is large and pages need to dropped from the cache). The user API for CFITSIO usually works by having the user allocate a memory buffer large enough to hold the image/table they want to read (or at least the -section they're interested in). There are some helper functions for -determining the appropriate amount of space to allocate. Then you just pass it -a pointer to your buffer and CFITSIO handles all the reading (usually using the -process described above), and copies the results into your user buffer. For -large reads it reads directly from the file into your buffer. Though if the +section they are interested in). There are some helper functions for +determining the appropriate amount of space to allocate. Then you pass in +a pointer to your buffer and CFITSIO handles all of the reading (usually using +the process described above), and copies the results into your user buffer. For +large reads, it reads directly from the file into your buffer, though if the data needs to be scaled it makes a stop in CFITSIO's own buffer first, then writes the rescaled values out to the user buffer (if rescaling has been -requested). Regardless, this means that if your program wishes to hold an +requested). Regardless, this means that if your program wishes to hold an entire image in memory at once it will use as much RAM as the size of the -data. For most applications it's better (and sufficient) to write it work on -smaller sections of the data, but this requires extra complexity. Using mmap -on the other hand makes managing this complexity simpler and more efficient. - -A very simple and informal test demonstrates this difference. This test was -performed on four simple FITS images (one of which is a cube) of dimensions -256x256, 1024x1024, 4096x4096, and 256x1024x1024. Each image was generated -before the test and filled with randomized 64-bit floating point values. A -similar test was performed using both `astropy.io.fits` and ``fitsio``: A -handle to the FITS file is opened using each library's basic semantics, and -then the entire data array of the files is copied into a temporary array in -memory (for example if we were blitting the image to a video buffer). For -Astropy the test is written: +data. For most applications it is better (and sufficient) to work on +smaller sections of the data, but this requires extra complexity. Using mmap +on the other hand makes managing this complexity more efficient. + +An informal test demonstrates this difference. This test was performed on four +simple FITS images (one of which is a cube) of dimensions 256x256, 1024x1024, +4096x4096, and 256x1024x1024. Each image was generated before the test and +filled with randomized 64-bit floating point values. A similar test was +performed using both `astropy.io.fits` and ``fitsio``. A handle to the FITS +file is opened using each library's basic semantics, and then the entire data +array of the files is copied into a temporary array in memory (for example, if +we were blitting the image to a video buffer). For ``astropy`` the test is +written: .. code:: python - def read_test_pyfits(filename): + def read_test_astropy(filename): with fits.open(filename, memmap=True) as hdul: data = hdul[0].data c = data.copy() @@ -707,10 +755,10 @@ using: for filename in filenames: print(filename) - %timeit read_test_pyfits(filename) + %timeit read_test_astropy(filename) where ``filenames`` is just a list of the aforementioned generated sample -files. The results were:: +files. The results were:: 256x256.fits 1000 loops, best of 3: 1.28 ms per loop @@ -730,7 +778,8 @@ For ``fitsio`` the test was: data = f[0].read() c = data.copy() -This was also run in a loop over all the sample files, producing the results:: +This was also run in a loop over all of the sample files, producing the +results:: 256x256.fits 1000 loops, best of 3: 476 Âĩs per loop @@ -742,59 +791,59 @@ This was also run in a loop over all the sample files, producing the results:: 1 loops, best of 3: 3.65 s per loop It should be made clear that the sample files were rewritten with new random -data between the Astropy test and the fitsio test, so they were not reading -the same data from the OS's page cache. Fitsio was much faster on the small +data between the ``astropy`` test and the fitsio test, so they were not reading +the same data from the OS's page cache. Fitsio was much faster on the small (256x256) image because in that case the time is dominated by parsing the -headers. As already explained this is much faster in CFITSIO. However, as +headers. As already explained, this is much faster in CFITSIO. However, as the data size goes up and the header parsing no longer dominates the time, -`astropy.io.fits` using mmap is roughly twice as fast. This discrepancy would -be almost entirely due to it requiring roughly half as many in-memory copies -to read the data, as explained earlier. That said, more extensive benchmarking +`astropy.io.fits` using mmap is roughly twice as fast. This discrepancy is +almost entirely due to it requiring roughly half as many in-memory copies +to read the data, as explained earlier. That said, more extensive benchmarking could be very interesting. -This is also not to say that `astropy.io.fits` does better in all cases. There -are some cases where it is currently blown away by fitsio. See the subsequent +This is also not to say that `astropy.io.fits` does better in all cases. There +are some cases where it is currently blown away by fitsio. See the subsequent question. -Why is fitsio so much faster than Astropy at reading tables? -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +Why is fitsio so much faster than ``astropy`` at reading tables? +---------------------------------------------------------------- -In many cases it isn't--there is either no difference, or it may be a little -faster in Astropy depending on what you're trying to do with the table and -what types of columns or how many columns the table has. There are some +In many cases it is not: there is either no difference, or it may be a little +faster in ``astropy`` depending on what you are trying to do with the table and +what types of columns or how many columns the table has. There are some cases, however, where ``fitsio`` can be radically faster, mostly for reasons explained above in "`Why is reading rows out of a FITS table so slow?`_" -In principle a table is no different from, say, an array of pixels. But +In principle a table is no different from, say, an array of pixels. But instead of pixels each element of the array is some kind of record structure -(for example two floats, a boolean, and a 20 character string field). Just as +(for example, two floats, a boolean, and a 20-character string field). Just as a 64-bit float is an 8 byte record in an array, a row in such a table can be -thought of as a 37 byte (in the case of the previous example) record in a 1-D -array of rows. So in principle everything that was explained in the answer to +thought of as a 37 byte (in the case of the previous example) record in a 1D +array of rows. So in principle everything that was explained in the answer to the question "`What performance differences are there between astropy.io.fits and fitsio?`_" applies just as well to tables as it does to any other array. However, FITS tables have many additional complexities that sometimes preclude streaming the data directly from disk, and instead require transformation from -the on-disk FITS format to a format more immediately useful to the user. A +the on-disk FITS format to a format more immediately useful to the user. A common example is how FITS represents boolean values in binary tables. -Another, significantly more complicated example, is variable length arrays. +Another significantly more complicated example, is variable length arrays. As explained in "`Why is reading rows out of a FITS table so slow?`_", -`astropy.io.fits`/PyFITS does not currently handle some of these cases as +`astropy.io.fits` does not currently handle some of these cases as efficiently as it could, in particular in cases where a user only wishes to -read a few rows out of a table. Fitsio, on the other hand, has a better +read a few rows out of a table. Fitsio, on the other hand, has a better interface for copying one row at a time out of a table and performing the necessary transformations on that row *only*, rather than on the entire column -or columns that the row is taken from. As such, for many cases ``fitsio`` gets +or columns that the row is taken from. As such, for many cases ``fitsio`` gets much better performance and should be preferred for many performance-critical table operations. Fitsio also exposes a microlanguage (implemented in CFITSIO) for making -efficient SQL-like queries of tables (single tables only though--no joins or -anything like that). This format, described in the `CFITSIO documentation -`_ can +efficient SQL-like queries of tables (single tables only though — no joins or +anything like that). This format, described in the `CFITSIO documentation +`_ can in some cases perform more efficient selections of rows than might be possible -with Numpy alone, which requires creating an intermediate mask array in order -to perform row selection. +with ``numpy`` alone, which requires creating an intermediate mask array in +order to perform row selection. diff --git a/docs/io/fits/appendix/header_transition.rst b/docs/io/fits/appendix/header_transition.rst index afb38903a7ee..9df6c08f5419 100644 --- a/docs/io/fits/appendix/header_transition.rst +++ b/docs/io/fits/appendix/header_transition.rst @@ -11,147 +11,140 @@ Header Interface Transition Guide This guide was originally included with the release of PyFITS 3.1, and still references PyFITS in many places, though the examples have been - updated for ``astropy.io.fits``. It is still useful here for informational + updated for ``astropy.io.fits``. It is still useful here for informational purposes, though Astropy has always used the PyFITS 3.1 Header interface. PyFITS v3.1 included an almost complete rewrite of the :class:`Header` -interface. Although the new interface is largely compatible with the old +interface. Although the new interface is largely compatible with the old interface (whether due to similarities in the design, or backwards-compatibility support), there are enough differences that a full explanation of the new interface is merited. -The Trac ticket discussing the initial motivation and changes to be made to the -:class:`Header` class is `#64`_. It may be worth reading for some of the -background to this work, though this document contains a more complete -description of the "final" product (which will continue to evolve). - -.. _#64: https://aeon.stsci.edu/ssb/trac/pyfits/ticket/64 - - Background ========== Prior to 3.1, PyFITS users interacted with FITS headers by way of three -different classes: :class:`Card`, :class:`CardList`, and :class:`Header`. +different classes: :class:`Card`, ``CardList``, and :class:`Header`. The Card class represents a single header card with a keyword, value, and -comment. It also contains all the machinery for parsing FITS header cards, -given the 80 character string, or "card image" read from the header. +comment. It also contains all of the machinery for parsing FITS header cards, +given the 80-character string, or "card image" read from the header. -The CardList class is actually a subclass of Python's `list` built-in. It was -meant to represent the actual list of cards that make up a header. That is, it +The CardList class is actually a subclass of Python's `list` built-in. It was +meant to represent the actual list of cards that make up a header. That is, it represents an ordered list of cards in the physical order that they appear in -the header. It supports the usual list methods for inserting and appending new -cards into the list. It also supports `dict`-like keyword access, where +the header. It supports the usual list methods for inserting and appending new +cards into the list. It also supports `dict`-like keyword access, where ``cardlist['KEYWORD']`` would return the first card in the list with the given keyword. A lot of the functionality for manipulating headers was actually buried in the -CardList class. The Header class was more of a wrapper around CardList that -added a little bit of abstraction. It also implemented a partial dict-like +CardList class. The Header class was more of a wrapper around CardList that +added a little bit of abstraction. It also implemented a partial dict-like interface, though for Headers a keyword lookup returned the header value -associated with that keyword, and not the Card object. Though almost every +associated with that keyword, not the Card object, and almost every method on the Header class was just performing some operations on the underlying CardList. -The problem is that there were certain things one could *only* do by directly -accessing the CardList, such as look up the comments on a card, or access cards -that have duplicate keywords, such as HISTORY. Another long-standing -misfeature that slicing a Header object actually returned a CardList object, -rather than a new Header. For all but the most simple use cases, working with -CardList objects was largely unavoidable. +The problem was that there were certain things a user could *only* do by +directly accessing the CardList, such as look up the comments on a card or +access cards that have duplicate keywords, such as HISTORY. Another +long-standing misfeature was that slicing a Header object actually returned a +CardList object, rather than a new Header. For all but the simplest use cases, +working with CardList objects was largely unavoidable. But it was realized that CardList is really an implementation detail not representing any element of a FITS file distinct from the header itself. -Users familiar with the FITS format know what a header is, but it's not clear +Users familiar with the FITS format know what a header is, but it is not clear how a "card list" is distinct from that, or why operations go through the Header object, while some have to be performed through the CardList. -So the primary goal of this redesign was eliminate the :class:`CardList` class +So the primary goal of this redesign was to eliminate the ``CardList`` class altogether, and make it possible for users to perform all header manipulations -directly through :class:`Header` objects. It also tries to present headers as -similar as possible to more a more familiar data structure--an ordered mapping +directly through :class:`Header` objects. It also tried to present headers as +similarly as possible to a more familiar data structure — an ordered mapping (or :class:`~collections.OrderedDict` in Python) for ease of use by new users -less familiar with the FITS format. Though there are still many added +less familiar with the FITS format, though there are still many added complexities for dealing with the idiosyncrasies of the FITS format. Deprecation Warnings ==================== -A few old methods on the :class:`Header` class have been marked as deprecated, +A few older methods on the :class:`Header` class have been marked as deprecated, either because they have been renamed to a more `PEP 8`_-compliant name, or -because have become redundant due to new features. To check if your code is +because have become redundant due to new features. To check if your code is using any deprecated methods or features, run your code with ``python -Wd``. This will output any deprecation warnings to the console. -Two of the most common deprecation warnings related to Headers are for: +Two of the most common deprecation warnings related to Headers are: -- :meth:``Header.has_key``--this has actually been deprecated since PyFITS 3.0, - just as Python's `dict.has_key` is deprecated. For checking a key's presence +- ``Header.has_key``: this has been deprecated since PyFITS 3.0, + just as Python's `dict.has_key` is deprecated. To check a key's presence in a mapping object like `dict` or :class:`Header`, use the ``key in d`` - syntax. This has long been the preference in Python. + syntax. This has long been the preference in Python. -- :meth:``Header.ascardlist`` and :attr:`Header.ascard`--these were used to - access the :class:`CardList` object underlying a header. They should still +- ``Header.ascardlist`` and ``Header.ascard``: these were used to + access the ``CardList`` object underlying a header. They should still work, and return a skeleton CardList implementation that should support most - of the old CardList functionality. But try removing as much of this as - possible. If direct access to the :class:`Card` objects making up a header + of the old CardList functionality. But try removing as much of this as + possible. If direct access to the :class:`Card` objects making up a header is necessary, use :attr:`Header.cards`, which returns an iterator over the - cards. More on that below. + cards. More on that below. -.. _PEP 8: http://www.python.org/dev/peps/pep-0008/ +.. _PEP 8: https://www.python.org/dev/peps/pep-0008/ New Header Design ================= The new :class:`Header` class is designed to work as a drop-in replacement for -a `dict` via `duck typing`_. That is, although it is not a subclass of `dict`, -it implements all the same methods and interfaces. In particular, it is +a `dict` via `duck typing`_. That is, although it is not a subclass of `dict`, +it implements all of the same methods and interfaces. In particular, it is similar to an :class:`~collections.OrderedDict` in that the order of insertions -is preserved. However, Header also supports many additional features and -behaviors specific to the FITS format. It should also be noted that while the +is preserved. However, Header also supports many additional features and +behaviors specific to the FITS format. It should also be noted that while the old Header implementation also had a dict-like interface, it did not implement the *entire* dict interface as the new Header does. Although the new Header is used like a dict/mapping in most cases, it also -supports a `list` interface. The list-like interface is a bit idiosyncratic in -that in some contexts the Header acts like a list of values, in some like a -list of keywords, and in a few contexts like a list of :class:`Card` objects. This -may be the most difficult aspect of the new design, but there is logic to it. +supports a `list` interface. The list-like interface is a bit idiosyncratic in +that in some contexts the Header acts like a list of values, in others like a +list of keywords, and in a few contexts like a list of :class:`Card` objects. +This may be the most difficult aspect of the new design, but there is a logic +to it. As with the old Header implementation, integer index access is supported: -``header[0]`` returns the value of the first keyword. However, the -:meth:`Header.index` method treats the header as though it's a list of -keywords, and returns the index of a given keyword. For example:: +``header[0]`` returns the value of the first keyword. However, the +:meth:`Header.index` method treats the header as though it is a list of +keywords and returns the index of a given keyword. For example:: >>> header.index('BITPIX') 2 -:meth:`Header.count` is similar to `list.count`, and also takes a keyword as +:meth:`Header.count` is similar to `list.count` and also takes a keyword as its argument:: >>> header.count('HISTORY') 20 -A good rule of thumb is that any item access using square brackets ``[]`` returns -*value* in the header, whether using keyword or index lookup. Methods like -:meth:`~Header.index` and :meth:`~Header.count` that deal with the order and -quantity of items in the Header generally work on keywords. Finally, methods -like :meth:`~Header.insert` and :meth:`~Header.append` that add new items to -the header work on cards. +A good rule of thumb is that any item access using square brackets ``[]`` +returns *value* in the header, whether using keyword or index lookup. Methods +like :meth:`~Header.index` and :meth:`~Header.count` that deal with the order +and quantity of items in the Header generally work on keywords. Finally, +methods like :meth:`~Header.insert` and :meth:`~Header.append` that add new +items to the header work on cards. Aside from the list-like methods, the new Header class works very similarly to the old implementation for most basic use cases and should not present too many -surprises. There are differences, however: +surprises. There are differences, however: - As before, the Header() initializer can take a list of :class:`Card` objects - with which to fill the header. However, now any iterable may be used. It is + with which to fill the header. However, now any iterable may be used. It is also important to note that *any* Header method that accepts :class:`Card` - objects can also accept 2-tuples or 3-tuples in place of Cards. That is, + objects can also accept 2-tuples or 3-tuples in place of Cards. That is, either a ``(keyword, value, comment)`` tuple or a ``(keyword, value)`` tuple (comment is assumed blank) may be used anywhere in place of a Card object. - This is even preferred, as it simply involves less typing. For example:: + This is even preferred, as it involves less typing. For example:: >>> from astropy.io import fits >>> header = fits.Header([('A', 1), ('B', 2), ('C', 3, 'A comment')]) @@ -160,26 +153,26 @@ surprises. There are differences, however: B = 2 C = 3 / A comment -- As demonstrated in the previous example, the ``repr()`` for a Header, that is +- As demonstrated in the previous example, the ``repr()`` for a Header (that is, the text that is displayed when entering a Header object in the Python - console as an expression, shows the header as it would appear in a FITS file. - This inserts newlines after each card so that it is easily readable - regardless of terminal width. It is *not* necessary to use ``print header`` - to view this. Simply entering ``header`` displays the header contents as it - would appear in the file (sans the END card). + console as an expression), shows the header as it would appear in a FITS file. + This inserts newlines after each card so that it is readable regardless of + terminal width. It is *not* necessary to use ``print header`` to view this. + Entering ``header`` displays the header contents as it would appear in the + file (sans the END card). - ``len(header)`` is now supported (previously it was necessary to do - ``len(header.ascard)``. This returns the total number of cards in the + ``len(header.ascard)``). This returns the total number of cards in the header, including blank cards, but excluding the END card. - FITS supports having duplicate keywords, although they are generally in error - except for commentary keywords like COMMENT and HISTORY. PyFITS now supports - reading, updating, and deleting duplicate keywords: Instead of using the - keyword by itself, use a ``(keyword, index)`` tuple. For example + except for commentary keywords like COMMENT and HISTORY. PyFITS now supports + reading, updating, and deleting duplicate keywords; instead of using the + keyword by itself, use a ``(keyword, index)`` tuple. For example, ``('HISTORY', 0)`` represents the first HISTORY card, ``('HISTORY', 1)`` - represents the second HISTORY card, and so on. In fact, when a keyword is - used by itself, it's really just shorthand for ``(keyword, 0)``. Its is now - possible to delete an accidental duplicate like so:: + represents the second HISTORY card, and so on. In fact, when a keyword is + used by itself, it is shorthand for ``(keyword, 0)``. It is now possible to + delete an accidental duplicate like so:: >>> del header[('NAXIS', 1)] @@ -187,22 +180,22 @@ surprises. There are differences, however: - Even if there are duplicate keywords, keyword lookups like ``header['NAXIS']`` will always return the value associated with the first - copy of that keyword, with one exception: Commentary keywords like COMMENT - and HISTORY are expected to have duplicates. So ``header['HISTORY']``, for + copy of that keyword, with one exception: commentary keywords like COMMENT + and HISTORY are expected to have duplicates. So ``header['HISTORY']``, for example, returns the whole sequence of HISTORY values in the correct order. - This list of values can be sliced arbitrarily. For example, to view the last - 3 history entries in a header:: + This list of values can be sliced arbitrarily. For example, to view the last + three history entries in a header:: >>> hdulist[0].header['HISTORY'][-3:] reference table oref$laf13367o_pct.fits reference table oref$laf13369o_apt.fits Heliocentric correction = 16.225 km/s -- Subscript assignment can now be used to add new keywords to the header. Just +- Subscript assignment can now be used to add new keywords to the header. Just as with a normal `dict`, ``header['NAXIS'] = 1`` will either update the NAXIS keyword if it already exists, or add a new NAXIS keyword with a value of - ``1`` if it does not exist. In the old interface this would return a - `~.exceptions.KeyError` if NAXIS did not exist, and the only way to add a new + ``1`` if it does not exist. In the old interface this would return a + `KeyError` if NAXIS did not exist, and the only way to add a new keyword was through the update() method. By default, new keywords added in this manner are added to the end of the @@ -211,13 +204,13 @@ surprises. There are differences, however: * If the header contains extra blank cards at the end, new keywords are added before the blanks. - * If the header ends with a list of commentary cards--for example a sequence - of HISTORY cards--those are kept at the end, and new keywords are inserted + * If the header ends with a list of commentary cards — for example, a sequence + of HISTORY cards — those are kept at the end, and new keywords are inserted before the commentary cards. * If the keyword is a commentary keyword like COMMENT or HISTORY (or an empty - string for blank keywords), a *new* commentary keyword is always added, and - appended to the last commentary keyword of the same type. For example, + string for blank keywords), a *new* commentary keyword is always added and + appended to the last commentary keyword of the same type. For example, HISTORY keywords are always placed after the last history keyword:: >>> header = fits.Header() @@ -232,10 +225,10 @@ surprises. There are differences, however: HISTORY History 2 These behaviors represent a sensible default behavior for keyword assignment, - and represents the same behavior as :meth:`~Header.update` in the old Header - implementation. The default behaviors may still be bypassed through the use - of other assignment methods like :meth:`Header.set` and :meth:`Header.append` - described later. + and the same behavior as :meth:`~Header.update` in the old Header + implementation. The default behaviors may still be bypassed through the use + of other assignment methods like the :meth:`Header.set` and + :meth:`Header.append` methods described later. - It is now also possible to assign a value and a comment to a keyword simultaneously using a tuple:: @@ -245,9 +238,10 @@ surprises. There are differences, however: This will update the value and comment of an existing keyword, or add a new keyword with the given value and comment. -- There is a new :attr:`Header.comments` attribute which lists all the comments - associated with keywords in the header (not to be confused with COMMENT - cards). This allows viewing and updating the comments on specific cards:: +- There is a new :attr:`Header.comments` attribute which lists all of the + comments associated with keywords in the header (not to be confused with + COMMENT cards). This allows viewing and updating the comments on specific + cards:: >>> header.comments['NAXIS'] Number of axis @@ -255,13 +249,13 @@ surprises. There are differences, however: >>> header.comments['NAXIS'] Number of axes -- When deleting a keyword from a header, don't assume that the keyword already - exists. In the old Header implementation this would just silently do - nothing. For backwards-compatibility it is still okay to delete a - non-existent keyword, but a warning will be raised. In the future this - *will* be changed so that trying to delete a non-existent keyword raises a - `~.exceptions.KeyError`. This is for consistency with the behavior of Python dicts. So - unless you know for certain that a keyword exists before deleting it, it's +- When deleting a keyword from a header, do not assume that the keyword already + exists. In the old Header implementation, this action would silently do + nothing. For backwards-compatibility, it is still okay to delete a + nonexistent keyword, but a warning will be raised. In the future this + *will* be changed so that trying to delete a nonexistent keyword raises a + `KeyError`. This is for consistency with the behavior of Python dicts. So + unless you know for certain that a keyword exists before deleting it, it is best to do something like:: >>> try: @@ -274,50 +268,50 @@ surprises. There are differences, however: >>> if 'BITPIX' in header: ... del header['BITPIX'] -- ``del header`` now supports slices. For example, to delete the last three +- ``del header`` now supports slices. For example, to delete the last three keywords from a header:: >>> del header[-3:] -- Two headers can now be compared for equality--previously no two Header - objects were the same. Now they compare as equal if they contain the exact - same content. That is, this requires strict equality. +- Two headers can now be compared for equality — previously no two Header + objects were the same. Now they compare as equal if they contain the exact + same content. That is, this requires strict equality. - Two headers can now be added with the '+' operator, which returns a copy of the left header extended by the right header with :meth:`~Header.extend`. Assignment addition is also possible. - The Header.update() method used commonly with the old Header API has been - renamed to :meth:`Header.set`. The primary reason for this change is very - simple: Header implements the `dict` interface, which already has a method + renamed to :meth:`Header.set`. The primary reason for this change is very + simple: Header implements the `dict` interface, which already has a method called update(), but that behaves differently from the old Header.update(). The details of the new update() can be read in the API docs, but it is very - similar to `dict.update`. It also supports backwards compatibility with the + similar to `dict.update`. It also supports backwards compatibility with the old update() by analysis of the arguments passed to it, so existing code will - not break immediately. However, this *will* cause a deprecation warning to - be output if they're enabled. It is best, for starters, to replace all - update() calls with set(). Recall, also, that direct assignment is now - possible for adding new keywords to a header. So by and large the only + not break immediately. However, this *will* cause a deprecation warning to + be output if they are enabled. It is best, for starters, to replace all + update() calls with set(). Recall, also, that direct assignment is now + possible for adding new keywords to a header. So by and large the only reason to prefer using :meth:`Header.set` is its capability of inserting or moving a keyword to a specific location using the ``before`` or ``after`` arguments. - Slicing a Header with a slice index returns a new Header containing only - those cards contained in the slice. As mentioned earlier, it used to be that - slicing a Header returned a card list--something of a misfeature. In + those cards contained in the slice. As mentioned earlier, it used to be that + slicing a Header returned a card list — something of a misfeature. In general, objects that support slicing ought to return an object of the same type when you slice them. - Likewise, wildcard keywords used to return a CardList object. Now they - return a new Header--similarly to a slice. For example:: + Likewise, wildcard keywords used to return a CardList object — now they + return a new Header similarly to a slice. For example:: >>> header['NAXIS*'] returns a new header containing only the NAXIS and NAXISn cards from the original header. -.. _duck typing: http://en.wikipedia.org/wiki/Duck_typing +.. _duck typing: https://en.wikipedia.org/wiki/Duck_typing Transition Tips @@ -328,97 +322,95 @@ to manipulate headers should not need to be updated, at least not immediately. The most common operations still work the same. As mentioned above, it would be helpful to run your code with ``python -Wd`` to -enable deprecation warnings--that should be a good idea of where to look to +enable deprecation warnings — that should be a good idea of where to look to update your code. If your code needs to be able to support older versions of PyFITS simultaneously with PyFITS 3.1, things are slightly trickier, but not by -much--the deprecated interfaces will not be removed for several more versions +much — the deprecated interfaces will not be removed for several more versions because of this. - The first change worth making, which is supported by any PyFITS version in - the last several years, is remove any use of :meth:``Header.has_key`` and - replace it with ``keyword in header`` syntax. It's worth making this change - for any dict as well, since `dict.has_key` is deprecated. Running the + the last several years, is to remove any use of ``Header.has_key`` and + replace it with ``keyword in header`` syntax. It is worth making this change + for any dict as well, since `dict.has_key` is deprecated. Running the following regular expression over your code may help with most (but not all) cases:: s/([^ ]+)\.has_key\(([^)]+)\)/\2 in \1/ - If possible, replace any calls to Header.update() with Header.set() (though - don't bother with this if you need to support older PyFITS versions). Also, + do not bother with this if you need to support older PyFITS versions). Also, if you have any calls to Header.update() that can be replaced with simple - subscript assignments (eg. ``header['NAXIS'] = (2, 'Number of axes')``) do + subscript assignments (e.g., ``header['NAXIS'] = (2, 'Number of axes')``) do that too, if possible. -- Find any code that uses ``header.ascard`` or ``header.ascardlist()``. First +- Find any code that uses ``header.ascard`` or ``header.ascardlist()``. First ascertain whether that code really needs to work directly on Card objects. If that is definitely the case, go ahead and replace those with - ``header.cards``--that should work without too much fuss. If you do need to + ``header.cards`` — that should work without too much fuss. If you do need to support older versions, you may keep using ``header.ascard`` for now. -- In the off chance that you have any code that slices a header, it's best to - take the result of that and create a new Header object from it. For +- In the off chance that you have any code that slices a header, it is best to + take the result of that and create a new Header object from it. For example:: >>> new_header = fits.Header(old_header[2:]) This avoids the problem that in PyFITS <= 3.0 slicing a Header returns a - CardList by using the result to initialize a new Header object. This will + CardList by using the result to initialize a new Header object. This will work in both cases (in PyFITS 3.1, initializing a Header with an existing - Header just copies it, a la `list`). + Header just copies it, à la `list`). -- As mentioned earlier, locate any code that deletes keywords with ``del``, and +- As mentioned earlier, locate any code that deletes keywords with ``del`` and make sure they either look before they leap (``if keyword in header:``) or ask forgiveness (``try/except KeyError:``). Other Gotchas ------------- -- As mentioned above it is not necessary to enter ``print header`` to display - a header in an interactive Python prompt. Simply entering ``>>> header`` - by itself is sufficient. Using ``print`` usually will *not* display the - header readably, because it does not include line-breaks between the header - cards. The reason is that Python has two types of string representations: - One is returned when one calls ``str(header)`` which happens automatically - when you ``print`` a variable. In the case of the Header class this actually +- As mentioned above, it is not necessary to enter ``print header`` to display + a header in an interactive Python prompt. Entering ``>>> header`` + by itself is sufficient. Using ``print`` usually will *not* display the + header readably, because it does not include line breaks between the header + cards. The reason is that Python has two types of string representations. + One is returned when a user calls ``str(header)``, which happens automatically + when you ``print`` a variable. In the case of the Header class this actually returns the string value of the header as it is written literally in the FITS file, which includes no line breaks. The other type of string representation happens when one calls - ``repr(header)``. The `repr` of an object is just meant to be a useful + ``repr(header)``. The `repr` of an object is meant to be a useful string "representation" of the object; in this case the contents of the - header but with linebreaks between the cards and with the END card and - padding trailing padding stripped off. This happens automatically when - one enters a variable at the Python prompt by itself without a ``print`` + header but with line breaks between the cards and with the END card and + trailing padding stripped off. This happens automatically when + a user enters a variable at the Python prompt by itself without a ``print`` call. - The current version of the FITS Standard (3.0) states in section 4.2.1 that trailing spaces in string values in headers are not significant and - should be ignored. PyFITS < 3.1 *did* treat treat trailing spaces as - significant. For example if a header contained: + should be ignored. PyFITS < 3.1 *did* treat trailing spaces as significant. + For example, if a header contained: KEYWORD1= 'Value ' then ``header['KEYWORD1']`` would return the string ``'Value '`` exactly, - with the trailing spaces intact. The new Header interface fixes this by + with the trailing spaces intact. The new Header interface fixes this by automatically stripping trailing spaces, so that ``header['KEYWORD1']`` would return just ``'Value'``. - There is, however, one convention used by the IRAF ccdmosaic task for - representing its `TNX World Coordinate System - `_ and `ZPX World - Coordinate System `_ - non-standard WCS' that uses a series of keywords in the form ``WATj_nnn`` - which store a text description of coefficients for a non-linear distortion - projection. It uses its own microformat for listing the coefficients as a + There is, however, one convention used by the IRAF CCD mosaic task for + representing its TNX World Coordinate System and ZPX World Coordinate System + nonstandard WCS that uses a series of keywords in the form ``WATj_nnn``, + which store a text description of coefficients for a nonlinear distortion + projection. It uses its own microformat for listing the coefficients as a string, but the string is long, and thus broken up into several of these - ``WATj_nnn`` keywords. Correct recombination of these keywords requires - treating all whitespace literally. This convention either overlooked or + ``WATj_nnn`` keywords. Correct recombination of these keywords requires + treating all whitespace literally. This convention either overlooked or predated the prescribed treatment of whitespace in the FITS standard. - To get around this issue a global variable ``fits.STRIP_HEADER_WHITESPACE`` - was introduced. Temporarily setting + To get around this issue, a global variable ``fits.STRIP_HEADER_WHITESPACE`` + was introduced. Temporarily setting ``fits.STRIP_HEADER_WHITESPACE.set(False)`` before reading keywords affected by this issue will return their values with all trailing whitespace intact. diff --git a/docs/io/fits/appendix/history.rst b/docs/io/fits/appendix/history.rst index 2498e5254581..695847648861 100644 --- a/docs/io/fits/appendix/history.rst +++ b/docs/io/fits/appendix/history.rst @@ -1,11 +1,11 @@ .. doctest-skip-all astropy.io.fits History -======================= +*********************** -Prior to its inclusion in Astropy, the `astropy.io.fits` package was a stand- -alone package called `PyFITS`_. Though for the time being active development -is continuing on PyFITS, that development is also being merged into Astropy. +Prior to its inclusion in Astropy, the `astropy.io.fits` package was a +stand-alone package called `PyFITS`_. PyFITS is no longer actively maintained, and +its development is now solely in Astropy. This page documents the release history of PyFITS prior to its merge into Astropy. @@ -14,11 +14,17 @@ Astropy. :local: -3.3.0 (unreleased) ------------------- +3.4.0 (2016-01-29) +================== + +This is the last released version of PyFITS as a standalone package. + + +3.3.0 (2014-07-17) +================== New Features -^^^^^^^^^^^^ +------------ - Added new verification options ``fix+ignore``, ``fix+warn``, ``fix+exception``, ``silentfix+ignore``, ``silentfix+warn``, and @@ -27,7 +33,7 @@ New Features the PyFITS documentation for more details. API Changes -^^^^^^^^^^^ +----------- - The ``pyfits.new_table`` function is now fully deprecated (though will not be removed for a long time, considering how widely it is used). @@ -105,7 +111,7 @@ API Changes v3.3.x bugfix releases, however). Other Changes and Additions -^^^^^^^^^^^^^^^^^^^^^^^^^^^ +--------------------------- - PyFITS has switched to a unified code base which supports Python 2.5 through 3.4 simultaneously without translation. This *shouldn't* have any @@ -124,7 +130,7 @@ Other Changes and Additions ``False`` otherwise. Bug Fixes -^^^^^^^^^ +--------- - Fixed a regression where it was not possible to save an empty "compressed" image to a file (in this case there is nothing to compress, hence the @@ -135,22 +141,16 @@ Bug Fixes GZIP_COMPRESSED_DATA column. (spacetelescope/#71) -3.2.4 (unreleased) ------------------- +3.2.4 (2014-06-02) +================== - Fixed a regression where multiple consecutive calls of the ``writeto`` method on the same HDU but to different files could lead to corrupt data or crashes on the subsequent calls after the first. (spacetelescope/PyFITS#40) -3.1.7 (unreleased) ------------------- - -- Nothing changed yet. - - 3.2.3 (2014-05-14) ------------------- +================== - Nominal support for Python 3.4. @@ -179,7 +179,7 @@ Bug Fixes 3.1.6 (2014-05-14) ------------------- +================== - Nominal support for Python 3.4. @@ -205,7 +205,7 @@ Bug Fixes 3.2.2 (2014-03-25) ------------------- +================== - Fixed a regression on deletion of record-valued keyword cards using the Header wildcard syntax. This was intended to be fixed before the @@ -213,7 +213,7 @@ Bug Fixes 3.1.5 (2014-03-25) ------------------- +================== - Fixed a regression on deletion of record-valued keyword cards using the Header wildcard syntax. This was intended to be fixed before the @@ -221,7 +221,7 @@ Bug Fixes 3.2.1 (2014-03-04) ------------------- +================== - Nominal support for the upcoming Python 3.4. @@ -297,7 +297,7 @@ Bug Fixes 3.1.4 (2014-03-04) ------------------- +================== - Added missing features from the ``Header.insert()`` method that were intended for inclusion in the original 3.1 release: In addition to @@ -352,7 +352,7 @@ Bug Fixes 3.0.13 (2014-03-04) -------------------- +=================== - Fixed a bug where writing a file with ``checksum=True`` did not add the checksum on new files. (Backported from 3.2.1) @@ -363,10 +363,10 @@ Bug Fixes 3.2 (2013-11-26) ----------------- +================ Highlights -^^^^^^^^^^ +---------- - Rewrote CFITSIO-based backend for handling tile compression of FITS files. It now uses a standard CFITSIO instead of heavily modified pieces of CFITSIO @@ -393,7 +393,7 @@ Highlights API Changes -^^^^^^^^^^^ +----------- - Assigning to values in ``ColDefs.names``, ``ColDefs.formats``, ``ColDefs.nulls`` and other attributes of ``ColDefs`` instances that return @@ -460,7 +460,7 @@ API Changes Other Changes and Additions -^^^^^^^^^^^^^^^^^^^^^^^^^^^ +--------------------------- - The new compression code also adds support for the ZQUANTIZ and ZDITHER0 keywords added in more recent versions of this FITS Tile Compression spec. @@ -496,7 +496,7 @@ Other Changes and Additions Bug Fixes -^^^^^^^^^ +--------- - Binary tables containing compressed images may, optionally, contain other columns unrelated to the tile compression convention. Although this is an @@ -515,7 +515,7 @@ Bug Fixes 3.1.3 (2013-11-26) ------------------- +================== - Disallowed assigning NaN and Inf floating point values as header values, since the FITS standard does not define a way to represent them in. Because @@ -526,7 +526,7 @@ Bug Fixes writing files greater than 2^32 bytes in size. (spacetelescope/PyFITS#28) - Fixed a long-standing issue where writing binary tables did not correctly - write the TFORMn keywords for variable-length array columns (they ommitted + write the TFORMn keywords for variable-length array columns (they omitted the max array length parameter of the format). This was thought fixed in v3.1.2, but it was only fixed there for compressed image HDUs and not for binary tables in general. @@ -536,7 +536,7 @@ Bug Fixes 3.0.12 (2013-11-26) -------------------- +=================== - Disallowed assigning NaN and Inf floating point values as header values, since the FITS standard does not define a way to represent them in. Because @@ -547,7 +547,7 @@ Bug Fixes writing files greater than 2^32 bytes in size. (Backported from 3.1.3) - Fixed a long-standing issue where writing binary tables did not correctly - write the TFORMn keywords for variable-length array columns (they ommitted + write the TFORMn keywords for variable-length array columns (they omitted the max array length parameter of the format). This was thought fixed in v3.1.2, but it was only fixed there for compressed image HDUs and not for binary tables in general. (Backported from 3.1.3) @@ -557,7 +557,7 @@ Bug Fixes 3.1.3 (unreleased) ------------------- +================== - Disallowed assigning NaN and Inf floating point values as header values, since the FITS standard does not define a way to represent them in. Because @@ -566,7 +566,7 @@ Bug Fixes 3.0.12 (unreleased) -------------------- +=================== - Disallowed assigning NaN and Inf floating point values as header values, since the FITS standard does not define a way to represent them in. Because @@ -578,7 +578,7 @@ Bug Fixes 3.1.2 (2013-04-22) ------------------- +================== - When an error occurs opening a file in fitsdiff the exception message will now at least mention which file had the error. (#168) @@ -651,7 +651,7 @@ Bug Fixes 3.0.11 (2013-04-17) -------------------- +=================== - Fixed support for opening gzipped FITS files by filename in a writeable mode (PyFITS has supported writing to gzip files for some time now, but only @@ -664,7 +664,7 @@ Bug Fixes - Fixed an (apparently long-standing) issue where writing compressed images did not correctly write the TFORMn keywords for variable-length array columns - (they ommitted the max array length parameter of the format). Backported from + (they omitted the max array length parameter of the format). Backported from 3.1.2. (#199) - Slightly refactored how tables containing variable-length array columns are @@ -701,12 +701,12 @@ Bug Fixes 3.1.1 (2013-01-02) ------------------- +================== This is a bug fix release for the 3.1.x series. Bug Fixes -^^^^^^^^^ +--------- - Improved handling of scaled images and pseudo-unsigned integer images in compressed image HDUs. They now work more transparently like normal image @@ -783,7 +783,7 @@ Bug Fixes 3.0.10 (2013-01-02) -------------------- +=================== - Improved handling of scaled images and pseudo-unsigned integer images in compressed image HDUs. They now work more transparently like normal image @@ -820,10 +820,10 @@ Bug Fixes 3.1 (2012-08-08) ----------------- +================ Highlights -^^^^^^^^^^ +---------- - The ``Header`` object has been significantly reworked, and ``CardList`` objects are now deprecated (their functionality folded into the ``Header`` @@ -837,7 +837,7 @@ Highlights Features below. API Changes -^^^^^^^^^^^ +----------- - The ``Header`` class has been rewritten, and the ``CardList`` class is deprecated. Most of the basic details of working with FITS headers are @@ -952,10 +952,9 @@ API Changes and removed when the file is saved. New Features -^^^^^^^^^^^^ +------------ -- Added support for the proposed "FITS" extension HDU type. See - http://listmgr.cv.nrao.edu/pipermail/fitsbits/2002-April/001094.html. FITS +- Added support for the proposed "FITS" extension HDU type. FITS HDUs contain an entire FITS file embedded in their data section. ``FitsHDU`` objects work like other HDU types in PyFITS. Their ``.data`` attribute returns the raw data array. However, they have a special ``.hdulist`` @@ -1000,7 +999,7 @@ New Features (#121) Changes in Behavior -^^^^^^^^^^^^^^^^^^^ +------------------- - Warnings from PyFITS are not output to stderr by default, instead of stdout as it has been for some time. This is contrary to most users' expectations @@ -1008,7 +1007,7 @@ Changes in Behavior desired output for their scripts. (r1319) Bug Fixes -^^^^^^^^^ +--------- - Fixed ``pyfits.tcreate()`` (now ``pyfits.tableload()``) to be more robust when encountering blank lines in a column definition file (#14) @@ -1022,7 +1021,7 @@ Bug Fixes This allowed for the implementation of ``HDUList.fromstring`` described above. (#90) -- Fixed a rare corner case where, in some use cases, (mildly, recoverably) +- Fixed a rare corner case where, in some use cases, (mildly, recoverable) malformatted float values in headers were not properly returned as floats. (#137) @@ -1046,12 +1045,12 @@ Bug Fixes 3.0.9 (2012-08-06) ------------------- +================== This is a bug fix release for the 3.0.x series. Bug Fixes -^^^^^^^^^ +--------- - Fixed ``Header.values()``/``Header.itervalues()`` and ``Header.items()``/ ``Header.iteritems()`` to correctly return the different values for @@ -1078,10 +1077,10 @@ Bug Fixes 3.0.8 (2012-06-04) ------------------- +================== Changes in Behavior -^^^^^^^^^^^^^^^^^^^ +------------------- - Prior to this release, image data sections did not work with scaled data--that is, images with non-trivial BSCALE and/or BZERO values. @@ -1091,7 +1090,7 @@ Changes in Behavior extends that support for general BSCALE+BZERO values. Bug Fixes -^^^^^^^^^ +--------- - Fixed a bug that prevented updates to values in boolean table columns from being saved. This turned out to be a symptom of a deeper problem that could @@ -1101,7 +1100,7 @@ Bug Fixes could, in some circumstances, cause headers (and the rest of the file after that point) to be misread. (#142) -- Fixed support for scaled image data and psuedo-unsigned ints in image data +- Fixed support for scaled image data and pseudo-unsigned ints in image data sections (``hdu.section``). Previously this was not supported at all. At some point support was supposedly added, but it was buggy and incomplete. Now the feature seems to work much better. (#143) @@ -1132,16 +1131,16 @@ Bug Fixes 3.0.7 (2012-04-10) ------------------- +================== Changes in Behavior -^^^^^^^^^^^^^^^^^^^ +------------------- - Slices of GroupData objects now return new GroupData objects instead of extended multi-row _Group objects. This is analogous to how PyFITS 3.0 fixed FITS_rec slicing, and should have been fixed for GroupData at the same time. The old behavior caused bugs where functions internal to Numpy expected that - slicing an ndarray would return a new ndarray. As this is a rare usecase + slicing an ndarray would return a new ndarray. As this is a rare use case with a rare feature most users are unlikely to be affected by this change. - The previously internal _Group object for representing individual group @@ -1154,7 +1153,7 @@ Changes in Behavior HDUs. It was unnecessary to modify this value. Bug Fixes -^^^^^^^^^ +--------- - Fixed GroupData objects to return new GroupData objects when sliced instead of _Group record objects. See "Changes in behavior" above for more details. @@ -1189,10 +1188,10 @@ Bug Fixes 3.0.6 (2012-02-29) ------------------- +================== Highlights -^^^^^^^^^^ +---------- The main reason for this release is to fix an issue that was introduced in PyFITS 3.0.5 where merely opening a file containing scaled data (that is, with @@ -1208,7 +1207,7 @@ This release also fixes a few Windows-specific bugs found through more extensive Windows testing, and other miscellaneous bugs. Bug Fixes -^^^^^^^^^ +--------- - More accurate error messages when opening files containing invalid header cards. (#109) @@ -1238,7 +1237,7 @@ Bug Fixes 3.0.5 (2012-01-30) ------------------- +================== - Fixed a crash that could occur when accessing image sections of files opened with memmap=True. (r1211) @@ -1286,7 +1285,7 @@ Bug Fixes 3.0.4 (2011-11-22) ------------------- +================== - Fixed a crash when writing HCOMPRESS compressed images that could happen on Python 2.5 and 2.6. (r1217) @@ -1330,7 +1329,7 @@ Bug Fixes 3.0.3 (2011-10-05) ------------------- +================== - Fixed several small bugs involving corner cases in record-valued keyword cards (#70) @@ -1348,7 +1347,7 @@ Bug Fixes 3.0.2 (2011-09-23) ------------------- +================== - The ``BinTableHDU.tcreate`` method and by extension the ``pyfits.tcreate`` function don't get tripped up by blank lines anymore (#14) @@ -1377,7 +1376,7 @@ Bug Fixes original file permissions (#79) - Fixed the handling of TDIMn keywords; 3.0 added support for them, but got - the axis order backards (they were treated as though they were row-major) + the axis order backwards (they were treated as though they were row-major) (#82) - Fixed a crash when a FITS file containing scaled data is opened and @@ -1389,7 +1388,7 @@ Bug Fixes 3.0.1 (2011-09-12) ------------------- +================== - Fixed a bug where updating a header card comment could cause the value to be lost if it had not already been read from the card image string. @@ -1416,7 +1415,7 @@ Bug Fixes 3.0.0 (2011-08-23) --------------------- +==================== - Contains major changes, bumping the version to 3.0 @@ -1488,8 +1487,8 @@ Bug Fixes - Added support for binary table fields with zero width (#42) -- Added support for wider integer types in ASCII tables; although this is non- - standard, some GEIS images require it (#45) +- Added support for wider integer types in ASCII tables; although this is + non-standard, some GEIS images require it (#45) - Fixed a bug that caused the index_of() method of HDULists to crash when the HDUList object is created from scratch (#48) @@ -1522,7 +1521,7 @@ Bug Fixes 2.4.0 (2011-01-10) --------------------- +==================== The following enhancements were added: - Checksum support now correctly conforms to the FITS standard. pyfits @@ -1598,7 +1597,7 @@ The following bugs were fixed: 2.3.1 (2010-06-03) --------------------- +==================== The following bugs were fixed: @@ -1608,7 +1607,7 @@ The following bugs were fixed: 2.3 (2010-05-11) ------------------- +================== The following enhancements were made: @@ -1956,7 +1955,7 @@ The following bugs were fixed: 2.2.2 (2009-10-12) --------------------- +==================== Updates described in this release are only supported in the NUMPY version of pyfits. @@ -1969,7 +1968,7 @@ The following bugs were fixed: 2.2.1 (2009-10-06) --------------------- +==================== Updates described in this release are only supported in the NUMPY version of pyfits. @@ -1985,7 +1984,7 @@ The following bugs were fixed: 2.2 (2009-09-23) ------------------- +================== Updates described in this release are only supported in the NUMPY version of pyfits. @@ -2059,7 +2058,7 @@ The following bugs were fixed: 2.1.1 (2009-04-22) -------------------- +=================== Updates described in this release are only supported in the NUMPY version of pyfits. @@ -2076,7 +2075,7 @@ The following bugs were fixed: 2.1 (2009-04-14) ------------------- +================== Updates described in this release are only supported in the NUMPY version of pyfits. @@ -2166,7 +2165,7 @@ The following bugs were fixed: 2.0.1 (2009-02-03) --------------------- +==================== Updates described in this release are only supported in the NUMPY version of pyfits. @@ -2178,7 +2177,7 @@ The following bugs were fixed: 2.0 (2009-01-30) ------------------- +================== Updates described in this release are only supported in the NUMPY version of pyfits. @@ -2335,11 +2334,11 @@ The following bugs were fixed: an ImageHDU header with a PCOUNT card that is missing or has a value other than 0. -.. _[1]: http://fits.gsfc.nasa.gov/registry/tilecompression.html +.. _[1]: https://fits.gsfc.nasa.gov/registry/tilecompression.html 1.4.1 (2008-11-04) --------------------- +==================== Updates described in this release are only supported in the NUMPY version of pyfits. @@ -2358,7 +2357,7 @@ The following bugs were fixed: 1.4 (2008-07-07) ------------------- +================== Updates described in this release are only supported in the NUMPY version of pyfits. @@ -2664,19 +2663,19 @@ The following enhancements were made: the appropriate card object given an input string. These two methods are also available as convenience functions: - >>> c1 = pyfits.RecordValuedKeywordCard.createCard('DP1','AUX: 1','comment) + >>> c1 = pyfits.RecordValuedKeywordCard.createCard('DP1','AUX: 1','comment') or - >>> c1 = pyfits.createCard('DP1','AUX: 1','comment) + >>> c1 = pyfits.createCard('DP1','AUX: 1','comment') >>> print type(c1) <'pyfits.NP_pyfits.RecordValuedKeywordCard'> - >>> c1 = pyfits.RecordValuedKeywordCard.createCard('DP1','AUX 1','comment) + >>> c1 = pyfits.RecordValuedKeywordCard.createCard('DP1','AUX 1','comment') or - >>> c1 = pyfits.createCard('DP1','AUX 1','comment) + >>> c1 = pyfits.createCard('DP1','AUX 1','comment') >>> print type(c1) <'pyfits.NP_pyfits.Card'> @@ -2731,7 +2730,7 @@ The following bugs were fixed: 1.3 (2008-02-22) ------------------- +================== Updates described in this release are only supported in the NUMPY version of pyfits. @@ -2824,11 +2823,11 @@ The following bugs were fixed: windows platform using a drive letter in the file specification caused a misleading IOError exception to be raised. -.. _[2]: http://stsdas.stsci.edu/stsci_python_sphinxdocs_2.13/tools/stpyfits.html +.. _[2]: https://stscitools.readthedocs.io/en/latest/stpyfits.html 1.1 (2007-06-15) ------------------- +================== - Modified to use either NUMPY or NUMARRAY. @@ -2847,7 +2846,7 @@ The following bugs were fixed: 1.0.1 (2006-03-24) --------------------- +==================== The changes to PyFITS were primarily to improve the docstrings and to reclassify some public functions and variables as private. Readgeis and @@ -2864,7 +2863,7 @@ stsci_python release. 1.0 (2005-11-01) ------------------- +================== Major Changes since v0.9.6: @@ -2921,7 +2920,7 @@ PyFITS Version 1.0 REQUIRES Python 2.3 or later. 0.9.6 (2004-11-11) --------------------- +==================== Major changes since v0.9.3: @@ -2942,11 +2941,11 @@ Some minor changes: 0.9.3 (2004-07-02) --------------------- +==================== Changes since v0.9.0: -- Lazy instanciation of full Headers/Cards for all HDU's when the file is +- Lazy instantiation of full Headers/Cards for all HDU's when the file is opened. At the open, only extracts vital info (e.g. NAXIS's) from the header parts. This change will speed up the performance if the user only needs to access one extension in a multi-extension FITS file. @@ -2962,7 +2961,7 @@ Changes since v0.9.0: 0.9 (2004-04-27) ------------------- +================== Changes since v0.8.0: @@ -2986,7 +2985,7 @@ Changes since v0.8.0: 0.8.0 (2003-08-19) --------------------- +==================== **NOTE:** This version will only work with numarray Version 0.6. In addition, earlier versions of PyFITS will not work with numarray 0.6. Therefore, both @@ -3028,7 +3027,7 @@ Changes since 0.7.5: order to make pyfits to work under Python 2.2. (2 occurrences) - Modify the "update" method in the Header class to use the "fixed-format" - card even if the card already exists. This is to avoid the mis-alignment as + card even if the card already exists. This is to avoid the misalignment as shown below: After running drizzle on ACS images it creates a CD matrix whose elements @@ -3079,7 +3078,7 @@ Changes since 0.7.5: 0.7.5 (2002-08-16) --------------------- +==================== Changes since v0.7.3: @@ -3104,7 +3103,7 @@ Changes since v0.7.3: 0.7.3 (2002-07-12) --------------------- +==================== Changes since v0.7.2: @@ -3136,7 +3135,7 @@ Changes since v0.7.2: the comment must be string type to avoid exception. 0.7.2.1 (2002-06-25) ----------------------- +====================== A couple of bugs were addressed in this version. @@ -3150,7 +3149,7 @@ A couple of bugs were addressed in this version. 0.7.2 (2002-06-19) --------------------- +==================== The two major improvements from Version 0.6.2 are: @@ -3208,7 +3207,7 @@ technical changes. 0.6.2 (2002-02-12) --------------------- +==================== This version requires numarray version 0.2. @@ -3239,4 +3238,4 @@ Things not yet supported but are part of future development: - Support for tables with TNULL values. This awaits an enhancement to numarray to support mask arrays (planned). (At least a couple of months off). -.. _PyFITS: http://www.stsci.edu/resources/software_hardware/pyfits +.. _PyFITS: https://github.com/spacetelescope/pyfits diff --git a/docs/io/fits/images/Blue.jpg b/docs/io/fits/images/Blue.jpg deleted file mode 100644 index ac9fa4c10609..000000000000 Binary files a/docs/io/fits/images/Blue.jpg and /dev/null differ diff --git a/docs/io/fits/images/Green.jpg b/docs/io/fits/images/Green.jpg deleted file mode 100644 index ed73ad5effe3..000000000000 Binary files a/docs/io/fits/images/Green.jpg and /dev/null differ diff --git a/docs/io/fits/images/Hs-2009-14-a-web.jpg b/docs/io/fits/images/Hs-2009-14-a-web.jpg deleted file mode 100644 index 42277a0dbdaa..000000000000 Binary files a/docs/io/fits/images/Hs-2009-14-a-web.jpg and /dev/null differ diff --git a/docs/io/fits/images/Red.jpg b/docs/io/fits/images/Red.jpg deleted file mode 100644 index 8a4e74c0a3c8..000000000000 Binary files a/docs/io/fits/images/Red.jpg and /dev/null differ diff --git a/docs/io/fits/index.rst b/docs/io/fits/index.rst index 0992129364f8..b6ac1ee6966c 100644 --- a/docs/io/fits/index.rst +++ b/docs/io/fits/index.rst @@ -1,11 +1,9 @@ -.. doctest-skip-all - .. currentmodule:: astropy.io.fits .. _astropy-io-fits: ************************************** -FITS File handling (`astropy.io.fits`) +FITS File Handling (`astropy.io.fits`) ************************************** Introduction @@ -15,6 +13,14 @@ The :mod:`astropy.io.fits` package provides access to FITS files. FITS (Flexible Image Transport System) is a portable file standard widely used in the astronomy community to store images and tables. +.. note:: + + If you want to read or write a single table in FITS format, the + recommended method is via the :ref:`table_io` interface. In particular + see the :ref:`Unified I/O FITS ` section. Likewise, for CCD image + data with a physical unit (e.g., ``electron``), see the :ref:`Unified Image Data` section. + + .. _tutorial: Getting Started @@ -22,97 +28,212 @@ Getting Started This section provides a quick introduction of using :mod:`astropy.io.fits`. The goal is to demonstrate the package's basic features without getting into too -much detail. If you are a first time user or have never used Astropy or PyFITS, -this is where you should start. See also the :ref:`FAQ ` for -answers to common questions/issues. +much detail. If you are a first time user or have never used ``astropy`` or +PyFITS, this is where you should start. See also the :ref:`FAQ ` +for answers to common questions and issues. + Reading and Updating Existing FITS Files ---------------------------------------- -Opening a FITS file +Opening a FITS File ^^^^^^^^^^^^^^^^^^^ -Once the `astropy.io.fits` package is loaded using the standard convention\ +.. note:: + + The ``astropy.io.fits.util.get_testdata_filepath()`` function, + used in the examples here, returns file path for test data shipped with ``astropy``. + To work with your own data instead, please use :func:`astropy.io.fits.open` or :ref:`io-fits-intro-convenience-functions`, + which take either the relative or absolute path as string or :term:`python:path-like object`. + +Once the `astropy.io.fits` package is loaded using the standard convention [#f1]_, we can open an existing FITS file:: >>> from astropy.io import fits - >>> hdulist = fits.open('input.fits') + >>> fits_image_filename = fits.util.get_testdata_filepath('test0.fits') + + >>> hdul = fits.open(fits_image_filename) The :func:`open` function has several optional arguments which will be discussed in a later chapter. The default mode, as in the above example, is -"readonly". The open function returns an object called an :class:`HDUList` +"readonly". The open function returns an object called an :class:`HDUList` which is a `list`-like collection of HDU objects. An HDU (Header Data Unit) is the highest level component of the FITS file structure, consisting of a header and (typically) a data array or table. -After the above open call, ``hdulist[0]`` is the primary HDU, ``hdulist[1]`` is -the first extension HDU, etc (if there are any extensions), and so on. It -should be noted that Astropy is using zero-based indexing when referring to +After the above open call, ``hdul[0]`` is the primary HDU, ``hdul[1]`` is +the first extension HDU, etc. (if there are any extensions), and so on. It +should be noted that ``astropy`` uses zero-based indexing when referring to HDUs and header cards, though the FITS standard (which was designed with -FORTRAN in mind) uses one-based indexing. +Fortran in mind) uses one-based indexing. The :class:`HDUList` has a useful method :meth:`HDUList.info`, which summarizes the content of the opened FITS file: - >>> hdulist.info() - Filename: test1.fits - No. Name Type Cards Dimensions Format - 0 PRIMARY PrimaryHDU 220 () int16 - 1 SCI ImageHDU 61 (800, 800) float32 - 2 SCI ImageHDU 61 (800, 800) float32 - 3 SCI ImageHDU 61 (800, 800) float32 - 4 SCI ImageHDU 61 (800, 800) float32 + >>> hdul.info() + Filename: ...test0.fits + No. Name Ver Type Cards Dimensions Format + 0 PRIMARY 1 PrimaryHDU 138 () + 1 SCI 1 ImageHDU 61 (40, 40) int16 + 2 SCI 2 ImageHDU 61 (40, 40) int16 + 3 SCI 3 ImageHDU 61 (40, 40) int16 + 4 SCI 4 ImageHDU 61 (40, 40) int16 After you are done with the opened file, close it with the :meth:`HDUList.close` method: - >>> hdulist.close() + >>> hdul.close() + +You can avoid closing the file manually by using :func:`open` as context +manager:: + + >>> with fits.open(fits_image_filename) as hdul: + ... hdul.info() + Filename: ...test0.fits + No. Name Ver Type Cards Dimensions Format + 0 PRIMARY 1 PrimaryHDU 138 () + 1 SCI 1 ImageHDU 61 (40, 40) int16 + 2 SCI 2 ImageHDU 61 (40, 40) int16 + 3 SCI 3 ImageHDU 61 (40, 40) int16 + 4 SCI 4 ImageHDU 61 (40, 40) int16 + +After exiting the ``with`` scope the file will be closed automatically. That is +(generally) the preferred way to open a file in Python, because it will close +the file even if an exception happens. -The headers will still be accessible after the HDUList is closed. The data may -or may not be accessible depending on whether the data are touched and if they -are memory-mapped, see later chapters for detail. +If the file is opened with ``lazy_load_hdus=False``, all of the headers will +still be accessible after the HDUList is closed. The headers and data may or +may not be accessible depending on whether the data are touched and if they +are memory-mapped; see later chapters for detail. + +.. _fits-large-files: Working with large files """""""""""""""""""""""" The :func:`open` function supports a ``memmap=True`` argument that allows the array data of each HDU to be accessed with mmap, rather than being read into -memory all at once. This is particularly useful for working with very large -arrays that cannot fit entirely into physical memory. +memory all at once. This is particularly useful for working with very large +arrays that cannot fit entirely into physical memory. Here ``memmap=True`` by +default, and this value is obtained from the configuration item ``astropy.io.fits.Conf.use_memmap``. This has minimal impact on smaller files as well, though some operations, such -as reading the array data sequentially, may incur some additional overhead. On -32-bit systems arrays larger than 2-3 GB cannot be mmap'd (which is fine, -because by that point you're likely to run out of physical memory anyways), but +as reading the array data sequentially, may incur some additional overhead. On +32-bit systems, arrays larger than 2 to 3 GB cannot be mmap'd (which is fine, +because by that point you are likely to run out of physical memory anyways), but 64-bit systems are much less limited in this respect. .. warning:: - When opening a file with ``memmap=True``, because of how mmap works this means that - when the HDU data is accessed (i.e. ``hdul[0].data``) another handle to the FITS file - is opened by mmap. This means that even after calling ``hdul.close()`` the mmap still - holds an open handle to the data so that it can still be accessed by unwary programs - that were built with the assumption that the .data attribute has all the data in-memory. - - In order to force the mmap to close either wait for the containing ``HDUList`` object to go - out of scope, or manually call ``del hdul[0].data`` (this works so long as there are no other - references held to the data array). + When opening a file with ``memmap=True``, because of how mmap works this + means that when the HDU data is accessed (i.e., ``hdul[0].data``) another + handle to the FITS file is opened by mmap. This means that even after + calling ``hdul.close()`` the mmap still holds an open handle to the data so + that it can still be accessed by unwary programs that were built with the + assumption that the .data attribute has all of the data in-memory. + + In order to force the mmap to close, either wait for the containing + ``HDUList`` object to go out of scope, or manually call + ``del hdul[0].data``. (This works so long as there are no other references + held to the data array.) + +.. _fits-cloud-files: + +Working with remote and cloud-hosted files +"""""""""""""""""""""""""""""""""""""""""" + +The :func:`open` function supports a ``use_fsspec`` argument which allows file +paths to be opened using |fsspec|. +The ``fsspec`` package supports a range of remote and distributed storage +backends such as Amazon and Google Cloud Storage. For example, you can access a +Hubble Space Telescope image located in the Hubble's public +Amazon S3 bucket as follows: + +.. doctest-requires:: fsspec + + >>> # Location of a large Hubble archive image in Amazon S3 (213 MB) + >>> uri = "s3://stpubdata/hst/public/j8pu/j8pu0y010/j8pu0y010_drc.fits" + ... + >>> # Extract a 10-by-20 pixel cutout image + >>> with fits.open(uri, use_fsspec=True, fsspec_kwargs={"anon": True}) as hdul: # doctest: +REMOTE_DATA + ... cutout = hdul[1].section[10:20, 30:50] + +Note that the example above obtains a cutout image using the `ImageHDU.section` +attribute rather than the traditional `ImageHDU.data` attribute. +The use of ``.section`` ensures that only the necessary parts of the FITS +image are transferred from the server, rather than downloading the entire data +array. This trick can significantly speed up your code if you require small +subsets of large FITS files located on slow (remote) storage systems. +See :ref:`fits_io_cloud` for additional information on working with remote FITS +files in this way. + +.. seealso:: :ref:`fits_io_cloud`. + + +Unsigned integers +""""""""""""""""" + +Due to the FITS format's Fortran origins, FITS does not natively support +unsigned integer data in images or tables. However, there is a common +convention to store unsigned integers as signed integers, along with a +*shift* instruction (a ``BZERO`` keyword with value ``2 ** (BITPIX - 1)``) to +shift up all signed integers to unsigned integers. For example, when writing +the value ``0`` as an unsigned 32-bit integer, it is stored in the FITS +file as ``-32768``, along with the header keyword ``BZERO = 32768``. + +``astropy`` recognizes and applies this convention by default, so that all data +that looks like it should be interpreted as unsigned integers is automatically +converted (this applies to both images and tables). + +Even with ``uint=False``, the ``BZERO`` shift is still applied, but the +returned array is of "float64" type. To disable scaling/shifting entirely, use +``do_not_scale_image_data=True`` (see :ref:`fits-scaled-data-faq` in the FAQ +for more details). Working with compressed files """"""""""""""""""""""""""""" -The :func:`open` function will seamlessly open FITS files that have been -compressed with gzip, bzip2 or pkzip. Note that in this context we're talking -about a fits file that has been compressed with one of these utilities - e.g. a -.fits.gz file. Files that use compressed HDUs within the FITS file are discussed -in :ref:`Compressed Image Data `. +.. note:: -There are some limitations with working with compressed files. For example with Zip -files that contain multiple compressed files, only the first file will be accessible. -Also bzip does not support the append or update access modes. + Files that use compressed HDUs within the FITS file are discussed + in :ref:`Compressed Image Data `. -When writing a file (e.g. with the :func:`writeto` function), compression will be -determined based on the filename extension given, or the compression used in a -pre-existing file that is being written to. + +The :func:`open` function will seamlessly open FITS files that have been +compressed with gzip, bzip2, pkzip, lzma or Unix's compress (LZW compression). +Note that in this context we are talking about a FITS file that has been +compressed with one of these utilities (e.g., a .fits.gz file). + +There are some limitations when working with compressed files. For example, +with Zip files that contain multiple compressed files, only the first file will +be accessible. Also bzip2 and lzma do not support the append or update access +mode and LZW-compressed files do not support any writing modes (including append +or update). + +When writing a file (e.g., with the :func:`writeto` function), compression will +be determined based on the filename extension given, or the compression used in +a pre-existing file that is being written to. + + +Working with non-standard files +""""""""""""""""""""""""""""""" +When `astropy.io.fits` reads a FITS file which does not conform to the FITS +standard it will try to make an educated interpretation of non-compliant fields. +This may not always succeed and may trigger warnings when accessing headers or +exceptions when writing to file. Verification of fields written to an output +file can be controlled with the ``output_verify`` parameter of :func:`open`. +Files opened for reading can be verified and fixed with method +``HDUList.verify``. This method is typically invoked after opening the file +but before accessing any headers or data:: + + >>> with fits.open(fits_image_filename) as hdul: + ... hdul.verify('fix') + ... data = hdul[1].data + +In the above example, the call to ``hdul.verify("fix")`` requests that `astropy.io.fits` +fix non-compliant fields and print informative messages. Other options in addition to ``"fix"`` +are described under FITS :ref:`fits_io_verification` + +.. seealso:: FITS :ref:`fits_io_verification`. Working with FITS Headers ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -122,73 +243,74 @@ As mentioned earlier, each element of an :class:`HDUList` is an HDU object with and data portions of the HDU. For those unfamiliar with FITS headers, they consist of a list of 80 byte -"cards", where a card contains a keyword, a value, and a comment. The keyword +"cards", where a card contains a keyword, a value, and a comment. The keyword and comment must both be strings, whereas the value can be a string or an -integer, floating point number, complex number, or `True`/`False`. Keywords +integer, floating point number, complex number, or ``True``/``False``. Keywords are usually unique within a header, except in a few special cases. -The header attribute is a Header instance, another Astropy object. To get the -value associated with a header keyword, simply do (a la Python dicts):: +The header attribute is a :class:`Header` instance, another ``astropy`` object. +To get the value associated with a header keyword, do (à la Python dicts):: - >>> hdulist[0].header['targname'] - 'NGC121' + >>> hdul = fits.open(fits_image_filename) + >>> hdul[0].header['DATE'] + '01/04/99' -to get the value of the keyword targname, which is a string 'NGC121'. +to get the value of the keyword "DATE", which is a string '01/04/99'. Although keyword names are always in upper case inside the FITS file, -specifying a keyword name with Astropy is case-insensitive, for the user's +specifying a keyword name with ``astropy`` is case-insensitive for the user's convenience. If the specified keyword name does not exist, it will raise a -`~.exceptions.KeyError` exception. +`KeyError` exception. -We can also get the keyword value by indexing (a la Python lists):: +We can also get the keyword value by indexing (à la Python lists):: - >>> hdulist[0].header[27] - 96 + >>> hdul[0].header[7] + 32768.0 -This example returns the 28th (like Python lists, it is 0-indexed) keyword's -value--an integer--96. +This example returns the eighth (like Python lists, it is 0-indexed) keyword's +value — a float — 32768.0. -Similarly, it is easy to update a keyword's value in Astropy, either through -keyword name or index:: +Similarly, it is possible to update a keyword's value in ``astropy``, either +through keyword name or index:: - >>> prihdr = hdulist[0].header - >>> prihdr['targname'] = 'NGC121-a' - >>> prihdr[27] = 99 + >>> hdr = hdul[0].header + >>> hdr['targname'] = 'NGC121-a' + >>> hdr[27] = 99 Please note however that almost all application code should update header -values via their keyword name and not via their positional index. This is +values via their keyword name and not via their positional index. This is because most FITS keywords may appear at any position in the header. It is also possible to update both the value and comment associated with a keyword by assigning them as a tuple:: - >>> prihdr = hdulist[0].header - >>> prihdr['targname'] = ('NGC121-a', 'the observation target') - >>> prihdr['targname'] + >>> hdr = hdul[0].header + >>> hdr['targname'] = ('NGC121-a', 'the observation target') + >>> hdr['targname'] 'NGC121-a' - >>> prihdr.comments['targname'] + >>> hdr.comments['targname'] 'the observation target' -Like a dict, one may also use the above syntax to add a new keyword/value pair -(and optionally a comment as well). In this case the new card is appended to -the end of the header (unless it's a commentary keyword such as COMMENT or +Like a dict, you may also use the above syntax to add a new keyword/value pair +(and optionally a comment as well). In this case the new card is appended to +the end of the header (unless it is a commentary keyword such as COMMENT or HISTORY, in which case it is appended after the last card with that keyword). Another way to either update an existing card or append a new one is to use the :meth:`Header.set` method:: - >>> prihdr.set('observer', 'Edwin Hubble') + >>> hdr.set('observer', 'Edwin Hubble') Comment or history records are added like normal cards, though in their case a new card is always created, rather than updating an existing HISTORY or COMMENT card:: - >>> prihdr['history'] = 'I updated this file 2/26/09' - >>> prihdr['comment'] = 'Edwin Hubble really knew his stuff' - >>> prihdr['comment'] = 'I like using HST observations' - >>> prihdr['history'] + >>> hdr['history'] = 'I updated this file 2/26/09' + >>> hdr['comment'] = 'Edwin Hubble really knew his stuff' + >>> hdr['comment'] = 'I like using HST observations' + >>> hdr['history'] I updated this file 2/26/09 - >>> prihdr['comment'] + >>> hdr['comment'] Edwin Hubble really knew his stuff I like using HST observations @@ -197,35 +319,38 @@ cards. To update existing COMMENT or HISTORY cards, reference them by index:: - >>> prihdr['history'][0] = 'I updated this file on 2/27/09' - >>> prihdr['history'] + >>> hdr['history'][0] = 'I updated this file on 2/27/09' + >>> hdr['history'] I updated this file on 2/27/09 - >>> prihdr['comment'][1] = 'I like using JWST observations' - >>> prihdr['comment'] + >>> hdr['comment'][1] = 'I like using JWST observations' + >>> hdr['comment'] Edwin Hubble really knew his stuff I like using JWST observations To see the entire header as it appears in the FITS file (with the END card and -padding stripped), simply enter the header object by itself, or ``print -repr(header)``:: +padding stripped), enter the header object by itself, or +``print(repr(hdr))``:: - >>> header + >>> hdr # doctest: +ELLIPSIS SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes - all cards are shown... - >>> print repr(header) - identical... + ... + >>> print(repr(hdr)) # doctest: +ELLIPSIS + SIMPLE = T / file does conform to FITS standard + BITPIX = 16 / number of bits per data pixel + NAXIS = 0 / number of data axes + ... -Entering simply ``print header`` will also work, but may not be very legible on -most displays, as this displays the header as it is written in the FITS file -itself, which means there are no linebreaks between cards. This is a common -confusion in new users. +Entering only ``print(hdr)`` will also work, but may not be very legible +on most displays, as this displays the header as it is written in the FITS file +itself, which means there are no line breaks between cards. This is a common +source of confusion for new users. -It's also possible to view a slice of the header:: +It is also possible to view a slice of the header:: - >>> header[:2] + >>> hdr[:2] SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel @@ -234,103 +359,170 @@ Only the first two cards are shown above. To get a list of all keywords, use the :meth:`Header.keys` method just as you would with a dict:: - >>> prihdr.keys() + >>> list(hdr.keys()) # doctest: +ELLIPSIS ['SIMPLE', 'BITPIX', 'NAXIS', ...] +.. note:: + + See also :ref:`io-fits-intro-convenience-functions`. + +.. _structural_keywords: + +Structural Keywords +""""""""""""""""""" + +FITS keywords mix up both metadata and critical information about the file structure +that is needed to parse the file. These *structural* keywords are managed internally by +:mod:`astropy.io.fits` and, in general, should not be touched by the user. Instead one +should use the related attributes of the `astropy.io.fits` classes (see examples below). + +The specific set of structural keywords used by the FITS standard varies with HDU type. +The following table lists which keywords are associated with each HDU type: + +.. csv-table:: Structural Keywords + :header: "HDU Type", "Structural Keywords" + :widths: 20, 20 + + "All", "``SIMPLE``, ``BITPIX``, ``NAXIS``" + ":class:`PrimaryHDU`", "``EXTEND``" + ":class:`ImageHDU`, :class:`TableHDU`, :class:`BinTableHDU`", "``PCOUNT``, ``GCOUNT``" + ":class:`GroupsHDU`", "``NAXIS1``, ``GCOUNT``, ``PCOUNT``, ``GROUPS``" + ":class:`TableHDU`, :class:`BinTableHDU`", "``TFIELDS``, ``TFORM``, ``TBCOL``" + +There are many other reserved keywords, for instance for the data scaling, or for table's column +attributes, as described in the `FITS Standard `__. +Most of these are accessible via attributes of the :class:`Column` or HDU objects, for instance +``hdu.name`` to set ``EXTNAME``, or ``hdu.ver`` for ``EXTVER``. Structural keywords are checked +and/or updated as a consequence of common operations. For example, when: + +1. Setting the data. The ``NAXIS*`` keywords are set from the data shape (``.data.shape``), and ``BITPIX`` + from the data type (``.data.dtype``). +2. Setting the header. Its keywords are updated based on the data properties (as above). +3. Writing a file. All the necessary keywords are deleted, updated or added to the header. +4. Calling an HDU's verify method (e.g., :func:`PrimaryHDU.verify`). Some keywords can be fixed automatically. + +In these cases any hand-written values users might assign to those keywords will be overwritten. Working with Image Data ^^^^^^^^^^^^^^^^^^^^^^^ -If an HDU's data is an image, the data attribute of the HDU object will return -a numpy `~numpy.ndarray` object. Refer to the numpy documentation for details -on manipulating these numerical arrays. +.. note:: + This section describes reading and writing image data in the FITS format using the + `~astropy.io.fits` package directly. For CCD image data with a unit, you should + consider using the :ref:`Unified Image Data` interface with the + :ref:`CCDData class `. This provides the capability to load data, + uncertainty and mask from a multi-extension FITS (MEF) file. -:: +If an HDU's data is an image, the data attribute of the HDU object will return +a ``numpy`` `~numpy.ndarray` object. Refer to the ``numpy`` documentation for +details on manipulating these numerical arrays:: - >>> scidata = hdulist[1].data + >>> data = hdul[1].data -Here, scidata points to the data object in the second HDU (the first HDU, -``hdulist[0]``, being the primary HDU) which corresponds to the 'SCI' +Here, ``data`` points to the data object in the second HDU (the first HDU, +``hdul[0]``, being the primary HDU) which corresponds to the 'SCI' extension. Alternatively, you can access the extension by its extension name (specified in the EXTNAME keyword):: - >>> scidata = hdulist['SCI'].data + >>> data = hdul['SCI'].data If there is more than one extension with the same EXTNAME, the EXTVER value -needs to be specified along with the EXTNAME as a tuple; e.g.:: +needs to be specified along with the EXTNAME as a tuple; for example:: - >>> scidata = hdulist['sci',2].data + >>> data = hdul['sci',2].data Note that the EXTNAME is also case-insensitive. -The returned numpy object has many attributes and methods for a user to get -information about the array, e.g. +The returned ``numpy`` object has many attributes and methods for a user to get +information about the array, for example:: -:: - - >>> scidata.shape - (800, 800) - >>> scidata.dtype.name - 'float32' + >>> data.shape + (40, 40) + >>> data.dtype.name + 'int16' -Since image data is a numpy object, we can slice it, view it, and perform +Since image data is a ``numpy`` object, we can slice it, view it, and perform mathematical operations on it. To see the pixel value at x=5, y=2:: - >>> print scidata[1, 4] + >>> print(data[1, 4]) + 348 -Note that, like C (and unlike FORTRAN), Python is 0-indexed and the indices -have the slowest axis first and fastest changing axis last; i.e. for a 2-D +Note that, like C (and unlike Fortran), Python is 0-indexed and the indices +have the slowest axis first and fastest changing axis last; that is, for a 2D image, the fast axis (X-axis) which corresponds to the FITS NAXIS1 keyword, is -the second index. Similarly, the 1-indexed sub-section of x=11 to 20 +the second index. Similarly, the 1-indexed subsection of x=11 to 20 (inclusive) and y=31 to 40 (inclusive) would be given in Python as:: - >>> scidata[30:40, 10:20] + >>> data[30:40, 10:20] + array([[350, 349, 349, 348, 349, 348, 349, 347, 350, 348], + [348, 348, 348, 349, 348, 349, 347, 348, 348, 349], + [348, 348, 347, 349, 348, 348, 349, 349, 349, 349], + [349, 348, 349, 349, 350, 349, 349, 347, 348, 348], + [348, 348, 348, 348, 349, 348, 350, 349, 348, 349], + [348, 347, 349, 349, 350, 348, 349, 348, 349, 347], + [347, 348, 347, 348, 349, 349, 350, 349, 348, 348], + [349, 349, 350, 348, 350, 347, 349, 349, 349, 348], + [349, 348, 348, 348, 348, 348, 349, 347, 349, 348], + [349, 349, 349, 348, 350, 349, 349, 350, 348, 350]], dtype='>i2') -To update the value of a pixel or a sub-section:: +To update the value of a pixel or a subsection:: - >>> scidata[30:40, 10:20] = scidata[1, 4] = 999 + >>> data[30:40, 10:20] = data[1, 4] = 999 -This example changes the values of both the pixel \[1, 4] and the sub-section -\[30:40, 10:20] to the new value of 999. See the `Numpy documentation`_ for +This example changes the values of both the pixel \[1, 4] and the subsection +\[30:40, 10:20] to the new value of 999. See the `Numpy documentation`_ for more details on Python-style array indexing and slicing. The next example of array manipulation is to convert the image data from counts to flux:: - >>> photflam = hdulist[1].header['photflam'] - >>> exptime = prihdr['exptime'] - >>> scidata *= photflam / exptime + >>> photflam = hdul[1].header['photflam'] + >>> exptime = hdr['exptime'] + >>> data = data * photflam / exptime + >>> hdul.close() Note that performing an operation like this on an entire image requires holding -the entire image in memory. This example performs the multiplication in-place +the entire image in memory. This example performs the multiplication in-place so that no copies are made, but the original image must first be able to fit in -main memory. For most observations this should not be an issue on modern +main memory. For most observations this should not be an issue on modern personal computers. -If at this point you want to preserve all the changes you made and write it to -a new file, you can use the :meth:`HDUList.writeto` method (see below). +If at this point you want to preserve all of the changes you made and write it +to a new file, you can use the :meth:`HDUList.writeto` method (see below). -.. _Numpy documentation: http://docs.scipy.org/doc/numpy/reference/arrays.indexing.html +.. _Numpy documentation: https://numpy.org/doc/stable/reference/routines.indexing.html +.. note:: -Working With Table Data + See more information in :doc:`/io/fits/usage/image`. + +Working with Table Data ^^^^^^^^^^^^^^^^^^^^^^^ -If you are familiar with numpy `~numpy.recarray` (record array) objects, you -will find the table data is basically a record array with some extra -properties. But familiarity with record arrays is not a prerequisite for this -guide. +.. note:: + This section describes reading and writing table data in the FITS format + using the `~astropy.io.fits` package directly. If you want to read or write a single + entire table in FITS format, the you should consider using the :ref:`table_io` + interface (e.g., ``QTable.read``). See the :ref:`Unified I/O FITS ` + section for details. Like images, the data portion of a FITS table extension is in the ``.data`` attribute:: - >>> hdulist = fits.open('table.fits') - >>> tbdata = hdulist[1].data # assuming the first extension is a table + >>> fits_table_filename = fits.util.get_testdata_filepath('tb.fits') + >>> hdul = fits.open(fits_table_filename) + >>> data = hdul[1].data # assuming the first extension is a table + >>> hdul.close() + +If you are familiar with ``numpy`` `~numpy.recarray` (record array) objects, you +will find the table data is basically a record array with some extra +properties. But familiarity with record arrays is not a prerequisite for this +guide. To see the first row of the table:: - >>> print tbdata[0] - (1, 'abc', 3.7000002861022949, 0) + >>> print(data[0]) + (np.int32(1), 'abc', np.float64(3.7000000715255736), np.False_) Each row in the table is a :class:`FITS_record` object which looks like a (Python) tuple containing elements of heterogeneous data types. In this @@ -338,88 +530,102 @@ example: an integer, a string, a floating point number, and a Boolean value. So the table data are just an array of such records. More commonly, a user is likely to access the data in a column-wise way. This is accomplished by using the :meth:`~FITS_rec.field` method. To get the first column (or "field" in -Numpy parlance--it is used here interchangeably with "column") of the table, +NumPy parlance — it is used here interchangeably with "column") of the table, use:: - >>> tbdata.field(0) - array([1, 2]) + >>> data.field(0) + array([1, 2]...) -A numpy object with the data type of the specified field is returned. +A ``numpy`` object with the data type of the specified field is returned. Like header keywords, a column can be referred either by index, as above, or by name:: - >>> tbdata.field('id') - array([1, 2]) + >>> data.field('c1') + array([1, 2]...) When accessing a column by name, dict-like access is also possible (and even preferable):: - >>> tbdata['id'] - array([1, 2]) + >>> data['c1'] + array([1, 2]...) In most cases it is preferable to access columns by their name, as the column -name is entirely independent of its physical order in the table. As with +name is entirely independent of its physical order in the table. As with header keywords, column names are case-insensitive. -But how do we know what columns we have in a table? First, let's introduce +But how do we know what columns we have in a table? First, we will introduce another attribute of the table HDU: the :attr:`~BinTableHDU.columns` attribute:: - >>> cols = hdulist[1].columns + >>> cols = hdul[1].columns This attribute is a :class:`ColDefs` (column definitions) object. If we use the :meth:`ColDefs.info` method from the interactive prompt:: >>> cols.info() - name: - ['c1', 'c2', 'c3', 'c4'] - format: - ['1J', '3A', '1E', '1L'] - unit: - ['', '', '', ''] - null: - [-2147483647, '', '', ''] - bscale: - ['', '', 3, ''] - bzero: - ['', '', 0.40000000000000002, ''] - disp: - ['I11', 'A3', 'G15.7', 'L6'] - start: - ['', '', '', ''] - dim: - ['', '', '', ''] + name: + ['c1', 'c2', 'c3', 'c4'] + format: + ['1J', '3A', '1E', '1L'] + unit: + ['', '', '', ''] + null: + [-2147483647, '', '', ''] + bscale: + ['', '', 3, ''] + bzero: + ['', '', 0.4, ''] + disp: + ['I11', 'A3', 'G15.7', 'L6'] + start: + ['', '', '', ''] + dim: + ['', '', '', ''] + coord_type: + ['', '', '', ''] + coord_unit: + ['', '', '', ''] + coord_ref_point: + ['', '', '', ''] + coord_ref_value: + ['', '', '', ''] + coord_inc: + ['', '', '', ''] + time_ref_pos: + ['', '', '', ''] it will show the attributes of all columns in the table, such as their names, formats, bscales, bzeros, etc. A similar output that will display the column names and their formats can be printed from within a script with:: - print hdulist[1].columns + >>> hdul[1].columns + ColDefs( + name = 'c1'; format = '1J'; null = -2147483647; disp = 'I11' + name = 'c2'; format = '3A'; disp = 'A3' + name = 'c3'; format = '1E'; bscale = 3; bzero = 0.4; disp = 'G15.7' + name = 'c4'; format = '1L'; disp = 'L6' + ) -We can also get these properties individually; -e.g. - -:: +We can also get these properties individually; for example:: >>> cols.names - ['ID', 'name', 'mag', 'flag'] + ['c1', 'c2', 'c3', 'c4'] returns a (Python) list of field names. -Since each field is a Numpy object, we'll have the entire arsenal of Numpy -tools to use. We can reassign (update) the values:: +Since each field is a ``numpy`` object, we will have the entire arsenal of +``numpy`` tools to use. We can reassign (update) the values:: - >>> tbdata['flag'][:] = 0 + >>> data['c4'][:] = 0 take the mean of a column:: - >>> tbdata['mag'].mean() - >>> 84.4 + >>> data['c3'].mean() # doctest: +FLOAT_CMP + np.float64(5.19999989271164) and so on. - Save File Changes ^^^^^^^^^^^^^^^^^ @@ -428,23 +634,25 @@ header or data, the user can use :meth:`HDUList.writeto` to save the changes. This takes the version of headers and data in memory and writes them to a new FITS file on disk. Subsequent operations can be performed to the data in memory and written out to yet another different file, all without recopying the -original data to (more) memory. +original data to (more) memory: -:: +.. code:: python - >>> hdulist.writeto('newimage.fits') + hdul.writeto('newtable.fits') will write the current content of ``hdulist`` to a new disk file newfile.fits. If a file was opened with the update mode, the :meth:`HDUList.flush` method can -also be used to write all the changes made since :func:`open`, back to the +also be used to write all of the changes made since :func:`open`, back to the original file. The :meth:`~HDUList.close` method will do the same for a FITS -file opened with update mode:: +file opened with update mode: - >>> f = fits.open('original.fits', mode='update') - ... # making changes in data and/or header - >>> f.flush() # changes are written back to original.fits - >>> f.close() # closing the file will also flush any changes and prevent - ... # further writing +.. code:: python + + with fits.open('original.fits', mode='update') as hdul: + # Change something in hdul. + hdul.flush() # changes are written back to original.fits + + # closing the file will also flush any changes and prevent further writing Creating a New FITS File @@ -454,29 +662,29 @@ Creating a New Image File ^^^^^^^^^^^^^^^^^^^^^^^^^ So far we have demonstrated how to read and update an existing FITS file. But -how about creating a new FITS file from scratch? Such tasks are very easy in -Astropy for an image HDU. We'll first demonstrate how to create a FITS file -consisting only the primary HDU with image data. +how about creating a new FITS file from scratch? Such tasks are very convenient +in ``astropy`` for an image HDU. We will first demonstrate how to create a FITS +file consisting of only the primary HDU with image data. -First, we create a numpy object for the data part:: +First, we create a ``numpy`` object for the data part:: >>> import numpy as np - >>> n = np.arange(100.0) # a simple sequence of floats from 0.0 to 99.9 + >>> data = np.arange(100.0) # a simple sequence of floats from 0.0 to 99.0 Next, we create a :class:`PrimaryHDU` object to encapsulate the data:: - >>> hdu = fits.PrimaryHDU(n) + >>> hdu = fits.PrimaryHDU(data=data) -We then create a HDUList to contain the newly created primary HDU, and write to +We then create an :class:`HDUList` to contain the newly created primary HDU, and write to a new file:: - >>> hdulist = fits.HDUList([hdu]) - >>> hdulist.writeto('new.fits') + >>> hdul = fits.HDUList([hdu]) + >>> hdul.writeto('new1.fits') -That's it! In fact, Astropy even provides a shortcut for the last two lines to -accomplish the same behavior:: +That is it! In fact, ``astropy`` even provides a shortcut for the last two +lines to accomplish the same behavior:: - >>> hdu.writeto('new.fits') + >>> hdu.writeto('new2.fits') This will write a single HDU to a FITS file without having to manually encapsulate it in an :class:`HDUList` object first. @@ -485,23 +693,48 @@ encapsulate it in an :class:`HDUList` object first. Creating a New Table File ^^^^^^^^^^^^^^^^^^^^^^^^^ -To create a table HDU is a little more involved than image HDU, because a +.. note:: + + If you want to create a **binary** FITS table with no other HDUs, + you can use :class:`~astropy.table.Table` instead and then write to FITS. + This is less complicated than "lower-level" FITS interface:: + + >>> from astropy.table import Table + >>> t = Table([[1, 2], [4, 5], [7, 8]], names=('a', 'b', 'c')) + >>> t.write('table1.fits', format='fits') + + The equivalent code using ``astropy.io.fits`` would look like this: + + >>> from astropy.io import fits + >>> import numpy as np + >>> c1 = fits.Column(name='a', array=np.array([1, 2]), format='K') + >>> c2 = fits.Column(name='b', array=np.array([4, 5]), format='K') + >>> c3 = fits.Column(name='c', array=np.array([7, 8]), format='K') + >>> t = fits.BinTableHDU.from_columns([c1, c2, c3]) + >>> t.writeto('table2.fits') + +To create a table HDU is a little more involved than an image HDU, because a table's structure needs more information. First of all, tables can only be an extension HDU, not a primary. There are two kinds of FITS table extensions: -ASCII and binary. We'll use binary table examples here. +ASCII and binary. We will use binary table examples here. To create a table from scratch, we need to define columns first, by constructing the :class:`Column` objects and their data. Suppose we have two columns, the first containing strings, and the second containing floating point numbers:: - >>> from astropy.io import fits >>> import numpy as np >>> a1 = np.array(['NGC1001', 'NGC1002', 'NGC1003']) >>> a2 = np.array([11.1, 12.3, 15.2]) >>> col1 = fits.Column(name='target', format='20A', array=a1) >>> col2 = fits.Column(name='V_mag', format='E', array=a2) +.. note:: + + It is not necessary to create a :class:`Column` object explicitly + if the data is stored in a + `structured array `_. + Next, create a :class:`ColDefs` (column-definitions) object for all columns:: >>> cols = fits.ColDefs([col1, col2]) @@ -509,81 +742,114 @@ Next, create a :class:`ColDefs` (column-definitions) object for all columns:: Now, create a new binary table HDU object by using the :func:`BinTableHDU.from_columns` function:: - >>> tbhdu = fits.BinTableHDU.from_columns(cols) + >>> hdu = fits.BinTableHDU.from_columns(cols) This function returns (in this case) a :class:`BinTableHDU`. -Of course, you can do this more concisely without creating intermediate +The data structure used to represent FITS tables is called a :class:`FITS_rec` +and is derived from the :class:`numpy.recarray` interface. When creating +a new table HDU the individual column arrays will be assembled into a single +:class:`FITS_rec` array. + +You can create a :class:`BinTableHDU` more concisely without creating intermediate variables for the individual columns and without manually creating a :class:`ColDefs` object:: - - >>> from astropy.io import fits - >>> tbhdu = fits.BinTableHDU.from_columns( + >>> hdu = fits.BinTableHDU.from_columns( ... [fits.Column(name='target', format='20A', array=a1), ... fits.Column(name='V_mag', format='E', array=a2)]) Now you may write this new table HDU directly to a FITS file like so:: - >>> tbhdu.writeto('table.fits') + >>> hdu.writeto('table3.fits') This shortcut will automatically create a minimal primary HDU with no data and -prepend it to the table HDU to create a valid FITS file. If you require +prepend it to the table HDU to create a valid FITS file. If you require additional data or header keywords in the primary HDU you may still create a :class:`PrimaryHDU` object and build up the FITS file manually using an -:class:`HDUList`. +:class:`HDUList`, as described in the next section. -For example, first create a new :class:`Header` object to encapsulate any -keywords you want to include in the primary HDU, then as before create a -:class:`PrimaryHDU`:: +Creating a Multi-Extension FITS (MEF) file +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - >>> prihdr = fits.Header() - >>> prihdr['OBSERVER'] = 'Edwin Hubble' - >>> prihdr['COMMENT'] = "Here's some commentary about this FITS file." - >>> prihdu = fits.PrimaryHDU(header=prihdr) +In the previous examples we created files with a single meaningful extension (a +:class:`PrimaryHDU` or :class:`BinTableHDU`). To create a file with multiple +extensions we need to create extension HDUs and append them to an :class:`HDUList`. -When we create a new primary HDU with a custom header as in the above example, -this will automatically include any additional header keywords that are -*required* by the FITS format (keywords such as ``SIMPLE`` and ``NAXIS`` for -example). In general, users should not have to manually manage such keywords, -and should only create and modify observation-specific informational keywords. +First, we create some data for Image extensions and we place the data into +separate :class:`PrimaryHDU` and :class:`ImageHDU` objects:: -We then create a HDUList containing both the primary HDU and the newly created -table extension, and write to a new file:: + >>> import numpy as np + >>> primary_hdu = fits.PrimaryHDU(data=np.ones((3, 3))) + >>> image_hdu = fits.ImageHDU(data=np.ones((100, 100)), name="MYIMAGE") + >>> image_hdu2 = fits.ImageHDU(data=np.ones((10, 10, 10)), name="MYCUBE") - >>> thdulist = fits.HDUList([prihdu, tbhdu]) - >>> thdulist.writeto('table.fits') +A multi-extension FITS file is not constrained to be only imaging or table data, we +can mix them. To show this we'll use the example from the previous section to make a +:class:`BinTableHDU`:: -Alternatively, we can append the table to the HDU list we already created in -the image file section:: + >>> c1 = fits.Column(name='a', array=np.array([1, 2]), format='K') + >>> c2 = fits.Column(name='b', array=np.array([4, 5]), format='K') + >>> c3 = fits.Column(name='c', array=np.array([7, 8]), format='K') + >>> table_hdu = fits.BinTableHDU.from_columns([c1, c2, c3]) - >>> hdulist.append(tbhdu) - >>> hdulist.writeto('image_and_table.fits') +Now when we create the :class:`HDUList` we list all extensions we want to +include:: -The data structure used to represent FITS tables is called a :class:`FITS_rec` -and is derived from the :class:`numpy.recarray` interface. When creating -a new table HDU the individual column arrays will be assembled into a single -:class:`FITS_rec` array. + >>> hdul = fits.HDUList([primary_hdu, image_hdu, table_hdu]) + +Because :class:`HDUList` acts like a :class:`list` we can also append, for example, +an :class:`ImageHDU` to an already existing :class:`HDUList`:: + + >>> hdul.append(image_hdu2) + +Multi-extension :class:`HDUList` are treated just like those with only a +:class:`PrimaryHDU`, so to save the file use :func:`HDUList.writeto` as shown above. + +.. note:: + + The FITS standard enforces all files to have exactly one :class:`PrimaryHDU` that + is the first HDU present in the file. This standard is enforced during the call to + :func:`HDUList.writeto` and an error will be raised if it is not met. See the + ``output_verify`` option in :func:`HDUList.writeto` for ways to fix or ignore these + warnings. + +In the previous example the :class:`PrimaryHDU` contained actual data. In some cases it +is desirable to have a minimal :class:`PrimaryHDU` with only basic header information. +To do this, first create a new :class:`Header` object to encapsulate any keywords you want +to include in the primary HDU, then as before create a :class:`PrimaryHDU`:: + + >>> hdr = fits.Header() + >>> hdr['OBSERVER'] = 'Edwin Hubble' + >>> hdr['COMMENT'] = "Here's some commentary about this FITS file." + >>> empty_primary = fits.PrimaryHDU(header=hdr) -So far, we have covered the most basic features of `astropy.io.fits`. In the -following chapters we'll show more advanced examples and explain options in -each class and method. +When we create a new primary HDU with a custom header as in the above example, +this will automatically include any additional header keywords that are +*required* by the FITS format (keywords such as ``SIMPLE`` and ``NAXIS`` for +example). In general, users should not have to manually manage such keywords, +and should only create and modify observation-specific informational keywords. +We then create an HDUList containing both the primary HDU and any other HDUs want:: + + >>> hdul = fits.HDUList([empty_primary, image_hdu2, table_hdu]) + +.. _io-fits-intro-convenience-functions: Convenience Functions --------------------- -`astropy.io.fits` also provides several high level ("convenience") functions. -Such a convenience function is a "canned" operation to achieve one simple task. +`astropy.io.fits` also provides several high-level ("convenience") functions. +Such a convenience function is a "canned" operation to achieve one task. By using these "convenience" functions, a user does not have to worry about -opening or closing a file, all the housekeeping is done implicitly. +opening or closing a file; all of the housekeeping is done implicitly. .. warning:: - These functions are useful for interactive Python sessions and simple + These functions are useful for interactive Python sessions and less complex analysis scripts, but should not be used for application code, as they - are highly inefficient. For example, each call to :func:`getval` - requires re-parsing the entire FITS file. Code that makes repeated use + are highly inefficient. For example, each call to :func:`getval` + requires re-parsing the entire FITS file. Code that makes repeated use of these functions should instead open the file with :func:`open` and access the data structures directly. @@ -593,31 +859,28 @@ for this function. The rest of the arguments are optional and flexible to specify which HDU the user wants to access:: >>> from astropy.io.fits import getheader - >>> getheader('in.fits') # get default HDU (=0), i.e. primary HDU's header - >>> getheader('in.fits', 0) # get primary HDU's header - >>> getheader('in.fits', 2) # the second extension - >>> getheader('in.fits', 'sci') # the first HDU with EXTNAME='SCI' - >>> getheader('in.fits', 'sci', 2) # HDU with EXTNAME='SCI' and EXTVER=2 - >>> getheader('in.fits', ('sci', 2)) # use a tuple to do the same - >>> getheader('in.fits', ext=2) # the second extension - >>> getheader('in.fits', extname='sci') # first HDU with EXTNAME='SCI' - >>> getheader('in.fits', extname='sci', extver=2) + >>> hdr = getheader(fits_image_filename) # get default HDU (=0), i.e. primary HDU's header + >>> hdr = getheader(fits_image_filename, ext=0) # get primary HDU's header + >>> hdr = getheader(fits_image_filename, ext=2) # the second extension + >>> hdr = getheader(fits_image_filename, extname='sci') # the first HDU with EXTNAME='SCI' + >>> hdr = getheader(fits_image_filename, extname='sci', extver=2) # HDU with EXTNAME='SCI' and EXTVER=2 + >>> hdr = getheader(fits_image_filename, ext=('sci', 2)) # use a tuple to do the same Ambiguous specifications will raise an exception:: - >>> getheader('in.fits', ext=('sci', 1), extname='err', extver=2) - ... - TypeError: Redundant/conflicting extension arguments(s): {'ext': ('sci', - 1), 'args': (), 'extver': 2, 'extname': 'err'} + >>> getheader(fits_image_filename, ext=('sci', 1), extname='err', extver=2) + Traceback (most recent call last): + ... + TypeError: Redundant/conflicting extension arguments(s): ... After you get the header, you can access the information in it, such as getting and modifying a keyword value:: - >>> from astropy.io.fits import getheader - >>> hdr = getheader('in.fits', 1) # get first extension's header - >>> filter = hdr['filter'] # get the value of the keyword "filter' - >>> val = hdr[10] # get the 11th keyword's value - >>> hdr['filter'] = 'FW555' # change the keyword value + >>> fits_image_2_filename = fits.util.get_testdata_filepath('o4sp040b0_raw.fits') + >>> hdr = getheader(fits_image_2_filename, ext=0) # get primary hdu's header + >>> filter = hdr['filter'] # get the value of the keyword "filter' + >>> val = hdr[10] # get the 11th keyword's value + >>> hdr['filter'] = 'FW555' # change the keyword value For the header keywords, the header is like a dictionary, as well as a list. The user can access the keywords either by name or by numeric index, as @@ -628,10 +891,24 @@ further simplify to just one call, instead of two as shown in the above examples:: >>> from astropy.io.fits import getval - >>> flt = getval('in.fits', 'filter', 1) # get 1st extension's keyword - # FILTER's value - >>> val = getval('in.fits', 10, 'sci', 2) # get the 2nd sci extension's - # 11th keyword's value + >>> # get 0th extension's keyword FILTER's value + >>> getval(fits_image_2_filename, 'filter', ext=0) + 'Clear' + + >>> # get the 2nd sci extension's 11th keyword's value + >>> getval(fits_image_2_filename, 10, extname='sci', extver=2) + False + +To edit a single header value in the header for extension 0, use the +:func:`setval` function. For example, to change the value of the "filter" +keyword:: + + >>> fits.setval(fits_image_2_filename, "filter", value="FW555") # doctest: +SKIP + +This can also be used to create a new keyword-value pair ("card" in FITS +lingo):: + + >>> fits.setval(fits_image_2_filename, "ANEWKEY", value="some value") # doctest: +SKIP The function :func:`getdata` gets the data of an HDU. Similar to :func:`getheader`, it only requires the input FITS file name while the @@ -640,53 +917,70 @@ optional argument header. If header is set to True, this function will return both data and header, otherwise only data is returned:: >>> from astropy.io.fits import getdata - >>> dat = getdata('in.fits', 'sci', 3) # get 3rd sci extension's data - ... # get 1st extension's data and header - >>> data, hdr = getdata('in.fits', 1, header=True) + >>> # get 3rd sci extension's data: + >>> data = getdata(fits_image_filename, extname='sci', extver=3) + >>> # get 1st extension's data AND header: + >>> data, hdr = getdata(fits_image_filename, ext=1, header=True) The functions introduced above are for reading. The next few functions demonstrate convenience functions for writing:: - >>> fits.writeto('out.fits', data, header) + >>> fits.writeto('out.fits', data, hdr) The :func:`writeto` function uses the provided data and an optional header to write to an output FITS file. :: - >>> fits.append('out.fits', data, header) + >>> fits.append('out.fits', data, hdr) The :func:`append` function will use the provided data and the optional header to append to an existing FITS file. If the specified output file does not exist, it will create one. -:: +.. code:: python - >>> from astropy.io.fits import update - >>> update(file, dat, hdr, 'sci') # update the 'sci' extension - >>> update(file, dat, 3) # update the 3rd extension - >>> update(file, dat, hdr, 3) # update the 3rd extension - >>> update(file, dat, 'sci', 2) # update the 2nd SCI extension - >>> update(file, dat, 3, header=hdr) # update the 3rd extension - >>> update(file, dat, header=hdr, ext=5) # update the 5th extension + from astropy.io.fits import update + update(filename, dat, hdr, 'sci') # update the 'sci' extension + update(filename, dat, 3) # update the 3rd extension + update(filename, dat, hdr, 3) # update the 3rd extension + update(filename, dat, 'sci', 2) # update the 2nd SCI extension + update(filename, dat, 3, header=hdr) # update the 3rd extension + update(filename, dat, header=hdr, ext=5) # update the 5th extension The :func:`update` function will update the specified extension with the input -data/header. The 3rd argument can be the header associated with the data. If -the 3rd argument is not a header, it (and other positional arguments) are +data/header. The third argument can be the header associated with the data. If +the third argument is not a header, it (and other positional arguments) are assumed to be the extension specification(s). Header and extension specs can also be keyword arguments. +The :func:`printdiff` function will print a difference report of two FITS files, +including headers and data. The first two arguments must be two FITS +filenames or FITS file objects with matching data types (i.e., if using strings +to specify filenames, both inputs must be strings). The third +argument is an optional extension specification, with the same call format +of :func:`getheader` and :func:`getdata`. In addition you can add any keywords +accepted by the :class:`FITSDiff` class. + +.. code:: python + + from astropy.io.fits import printdiff + # get a difference report of ext 2 of inA and inB + printdiff('inA.fits', 'inB.fits', ext=2) + # ignore HISTORY and COMMENT keywords + printdiff('inA.fits', 'inB.fits', ignore_keywords=('HISTORY','COMMENT') + Finally, the :func:`info` function will print out information of the specified FITS file:: - >>> fits.info('test0.fits') - Filename: test0.fits - No. Name Type Cards Dimensions Format - 0 PRIMARY PrimaryHDU 138 () Int16 - 1 SCI ImageHDU 61 (400, 400) Int16 - 2 SCI ImageHDU 61 (400, 400) Int16 - 3 SCI ImageHDU 61 (400, 400) Int16 - 4 SCI ImageHDU 61 (400, 400) Int16 + >>> fits.info(fits_image_filename) + Filename: ...test0.fits + No. Name Ver Type Cards Dimensions Format + 0 PRIMARY 1 PrimaryHDU 138 () + 1 SCI 1 ImageHDU 61 (40, 40) int16 + 2 SCI 2 ImageHDU 61 (40, 40) int16 + 3 SCI 3 ImageHDU 61 (40, 40) int16 + 4 SCI 4 ImageHDU 61 (40, 40) int16 This is one of the most useful convenience functions for getting an overview of what a given file contains without looking at any of the details. @@ -704,7 +998,27 @@ Using `astropy.io.fits` usage/unfamiliar usage/scripts usage/misc - usage/examples + usage/cloud + +Command-Line Utilities +====================== + +For convenience, several of ``astropy``'s sub-packages install utility programs +on your system which allow common tasks to be performed without having +to open a Python interpreter. These utilities include: + +- `~astropy.io.fits.scripts.fitsheader`: prints the headers of a FITS file. + +- `~astropy.io.fits.scripts.fitscheck`: verifies and optionally rewrites + the CHECKSUM and DATASUM keywords of a FITS file. + +- :ref:`fitsdiff`: compares two FITS files and reports the differences. + +- :ref:`fits2bitmap`: converts FITS images to bitmaps, including scaling and + stretching. + +- :ref:`wcslint `: checks the :ref:`WCS ` keywords in a + FITS file for compliance against the standards. Other Information ================= @@ -716,23 +1030,21 @@ Other Information appendix/header_transition appendix/history +.. note that if this section gets too long, it should be moved to a separate + doc page - see the top of performance.inc.rst for the instructions on how to do + that + +.. include:: performance.inc.rst + Reference/API ============= .. automodule:: astropy.io.fits .. toctree:: - :maxdepth: 3 - - api/files.rst - api/hdulists.rst - api/hdus.rst - api/headers.rst - api/cards.rst - api/tables.rst - api/images.rst - api/diff.rst - api/verification.rst + :maxdepth: 2 + + api/index.rst .. rubric:: Footnotes diff --git a/docs/io/fits/performance.inc.rst b/docs/io/fits/performance.inc.rst new file mode 100644 index 000000000000..cb3827a8dae8 --- /dev/null +++ b/docs/io/fits/performance.inc.rst @@ -0,0 +1,52 @@ +.. note that if this is changed from the default approach of using an *include* + (in index.rst) to a separate performance page, the header needs to be changed + from === to ***, the filename extension needs to be changed from .inc.rst to + .rst, and a link needs to be added in the subpackage toctree + +.. _astropy-io-fits-performance: + +Performance Tips +================ + +Use dask for lazy compute +------------------------- + +It is possible to set the data array for :class:`~astropy.io.fits.PrimaryHDU` +and :class:`~astropy.io.fits.ImageHDU` to a `dask `_ array. +If this is written to disk, the dask array will be computed as it is being +written, which will avoid using excessive memory: + +.. doctest-requires:: dask + + >>> import dask.array as da + >>> array = da.random.random((1000, 1000)) + >>> from astropy.io import fits + >>> hdu = fits.PrimaryHDU(data=array) + >>> hdu.writeto('test_dask.fits') + +Arbitrary padding end of file will degrade performance +------------------------------------------------------ + +As discussed in detail in +`GitHub issue 19296 `_, +arbitrary padding at the end of a FITS file might cause the parser +to inefficiently search for the END card of the next header. +Therefore, we recommend that astropy users to not blindly open +untrusted large FITS files without independently verifying their fidelity +first. + +.. TODO: determine whether the following is quantitatively true, and either +.. uncomment or remove. + +.. Turn off memmap to run faster but use more memory +.. ------------------------------------------------- +.. +.. By default, :func:`astropy.io.fits.open` will open files using memory-mapping, +.. which means that the data is not necessarily read into memory until it is +.. needed. While memory-efficient, if memory is not a concern for you, you may +.. find that you can get better performance by turning memory mapping off, which +.. forces the data to be read into memory immediately: +.. +.. .. doctest-skip:: +.. +.. >>> fits.open('example.fits', memmap=False) diff --git a/docs/io/fits/usage/cloud.rst b/docs/io/fits/usage/cloud.rst new file mode 100644 index 000000000000..92ac44b87eb7 --- /dev/null +++ b/docs/io/fits/usage/cloud.rst @@ -0,0 +1,265 @@ +.. currentmodule:: astropy.io.fits + +.. _fits_io_cloud: + +Obtaining subsets from cloud-hosted FITS files +********************************************** + +Astropy offers support for extracting data from FITS files stored in the cloud. +Specifically, the `astropy.io.fits.open` function accepts the ``use_fsspec`` +and ``fsspec_kwargs`` parameters, which allow remote files to be accessed in an +efficient way using the |fsspec| package. + +``fsspec`` is an optional dependency of Astropy which supports reading +files from a range of remote and distributed storage backends, such as Amazon +and Google Cloud Storage. This chapter explains its use. + +.. note:: + + The examples in this chapter require ``fsspec`` which is an optional + dependency of Astropy. See :ref:`installing-astropy` for details on + installing optional dependencies. + + +Subsetting FITS files hosted on an HTTP web server +================================================== + +A common use case for ``fsspec`` is to read subsets of FITS data from a web +server which supports serving partial files via the +`Range Requests `__ feature of the +HTTP protocol. Most web servers support serving portions of files in this way. + +For example, let's assume you want to retrieve data from a large image obtained +by the Hubble Space Telescope available at the following url:: + + >>> # Download link for a large Hubble archive image (213 MB) + >>> url = "https://mast.stsci.edu/api/v0.1/Download/file/?uri=mast:HST/product/j8pu0y010_drc.fits" + +This file can be opened by passing the url to `astropy.io.fits.open`. +By default, Astropy will download the entire file to local disc before opening +it. This works fine for small files but tends to require a lot of time and +memory for large files. + +You can improve the performance for large files by passing the parameter +``use_fsspec=True`` to `open`. This will make Astropy use ``fsspec`` +to download only the necessary parts of the FITS file. +For example: + +.. doctest-requires:: fsspec + + >>> from astropy.io import fits + ... + >>> # `fits.open` will download the primary header + >>> with fits.open(url, use_fsspec=True) as hdul: # doctest: +REMOTE_DATA + ... + ... # Download a single header + ... header = hdul[1].header + ... + ... # Download a single data array + ... image = hdul[1].data + ... + ... # Download a 10-by-20 pixel cutout by using .section + ... cutout = hdul[2].section[10:20, 30:50] + +The example above requires less time and memory than would be required to +download the entire file. This is because ``fsspec`` is able to leverage +two *lazy data loading* features available in Astropy: + +1. The ``lazy_load_hdus`` parameter offered by `open` takes care of loading HDU + header and data attributes on demand rather than reading all HDUs at once. + This parameter is set to ``True`` by default. You do not need to pass it + explicitly, unless you changed its default value in the + :ref:`astropy:astropy_config`. +2. The `ImageHDU.section` and `CompImageHDU.section` properties enables a + subset of a data array to be read into memory without downloading the entire + image or cube. See the :ref:`astropy:data-sections` part of the documentation + for more details. + +Additional tips for achieving good performance when working with remote files +are provided in the :ref:`astropy:optimizing_fsspec` section further down +this page. + +.. note:: + + The `ImageHDU.section` and `CompImageHDU.section` feature is only efficient + for files that are not externally compressed (such as ``.fits.gz`` files). + Files that are compressed using internal tile compression should work properly. + Use ``.section`` on an externally compressed image will cause the whole FITS + file to be downloaded. + + +Subsetting FITS files hosted in Amazon S3 cloud storage +======================================================= + +The FITS file used in the example above also happens to be available via +Amazon cloud storage, where it is stored in a `public S3 bucket +`__ at the following location:: + + >>> s3_uri = "s3://stpubdata/hst/public/j8pu/j8pu0y010/j8pu0y010_drc.fits" + +With ``use_fsspec`` enabled, you can obtain a small cutout from a file stored +in Amazon S3 cloud storage in the same way as above. When opening paths with +prefix ``s3://`` (Amazon S3 Storage) or ``gs://`` (Google Cloud Storage), +`open` will automatically default to ``use_fsspec=True`` for convenience. +For example: + +.. doctest-requires:: fsspec + + >>> # Download a small 10-by-20 pixel cutout from a FITS file stored in Amazon S3 + >>> with fits.open(s3_uri, fsspec_kwargs={"anon": True}) as hdul: # doctest: +REMOTE_DATA + ... cutout = hdul[1].section[10:20, 30:50] + +Obtaining cutouts from Amazon S3 in this way may be particularly performant if +your code is running on a server in the same Amazon cloud region as the data. + +.. note:: + + To open paths with prefix ``s3://``, fsspec requires an optional dependency + called |s3fs|. A ``ModuleNotFoundError`` will be raised if this dependency + is missing. See :ref:`installing-astropy` for details on installing optional + dependencies. + + +Working with Amazon S3 access credentials +----------------------------------------- + +In the example above, we passed ``fsspec_kwargs={"anon": True}`` to enable the +data to be retrieved in an anonymous way without providing Amazon cloud access +credentials. This is possible because the data is located in a public S3 +bucket which has been configured to allow anonymous access. + +In some cases you may want to access data stored in an Amazon S3 data bucket +that is private or uses the "Requester Pays" feature. You will have to provide +a secret access key in this case to avoid encountering a ``NoCredentialsError``. +You can use the ``fsspec_kwargs`` parameter to pass extra arguments, such as +access keys, to the `fsspec.open` function as follows: + +.. doctest-skip:: + + >>> fsspec_kwargs = {"key": "YOUR-SECRET-KEY-ID", + ... "secret": "YOUR-SECRET-KEY"} + >>> with fits.open(s3_uri, fsspec_kwargs=fsspec_kwargs) as hdul: + ... cutout = hdul[2].section[10:20, 30:50] + +.. warning:: + + Including secret access keys inside Python code is dangerous because you + may accidentally end up revealing your keys when you share your code with + others. A better practice is to store your access keys via a configuration + file or environment variables. See the |s3fs| documentation for guidance. + + +Using :class:`~astropy.nddata.Cutout2D` with cloud-hosted FITS files +==================================================================== + +The examples above used the `ImageHDU.section` feature to download +small cutouts given a set of pixel coordinates. For astronomical images it is +often more convenient to obtain cutouts based on a sky position and angular +size rather than array coordinates. For this reason, Astropy provides the +`astropy.nddata.Cutout2D` tool which makes it easy to obtain cutouts informed +by an image's World Coordinate System (`~astropy.wcs.WCS`). + +This cutout tool can be used in combination with ``fsspec`` and ``.section``. +For example, assume you happen to know that the image we opened above contains +a nice edge-on galaxy at the following position:: + + >>> # Approximate location of an edge-on galaxy + >>> from astropy.coordinates import SkyCoord + >>> position = SkyCoord('10h01m41.13s 02d25m20.58s') + +We also know that the radius of the galaxy is approximately 5 arcseconds:: + + >>> # Approximate size of the galaxy + >>> from astropy import units as u + >>> size = 5*u.arcsec + +Given this sky position and radius, we can use `~astropy.nddata.Cutout2D` +in combination with ``use_fsspec=True`` and ``.section`` as follows: + +.. doctest-requires:: fsspec + + >>> from astropy.nddata import Cutout2D + >>> from astropy.wcs import WCS + ... + >>> with fits.open(s3_uri, use_fsspec=True, fsspec_kwargs={"anon": True}) as hdul: # doctest: +REMOTE_DATA + ... wcs = WCS(hdul[1].header) + ... cutout = Cutout2D(hdul[1].section, # use `.section` rather than `.data`! + ... position=position, + ... size=size, + ... wcs=wcs) + +See :ref:`cutout_images` for more details on this feature. + + +.. _optimizing_fsspec: + +Performance improvement tips for subsetting remote FITS files +============================================================= + +In the examples above we explained that it is important to use the +``use_fsspec=True`` feature in combination with the ``lazy_load_hdus=True`` +parameter and the ``ImageHDU.section`` feature to obtain good performance. + +There are two additional factors which significantly impact the performance +you will encounter, namely: (i) the structure of the FITS file, and (ii) the caching +and block size configuration of ``fsspec``. The remainder of this section +briefly explains these two factors. + +Matching the FITS file structure to the data slicing patterns +------------------------------------------------------------- + +The order in which multi-dimensional data is organized inside FITS files plays +a major role in the subsetting performance. + +Astropy uses the row-major order for indexing FITS data. This means that the +right-most axis is the one that varies the fastest inside the file. +Put differently, the data for the right-most dimension tends to be located in +contiguous regions of the file and is therefore the easiest to extract. + +For example, in the case of a 2D image, the slice ``.section[0, :]`` can be +obtained by downloading one contiguous region of bytes from the file. +In contrast, the slice ``.section[:, 0]`` requires accessing bytes spread +across the entire image array. The same is true for higher dimensions, +for example, obtaining the slice ``.section[0, :, :]`` from a 3D cube +will tend to be much faster than requesting ``.section[:, :, 0]``. + +Obtaining slices of data that are well matched to the internal layout of +the FITS file generally yields the best performance. +If subsetting performance is important to you, you may have to consider +modifying your FITS files to ensure that the ordering of the dimensions +is well-matched to your data slicing patterns. + +Configuring the ``fsspec`` block size and download strategy +----------------------------------------------------------- + +The ``fsspec`` package supports different data reading and caching strategies +which aim to find a balance between the number of network requests on one hand +and the total amount of data transferred on the other hand. By default, +``fsspec`` will attempt to download data in large contiguous blocks using a +buffered *read ahead* strategy, similar to the strategy that is employed +when operating systems load local files into memory. + +You can tune the performance of ``fsspec``'s buffering strategy by passing custom +``block_size`` and ``cache_type`` parameters to `fsspec.filesystem`, and passing +the filesystem into the ``fsspec_filesystem`` keyword argument in `astropy.io.fits.open`. +For example, we can configure fsspec to make buffered reads with a minimum +``block_size`` of 1 MB as follows: + +.. doctest-requires:: fsspec + + >>> import fsspec + >>> fsspec_fs = fsspec.filesystem( + ... protocol="s3", + ... anon=True, + ... block_size=1_000_000, + ... cache_type="bytes" + ... ) + >>> with fits.open( + ... s3_uri, fsspec_filesystem=fsspec_fs, fsspec_kwargs={"anon": True} + ... ) as hdul: # doctest: +REMOTE_DATA + ... cutout = hdul[1].section[10:20, 30:50] + +The ideal configuration will depend on the latency and throughput of the +network, as well as the exact shape and volume of the data you seek to obtain. + +See the |fsspec| documentation for more information on its options. diff --git a/docs/io/fits/usage/examples.rst b/docs/io/fits/usage/examples.rst deleted file mode 100644 index 8512521effc7..000000000000 --- a/docs/io/fits/usage/examples.rst +++ /dev/null @@ -1,69 +0,0 @@ -Examples --------- - -Converting a 3-color image (JPG) to separate FITS images -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. figure:: ../images/Hs-2009-14-a-web.jpg - :scale: 100 % - :align: center - :alt: Starting image - -.. container:: figures - - .. figure:: ../images/Red.jpg - :scale: 50 - :alt: Red color information - - Red color information - - .. figure:: ../images/Green.jpg - :scale: 50 - :alt: Green color information - - Green color information - - .. figure:: ../images/Blue.jpg - :scale: 50 - :alt: Blue color information - - Blue color information - -:: - - #!/usr/bin/env python - import numpy - import Image - - from astropy.io import fits - - #get the image and color information - image = Image.open('hs-2009-14-a-web.jpg') - #image.show() - xsize, ysize = image.size - r, g, b = image.split() - rdata = r.getdata() # data is now an array of length ysize\*xsize - gdata = g.getdata() - bdata = b.getdata() - - # create numpy arrays - npr = numpy.reshape(rdata, (ysize, xsize)) - npg = numpy.reshape(gdata, (ysize, xsize)) - npb = numpy.reshape(bdata, (ysize, xsize)) - - # write out the fits images, the data numbers are still JUST the RGB - # scalings; don't use for science - red = fits.PrimaryHDU(data=npr) - red.header['LATOBS'] = "32:11:56" # add spurious header info - red.header['LONGOBS'] = "110:56" - red.writeto('red.fits') - - green = fits.PrimaryHDU(data=npg) - green.header['LATOBS'] = "32:11:56" - green.header['LONGOBS'] = "110:56" - green.writeto('green.fits') - - blue = fits.PrimaryHDU(data=npb) - blue.header['LATOBS'] = "32:11:56" - blue.header['LONGOBS'] = "110:56" - blue.writeto('blue.fits') diff --git a/docs/io/fits/usage/headers.rst b/docs/io/fits/usage/headers.rst index 54edf555184e..8616c31d2b8e 100644 --- a/docs/io/fits/usage/headers.rst +++ b/docs/io/fits/usage/headers.rst @@ -1,86 +1,90 @@ -.. doctest-skip-all - .. currentmodule:: astropy.io.fits FITS Headers ------------- +************ -In the next three chapters, more detailed information as well as examples will +In the next three chapters, more detailed information including examples will be explained for manipulating FITS headers, image/array data, and table data respectively. Header of an HDU -^^^^^^^^^^^^^^^^ +================ -Every HDU normally has two components: header and data. In Astropy these two -components are accessed through the two attributes of the HDU, -``hdu.header`` and ``hdu.data``. +Every Header Data Unit (HDU) normally has two components: header and data. In +``astropy`` these two components are accessed through the two attributes of the +HDU, ``hdu.header`` and ``hdu.data``. -While an HDU may have empty data, i.e. the ``.data`` attribute is `None`, any -HDU will always have a header. When an HDU is created with a constructor, e.g. -``hdu = PrimaryHDU(data, header)``, the user may supply the header value from -an existing HDU's header and the data value from a numpy array. If the +While an HDU may have empty data (i.e., the ``.data`` attribute is `None`), any +HDU will always have a header. When an HDU is created with a constructor (e.g., +``hdu = PrimaryHDU(data, header)``), the user may supply the header value from +an existing HDU's header and the data value from a ``numpy`` array. If the defaults (None) are used, the new HDU will have the minimal required keywords for an HDU of that type:: + >>> from astropy.io import fits >>> hdu = fits.PrimaryHDU() >>> hdu.header # show the all of the header cards - SIMPLE = T / conforms to FITS standard - BITPIX = 8 / array data type - NAXIS = 0 / number of array dimensions - EXTEND = T + SIMPLE = T / conforms to FITS standard + BITPIX = 8 / array data type + NAXIS = 0 / number of array dimensions + EXTEND = T -A user can use any header and any data to construct a new HDU. Astropy will +A user can use any header and any data to construct a new HDU. ``astropy`` will strip any keywords that describe the data structure leaving only your -informational keywords. Later it will add back in the required structural -keywords for compatibility with the new HDU and any data added to it. So, a +informational keywords. Later it will add back in the required structural +keywords for compatibility with the new HDU and any data added to it. So, a user can use a table HDU's header to construct an image HDU and vice versa. The constructor will also ensure the data type and dimension information in the header agree with the data. The Header Attribute -^^^^^^^^^^^^^^^^^^^^ +==================== Value Access, Updating, and Creating -"""""""""""""""""""""""""""""""""""" +------------------------------------ As shown in the :ref:`Getting Started ` tutorial, keyword values can -be accessed via keyword name or index of an HDU's header attribute. Here is a -quick summary:: - - >>> hdulist = fits.open('input.fits') # open a FITS file - >>> prihdr = hdulist[0].header # the primary HDU header - >>> print prihdr[3] # get the 4th keyword's value - 10 - >>> prihdr[3] = 20 # change its value - >>> prihdr['DARKCORR'] # get the value of the keyword 'darkcorr' +be accessed via keyword name or index of an HDU's header attribute. You can +also use the wildcard character ``*`` to get the keyword value pairs that match +your search string. Here is a quick summary:: + + >>> fits_image_filename = fits.util.get_testdata_filepath('test0.fits') + >>> hdul = fits.open(fits_image_filename) # open a FITS file + >>> hdr = hdul[0].header # the primary HDU header + >>> print(hdr[34]) # get the 2nd keyword's value + 96 + >>> hdr[34] = 20 # change its value + >>> hdr['DARKCORR'] # get the value of the keyword 'darkcorr' 'OMIT' - >>> prihdr['darkcorr'] = 'PERFORM' # change darkcorr's value + >>> hdr['DARKCOR*'] # get keyword values using wildcard matching + DARKCORR= 'OMIT ' / Do dark correction: PERFORM, OMIT, COMPLETE + >>> hdr['darkcorr'] = 'PERFORM' # change darkcorr's value Keyword names are case-insensitive except in a few special cases (see the -sections on HIERARCH card and record-valued cards). Thus, ``prihdr['abc']``, -``prihdr['ABC']``, or ``prihdr['aBc']`` are all equivalent. +sections on HIERARCH card and record-valued cards). Thus, ``hdr['abc']``, +``hdr['ABC']``, or ``hdr['aBc']`` are all equivalent. -Like with Python's :class:`dict` type, new keywords can also be added to the +As with Python's :class:`dict` type, new keywords can also be added to the header using assignment syntax:: - >>> 'DARKCORR' in header # Check for existence + >>> hdr = hdul[1].header + >>> 'DARKCORR' in hdr # Check for existence False - >>> header['DARKCORR'] = 'OMIT' # Add a new DARKCORR keyword + >>> hdr['DARKCORR'] = 'OMIT' # Add a new DARKCORR keyword You can also add a new value *and* comment by assigning them as a tuple:: - >>> header['DARKCORR'] = ('OMIT', 'Dark Image Subtraction') + >>> hdr['DARKCORR'] = ('OMIT', 'Dark Image Subtraction') .. note:: An important point to note when adding new keywords to a header is that by default they are not appended *immediately* to the end of the file. - Rather, they are appended to the last non-commentary keyword. This is in + Rather, they are appended to the last non-commentary keyword. This is in order to support the common use case of always having all HISTORY keywords - grouped together at the end of a header. A new non-commentary keyword will + grouped together at the end of a header. A new non-commentary keyword will be added at the end of the existing keywords, but before any HISTORY/COMMENT keywords at the end of the header. @@ -88,65 +92,66 @@ You can also add a new value *and* comment by assigning them as a tuple:: * Use the :meth:`Header.append` method with the ``end=True`` argument: - >>> header.append(('DARKCORR', 'OMIT', 'Dark Image Subtraction'), - end=True) + >>> hdr.append(('DARKCORR', 'OMIT', 'Dark Image Subtraction'), end=True) This forces the new keyword to be added at the actual end of the header. * The :meth:`Header.insert` method will always insert a new keyword exactly where you ask for it: - >>> header.insert(20, ('DARKCORR', 'OMIT', 'Dark Image Subtraction')) + >>> del hdr['DARKCORR'] # Delete previous insertion for doctest + >>> hdr.insert(20, ('DARKCORR', 'OMIT', 'Dark Image Subtraction')) - This inserts the DARKCORR keyword before the 20th keyword in the header - no matter what it is. + This inserts the DARKCORR keyword before the 20th keyword in the + header no matter what it is. A keyword (and its corresponding card) can be deleted using the same index/name syntax:: - >>> del prihdr[3] # delete the 2nd keyword - >>> del prihdr['abc'] # get the value of the keyword 'abc' + >>> del hdr[3] # delete the 2nd keyword + >>> del hdr['DARKCORR'] # delete the value of the keyword 'DARKCORR' Note that, like a regular Python list, the indexing updates after each delete, -so if ``del prihdr[3]`` is done two times in a row, the 4th and 5th keywords -are removed from the original header. Likewise, ``del prihdr[-1]`` will delete +so if ``del hdr[3]`` is done two times in a row, the fourth and fifth keywords +are removed from the original header. Likewise, ``del hdr[-1]`` will delete the last card in the header. It is also possible to delete an entire range of cards using the slice syntax:: - >>> del prihdr[3:5] + >>> del hdr[3:5] -The method :meth:`Header.set` is another way to update they value or comment -associated with an existing keyword, or to create a new keyword. Most of its -functionality can be duplicated with the dict-like syntax shown above. But in -some cases it might be more clear. It also has the advantage of allowing one -to either move cards within the header, or specify the location of a new card +The method :meth:`Header.set` is another way to update the value or comment +associated with an existing keyword, or to create a new keyword. Most of its +functionality can be duplicated with the dict-like syntax shown above. But in +some cases it might be more clear. It also has the advantage of allowing a user +to either move cards within the header or specify the location of a new card relative to existing cards:: - >>> prihdr.set('target', 'NGC1234', 'target name') + >>> hdr.set('target', 'NGC1234', 'target name') >>> # place the next new keyword before the 'TARGET' keyword - >>> prihdr.set('newkey', 666, before='TARGET') # comment is optional + >>> hdr.set('newkey', 666, before='TARGET') # comment is optional >>> # place the next new keyword after the 21st keyword - >>> prihdr.set('newkey2', 42.0, 'another new key', after=20) + >>> hdr.set('newkey2', 42.0, 'another new key', after=20) In FITS headers, each keyword may also have a comment associated with it -explaining its purpose. The comments associated with each keyword are accessed +explaining its purpose. The comments associated with each keyword are accessed through the :attr:`~Header.comments` attribute:: - >>> header['NAXIS'] + >>> hdr['NAXIS'] 2 - >>> header.comments['NAXIS'] - the number of image axes - >>> header.comments['NAXIS'] = 'The number of image axes' # Update - -Comments can be accessed in all the same ways that values are accessed, whether -by keyword name or card index. Slices are also possible. The only difference -is that you go through ``header.comments`` instead of just ``header`` by + >>> hdr.comments['NAXIS'] + 'number of data axes' + >>> hdr.comments['NAXIS'] = 'The number of image axes' # Update + >>> hdul.close() # close the HDUList again + +Comments can be accessed in all of the same ways that values are accessed, +whether by keyword name or card index. Slices are also possible. The only +difference is that you go through ``hdr.comments`` instead of just ``hdr`` by itself. COMMENT, HISTORY, and Blank Keywords -"""""""""""""""""""""""""""""""""""" +------------------------------------ Most keywords in a FITS header have unique names. If there are more than two cards sharing the same name, it is the first one accessed when referred by @@ -157,20 +162,32 @@ to as commentary cards), which commonly appear in FITS headers more than once. They are (1) blank keyword, (2) HISTORY, and (3) COMMENT. Unlike other keywords, when accessing these keywords they are returned as a list:: - >>> prihdr['HISTORY'] + >>> filename = fits.util.get_testdata_filepath('history_header.fits') + >>> with fits.open(filename) as hdul: # open a FITS file + ... hdr = hdul[0].header + + >>> hdr['HISTORY'] I updated this file on 02/03/2011 I updated this file on 02/04/2011 - .... -These lists can be sliced like any other list. For example, to display just the -last HISTORY entry, use ``prihdr['history'][-1]``. Existing commentary cards +These lists can be sliced like any other list. For example, to display just the +last HISTORY entry, use ``hdr['history'][-1]``. Existing commentary cards can also be updated by using the appropriate index number for that card. New commentary cards can be added like any other card by using the dict-like -keyword assignment syntax, or by using the :meth:`Header.set` method. However, +keyword assignment syntax, or by using the :meth:`Header.set` method. However, unlike with other keywords, a new commentary card is always added and appended to the last commentary card with the same keyword, rather than to the end of -the header. Here is an example:: +the header. + +Example +^^^^^^^ + +.. + EXAMPLE START + Manipulating FITS Headers in astropy.io.fits + +To add a new commentary card:: >>> hdu.header['HISTORY'] = 'history 1' >>> hdu.header[''] = 'blank 1' @@ -199,31 +216,63 @@ commentary card by using the :meth:`Header.insert` method. Ironically, there is no comment in a commentary card, only a string value. +.. + EXAMPLE END + +Undefined Values +---------------- + +FITS headers can have undefined values and these are represented in Python +with the special value `None`. `None` can be used when assigning values +to a `~astropy.io.fits.Header` or `~astropy.io.fits.Card`. + + >>> hdr = fits.Header() + >>> hdr['UNDEF'] = None + >>> hdr['UNDEF'] is None + True + >>> repr(hdr) + 'UNDEF = ' + >>> hdr.append('UNDEF2') + >>> hdr['UNDEF2'] is None + True + >>> hdr.append(('UNDEF3', None, 'Undefined value')) + >>> str(hdr.cards[-1]) + 'UNDEF3 = / Undefined value ' + Card Images -^^^^^^^^^^^ +=========== A FITS header consists of card images. A card image in a FITS header consists of a keyword name, a value, and -optionally a comment. Physically, it takes 80 columns (bytes)--without carriage -return--in a FITS file's storage format. In Astropy, each card image is +optionally a comment. Physically, it takes 80 columns (bytes) — without carriage +return — in a FITS file's storage format. In ``astropy``, each card image is manifested by a :class:`Card` object. There are also special kinds of cards: commentary cards (see above) and card images taking more than one 80-column -card image. The latter will be discussed later. +card image. The latter will be discussed later. Most of the time the details of dealing with cards are handled by the :class:`Header` object, and it is not necessary to directly manipulate cards. In fact, most :class:`Header` methods that accept a ``(keyword, value)`` or ``(keyword, value, comment)`` tuple as an argument can also take a -:class:`Card` object as an argument. :class:`Card` objects are just wrappers +:class:`Card` object as an argument. :class:`Card` objects are just wrappers around such tuples that provide the logic for parsing and formatting individual -cards in a header. But there's usually nothing gained by manually using a +cards in a header. There is usually nothing gained by manually using a :class:`Card` object, except to examine how a card might appear in a header before actually adding it to the header. A new Card object is created with the :class:`Card` constructor: -``Card(key, value, comment)``. For example:: +``Card(key, value, comment)``. + +Example +------- + +.. + EXAMPLE START + Card Images in FITS Headers in astropy.io.fits + +To create a new Card object:: >>> c1 = fits.Card('TEMP', 80.0, 'temperature, floating value') >>> c2 = fits.Card('DETECTOR', 1) # comment is optional @@ -232,12 +281,12 @@ A new Card object is created with the :class:`Card` constructor: >>> c4 = fits.Card('ABC', 2+3j, 'complex value') >>> c5 = fits.Card('OBSERVER', 'Hubble', 'string value') - >>> print c1; print c2; print c3; print c4; print c5 # show the cards - TEMP = 80.0 / temperature, floating value - DETECTOR= 1 / - MIR_REVR= T / mirror reversed? Boolean value - ABC = (2.0, 3.0) / complex value - OBSERVER= 'Hubble ' / string value + >>> print(c1); print(c2); print(c3); print(c4); print(c5) # show the cards + TEMP = 80.0 / temperature, floating value + DETECTOR= 1 + MIR_REVR= T / mirror reversed? Boolean value + ABC = (2.0, 3.0) / complex value + OBSERVER= 'Hubble ' / string value Cards have the attributes ``.keyword``, ``.value``, and ``.comment``. Both ``.value`` and ``.comment`` can be changed but not the ``.keyword`` attribute. @@ -247,130 +296,151 @@ keyword. The :meth:`Card` constructor will check if the arguments given are conforming to the FITS standard and has a fixed card image format. If the user wants to create a card with a customized format or even a card which is not conforming -to the FITS standard (e.g. for testing purposes), the :meth:`Card.fromstring` +to the FITS standard (e.g., for testing purposes), the :meth:`Card.fromstring` class method can be used. -Cards can be verified with :meth:`Card.verify`. The non-standard card ``c2`` in +Cards can be verified with :meth:`Card.verify`. The nonstandard card ``c2`` in the example below is flagged by such verification. More about verification in -Astropy will be discussed in a later chapter. +``astropy`` will be discussed in a later chapter. :: - >>> c1 = fits.Card.fromstring('ABC = 3.456D023') + >>> c1 = fits.Card.fromstring('ABC = 3.456D023') >>> c2 = fits.Card.fromstring("P.I. ='Hubble'") - >>> print c1; print c2 - ABC = 3.456D023 + >>> print(c1) + ABC = 3.456D023 + >>> print(c2) # doctest: +SKIP P.I. ='Hubble' - >>> c2.verify() + >>> c2.verify() # doctest: +SKIP Output verification result: Unfixable error: Illegal keyword name 'P.I.' A list of the :class:`Card` objects underlying a :class:`Header` object can be -accessed with the :attr:`Header.cards` attribute. This list is only meant for -observing, and should not be directly manipulated. In fact, it is only a -copy--modifications to it will not affect the header it came from. Use the -methods provided by the :class:`Header` class instead. +accessed with the :attr:`Header.cards` attribute. This list is only meant for +observing, and should not be directly manipulated. In fact, it is only a +copy — modifications to it will not affect the header from which it came. Use +the methods provided by the :class:`Header` class instead. + +.. + EXAMPLE END CONTINUE Cards -^^^^^^^^^^^^^^ +============== + +The fact that the FITS standard only allows up to eight characters for the +keyword name and 80 characters to contain the keyword, the value, and the +comment is restrictive for certain applications. To allow long string values +for keywords, a proposal was made in: + + https://heasarc.gsfc.nasa.gov/docs/heasarc/ofwg/docs/ofwg_recomm/r13.html -The fact that the FITS standard only allows up to 8 characters for the keyword -name and 80 characters to contain the keyword, the value, and the comment is -restrictive for certain applications. To allow long string values for keywords, -a proposal was made in: +by using the CONTINUE keyword after the regular 80 column containing the +keyword. ``astropy`` does support this convention, which is a part of the FITS +standard since version 4.0. - http://legacy.gsfc.nasa.gov/docs/heasarc/ofwg/docs/ofwg_recomm/r13.html +Examples +-------- -by using the CONTINUE keyword after the regular 80-column containing the -keyword. Astropy does support this convention, even though it is not a FITS -standard. The examples below show the use of CONTINUE is automatic for long +.. + EXAMPLE START + CONTINUE Cards for Long String Values in astropy.io.fits + +The examples below show that the use of CONTINUE is automatic for long string values:: - >>> header = fits.Header() - >>> header['abc'] = 'abcdefg' * 20 - >>> header - ABC = 'abcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcd&' - CONTINUE 'efgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefga&' - CONTINUE 'bcdefg&' - >>> header['abc'] + >>> hdr = fits.Header() + >>> hdr['abc'] = 'abcdefg' * 20 + >>> hdr + ABC = 'abcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcd&' + CONTINUE 'efgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefga&' + CONTINUE 'bcdefg' + >>> hdr['abc'] 'abcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefg' >>> # both value and comments are long - >>> header['abc'] = ('abcdefg' * 10, 'abcdefg' * 10) - >>> header - ABC = 'abcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcd&' - CONTINUE 'efg&' - CONTINUE '&' / abcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefga - CONTINUE '&' / bcdefg - -Note that when a CONTINUE card is used, at the end of each 80-characters card + >>> hdr['abc'] = ('abcdefg' * 10, 'abcdefg' * 10) + >>> hdr + ABC = 'abcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcd&' + CONTINUE 'efg&' + CONTINUE '&' / abcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefga + CONTINUE '' / bcdefg + +Note that when a CONTINUE card is used, at the end of each 80-character card image, an ampersand is present. The ampersand is not part of the string value. -Also, there is no "=" at the 9th column after CONTINUE. In the first example, -the entire 240 characters is treated by Astropy as a single card. So, if it is -the nth card in a header, the (n+1)th card refers to the next keyword, not the -next CONTINUE card. As such, CONTINUE cards are transparently handled by -Astropy as a single logical card, and it's generally not necessary to worry -about the details of the format. Keywords that resolve to a set of CONTINUE +Also, there is no "=" at the ninth column after CONTINUE. In the first example, +the entire 240 characters is treated by ``astropy`` as a single card. So, if it +is the nth card in a header, the (n+1)th card refers to the next keyword, not +the next CONTINUE card. As such, CONTINUE cards are transparently handled by +``astropy`` as a single logical card, and it is generally not necessary to worry +about the details of the format. Keywords that resolve to a set of CONTINUE cards can be accessed and updated just like regular keywords. +.. + EXAMPLE END + HIERARCH Cards -^^^^^^^^^^^^^^ - -For keywords longer than 8 characters, there is a convention originated at ESO -to facilitate such use. It uses a special keyword HIERARCH with the actual long -keyword following. Astropy supports this convention as well. - -If a keyword contains more than 8 characters Astropy will automatically use a -HIERARCH card, but will also issue a warning in case this is in error. -However, one may explicitly request a HIERARCH card by prepending the keyword -with 'HIERARCH ' (just as it would appear in the header). For example, -``header['HIERARCH abcdefghi']`` will create the keyword ``abcdefghi`` without -displaying a warning. Once created, HIERARCH keywords can be accessed like any -other: ``header['abcdefghi']``, without prepending 'HIERARCH' to the keyword. -HIEARARCH keywords also differ from normal FITS keywords in that they are -case-sensitive. - -Examples follow:: - - >>> c = fits.Card('abcdefghi', 10) - Keyword name 'abcdefghi' is greater than 8 characters; a HIERARCH card will - be created. - >>> print c +============== + +For keywords longer than eight characters, there is a convention originated at +the European Southern Observatory (ESO) to facilitate such use. It uses a +special keyword HIERARCH with the actual long keyword following. ``astropy`` +supports this convention as well. + +If a keyword contains more than eight characters ``astropy`` will automatically +use a HIERARCH card, but will also issue a warning in case this is in error. +However, you may explicitly request a HIERARCH card by prepending the keyword +with 'HIERARCH ' (just as it would appear in the header). For example, +``hdr['HIERARCH abcdefghi']`` will create the keyword ``abcdefghi`` without +displaying a warning. Once created, HIERARCH keywords can be accessed like any +other: ``hdr['abcdefghi']``, without prepending 'HIERARCH' to the keyword. + +Examples +-------- + +.. + EXAMPLE START + HIERARCH Cards for Keywords Longer than Eight Characters in astropy.io.fits + +``astropy`` will use a HIERARCH card and issue a warning when keywords contain +more than eight characters:: + + >>> # this will result in a Warning because a HIERARCH card is implicitly created + >>> c = fits.Card('abcdefghi', 10) # doctest: +SKIP + >>> print(c) # doctest: +SKIP HIERARCH abcdefghi = 10 >>> c = fits.Card('hierarch abcdefghi', 10) - >>> print c + >>> print(c) HIERARCH abcdefghi = 10 - >>> h = fits.PrimaryHDU() - >>> h.header['hierarch abcdefghi'] = 99 - >>> h.header['abcdefghi'] + >>> hdu = fits.PrimaryHDU() + >>> hdu.header['hierarch abcdefghi'] = 99 + >>> hdu.header['abcdefghi'] 99 - >>> h.header['abcdefghi'] = 10 - >>> h.header['abcdefghi'] + >>> hdu.header['abcdefghi'] = 10 + >>> hdu.header['abcdefghi'] 10 - >>> h.header['ABCDEFGHI'] - Traceback (most recent call last): - ... - KeyError: "Keyword 'ABCDEFGI.' not found." - >>> h.header - SIMPLE = T / conforms to FITS standard - BITPIX = 8 / array data type - NAXIS = 0 / number of array dimensions - EXTEND = T - HIERARCH abcdefghi = 1000 + >>> hdu.header + SIMPLE = T / conforms to FITS standard + BITPIX = 8 / array data type + NAXIS = 0 / number of array dimensions + EXTEND = T + HIERARCH abcdefghi = 10 + +.. + EXAMPLE END .. note:: A final point to keep in mind about the :class:`Header` class is that much of its design is intended to abstract away quirks about the FITS format. - This is why, for example, it will automatically created CONTINUE and - HIERARCH cards. The Header is just a data structure, and as user you - shouldn't have to worry about how it ultimately gets serialized to a header + This is why, for example, it will automatically create CONTINUE and + HIERARCH cards. The Header is just a data structure, and as a user you + should not have to worry about how it ultimately gets serialized to a header in a FITS file. - Though there are some areas where it's almost impossible to hide away the - quirks of the FITS format, Astropy tries to make it so that you have to - think about it as little as possible. If there are any areas where you - have concern yourself unnecessarily about how the header is constructed, - then let help@stsci.edu know, as there are probably areas where this can be + Though there are some areas where it is almost impossible to hide away the + quirks of the FITS format, ``astropy`` tries to make it so that you have to + think about it as little as possible. If there are any areas that are left + vague or difficult to understand about how the header is constructed, please + let us know, as there are probably areas where this can be improved on even more. diff --git a/docs/io/fits/usage/image.rst b/docs/io/fits/usage/image.rst index 2352106492df..c3bec5a536da 100644 --- a/docs/io/fits/usage/image.rst +++ b/docs/io/fits/usage/image.rst @@ -1,22 +1,20 @@ -.. doctest-skip-all - .. currentmodule:: astropy.io.fits Image Data ----------- +********** -In this chapter, we'll discuss the data component in an image HDU. +In this chapter, we will discuss the data component in an image HDU. Image Data as an Array -^^^^^^^^^^^^^^^^^^^^^^ +====================== A FITS primary HDU or an image extension HDU may contain image data. The -following discussions apply to both of these HDU classes. In Astropy, for most -cases, it is just a simple numpy array, having the shape specified by the NAXIS -keywords and the data type specified by the BITPIX keyword - unless the data is -scaled, see next section. Here is a quick cross reference between allowed -BITPIX values in FITS images and the numpy data types: +following discussions apply to both of these HDU classes. For most cases in +``astropy``, it is a ``numpy`` array, having the shape specified by the NAXIS +keywords and the data type specified by the BITPIX keyword — unless the data is +scaled, in which case see the next section. Here is a quick cross reference +between allowed BITPIX values in FITS images and the ``numpy`` data types: .. parsed-literal:: @@ -24,39 +22,72 @@ BITPIX values in FITS images and the numpy data types: 8 numpy.uint8 (note it is UNsigned integer) 16 numpy.int16 32 numpy.int32 + 64 numpy.int64 -32 numpy.float32 -64 numpy.float64 -To recap the fact that in numpy the arrays are 0-indexed and the axes are +To recap, in ``numpy`` the arrays are 0-indexed and the axes are ordered from slow to fast. So, if a FITS image has NAXIS1=300 and NAXIS2=400, -the numpy array of its data will have the shape of (400, 300). +the ``numpy`` array of its data will have the shape of (400, 300). + +Examples +-------- + +.. note:: + + The ``astropy.io.fits.util.get_testdata_filepath()`` function, + used in the examples here, returns file path for test data shipped with ``astropy``. + To work with your own data instead, please use :func:`astropy.io.fits.open` or :ref:`io-fits-intro-convenience-functions`, + which take either the relative or absolute path as string or :term:`python:path-like object`. + +.. + EXAMPLE START + Image Data as an Array in astropy.io.fits Here is a summary of reading and updating image data values:: - >>> f = fits.open('image.fits') # open a FITS file - >>> scidata = f[1].data # assume the first extension is an image - >>> print scidata[1,4] # get the pixel value at x=5, y=2 - >>> scidata[30:40, 10:20] # get values of the subsection - ... # from x=11 to 20, y=31 to 40 (inclusive) - >>> scidata[1,4] = 999 # update a pixel value - >>> scidata[30:40, 10:20] = 0 # update values of a subsection - >>> scidata[3] = scidata[2] # copy the 3rd row to the 4th row + >>> from astropy.io import fits + >>> fits_image_filename = fits.util.get_testdata_filepath('test0.fits') + + >>> with fits.open(fits_image_filename) as hdul: # open a FITS file + ... data = hdul[1].data # assume the first extension is an image + >>> print(data[1, 4]) # get the pixel value at x=5, y=2 + 313 + >>> # get values of the subsection from x=11 to 20, y=31 to 40 (inclusive) + >>> data[30:40, 10:20] + array([[314, 314, 313, 312, 313, 313, 313, 313, 313, 312], + [314, 314, 312, 313, 313, 311, 313, 312, 312, 314], + [314, 315, 313, 313, 313, 313, 315, 312, 314, 312], + [314, 313, 313, 314, 311, 313, 313, 313, 313, 313], + [313, 314, 312, 314, 312, 314, 314, 315, 313, 313], + [312, 311, 311, 312, 312, 312, 312, 313, 311, 312], + [314, 314, 314, 314, 312, 313, 314, 314, 314, 311], + [314, 313, 312, 313, 313, 314, 312, 312, 311, 314], + [313, 313, 313, 314, 313, 313, 315, 313, 312, 313], + [314, 313, 313, 314, 313, 312, 312, 314, 310, 314]], dtype='>i2') + >>> data[1,4] = 999 # update a pixel value + >>> data[30:40, 10:20] = 0 # update values of a subsection + >>> data[3] = data[2] # copy the 3rd row to the 4th row Here are some more complicated examples by using the concept of the "mask -array". The first example is to change all negative pixel values in scidata to +array." The first example is to change all negative pixel values in ``data`` to zero. The second one is to take logarithm of the pixel values which are positive:: - >>> scidata[scidata < 0] = 0 - >>> scidata[scidata > 0] = numpy.log(scidata[scidata > 0]) + >>> data[data < 0] = 0 + >>> import numpy as np + >>> data[data > 0] = np.log(data[data > 0]) + +These examples show the concise nature of ``numpy`` array operations. -These examples show the concise nature of numpy array operations. +.. + EXAMPLE END Scaled Data -^^^^^^^^^^^ +=========== -Sometimes an image is scaled, i.e. the data stored in the file is not the +Sometimes an image is scaled; that is, the data stored in the file is not the image's physical (true) values, but linearly transformed according to the equation: @@ -65,106 +96,138 @@ equation: physical value = BSCALE \* (storage value) + BZERO BSCALE and BZERO are stored as keywords of the same names in the header of the -same HDU. The most common use of scaled image is to store unsigned 16-bit -integer data because FITS standard does not allow it. In this case, the stored -data is signed 16-bit integer (BITPIX=16) with BZERO=32768 (:math:`2^{15}`), -BSCALE=1. +same HDU. The most common use of a scaled image is to store unsigned 16-bit +integer data because the FITS standard does not allow it. In this case, the +stored data is signed 16-bit integer (BITPIX=16) with BZERO=32768 +(:math:`2^{15}`), BSCALE=1. Reading Scaled Image Data -""""""""""""""""""""""""" +------------------------- Images are scaled only when either of the BSCALE/BZERO keywords are present in the header and either of their values is not the default value (BSCALE=1, BZERO=0). -For unscaled data, the data attribute of an HDU in Astropy is a numpy array of -the same data type specified by the BITPIX keyword. For scaled image, the -``.data`` attribute will be the physical data, i.e. already transformed from -the storage data and may not be the same data type as prescribed in BITPIX. -This means an extra step of copying is needed and thus the corresponding memory -requirement. This also means that the advantage of memory mapping is reduced -for scaled data. +For unscaled data, the data attribute of an HDU in ``astropy`` is a ``numpy`` +array of the same data type specified by the BITPIX keyword. For a scaled +image, the ``.data`` attribute will be the physical data (i.e., already +transformed from the storage data and may not be the same data type as +prescribed in BITPIX). This means an extra step of copying is needed and thus +the corresponding memory requirement. This also means that the advantage of +memory mapping is reduced for scaled data. For floating point storage data, the scaled data will have the same data type. For integer data type, the scaled data will always be single precision floating -point (``numpy.float32``). Here is an example of what happens to such a file, -before and after the data is touched:: - - >>> f = fits.open('scaled_uint16.fits') - >>> hdu = f[1] - >>> print hdu.header['bitpix'], hdu.header['bzero'] - 16 32768 - >>> print hdu.data # once data is touched, it is scaled - [ 11. 12. 13. 14. 15.] +point (``numpy.float32``). + +Example +^^^^^^^ + +.. + EXAMPLE START + Reading Scaled Image Data with astropy.io.fits + +Here is an example of what happens to scaled data, before and after the data is +touched:: + + >>> fits_scaledimage_filename = fits.util.get_testdata_filepath('scale.fits') + + >>> hdul = fits.open(fits_scaledimage_filename) + >>> hdu = hdul[0] + >>> hdu.header['bitpix'] + 16 + >>> hdu.header['bzero'] + 1500.0 + >>> hdu.data[0, 0] # once data is touched, it is scaled # doctest: +FLOAT_CMP + np.float32(557.7563) >>> hdu.data.dtype.name 'float32' - >>> print hdu.header['bitpix'] # BITPIX is also updated + >>> hdu.header['bitpix'] # BITPIX is also updated -32 >>> # BZERO and BSCALE are removed after the scaling - >>> print hdu.header['bzero'] - KeyError: "Keyword 'bzero' not found." + >>> hdu.header['bzero'] + Traceback (most recent call last): + ... + KeyError: "Keyword 'BZERO' not found." .. warning:: - An important caveat to be aware of when dealing with scaled data in PyFITS, - is that when accessing the data via the ``.data`` attribute, the data is - automatically scaled with the BZERO and BSCALE parameters. If the file was - opened in "update" mode, it will be saved with the rescaled data. This - surprising behavior is a compromise to err on the side of not losing data: - If some floating point calculations were made on the data, rescaling it - when saving could result in a loss of information. + An important caveat to be aware of when dealing with scaled data in + ``astropy``, is that when accessing the data via the ``.data`` attribute, + the data is automatically scaled with the BZERO and BSCALE parameters. If + the file was opened in "update" mode, it will be saved with the rescaled + data. This surprising behavior is a compromise to err on the side of not + losing data: if some floating point calculations were made on the data, + rescaling it when saving could result in a loss of information. To prevent this automatic scaling, open the file with the - ``do_not_scale_image_data=True`` argument to ``fits.open()``. This is + ``do_not_scale_image_data=True`` argument to ``fits.open()``. This is especially useful for updating some header values, while ensuring that the data is not modified. - One may also manually reapply scale parameters by using ``hdu.scale()`` - (see below). Alternately, one may open files with the ``scale_back=True`` - argument. This assures that the original scaling is preserved when saving - even when the physical values are updated. In other words, it reapplies + You may also manually reapply scale parameters by using ``hdu.scale()`` + (see below). Alternately, you may open files with the ``scale_back=True`` + argument. This assures that the original scaling is preserved when saving + even when the physical values are updated. In other words, it reapplies the scaling to the new physical values upon saving. +.. + EXAMPLE END + Writing Scaled Image Data -""""""""""""""""""""""""" +------------------------- -With the extra processing and memory requirement, we discourage use of scaled -data as much as possible. However, Astropy does provide ways to write scaled -data with the `~ImageHDU.scale` method. Here are a few examples:: +With the extra processing and memory requirement, we discourage the use of +scaled data as much as possible. However, ``astropy`` does provide ways to +write scaled data with the `~ImageHDU.scale` method. + +Examples +^^^^^^^^ + +.. + EXAMPLE START + Writing Scaled Image Data in astropy.io.fits + +To write scaled data with the `~ImageHDU.scale` method:: >>> # scale the data to Int16 with user specified bscale/bzero >>> hdu.scale('int16', bzero=32768) - >>> # scale the data to Int32 with the min/max of the data range - >>> hdu.scale('int32', 'minmax') - >>> # scale the data, using the original BSCALE/BZERO - >>> hdu.scale('int32', 'old') + >>> # scale the data to Int32 with the min/max of the data range, emits + >>> # RuntimeWarning: overflow encountered in short_scalars + >>> hdu.scale('int32', 'minmax') # doctest: +SKIP + >>> # scale the data, using the original BSCALE/BZERO, emits + >>> # RuntimeWarning: invalid value encountered in add + >>> hdu.scale('int32', 'old') # doctest: +SKIP + >>> hdul.close() The first example above shows how to store an unsigned short integer array. -Great caution must be exercised when using the :meth:`~ImageHDU.scale` method. +Caution must be exercised when using the :meth:`~ImageHDU.scale` method. The :attr:`~ImageHDU.data` attribute of an image HDU, after the :meth:`~ImageHDU.scale` call, will become the storage values, not the physical values. So, only call :meth:`~ImageHDU.scale` just before writing out to FITS -files, i.e. calls of :meth:`~HDUList.writeto`, :meth:`~HDUList.flush`, or -:meth:`~HDUList.close`. No further use of the data should be exercised. Here is +files (i.e., calls of :meth:`~HDUList.writeto`, :meth:`~HDUList.flush`, or +:meth:`~HDUList.close`). No further use of the data should be exercised. Here is an example of what happens to the :attr:`~ImageHDU.data` attribute after the :meth:`~ImageHDU.scale` call:: - >>> hdu = fits.PrimaryHDU(numpy.array([0., 1, 2, 3])) - >>> print hdu.data - [ 0. 1. 2. 3.] + >>> hdu = fits.PrimaryHDU(np.array([0., 1, 2, 3])) + >>> print(hdu.data) # doctest: +FLOAT_CMP + [0. 1. 2. 3.] >>> hdu.scale('int16', bzero=32768) - >>> print hdu.data # now the data has storage values + >>> print(hdu.data) # now the data has storage values [-32768 -32767 -32766 -32765] >>> hdu.writeto('new.fits') +.. + EXAMPLE END .. _data-sections: Data Sections -^^^^^^^^^^^^^ +============= When a FITS image HDU's :attr:`~ImageHDU.data` is accessed, either the whole data is copied into memory (in cases of NOT using memory mapping or if the data @@ -173,37 +236,59 @@ is scaled) or a virtual memory space equivalent to the data size is allocated large image HDUs being accessed at the same time, the system may run out of memory. -If a user does not need the entire image(s) at the same time, e.g. processing -images(s) ten rows at a time, the :attr:`~ImageHDU.section` attribute of an +If a user does not need the entire image(s) at the same time (e.g., processing +the images(s) ten rows at a time), the :attr:`~ImageHDU.section` attribute of an HDU can be used to alleviate such memory problems. -With PyFITS' improved support for memory-mapping, the sections feature is not -as necessary as it used to be for handling very large images. However, if the -image's data is scaled with non-trivial BSCALE/BZERO values, accessing the data -in sections may still be necessary under the current implementation. Memmap is -also insufficient for loading images larger than 2 to 4 GB on a 32-bit -system--in such cases it may be necessary to use sections. - -Here is an example of getting the median image from 3 input images of the size -5000x5000:: - - >>> f1 = fits.open('file1.fits') - >>> f2 = fits.open('file2.fits') - >>> f3 = fits.open('file3.fits') - >>> output = numpy.zeros(5000 * 5000) - >>> for i in range(50): - ... j = i * 100 - ... k = j + 100 - ... x1 = f1[1].section[j:k,:] - ... x2 = f2[1].section[j:k,:] - ... x3 = f3[1].section[j:k,:] - ... # use scipy.stsci.image's median function - ... output[j:k] = image.median([x1, x2, x3]) +With ``astropy``'s improved support for memory-mapping, the sections feature is +not as necessary as it used to be for handling large images stored in local files. +However, it remains very useful in the following circumstances: + +* If the image's data is scaled with non-trivial BSCALE/BZERO values, accessing the + data in sections may still be necessary under the current implementation. +* Memory mapping is insufficient for loading images larger than 2 to 4 GB on a 32-bit + system — in such cases it may be necessary to use sections. +* Memory mapping does not work for accessing remote FITS files. + In this case sections may be your only option. See :ref:`astropy:fits_io_cloud`. + +In addition, for compressed FITS files, :attr:`CompImageHDU.section` can be used +to access and decompress only parts of the data, and can provide a significant +speedup. Note however that accessing data using :attr:`CompImageHDU.section` will +always load tiles one at a time from disk, and therefore when accessing a large +fraction of the data (or slicing it in a way that would cause most tiles to be +loaded) you may obtain better performance by using :attr:`CompImageHDU.data`. + +Example +------- + +.. + EXAMPLE START + Data Sections in astropy.io.fits + +Here is an example of getting the median image from three input images of the +size 5000x5000. + +.. code:: python + + hdul1 = fits.open('file1.fits') + hdul2 = fits.open('file2.fits') + hdul3 = fits.open('file3.fits') + output = np.zeros((5000, 5000)) + for i in range(50): + j = i * 100 + k = j + 100 + x1 = hdul1[0].section[j:k,:] + x2 = hdul2[0].section[j:k,:] + x3 = hdul3[0].section[j:k,:] + output[j:k, :] = np.median([x1, x2, x3], axis=0) Data in each :attr:`~ImageHDU.section` does not need to be contiguous for -memory savings to be possible. PyFITS will do its best to join together +memory savings to be possible. ``astropy`` will do its best to join together discontiguous sections of the array while reading as little as possible into main memory. -Sections cannot currently be assigned to. Any modifications made to a data +Sections cannot currently be assigned. Any modifications made to a data section are not saved back to the original file. + +.. + EXAMPLE END diff --git a/docs/io/fits/usage/misc.rst b/docs/io/fits/usage/misc.rst index bf598318c6f8..8552477c56a2 100644 --- a/docs/io/fits/usage/misc.rst +++ b/docs/io/fits/usage/misc.rst @@ -1,12 +1,14 @@ .. currentmodule:: astropy.io.fits Miscellaneous Features ----------------------- +********************** This section describes some of the miscellaneous features of :mod:`astropy.io.fits`. +.. _io-fits-differs: + Differs -^^^^^^^ +======= The :mod:`astropy.io.fits.diff` module contains several facilities for generating and reporting the differences between two FITS files, or two @@ -17,15 +19,22 @@ differences between either two FITS files on disk, or two existing :class:`HDUList` objects (or some combination thereof). Likewise, the :class:`HeaderDiff` class can be used to find the differences -just between two :class:`Header` objects. Other available differs include +just between two :class:`Header` objects. Other available differs include :class:`HDUDiff`, :class:`ImageDataDiff`, :class:`TableDataDiff`, and :class:`RawDataDiff`. Each of these classes are instantiated with two instances of the objects that -they diff. The returned diff instance has a number of attributes starting with +they diff. The returned diff instance has a number of attributes starting with ``.diff_`` that describe differences between the two objects. -For example the :class:`HeaderDiff` class cam be used to find the differences +Example +------- + +.. + EXAMPLE START + Generating Differences Between FITS Files Using astropy.io.fits.diff + +The :class:`HeaderDiff` class can be used to find the differences between two :class:`Header` objects like so:: >>> from astropy.io import fits @@ -37,6 +46,9 @@ between two :class:`Header` objects like so:: >>> diff.diff_keywords (['KEY_B'], ['KEY_C']) >>> diff.diff_keyword_values - defaultdict( at ...>, {'KEY_A': [(1, 3)]}) + defaultdict(..., {'KEY_A': [(1, 3)]}) See the API documentation for details on the different differ classes. + +.. + EXAMPLE END diff --git a/docs/io/fits/usage/scripts.rst b/docs/io/fits/usage/scripts.rst index ec19c1bc79a4..dd08dac63d3d 100644 --- a/docs/io/fits/usage/scripts.rst +++ b/docs/io/fits/usage/scripts.rst @@ -1,31 +1,35 @@ Executable Scripts ------------------- +****************** -Astropy installs a couple of useful utility programs on your system that are -built with Astropy. +``astropy`` installs a couple of useful utility programs on your system that are +built with ``astropy``. + +fitsinfo +======== +.. automodule:: astropy.io.fits.scripts.fitsinfo fitsheader -^^^^^^^^^^ +========== .. automodule:: astropy.io.fits.scripts.fitsheader fitscheck -^^^^^^^^^ +========= .. automodule:: astropy.io.fits.scripts.fitscheck -With Astropy installed, please run ``fitscheck --help`` to see the full program -usage documentation. +With ``astropy`` installed, please run ``fitscheck --help`` to see the full +program usage documentation. .. _fitsdiff: fitsdiff -^^^^^^^^ +======== .. currentmodule:: astropy.io.fits ``fitsdiff`` provides a thin command-line wrapper around the :class:`FITSDiff` -interface--it outputs the report from a :class:`FITSDiff` of two FITS files, +interface. It outputs the report from a :class:`FITSDiff` of two FITS files, and like common diff-like commands returns a 0 status code if no differences were found, and 1 if differences were found: -With Astropy installed, please run ``fitscheck --help`` to see the full program -usage documentation. +With ``astropy`` installed, please run ``fitsdiff --help`` to see the full +program usage documentation. diff --git a/docs/io/fits/usage/table.rst b/docs/io/fits/usage/table.rst index 84b5e82014db..80ccce9de953 100644 --- a/docs/io/fits/usage/table.rst +++ b/docs/io/fits/usage/table.rst @@ -1,183 +1,266 @@ -.. doctest-skip-all .. currentmodule:: astropy.io.fits Table Data ----------- +********** -In this chapter, we'll discuss the data component in a table HDU. A table will +In this chapter, we will discuss the data component in a table HDU. A table will always be in an extension HDU, never in a primary HDU. -There are two kinds of table in the FITS standard: binary tables and ASCII +There are two kinds of tables in the FITS standard: binary tables and ASCII tables. Binary tables are more economical in storage and faster in data access and manipulation. ASCII tables store the data in a "human readable" form and therefore take up more storage space as well as more processing time since the ASCII text needs to be parsed into numerical values. +.. note:: + + If you want to read or write a single table in FITS format then the + most convenient method is often via the high-level :ref:`table_io`. In + particular see the :ref:`Unified I/O FITS ` section. Table Data as a Record Array -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +============================ What is a Record Array? -""""""""""""""""""""""" +----------------------- -A record array is an array which contains records (i.e. rows) of heterogeneous -data types. Record arrays are available through the records module in the numpy -library. Here is a simple example of record array:: +A record array is an array which contains records (i.e., rows) of heterogeneous +data types. Record arrays are available through the records module in the NumPy +library. - >>> from numpy import rec - >>> bright = rec.array([(1,'Sirius', -1.45, 'A1V'), - ... (2,'Canopus', -0.73, 'F0Ib'), - ... (3,'Rigil Kent', -0.1, 'G2V')], - ... formats='int16,a20,float32,a10', - ... names='order,name,mag,Sp') +Here is a sample record array:: + + >>> import numpy as np + >>> bright = np.rec.array([(1,'Sirius', -1.45, 'A1V'), + ... (2,'Canopus', -0.73, 'F0Ib'), + ... (3,'Rigil Kent', -0.1, 'G2V')], + ... formats='int16,S20,float32,S10', + ... names='order,name,mag,Sp') -In this example, there are 3 records (rows) and 4 fields (columns). The first -field is a short integer, second a character string (of length 20), third a -floating point number, and fourth a character string (of length 10). Each -record has the same (heterogeneous) data structure. +In this example, there are three records (rows) and four fields (columns). The +first field is a short integer, the second a character string (of length 20), +the third a floating point number, and the fourth a character string (of length +10). Each record has the same (heterogeneous) data structure. The underlying data structure used for FITS tables is a class called -:class:`FITS_rec` which is a specialized subclass of `numpy.recarray`. A -:class:`FITS_rec` can be instantiated directly using the same initialization -format presented for plain recarrays as in the example above. One may also -instantiate a new :class:`FITS_rec` from a list of PyFITS `Column` objects -using the :meth:`FITS_rec.from_columns` class method. This has the exact same -semantics as :meth:`BinTableHDU.from_columns` and +:class:`FITS_rec` which is a specialized subclass of `numpy.recarray`. A +:class:`FITS_rec` can be instantiated directly from a +numpy recarray:: + + >>> from astropy.io import fits + >>> data = fits.FITS_rec(bright) + +You may also instantiate a new :class:`FITS_rec` from a list of `astropy.io.fits.Column` +objects using the :meth:`FITS_rec.from_columns` class method. This has the +exact same semantics as :meth:`BinTableHDU.from_columns` and :meth:`TableHDU.from_columns`, except that it only returns an actual FITS_rec array and not a whole HDU object. Metadata of a Table -""""""""""""""""""" +------------------- -The data in a FITS table HDU is basically a record array, with added -attributes. The metadata, i.e. information about the table data, are stored in +The data in a FITS table HDU is basically a record array with added +attributes. The metadata (i.e., information about the table data) are stored in the header. For example, the keyword TFORM1 contains the format of the first field, TTYPE2 the name of the second field, etc. NAXIS2 gives the number of -records(rows) and TFIELDS gives the number of fields (columns). For FITS +records (rows) and TFIELDS gives the number of fields (columns). For FITS tables, the maximum number of fields is 999. The data type specified in TFORM -is represented by letter codes for binary tables and a FORTRAN-like format +is represented by letter codes for binary tables and a Fortran-like format string for ASCII tables. Note that this is different from the format specifications when constructing a record array. Reading a FITS Table -"""""""""""""""""""" +-------------------- Like images, the ``.data`` attribute of a table HDU contains the data of the -table. To recap, the simple example in the Quick Tutorial:: - - >>> f = fits.open('bright_stars.fits') # open a FITS file - >>> tbdata = f[1].data # assume the first extension is a table - >>> print tbdata[:2] # show the first two rows - [(1, 'Sirius', -1.4500000476837158, 'A1V'), - (2, 'Canopus', -0.73000001907348633, 'F0Ib')] - - >>> print tbdata['mag'] # show the values in field "mag" - [-1.45000005 -0.73000002 -0.1 ] - >>> print tbdata.field(1) # columns can be referenced by index too - ['Sirius' 'Canopus' 'Rigil Kent'] - -Note that in Astropy, when using the ``field()`` method, it is 0-indexed while -the suffixes in header keywords, such as TFORM is 1-indexed. So, -``tbdata.field(0)`` is the data in the column with the name specified in TTYPE1 +table. + +Example +^^^^^^^ + +.. note:: + + The ``astropy.io.fits.util.get_testdata_filepath()`` function, + used in the examples here, returns file path for test data shipped with ``astropy``. + To work with your own data instead, please use :func:`astropy.io.fits.open` or :ref:`io-fits-intro-convenience-functions`, + which take either the relative or absolute path as string or :term:`python:path-like object`. + +.. + EXAMPLE START + Reading a FITS Table with astropy.io.fits + +To read a FITS Table:: + + + >>> from astropy.io import fits + >>> fits_table_filename = fits.util.get_testdata_filepath('btable.fits') + + >>> hdul = fits.open(fits_table_filename) # open a FITS file + >>> data = hdul[1].data # assume the first extension is a table + >>> # show the first two rows + >>> first_two_rows = data[:2] + >>> first_two_rows # doctest: +SKIP + [(1, 'Sirius', -1.45000005, 'A1V') (2, 'Canopus', -0.73000002, 'F0Ib')] + >>> # show the values in field "mag" + >>> magnitudes = data['mag'] + >>> magnitudes # doctest: +SKIP + array([-1.45000005, -0.73000002, -0.1 ], dtype=float32) + >>> # columns can be referenced by index too + >>> names = data.field(1) + >>> names.tolist() # doctest: +SKIP + ['Sirius', 'Canopus', 'Rigil Kent'] + >>> hdul.close() + +Note that in ``astropy``, when using the ``field()`` method, it is 0-indexed +while the suffixes in header keywords such as TFORM is 1-indexed. So, +``data.field(0)`` is the data in the column with the name specified in TTYPE1 and format in TFORM1. .. warning:: The FITS format allows table columns with a zero-width data format, such as - ``'0D'``. This is probably intended as a space-saving measure on files in - which that column contains no data. In such files, the zero-width columns - are ommitted when accessing the table data, so the indexes of fields might - change when using the ``field()`` method. For this reason, if you expect + ``'0D'``. This is probably intended as a space-saving measure on files in + which that column contains no data. In such files, the zero-width columns + are omitted when accessing the table data, so the indexes of fields might + change when using the ``field()`` method. For this reason, if you expect to encounter files containing zero-width columns it is recommended to access fields by name rather than by index. +.. + EXAMPLE END + Table Operations -^^^^^^^^^^^^^^^^ +================ Selecting Records in a Table -"""""""""""""""""""""""""""" +---------------------------- Like image data, we can use the same "mask array" idea to pick out desired records from a table and make a new table out of it. -In the next example, assuming the table's second field having the name -'magnitude', an output table containing all the records of magnitude > 5 from -the input table is generated:: +Examples +^^^^^^^^ - >>> from astropy.io import fits - >>> t = fits.open('table.fits') - >>> tbdata = t[1].data - >>> mask = tbdata.['magnitude'] > 5 - >>> newtbdata = tbdata[mask] - >>> hdu = fits.BinTableHDU(data=newtbdata) - >>> hdu.writeto('newtable.fits') +.. + EXAMPLE START + Selecting Records in a Table Using a "Mask Array" + +Assuming the table's second field as having the name 'magnitude', an output +table containing all the records of magnitude > -0.5 from the input table is +generated:: + >>> with fits.open(fits_table_filename) as hdul: + ... data = hdul[1].data + ... mask = data['mag'] > -0.5 + ... newdata = data[mask] + ... hdu = fits.BinTableHDU(data=newdata) + ... hdu.writeto('newtable.fits') + +It is also possible to update the data from the HDU object in-place:: + + >>> with fits.open(fits_table_filename) as hdul: + ... hdu = hdul[1] + ... mask = hdu.data['mag'] > -0.5 + ... hdu.data = hdu.data[mask] + ... hdu.writeto('newtable2.fits') + +.. + EXAMPLE END Merging Tables -"""""""""""""" +-------------- + +Merging different tables is very convenient in ``astropy``. + +Examples +^^^^^^^^ + +.. + EXAMPLE START + Merging FITS Tables -Merging different tables is straightforward in Astropy. Simply merge the column -definitions of the input tables:: +To merge the column definitions of the input tables:: - >>> t1 = fits.open('table1.fits') - >>> t2 = fits.open('table2.fits') - >>> new_columns = t1[1].columns + t2[1].columns - >>> hdu = fits.BinTableHDU.from_columns(new_columns) - >>> hdu.writeto('newtable.fits') + >>> fits_other_table_filename = fits.util.get_testdata_filepath('table.fits') + + >>> with fits.open(fits_table_filename) as hdul1: + ... with fits.open(fits_other_table_filename) as hdul2: + ... new_columns = hdul1[1].columns + hdul2[1].columns + ... new_hdu = fits.BinTableHDU.from_columns(new_columns) + >>> new_columns + ColDefs( + name = 'order'; format = 'I' + name = 'name'; format = '20A' + name = 'mag'; format = 'E' + name = 'Sp'; format = '10A' + name = 'target'; format = '20A' + name = 'V_mag'; format = 'E' + ) The number of fields in the output table will be the sum of numbers of fields -of the input tables. Users have to make sure the input tables don't share any +of the input tables. Users have to make sure the input tables do not share any common field names. The number of records in the output table will be the largest number of records of all input tables. The expanded slots for the originally shorter table(s) will be zero (or blank) filled. -A simpler version of this example can be used to append a new column to a -table. Updating an existing table with a new column is generally more -difficult than it's worth, but one can "append" a column to a table by creating +Another version of this example can be used to append a new column to a +table. Updating an existing table with a new column is generally more +difficult than it is worth, but you can "append" a column to a table by creating a new table with columns from the existing table plus the new column(s):: - >>> orig_table = fits.open('table.fits')[1].data - >>> orig_cols = orig_table.columns + >>> with fits.open(fits_table_filename) as hdul: + ... orig_table = hdul[1].data + ... orig_cols = orig_table.columns >>> new_cols = fits.ColDefs([ ... fits.Column(name='NEWCOL1', format='D', ... array=np.zeros(len(orig_table))), ... fits.Column(name='NEWCOL2', format='D', ... array=np.zeros(len(orig_table)))]) >>> hdu = fits.BinTableHDU.from_columns(orig_cols + new_cols) - >>> hdu.writeto('newtable.fits') Now ``newtable.fits`` contains a new table with the original table, plus the two new columns filled with zeros. +.. + EXAMPLE END Appending Tables -"""""""""""""""" +---------------- Appending one table after another is slightly trickier, since the two tables -may have different field attributes. Here are two examples. The first is to -append by field indices, the second one is to append by field names. In both -cases, the output table will inherit column attributes (name, format, etc.) of -the first table:: - - >>> t1 = fits.open('table1.fits') - >>> t2 = fits.open('table2.fits') - >>> nrows1 = t1[1].data.shape[0] - >>> nrows2 = t2[1].data.shape[0] - >>> nrows = nrows1 + nrows2 - >>> hdu = fits.BinTableHDU.from_columns(t1[1].columns, nrows=nrows) - >>> for colname in t1[1].columns.names: - ... hdu.data[colname][nrows1:] = t2[1].data[colname] - >>> hdu.writeto('newtable.fits') +may have different field attributes. + +Examples +^^^^^^^^ + +.. + EXAMPLE START + Appending to FITS Tables +Here, the first example is to append by field indices, and the second one is to +append by field names. In both cases, the output table will inherit the column +attributes (name, format, etc.) of the first table:: + + >>> with fits.open(fits_table_filename) as hdul1: + ... with fits.open(fits_table_filename) as hdul2: + ... nrows1 = hdul1[1].data.shape[0] + ... nrows2 = hdul2[1].data.shape[0] + ... nrows = nrows1 + nrows2 + ... hdu = fits.BinTableHDU.from_columns(hdul1[1].columns, nrows=nrows) + ... for colname in hdul1[1].columns.names: + ... hdu.data[colname][nrows1:] = hdul2[1].data[colname] + +.. + EXAMPLE END Scaled Data in Tables -^^^^^^^^^^^^^^^^^^^^^ +===================== A table field's data, like an image, can also be scaled. Scaling in a table has a more generalized meaning than in images. In images, the physical data is a @@ -190,17 +273,18 @@ All scaled fields, like the image case, will take extra memory space as well as processing. So, if high performance is desired, try to minimize the use of scaled fields. -All the scalings are done for the user, so the user only sees the physical -data. Thus, this no need to worry about scaling back and forth between the +All of the scalings are done for the user, so the user only sees the physical +data. Thus, there is no need to worry about scaling back and forth between the physical and storage column values. Creating a FITS Table -^^^^^^^^^^^^^^^^^^^^^ +===================== +.. _column_creation: Column Creation -""""""""""""""" +--------------- To create a table from scratch, it is necessary to create individual columns first. A :class:`Column` constructor needs the minimal information of column @@ -215,18 +299,18 @@ name and format. Here is a summary of all allowed formats for a binary table: B Unsigned byte 1 I 16-bit integer 2 J 32-bit integer 4 - K 64-bit integer 4 + K 64-bit integer 8 A character 1 - E single precision floating point 4 - D double precision floating point 8 + E single precision float (32-bit) 4 + D double precision float (64-bit) 8 C single precision complex 8 M double precision complex 16 P array descriptor 8 Q array descriptor 16 -We'll concentrate on binary tables in this chapter. ASCII tables will be +We will concentrate on binary tables in this chapter. ASCII tables will be discussed in a later chapter. The less frequently used X format (bit array) and -P format (used in variable length tables) will also be discussed in a later +P format (used in :ref:`variable_length_array_tables`) will also be discussed in a later chapter. Besides the required name and format arguments in constructing a @@ -248,103 +332,346 @@ header keywords and descriptions: disp TDISP display format dim TDIM multi-dimensional array spec start TBCOL starting position for ASCII table + coord_type TCTYP coordinate/axis type + coord_unit TCUNI coordinate/axis unit + coord_ref_point TCRPX pixel coordinate of the reference point + coord_ref_value TCRVL coordinate value at reference point + coord_inc TCDLT coordinate increment at reference point + time_ref_pos TRPOS reference position for a time coordinate column + ascii specifies a column for an ASCII table array the data of the column +Examples +^^^^^^^^ -Here are a few Columns using various combination of these arguments: +.. + EXAMPLE START + Creating a FITS Table + +Here are a few Columns using various combinations of the optional arguments:: - >>> import numpy as np - >>> from fits import Column >>> counts = np.array([312, 334, 308, 317]) >>> names = np.array(['NGC1', 'NGC2', 'NGC3', 'NGC4']) - >>> c1 = Column(name='target', format='10A', array=names) - >>> c2 = Column(name='counts', format='J', unit='DN', array=counts) - >>> c3 = Column(name='notes', format='A10') - >>> c4 = Column(name='spectrum', format='1000E') - >>> c5 = Column(name='flag', format='L', array=[True, False, True, True]) + >>> values = np.arange(2*2*4).reshape(4, 2, 2) + >>> col1 = fits.Column(name='target', format='10A', array=names) + >>> col2 = fits.Column(name='counts', format='J', unit='count', array=counts) + >>> col3 = fits.Column(name='notes', format='A10') + >>> col4 = fits.Column(name='spectrum', format='10E') + >>> col5 = fits.Column(name='flag', format='L', array=[True, False, True, True]) + >>> col6 = fits.Column(name='intarray', format='4I', dim='(2, 2)', array=values) In this example, formats are specified with the FITS letter codes. When there is a number (>1) preceding a (numeric type) letter code, it means each cell in -that field is a one-dimensional array. In the case of column c4, each cell is -an array (a numpy array) of 1000 elements. - -For character string fields, the number be to the *left* of the letter 'A' when -creating binary tables, and should be to the *right* when creating ASCII -tables. However, as this is a common confusion both formats are understood -when creating binary tables (note, however, that upon writing to a file the -correct format will be written in the header). So, for columns c1 and c3, they -both have 10 characters in each of their cells. For numeric data type, the -dimension number must be before the letter code, not after. +that field is a one-dimensional array. In the case of column "col4", each cell +is an array (a NumPy array) of 10 elements. And in the case of column "col6", +with the use of the "dim" argument, each cell is a multi-dimensional array of +2x2 elements. + +For character string fields, the number should be to the *left* of the letter +'A' when creating binary tables, and should be to the *right* when creating +ASCII tables. However, as this is a common confusion, both formats are +understood when creating binary tables (note, however, that upon writing to a +file the correct format will be written in the header). So, for columns "col1" +and "col3", they both have 10 characters in each of their cells. For numeric +data type, the dimension number must be before the letter code, not after. After the columns are constructed, the :meth:`BinTableHDU.from_columns` class method can be used to construct a table HDU. We can either go through the column definition object:: - >>> coldefs = fits.ColDefs([c1, c2, c3, c4, c5]) - >>> tbhdu = fits.BinTableHDU.from_columns(coldefs) + >>> coldefs = fits.ColDefs([col1, col2, col3, col4, col5, col6]) + >>> hdu = fits.BinTableHDU.from_columns(coldefs) + >>> coldefs + ColDefs( + name = 'target'; format = '10A' + name = 'counts'; format = 'J'; unit = 'count' + name = 'notes'; format = '10A' + name = 'spectrum'; format = '10E' + name = 'flag'; format = 'L' + name = 'intarray'; format = '4I'; dim = '(2, 2)' + ) or directly use the :meth:`BinTableHDU.from_columns` method:: - >>> tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4, c5]) + >>> hdu = fits.BinTableHDU.from_columns([col1, col2, col3, col4, col5, col6]) + >>> hdu.columns + ColDefs( + name = 'target'; format = '10A' + name = 'counts'; format = 'J'; unit = 'count' + name = 'notes'; format = '10A' + name = 'spectrum'; format = '10E' + name = 'flag'; format = 'L' + name = 'intarray'; format = '4I'; dim = '(2, 2)' + ) .. note:: - Users familiar with older versions of PyFITS or Astropy will wonder what - happened to :func:`~astropy.io.fits.new_table`. It is still there, but is - deprecated. :meth:`BinTableHDU.from_columns` and its companion for ASCII - tables :meth:`TableHDU.from_columns` are the same as - :func:`~astropy.io.fits.new_table` in the arguments they accept and their - behavior. They just make it more explicit what type of table HDU they - create. + Users familiar with older versions of ``astropy`` will wonder what + happened to ``astropy.io.fits.new_table``. :meth:`BinTableHDU.from_columns` + and its companion for ASCII tables :meth:`TableHDU.from_columns` are the + same in the arguments they accept and their behavior, but make it + more explicit as to what type of table HDU they create. -A look of the newly created HDU's header will show that relevant keywords are +A look at the newly created HDU's header will show that relevant keywords are properly populated:: - >>> tbhdu.header - XTENSION = 'BINTABLE' / binary table extension - BITPIX = 8 / array data type - NAXIS = 2 / number of array dimensions - NAXIS1 = 4025 / length of dimension 1 - NAXIS2 = 4 / length of dimension 2 - PCOUNT = 0 / number of group parameters - GCOUNT = 1 / number of groups - TFIELDS = 5 / number of table fields - TTYPE1 = 'target ' - TFORM1 = '10A ' - TTYPE2 = 'counts ' - TFORM2 = 'J ' - TUNIT2 = 'DN ' - TTYPE3 = 'notes ' - TFORM3 = '10A ' - TTYPE4 = 'spectrum' - TFORM4 = '1000E ' - TTYPE5 = 'flag ' - TFORM5 = 'L ' + >>> hdu.header + XTENSION= 'BINTABLE' / binary table extension + BITPIX = 8 / array data type + NAXIS = 2 / number of array dimensions + NAXIS1 = 73 / length of dimension 1 + NAXIS2 = 4 / length of dimension 2 + PCOUNT = 0 / number of group parameters + GCOUNT = 1 / number of groups + TFIELDS = 6 / number of table fields + TTYPE1 = 'target ' + TFORM1 = '10A ' + TTYPE2 = 'counts ' + TFORM2 = 'J ' + TUNIT2 = 'count ' + TTYPE3 = 'notes ' + TFORM3 = '10A ' + TTYPE4 = 'spectrum' + TFORM4 = '10E ' + TTYPE5 = 'flag ' + TFORM5 = 'L ' + TTYPE6 = 'intarray' + TFORM6 = '4I ' + TDIM6 = '(2, 2) ' .. warning:: It should be noted that when creating a new table with :meth:`BinTableHDU.from_columns`, an in-memory copy of all of the input - column arrays is created. This is because it is not guaranteed that the + column arrays is created. This is because it is not guaranteed that the columns are arranged contiguously in memory in row-major order (in fact, they are most likely not), so they have to be combined into a new array. However, if the array data *is* already contiguous in memory, such as in an existing record array, a kludge can be used to create a new table HDU without -any copying. First, create the Columns as before, but without using the +any copying. First, create the Columns as before, but without using the ``array=`` argument:: - >>> c1 = Column(name='target', format='10A') + >>> col1 = fits.Column(name='target', format='10A') Then call :meth:`BinTableHDU.from_columns`:: - >>> tbhdu = fits.BinTableHDU.from_columns([c1, c2, c3, c4, c5]) + >>> hdu = fits.BinTableHDU.from_columns([col1, col2, col3, col4, col5]) This will create a new table HDU as before, with the correct column -definitions, but an empty data section. Now simply assign your array directly -to the HDU's data attribute:: +definitions, but an empty data section. Now you can assign your array directly +to the HDU's data attribute: + +.. doctest-skip:: + + >>> hdu.data = mydata + +In a future version of ``astropy``, table creation will be simplified and this +process will not be necessary. + +.. + EXAMPLE END + +.. _fits_time_column: + +FITS Tables with Time Columns +============================= + +The `FITS Time standard paper +`_ defines the formats +and keywords used to represent timing information in FITS files. The ``astropy`` +FITS package provides support for reading and writing native +`~astropy.time.Time` columns and objects using this format. This is done +within the :ref:`table_io_fits` unified I/O interface and examples of usage can +be found in the :ref:`fits_astropy_native` section. The support is not +complete and only a subset of the full standard is implemented. + +Example +------- + +.. + EXAMPLE START + FITS Tables with Time Columns + +The following is an example of a Header extract of a binary table (event list) +with a time column: + +.. parsed-literal:: + + COMMENT ---------- Globally valid key words ---------------- + TIMESYS = ’TT ’ / Time system + MJDREF = 50814.000000000000 / MJD zero point for (native) TT (= 1998-01-01) + MJD-OBS = 53516.257939301īŋŧīŋŧ / MJD for observation in (native) TT + + COMMENT ---------- Time Column ----------------------- + TTYPE1 = ’Time ’ / S/C TT corresponding to mid-exposure + TFORM1 = ’2D ’ / format of field + TUNIT1 = ’s ’ + TCTYP1 = ’TT ’ + TCNAM1 = ’Terrestrial Time’ / This is TT + TCUNI1 = ’s ’ + +.. + EXAMPLE END + +However, the FITS standard and the ``astropy`` Time object are not perfectly +mapped and some compromises must be made. To help the user understand how the +``astropy`` code deals with these situations, the following text describes the +approach that ``astropy`` takes in some detail. + +To create FITS columns which adhere to the FITS Time standard, we have taken +into account the following important points stated in the `FITS Time paper +`_. + +The strategy used to store `~astropy.time.Time` columns in FITS tables is to +create a `~astropy.io.fits.Header` with the appropriate time coordinate +global reference keywords and the column-specific override keywords. The +module ``astropy.io.fits.fitstime`` deals with the reading and writing of +Time columns. + +The following keywords set the Time Coordinate Frame: + +* TIME SCALE + + The most important of all of the metadata is the time scale which is a + specification for measuring time. + + .. parsed-literal:: + + **TIMESYS** (string-valued) + Time scale; default UTC + + **TCTYPn** (string-valued) + Column-specific override keyword + + The global time scale may be overridden by a time scale recorded in the table + equivalent keyword ``TCTYPn`` for time coordinates in FITS table columns. + ``TCTYna`` is used for alternate coordinates. + +* TIME REFERENCE + + The reference point in time to which all times in the HDU are relative. + Since there are no context-specific reference times in case there are + multiple time columns in the same table, we need to adjust the reference + times for the columns using some other keywords. + + The reference point in time shall be specified through one of the three + following keywords, which are listed in decreasing order of preference: + + .. parsed-literal:: + + **MJDREF** (floating-valued) + Reference time in MJD + + **JDREF** (floating-valued) + Reference time in JD + + **DATEREF** (datetime-valued) + Reference time in ISO-8601 + + The time reference keywords (MJDREF, JDREF, DATEREF) are interpreted using the + time scale specified in ``TIMESYS``. + + .. note:: + + If none of the three keywords are present, there is no problem as long as + all times in the HDU are expressed in ISO-8601 ``Datetime Strings`` format: + ``CCYY-MM-DD[Thh:mm:ss[.s...]]`` (e.g., ``"2015-04-05T12:22:33.8"``); + otherwise MJDREF = 0.0 must be assumed. + + The value of the reference time has global validity for all time values, + but it does not have a particular time scale associated with it. Thus we + need to use ``TCRVLn`` (time coordinate reference value) keyword to + compensate for the time scale differences. + +* TIME REFERENCE POSITION + + The reference position, specified by the keyword ``TREFPOS``, specifies the + spatial location at which the time is valid, either where the observation was + made or the point in space for which light-time corrections have been applied. + This may be a standard location (such as ``GEOCENTER`` or ``TOPOCENTER``) or + a point in space defined by specific coordinates. + + .. parsed-literal:: + + **TREFPOS** (string-valued) + Time reference position; default TOPOCENTER + + **TRPOSn** (string-valued) + Column-specific override keyword + + .. note:: + + For TOPOCENTER, we need to specify the observatory location + (ITRS Cartesian coordinates or geodetic latitude/longitude/height) in the + ``OBSGEO-*`` keywords. + +* TIME REFERENCE DIRECTION + + If any pathlength corrections have been applied to the time stamps (i.e., if + the reference position is not ``TOPOCENTER`` for observational data), the + reference direction that is used in calculating the pathlength delay should + be provided in order to maintain a proper analysis trail of the data. + However, this is useful only if there is also information available on the + location from where the observation was made (the observatory location). + + The reference direction is indicated through a reference to specific keywords. + These keywords may explicitly hold the direction or indicate columns holding + the coordinates. + + .. parsed-literal:: + + **TREFDIR** (string-valued) + Pointer to time reference direction + + **TRDIRn** (string-valued) + Column-specific override keyword + +* TIME UNIT + + The FITS standard recommends the time unit to be one of the allowed ones + in the specification. + + .. parsed-literal:: + + **TIMEUNIT** (string-valued) + Time unit; default s + + **TCUNIn** (string-valued) + Column-specific override + +* TIME OFFSET + + It is sometimes convenient to be able to apply a uniform clock correction + in bulk by putting that number in a single keyword. A second use + for a time offset is to set a zero offset to a relative time series, + allowing zero-relative times, or higher precision, in the time stamps. + Its default value is zero. + + .. parsed-literal:: + + **TIMEOFFS** (floating-valued) + This has global validity + +* The absolute, relative errors and time resolution, time binning can be used + when needed. + + +The following keywords define the global time informational keywords: + +* DATE and DATE-* keywords + + These define the date of HDU creation and observation in ISO-8601. + ``DATE`` is in UTC if the file is constructed on the Earth’s surface + and others are in the time scale given by ``TIMESYS``. + +* MJD-* keywords - >>> tbhdu.data = mydata + These define the same as above, but in ``MJD`` (Modified Julian Date). -In a future version of Astropy table creation will be simplified and this -process won't be necessary. +The implementation writes a subset of the above FITS keywords, which map +to the Time metadata. Time is intrinsically a coordinate and hence shares +keywords with the ``World Coordinate System`` specification for spatial +coordinates. Therefore, while reading FITS tables with time columns, +the verification that a coordinate column is indeed time is done using +the FITS WCS standard rules and suggestions. diff --git a/docs/io/fits/usage/unfamiliar.rst b/docs/io/fits/usage/unfamiliar.rst index 64769cac8cb9..b4da5f01ec2c 100644 --- a/docs/io/fits/usage/unfamiliar.rst +++ b/docs/io/fits/usage/unfamiliar.rst @@ -1,72 +1,72 @@ -.. doctest-skip-all - .. currentmodule:: astropy.io.fits Less Familiar Objects ---------------------- +********************* -In this chapter, we'll discuss less frequently used FITS data structures. They +In this chapter, we will discuss less frequently used FITS data structures. They include ASCII tables, variable length tables, and random access group FITS files. ASCII Tables -^^^^^^^^^^^^ +============ -FITS standard supports both binary and ASCII tables. In ASCII tables, all the -data are stored in a human readable text form, so it takes up more space and -extra processing to parse the text for numeric data. Depending on how the +FITS standard supports both binary and ASCII tables. In ASCII tables, all of the +data are stored in a human-readable text form, so it takes up more space and +extra processing to parse the text for numeric data. Depending on how the columns are formatted, floating point data may also lose precision. -In Astropy, the interface for ASCII tables and binary tables is basically the -same, i.e. the data is in the ``.data`` attribute and the ``field()`` method -is used to refer to the columns and returns a numpy array. When reading the -table, Astropy will automatically detect what kind of table it is. +In ``astropy``, the interface for ASCII tables and binary tables is basically +the same (i.e., the data is in the ``.data`` attribute and the ``field()`` +method is used to refer to the columns and returns a ``numpy`` array). When +reading the table, ``astropy`` will automatically detect what kind of table it +is. :: >>> from astropy.io import fits - >>> hdus = fits.open('ascii_table.fits') - >>> hdus[1].data[:1] - FITS_rec( - ... [(10.123000144958496, 37)], - ... dtype=[('a', '>f4'),('b','>i4')]) - >>> hdus[1].data['a'] - array([ 10.12300014, 5.19999981, 15.60999966, 0. , - 345. ], dtype=float32) - >>> hdus[1].data.formats + >>> filename = fits.util.get_testdata_filepath('ascii.fits') + >>> hdul = fits.open(filename) + >>> hdul[1].data[:1] # doctest: +SKIP + FITS_rec([(10.123, 37)], + dtype=(numpy.record, {'names':['a','b'], 'formats':['S10','S5'], 'offsets':[0,11], 'itemsize':16})) + >>> hdul[1].data['a'] + array([ 10.123, 5.2 , 15.61 , nan, 345. ]) + >>> hdul[1].data.formats ['E10.4', 'I5'] + >>> hdul.close() Note that the formats in the record array refer to the raw data which are ASCII strings (therefore 'a11' and 'a5'), but the ``.formats`` attribute of data retains the original format specifications ('E10.4' and 'I5'). +.. _creating_ascii_table: Creating an ASCII Table -""""""""""""""""""""""" +----------------------- Creating an ASCII table from scratch is similar to creating a binary table. The difference is in the Column definitions. The columns/fields in an ASCII table are more limited than in a binary table. It does not allow more than one -numerical value in a cell. Also, it only supports a subset of what allowed in a -binary table, namely character strings, integer, and (single and double +numerical value in a cell. Also, it only supports a subset of what is allowed +in a binary table, namely character strings, integer, and (single and double precision) floating point numbers. Boolean and complex numbers are not allowed. The format syntax (the values of the TFORM keywords) is different from that of a -binary table, they are: +binary table. They are: .. parsed-literal:: Aw Character string Iw (Decimal) Integer - Fw.d Single precision real - Ew.d Single precision real, in exponential notation + Fw.d Double precision real + Ew.d Double precision real, in exponential notation Dw.d Double precision real, in exponential notation -where, w is the width, and d the number of digits after the decimal point. The +where w is the width, and d the number of digits after the decimal point. The syntax difference between ASCII and binary tables can be confusing. For example, -a field of 3-character string is specified '3A' in a binary table and as 'A3' in -an ASCII table. +a field of 3-character string is specified as '3A' in a binary table and as +'A3' in an ASCII table. The other difference is the need to specify the table type when using the :meth:`TableHDU.from_columns` method, and that `Column` should be provided the @@ -75,48 +75,50 @@ The other difference is the need to specify the table type when using the .. note:: Although binary tables are more common in most FITS files, earlier versions - of the FITS format only supported ASCII tables. That is why the class + of the FITS format only supported ASCII tables. That is why the class :class:`TableHDU` is used for representing ASCII tables specifically, whereas :class:`BinTableHDU` is more explicit that it represents a binary - table. These names come from the value ``XTENSION`` keyword in the tables' + table. These names come from the value ``XTENSION`` keyword in the tables' headers, which is ``TABLE`` for ASCII tables and ``BINTABLE`` for binary tables. :meth:`TableHDU.from_columns` can be used like so:: - >>> import numpy as np - >>> from astropy.io import fits - >>> a1 = np.array(['abcd', 'def']) - >>> r1 = np.array([11., 12.]) - >>> c1 = fits.Column(name='abc', format='A3', array=a1, ascii=True) - >>> c2 = fits.Column(name='def', format='E', array=r1, bscale=2.3, - ... bzero=0.6, ascii=True) - >>> c3 = fits.Column(name='t1', format='I', array=[91, 92, 93], - ... ascii=True) - >>> hdu = fits.TableHDU.from_columns([c1, c2, c3]) - >>> hdu.writeto('ascii.fits') - >>> hdu.data - FITS_rec([('abcd', 11.0, 91), ('def', 12.0, 92), ('', 0.0, 93)], - dtype=[('abc', '|S3'), ('def', '|S14'), ('t1', '|S10')]) + >>> import numpy as np + + >>> a1 = np.array(['abcd', 'def']) + >>> r1 = np.array([11., 12.]) + >>> col1 = fits.Column(name='abc', format='A3', array=a1, ascii=True) + >>> col2 = fits.Column(name='def', format='E', array=r1, bscale=2.3, + ... bzero=0.6, ascii=True) + >>> col3 = fits.Column(name='t1', format='I', array=[91, 92, 93], ascii=True) + >>> hdu = fits.TableHDU.from_columns([col1, col2, col3]) + >>> hdu.data + FITS_rec([('abc', np.float64(11.0), np.int32(91)), + ('def', np.float64(12.0), np.int32(92)), + ('', np.float64(0.0), np.int32(93))], + dtype=(numpy.record, [('abc', 'S3'), ('def', 'S15'), ('t1', 'S10')])) It should be noted that when the formats of the columns are unambiguously specific to ASCII tables it is not necessary to specify ``ascii=True`` in -the :class:`ColDefs` constructor. In this case there *is* ambiguity because +the :class:`ColDefs` constructor. In this case there *is* ambiguity because the format code ``'I'`` represents a 16-bit integer in binary tables, while in -ASCII tables it is not technically a valid format. ASCII table format codes +ASCII tables it is not technically a valid format. ASCII table format codes technically require a character width for each column, such as ``'I10'`` to create a column that can hold integers up to 10 characters wide. -However, PyFITS allows the width specification to be omitted in some cases. -When it is ommitted from ``'I'`` format columns the minimum width needed to -accurately represent all integers in the column is used. The only problem with +However, ``astropy`` allows the width specification to be omitted in some cases. +When it is omitted from ``'I'`` format columns the minimum width needed to +accurately represent all integers in the column is used. The only problem with using this shortcut is its ambiguity with the binary table ``'I'`` format, so -specifying ``ascii=True`` is a good practice (though PyFITS will still figure -out what you meant in most cases). +specifying ``ascii=True`` is a good practice (though ``astropy`` will still +figure out what you meant in most cases). +.. _variable_length_array_tables: + Variable Length Array Tables -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +============================ The FITS standard also supports variable length array tables. The basic idea is that sometimes it is desirable to have tables with cells in the same field @@ -127,95 +129,135 @@ different cells. A variable length array table can have one or more fields (columns) which are variable length. The rest of the fields (columns) in the same table can still -be regular, fixed-length ones. Astropy will automatically detect what kind of -field it is during reading; no special action is needed from the user. The data -type specification (i.e. the value of the TFORM keyword) uses an extra letter -'P' and the format is +be regular, fixed-length ones. +The data for the variable-length arrays in a table are not +stored in the main data table; they are stored in a supplemental +data area, the heap, following the main data table. +``astropy`` will automatically detect what kind +of field it is during reading; no special action is needed from the user. The +data type specification (i.e., the value of the TFORM keyword) uses an extra +letter 'P' (or 'Q') and the format is: .. parsed-literal:: rPt(max) -where r is 0, 1, or absent, t is one of the letter code for regular table data -type (L, B, X, I, J, etc. currently, the X format is not supported for variable -length array field in Astropy), and max is the maximum number of elements. So, -for a variable length field of int32, The corresponding format spec is, -e.g. 'PJ(100)':: +where ``r`` may be 0 or 1 (typically omitted, as it is not applicable to +variable length arrays), ``t`` is one of the letter codes for basic data types +(L, B, I, J, etc.; currently, the X format is not supported for variable length +array field in ``astropy``), and ``max`` is the maximum number of elements of +any array in the column. So, for a variable length field of int16, the +corresponding format spec is, for example, 'PJ(100)'. +What is stored in the main data table field is an array descriptor. +This consists of two 32-bit signed integer values in the case of ’P’ format, +(or two 64-bit signed integer values in the case of ’Q’ format): +the number of elements (array length) of the stored array, +followed by the zero-indexed byte offset of the first +element of the array, measured from the start of the heap area. + +.. note:: + While P format uses 32-bit signed integers, the FITS standard does not define + the meaning for negative values. P format indexes from byte 0 to + :math:`2^{31} - 1`. + Depending on the format of the variable arrays (int or float or double) and + the number of rows it might be necessary to use the Q format to allocate enough + heap space. + +Example +------- + +.. + EXAMPLE START + Accessing Variable Length Array Columns in FITS Tables + +This example shows a variable length array field of data type int16:: - >>> f = fits.open('variable_length_table.fits') - >>> print f[1].header['tform5'] - 1PI(20) - >>> print f[1].data.field(4)[:3] - [array([1], dtype=int16) array([88, 2], dtype=int16) - array([ 1, 88, 3], dtype=int16)] + >>> filename = fits.util.get_testdata_filepath('variable_length_table.fits') + >>> hdul = fits.open(filename) + >>> hdul[1].header['tform1'] + 'PI(3)' + >>> print(hdul[1].data.field(0)) + [array([45, 56], dtype=int16) array([11, 12, 13], dtype=int16)] + >>> hdul.close() -The above example shows a variable length array field of data type int16 and its -first row has one element, second row has 2 elements etc. Accessing variable -length fields is almost identical to regular fields, except that operations on -the whole filed are usually not possible. A user has to process the field row by -row. +In this field the first row has one element, the second row has two elements, +etc. Accessing variable length fields is almost identical to regular fields, +except that operations on the whole field simultaneously are usually not +possible. A user has to process the field row by row as though they are +independent arrays. + +.. + EXAMPLE END Creating a Variable Length Array Table -"""""""""""""""""""""""""""""""""""""" +-------------------------------------- Creating a variable length table is almost identical to creating a regular table. The only difference is in the creation of field definitions which are variable length arrays. First, the data type specification will need the 'P' letter, and secondly, the field data must be an objects array (as included in -the numpy module). Here is an example of creating a table with two fields, one -is regular and the other variable length array:: - - >>> from astropy.io import fits - >>> import numpy as np - >>> c1 = fits.Column(name='var', format='PJ()', - ... array=np.array([[45., 56] - ... [11, 12, 13]], - ... dtype=np.object)) - >>> c2 = fits.Column(name='xyz', format='2I', array=[[11, 3], [12, 4]]) - >>> tbhdu = fits.BinTableHDU.from_columns([c1, c2]) - >>> print tbhdu.data - FITS_rec([(array([45, 56]), array([11, 3], dtype=int16)), - (array([11, 12, 13]), array([12, 4], dtype=int16))], - dtype=[('var', '>> tbhdu.writeto('var_table.fits') - >>> hdu = fits.open('var_table.fits') - >>> hdu[1].header - XTENSION= 'BINTABLE' / binary table extension - BITPIX = 8 / array data type - NAXIS = 2 / number of array dimensions - NAXIS1 = 12 / length of dimension 1 - NAXIS2 = 2 / length of dimension 2 - PCOUNT = 20 / number of group parameters - GCOUNT = 1 / number of groups - TFIELDS = 2 / number of table fields - TTYPE1 = 'var ' - TFORM1 = 'PJ(3) ' - TTYPE2 = 'xyz ' - TFORM2 = '2I ' - +the ``numpy`` module). + +Example +^^^^^^^ + +.. + EXAMPLE START + Creating a Variable Length Array Column in a FITS Table + +Here is an example of creating a table with two fields; one is regular and the +other a variable length array:: + + >>> col1 = fits.Column( + ... name='var', format='PI()', + ... array=np.array([[45, 56], [11, 12, 13]], dtype=np.object_)) + >>> col2 = fits.Column(name='xyz', format='2I', array=[[11, 3], [12, 4]]) + >>> hdu = fits.BinTableHDU.from_columns([col1, col2]) + >>> data = hdu.data + >>> data # doctest: +SKIP + FITS_rec([([45, 56], [11, 3]), ([11, 12, 13], [12, 4])], + dtype=(numpy.record, [('var', '>> hdu.writeto('variable_length_table.fits') + >>> with fits.open('variable_length_table.fits') as hdul: + ... print(repr(hdul[1].header)) + XTENSION= 'BINTABLE' / binary table extension + BITPIX = 8 / array data type + NAXIS = 2 / number of array dimensions + NAXIS1 = 12 / length of dimension 1 + NAXIS2 = 2 / length of dimension 2 + PCOUNT = 10 / number of group parameters + GCOUNT = 1 / number of groups + TFIELDS = 2 / number of table fields + TTYPE1 = 'var ' + TFORM1 = 'PI(3) ' + TTYPE2 = 'xyz ' + TFORM2 = '2I ' + +.. + EXAMPLE END .. _random-groups: Random Access Groups -^^^^^^^^^^^^^^^^^^^^ +==================== Another less familiar data structure supported by the FITS standard is the random access group. This convention was established before the binary table extension was introduced. In most cases its use can now be superseded by the binary table. It is mostly used in radio interferometry. -Like Primary HDUs, a Random Access Group HDU is always the first HDU of a FITS +Like primary HDUs, a Random Access Group HDU is always the first HDU of a FITS file. Its data has one or more groups. Each group may have any number (including 0) of parameters, together with an image. The parameters and the image have the same data type. -All groups in the same HDU have the same data structure, i.e. same data type +All groups in the same HDU have the same data structure, that is, same data type (specified by the keyword BITPIX, as in image HDU), same number of parameters (specified by PCOUNT), and the same size and shape (specified by NAXISn keywords) of the image data. The number of groups is specified by GCOUNT and the keyword NAXIS1 is always 0. Thus the total data size for a Random Access -Group HDU is +Group HDU is: .. parsed-literal:: @@ -223,87 +265,91 @@ Group HDU is Header and Summary -"""""""""""""""""" +------------------ Accessing the header of a Random Access Group HDU is no different from any -other HDU. Just use the .header attribute. +other HDU; you can use the .header attribute. The content of the HDU can similarly be summarized by using the :meth:`HDUList.info` method:: - >>> f = fits.open('random_group.fits') - >>> print f[0].header['groups'] + >>> filename = fits.util.get_testdata_filepath('group.fits') + >>> hdul = fits.open(filename) + >>> hdul[0].header['groups'] True - >>> print f[0].header['gcount'] - 7956 - >>> print f[0].header['pcount'] - 6 - >>> f.info() - Filename: random_group.fits - No. Name Type Cards Dimensions Format - 0 AN GroupsHDU 158 (3, 4, 1, 1, 1) Float32 7956 Groups - 6 Parameters + >>> hdul[0].header['gcount'] + 10 + >>> hdul[0].header['pcount'] + 3 + >>> hdul.info() + Filename: ...group.fits + No. Name Ver Type Cards Dimensions Format + 0 PRIMARY 1 GroupsHDU 15 (5, 3, 1, 1) float32 10 Groups 3 Parameters Data: Group Parameters -"""""""""""""""""""""" +---------------------- -The data part of a random access group HDU is, like other HDUs, in the +The data part of a Random Access Group HDU is, like other HDUs, in the ``.data`` attribute. It includes both parameter(s) and image array(s). -Show the data in 100th group, including parameters and data:: +Examples +^^^^^^^^ + +.. + EXAMPLE START + Group Parameters in Random Access Group HDUs - >>> print f[0].data[99] - (-8.1987486677035799e-06, 1.2010923615889215e-05, - -1.011189139244005e-05, 258.0, 2445728., 0.10, array([[[[[ 12.4308672 , - 0.56860745, 3.99993873], - [ 12.74043655, 0.31398511, 3.99993873], - [ 0. , 0. , 3.99993873], - [ 0. , 0. , 3.99993873]]]]], dtype=float32)) +To show the contents of the third group, including parameters and data:: -The data first lists all the parameters, then the image array, for the + >>> hdul[0].data[2] # doctest: +FLOAT_CMP + (np.float32(2.1), np.float32(42.0), np.float32(42.0), array([[[[30., 31., 32., 33., 34.], + [35., 36., 37., 38., 39.], + [40., 41., 42., 43., 44.]]]], dtype='>f4')) + +The data first lists all of the parameters, then the image array, for the specified group(s). As a reminder, the image data in this file has the shape of -(1,1,1,4,3) in Python or C convention, or (3,4,1,1,1) in IRAF or FORTRAN +(1,1,1,4,3) in Python or C convention, or (3,4,1,1,1) in IRAF or Fortran convention. To access the parameters, first find out what the parameter names are, with the -.parnames attribute:: +``.parnames`` attribute:: - >>> f[0].data.parnames # get the parameter names - ['uu--', 'vv--', 'ww--', 'baseline', 'date', 'date'] + >>> hdul[0].data.parnames # get the parameter names + ['abc', 'xyz', 'xyz'] The group parameter can be accessed by the :meth:`~GroupData.par` method. Like the table :meth:`~FITS_rec.field` method, the argument can be either index or name:: - >>> print f[0].data.par(0)[99] # Access group parameter by name or by index - -8.1987486677035799e-06 - >>> print f[0].data.par('uu--')[99] - -8.1987486677035799e-06 + >>> hdul[0].data.par(0)[8] # Access group parameter by name or by index # doctest: +FLOAT_CMP + np.float32(8.1) + >>> hdul[0].data.par('abc')[8] # doctest: +FLOAT_CMP + np.float32(8.1) -Note that the parameter name 'date' appears twice. This is a feature in the +Note that the parameter name 'xyz' appears twice. This is a feature in the random access group, and it means to add the values together. Thus:: - >>> f[0].data.parnames # get the parameter names - ['uu--', 'vv--', 'ww--', 'baseline', 'date', 'date'] - >>> print f[0].data.par(4)[99] # Duplicate parameter name 'date' - 2445728.0 - >>> print f[0].data.par(5)[99] - 0.10 + >>> hdul[0].data.parnames # get the parameter names + ['abc', 'xyz', 'xyz'] + >>> hdul[0].data.par(1)[8] # Duplicate parameter name 'xyz' + np.float32(42.0) + >>> hdul[0].data.par(2)[8] + np.float32(42.0) >>> # When accessed by name, it adds the values together if the name is >>> # shared by more than one parameter - >>> print f[0].data.par('date')[99] - 2445728.10 + >>> hdul[0].data.par('xyz')[8] + np.float64(84.0) The :meth:`~GroupData.par` is a method for either the entire data object or one data item (a group). So there are two possible ways to get a group parameter for a certain group, this is similar to the situation in table data (with its :meth:`~FITS_rec.field` method):: - >>> print f[0].data.par(0)[99] - -8.1987486677035799e-06 - >>> print f[0].data[99].par(0) - -8.1987486677035799e-06 + >>> hdul[0].data.par(0)[8] # doctest: +FLOAT_CMP + np.float32(8.1) + >>> hdul[0].data[8].par(0) # doctest: +FLOAT_CMP + np.float32(8.1) On the other hand, to modify a group parameter, we can either assign the new value directly (if accessing the row/group number last) or use the @@ -312,224 +358,345 @@ method :meth:`~Group.setpar` is also needed for updating by name if the parameter is shared by more than one parameters:: >>> # Update group parameter when selecting the row (group) number last - >>> f[0].data.par(0)[99] = 99. + >>> hdul[0].data.par(0)[8] = 99. >>> # Update group parameter when selecting the row (group) number first - >>> f[0].data[99].setpar(0, 99.) # or setpar('uu--', 99.) - >>> + >>> hdul[0].data[8].setpar(0, 99.) # or: + >>> hdul[0].data[8].setpar('abc', 99.) >>> # Update group parameter by name when the name is shared by more than >>> # one parameters, the new value must be a tuple of constants or >>> # sequences - >>> f[0].data[99].setpar('date', (2445729., 0.3)) - >>> f[0].data[:3].setpar('date', (2445729., [0.11, 0.22, 0.33])) - >>> f[0].data[:3].par('date') - array([ 2445729.11 , 2445729.22 , 2445729.33000001]) + >>> hdul[0].data[8].setpar('xyz', (2445729., 0.3)) + >>> hdul[0].data[8:].par('xyz') # doctest: +FLOAT_CMP + array([2.44572930e+06, 8.40000000e+01]) +.. + EXAMPLE END Data: Image Data -"""""""""""""""" +---------------- The image array of the data portion is accessible by the -:attr:`~GroupData.data` attribute of the data object. A numpy array is +:attr:`~GroupData.data` attribute of the data object. A ``numpy`` array is returned:: - >>> print f[0].data.data[99] - array([[[[[ 12.4308672 , 0.56860745, 3.99993873], - [ 12.74043655, 0.31398511, 3.99993873], - [ 0. , 0. , 3.99993873], - [ 0. , 0. , 3.99993873]]]]], type=float32) + >>> print(hdul[0].data.data[8]) # doctest: +FLOAT_CMP + [[[[120. 121. 122. 123. 124.] + [125. 126. 127. 128. 129.] + [130. 131. 132. 133. 134.]]]] + >>> hdul.close() Creating a Random Access Group HDU -"""""""""""""""""""""""""""""""""" +---------------------------------- -To create a random access group HDU from scratch, use :class:`GroupData` to +To create a Random Access Group HDU from scratch, use :class:`GroupData` to encapsulate the data into the group data structure, and use :class:`GroupsHDU` -to create the HDU itself:: +to create the HDU itself. + +Example +^^^^^^^ + +.. + EXAMPLE START + Creating a Random Access Group HDU in a FITS File + +To create a Random Access Group HDU:: >>> # Create the image arrays. The first dimension is the number of groups. - >>> imdata = numpy.arange(100.0, shape=(10, 1, 1, 2, 5)) + >>> imdata = np.arange(150.0).reshape(10, 1, 1, 3, 5) >>> # Next, create the group parameter data, we'll have two parameters. >>> # Note that the size of each parameter's data is also the number of >>> # groups. >>> # A parameter's data can also be a numeric constant. - >>> pdata1 = numpy.arange(10) + 0.1 + >>> pdata1 = np.arange(10) + 0.1 >>> pdata2 = 42 >>> # Create the group data object, put parameter names and parameter data >>> # in lists assigned to their corresponding arguments. >>> # If the data type (bitpix) is not specified, the data type of the >>> # image will be used. - >>> x = fits.GroupData(imdata, parnames=['abc', 'xyz'], - ... pardata=[pdata1, pdata2], bitpix=-32) + >>> x = fits.GroupData(imdata, bitpix=-32, + ... parnames=['abc', 'xyz', 'xyz'], + ... pardata=[pdata1, pdata2, pdata2]) >>> # Now, create the GroupsHDU and write to a FITS file. >>> hdu = fits.GroupsHDU(x) >>> hdu.writeto('test_group.fits') >>> hdu.header - SIMPLE = T / conforms to FITS standard - BITPIX = -32 / array data type - NAXIS = 5 / number of array dimensions - NAXIS1 = 0 - NAXIS2 = 5 - NAXIS3 = 2 - NAXIS4 = 1 - NAXIS5 = 1 - EXTEND = T - GROUPS = T / has groups - PCOUNT = 2 / number of parameters - GCOUNT = 10 / number of groups - PTYPE1 = 'abc ' - PTYPE2 = 'xyz ' - >>> print hdu.data[:2] - FITS_rec[ - (0.10000000149011612, 42.0, array([[[[ 0., 1., 2., 3., 4.], - [ 5., 6., 7., 8., 9.]]]], dtype=float32)), - (1.1000000238418579, 42.0, array([[[[ 10., 11., 12., 13., 14.], - [ 15., 16., 17., 18., 19.]]]], dtype=float32)) - ] + SIMPLE = T / conforms to FITS standard + BITPIX = -32 / array data type + NAXIS = 5 / number of array dimensions + NAXIS1 = 0 + NAXIS2 = 5 + NAXIS3 = 3 + NAXIS4 = 1 + NAXIS5 = 1 + EXTEND = T + GROUPS = T / has groups + PCOUNT = 3 / number of parameters + GCOUNT = 10 / number of groups + PTYPE1 = 'abc ' + PTYPE2 = 'xyz ' + PTYPE3 = 'xyz ' + >>> data = hdu.data + >>> hdu.data # doctest: +SKIP + GroupData([ (0.1 , 42., 42., [[[[ 0., 1., 2., 3., 4.], [ 5., 6., 7., 8., 9.], [ 10., 11., 12., 13., 14.]]]]), + (1.10000002, 42., 42., [[[[ 15., 16., 17., 18., 19.], [ 20., 21., 22., 23., 24.], [ 25., 26., 27., 28., 29.]]]]), + (2.0999999 , 42., 42., [[[[ 30., 31., 32., 33., 34.], [ 35., 36., 37., 38., 39.], [ 40., 41., 42., 43., 44.]]]]), + (3.0999999 , 42., 42., [[[[ 45., 46., 47., 48., 49.], [ 50., 51., 52., 53., 54.], [ 55., 56., 57., 58., 59.]]]]), + (4.0999999 , 42., 42., [[[[ 60., 61., 62., 63., 64.], [ 65., 66., 67., 68., 69.], [ 70., 71., 72., 73., 74.]]]]), + (5.0999999 , 42., 42., [[[[ 75., 76., 77., 78., 79.], [ 80., 81., 82., 83., 84.], [ 85., 86., 87., 88., 89.]]]]), + (6.0999999 , 42., 42., [[[[ 90., 91., 92., 93., 94.], [ 95., 96., 97., 98., 99.], [100., 101., 102., 103., 104.]]]]), + (7.0999999 , 42., 42., [[[[105., 106., 107., 108., 109.], [110., 111., 112., 113., 114.], [115., 116., 117., 118., 119.]]]]), + (8.10000038, 42., 42., [[[[120., 121., 122., 123., 124.], [125., 126., 127., 128., 129.], [130., 131., 132., 133., 134.]]]]), + (9.10000038, 42., 42., [[[[135., 136., 137., 138., 139.], [140., 141., 142., 143., 144.], [145., 146., 147., 148., 149.]]]])], + dtype=(numpy.record, [('abc', '>> f = fits.open('compressed_image.fits') - >>> print f[1].header - XTENSION= 'IMAGE ' / extension type - BITPIX = 16 / array data type - NAXIS = 2 / number of array dimensions - NAXIS1 = 512 / length of data axis - NAXIS2 = 512 / length of data axis - PCOUNT = 0 / number of parameters - GCOUNT = 1 / one data group (required keyword) - EXTNAME = 'COMPRESSED' / name of this binary table extension +.. + EXAMPLE START + Accessing Compressed FITS Image HDU Headers -The contents of the corresponding binary table HDU may be accessed using the -hidden ``._header`` attribute. However, all user interface with the HDU header -should be accomplished through the image header (the ``.header`` attribute):: +The content of the decompressed HDU header may be accessed using the ``.header`` attribute:: - >>> f = fits.open('compressed_image.fits') - >>> print f[1]._header - XTENSION= 'BINTABLE' / binary table extension - BITPIX = 8 / 8-bit bytes - NAXIS = 2 / 2-dimensional binary table - NAXIS1 = 8 / width of table in bytes - NAXIS2 = 512 / number of rows in table - PCOUNT = 157260 / size of special data area - GCOUNT = 1 / one data group (required keyword) - TFIELDS = 1 / number of fields in each row - TTYPE1 = 'COMPRESSED_DATA' / label for field 1 - TFORM1 = '1PB(384)' / data format of field: variable length array - ZIMAGE = T / extension contains compressed image - ZBITPIX = 16 / data type of original image - ZNAXIS = 2 / dimension of original image - ZNAXIS1 = 512 / length of original image axis - ZNAXIS2 = 512 / length of original image axis - ZTILE1 = 512 / size of tiles to be compressed - ZTILE2 = 1 / size of tiles to be compressed - ZCMPTYPE= 'RICE_1 ' / compression algorithm - ZNAME1 = 'BLOCKSIZE' / compression block size - ZVAL1 = 32 / pixels per block - EXTNAME = 'COMPRESSED' / name of this binary table extension + >>> filename = fits.util.get_testdata_filepath('compressed_image.fits') + + >>> hdul = fits.open(filename) + >>> hdul[1].header + XTENSION= 'IMAGE ' / Image extension + BITPIX = 16 / data type of original image + NAXIS = 2 / dimension of original image + NAXIS1 = 10 / length of original image axis + NAXIS2 = 10 / length of original image axis + PCOUNT = 0 / number of parameters + GCOUNT = 1 / number of groups The contents of the HDU can be summarized by using either the :func:`info` convenience function or method:: - >>> fits.info('compressed_image.fits') - Filename: compressed_image.fits - No. Name Type Cards Dimensions Format - 0 PRIMARY PrimaryHDU 6 () int16 - 1 COMPRESSED CompImageHDU 52 (512, 512) int16 - >>> - >>> f = fits.open('compressed_image.fits') - >>> f.info() - Filename: compressed_image.fits - No. Name Type Cards Dimensions Format - 0 PRIMARY PrimaryHDU 6 () int16 - 1 COMPRESSED CompImageHDU 52 (512, 512) int16 - >>> + >>> fits.info(filename) + Filename: ...compressed_image.fits + No. Name Ver Type Cards Dimensions Format + 0 PRIMARY 1 PrimaryHDU 4 () + 1 COMPRESSED_IMAGE 1 CompImageHDU 7 (10, 10) int16 + >>> hdul.info() + Filename: ...compressed_image.fits + No. Name Ver Type Cards Dimensions Format + 0 PRIMARY 1 PrimaryHDU 4 () + 1 COMPRESSED_IMAGE 1 CompImageHDU 7 (10, 10) int16 + +.. + EXAMPLE END Data -"""" +---- As with the header, the data of a compressed image HDU appears to the user as -standard uncompressed image data. The actual data is stored in the fits file -as Binary Table data containing at least one column (COMPRESSED_DATA). Each -row of this variable-length column contains the byte stream that was generated -as a result of compressing the corresponding image tile. Several optional -columns may also appear. These include, UNCOMPRESSED_DATA to hold the -uncompressed pixel values for tiles that cannot be compressed, ZSCALE and ZZERO -to hold the linear scale factor and zero point offset which may be needed to -transform the raw uncompressed values back to the original image pixel values, -and ZBLANK to hold the integer value used to represent undefined pixels (if -any) in the image. +standard uncompressed image data. The actual data is stored in the FITS file +as binary table data containing at least one column (COMPRESSED_DATA). Each +row of this variable length column contains the byte stream that was generated +as a result of compressing the corresponding image tile. Several optional +columns may also appear. These include GZIP_COMPRESSED_DATA to hold the +gzip-compressed data for tiles that cannot be compressed by the selected +algorithm, as well as ZSCALE and ZZERO to hold the linear scale factor and zero +point offset which may be needed to transform the raw uncompressed values back +to the original image pixel values, and ZBLANK to hold the integer value used to +represent undefined pixels (if any) in the image. + +Example +^^^^^^^ + +.. + EXAMPLE START + Accessing Compressed FITS Image HDU Data The contents of the uncompressed HDU data may be accessed using the ``.data`` attribute:: - >>> f = fits.open('compressed_image.fits') - >>> f[1].data - array([[38, 43, 35, ..., 45, 43, 41], - [36, 41, 37, ..., 42, 41, 39], - [38, 45, 37, ..., 42, 35, 43], - ..., - [49, 52, 49, ..., 41, 35, 39], - [57, 52, 49, ..., 40, 41, 43], - [53, 57, 57, ..., 39, 35, 45]], dtype=int16) + >>> hdul[1].data + array([[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], + [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], + [30, 31, 32, 33, 34, 35, 36, 37, 38, 39], + [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], + [50, 51, 52, 53, 54, 55, 56, 57, 58, 59], + [60, 61, 62, 63, 64, 65, 66, 67, 68, 69], + [70, 71, 72, 73, 74, 75, 76, 77, 78, 79], + [80, 81, 82, 83, 84, 85, 86, 87, 88, 89], + [90, 91, 92, 93, 94, 95, 96, 97, 98, 99]], dtype=int16) + >>> hdul.close() The compressed data can be accessed via the ``.compressed_data`` attribute, but -this rarely need be accessed directly. It may be useful for performing direct +this rarely needs be accessed directly. It may be useful for performing direct copies of the compressed data without needing to decompress it first. +.. + EXAMPLE END + Creating a Compressed Image HDU -""""""""""""""""""""""""""""""" +------------------------------- -To create a compressed image HDU from scratch, simply construct a +To create a compressed image HDU from scratch, construct a :class:`CompImageHDU` object from an uncompressed image data array and its -associated image header. From there, the HDU can be treated just like any -other image HDU:: +associated image header. From there, the HDU can be treated just like any +other image HDU. +Example +^^^^^^^ + +.. + EXAMPLE START + Creating a Compressed FITS Image HDU + +To create a compressed image HDU:: + + >>> imageData = np.arange(100).astype('i2').reshape(10, 10) + >>> imageHeader = fits.Header() >>> hdu = fits.CompImageHDU(imageData, imageHeader) >>> hdu.writeto('compressed_image.fits') The API documentation for the :class:`CompImageHDU` initializer method describes the possible options for constructing a :class:`CompImageHDU` object. + +.. + EXAMPLE END + + +Supported Integer Data Types +---------------------------- + +Not every compression algorithm can be used with every integer data type. The +table below summarizes which combinations work, including the cases where +``astropy`` accepts the input only when the values lie within a more +restricted range. + +.. list-table:: + :header-rows: 1 + :stub-columns: 1 + + * - Compression + - ``int16`` + - ``int32`` + - ``int64`` + - ``uint8`` + - ``uint16`` + - ``uint32`` + - ``uint64`` + * - ``GZIP_1`` + - ✅ + - ✅ + - ⚠ī¸ [1]_ + - ✅ + - ✅ + - ✅ + - ⚠ī¸ [1]_ + * - ``GZIP_2`` + - ✅ + - ✅ + - ⚠ī¸ [1]_ + - ✅ + - ✅ + - ✅ + - ⚠ī¸ [1]_ + * - ``RICE_1`` + - ✅ + - ✅ + - 🟡 [2]_ + - ✅ + - ✅ + - ✅ + - 🟡 [2]_ + * - ``HCOMPRESS_1`` + - ✅ + - ✅ + - 🟡 [2]_ + - ✅ + - ✅ + - ✅ + - 🟡 [2]_ + * - ``PLIO_1`` + - 🟡 [3]_ + - 🟡 [3]_ + - 🟡 [2]_ [3]_ + - ✅ + - ❌ [4]_ + - ❌ [4]_ + - ❌ [4]_ + * - ``NOCOMPRESS`` + - ✅ + - ✅ + - ✅ + - ✅ + - ✅ + - ✅ + - ✅ + +Legend: + +* ✅ Full support: any value within the type's range round-trips losslessly. +* 🟡 Partial support: works only when input values satisfy the numeric + restriction in the corresponding footnote; a ``ValueError`` is raised + otherwise. +* ⚠ī¸ Caveat: round-trips correctly within ``astropy``, but the resulting + file may not be readable by other FITS libraries (see footnote). +* ❌ Not supported: writing the data raises a ``ValueError``. + +.. [1] ``astropy`` writes a standards-compliant file, but ``cfitsio`` and + tools built on top of it (including ``funpack``, ``fitsio``, and DS9) do + not currently support reading 64-bit integer images compressed with + ``GZIP_1`` or ``GZIP_2``. The file round-trips correctly when read by + ``astropy`` itself. + +.. [2] 64-bit integer input is converted to a 32-bit type on write. The + conversion succeeds only if every input value fits in the corresponding + 32-bit range: ``[-2**31, 2**31 - 1]`` for signed and ``[0, 2**32 - 1]`` + for unsigned. Otherwise a ``ValueError`` is raised. When the conversion + succeeds an ``AstropyUserWarning`` is emitted to signal the precision + change. + +.. [3] ``PLIO_1`` is designed for pixel masks and supports only non-negative + integer values up to ``2**24 - 1`` (``16777215``). Negative values or + values above this limit cause a ``ValueError`` at write time. For + ``int64`` input both this restriction and the 32-bit conversion in + footnote [2]_ apply. + +.. [4] ``PLIO_1`` cannot store unsigned 16-, 32-, or 64-bit integers. Use + ``RICE_1``, ``HCOMPRESS_1``, ``GZIP_1``, or ``GZIP_2`` for unsigned data + that does not fit in ``uint8``. diff --git a/docs/io/fits/usage/verification.rst b/docs/io/fits/usage/verification.rst index cf6b610d6a5a..6ab9beb8c199 100644 --- a/docs/io/fits/usage/verification.rst +++ b/docs/io/fits/usage/verification.rst @@ -1,15 +1,15 @@ -.. doctest-skip-all - .. currentmodule:: astropy.io.fits +.. _fits_io_verification: + Verification ------------- +************ -Astropy has built in a flexible scheme to verify FITS data being conforming to -the FITS standard. The basic verification philosophy in Astropy is to be -tolerant in input and strict in output. +``astropy`` has built in a flexible scheme to verify FITS data conforming to +the FITS standard. The basic verification philosophy in ``astropy`` is to be +tolerant with input and strict with output. -When Astropy reads a FITS file which is not conforming to FITS standard, it +When ``astropy`` reads a FITS file which does not conform to FITS standard, it will not raise an error and exit. It will try to make the best educated interpretation and only gives up when the offending data is accessed and no unambiguous interpretation can be reached. @@ -21,19 +21,19 @@ not be held up because of a minor standard violation. FITS Standard -^^^^^^^^^^^^^ +============= Since FITS standard is a "loose" standard, there are many places the violation can occur and to enforce them all will be almost impossible. It is not uncommon for major observatories to generate data products which are not 100% FITS -compliant. Some observatories have also developed their own sub-standard -(dialect?) and some of these become so prevalent that they become de facto +compliant. Some observatories have also developed their own nonstandard +dialect and some of these are so prevalent that they have become de facto standards. Examples include the long string value and the use of the CONTINUE card. The violation of the standard can happen at different levels of the data -structure. Astropy's verification scheme is developed on these hierarchical -levels. Here are the 3 Astropy verification levels: +structure. ``astropy``'s verification scheme is developed on these hierarchical +levels. Here are the three ``astropy`` verification levels: 1. The HDU List @@ -42,30 +42,30 @@ levels. Here are the 3 Astropy verification levels: 3. Each Card in the HDU Header These three levels correspond to the three categories of objects: -:class:`HDUList`, any HDU (e.g. :class:`PrimaryHDU`, :class:`ImageHDU`, etc.), +:class:`HDUList`, any HDU (e.g., :class:`PrimaryHDU`, :class:`ImageHDU`, etc.), and :class:`Card`. They are the only objects having the ``verify()`` method. -Most other classes in astropy.io.fits do not have a ``verify()`` method. +Most other classes in `astropy.io.fits` do not have a ``verify()`` method. If ``verify()`` is called at the HDU List level, it verifies standard compliance at all three levels, but a call of ``verify()`` at the Card level -will only check the compliance of that Card. Since Astropy is tolerant when +will only check the compliance of that Card. Since ``astropy`` is tolerant when reading a FITS file, no ``verify()`` is called on input. On output, ``verify()`` is called with the most restrictive option as the default. Verification Options -^^^^^^^^^^^^^^^^^^^^ +==================== -There are several options accepted by all verify(option) calls in Astropy. In -addition, they available for the ``output_verify`` argument of the following +There are several options accepted by all verify(option) calls in ``astropy``. +In addition, they available for the ``output_verify`` argument of the following methods: ``close()``, ``writeto()``, and ``flush()``. In these cases, they are passed to a ``verify()`` call within these methods. The available options are: **exception** -This option will raise an exception, if any FITS standard is violated. This is -the default option for output (i.e. when ``writeto()``, ``close()``, or -``flush()`` is called. If a user wants to overwrite this default on output, the +This option will raise an exception if any FITS standard is violated. This is +the default option for output (i.e., when ``writeto()``, ``close()``, or +``flush()`` is called). If a user wants to overwrite this default on output, the other options listed below can be used. **warn** @@ -81,11 +81,11 @@ to the FITS standard. The ignore option is useful in the following situations: -1. An input FITS file with non-standard formatting is read and the user wants - to copy or write out to an output file. The non-standard formatting will be +1. An input FITS file with nonstandard formatting is read and the user wants + to copy or write out to an output file. The nonstandard formatting will be preserved in the output file. -2. A user wants to create a non-standard FITS file on purpose, possibly for +2. A user wants to create a nonstandard FITS file on purpose, possibly for testing or consistency. No warning message will be printed out. This is like a silent warning option @@ -96,7 +96,7 @@ No warning message will be printed out. This is like a silent warning option This option will try to fix any FITS standard violations. It is not always possible to fix such violations. In general, there are two kinds of FITS standard violations: fixable and non-fixable. For example, if a keyword has a -floating number with an exponential notation in lower case 'e' (e.g. 1.23e11) +floating number with an exponential notation in lower case 'e' (e.g., 1.23e11) instead of the upper case 'E' as required by the FITS standard, it is a fixable violation. On the other hand, a keyword name like 'P.I.' is not fixable, since it will not know what to use to replace the disallowed periods. If a violation @@ -104,11 +104,11 @@ is fixable, this option will print out a message noting it is fixed. If it is not fixable, it will throw an exception. The principle behind fixing is to do no harm. For example, it is plausible to -'fix' a Card with a keyword name like 'P.I.' by deleting it, but Astropy will -not take such action to hurt the integrity of the data. +'fix' a Card with a keyword name like 'P.I.' by deleting it, but ``astropy`` +will not take such action to hurt the integrity of the data. -Not all fixes may be the "correct" fix, but at least Astropy will try to make -the fix in such a way that it will not throw off other FITS readers. +Not all fixes may be the "correct" fix, but at least ``astropy`` will try to +make the fix in such a way that it will not throw off other FITS readers. **silentfix** @@ -116,8 +116,7 @@ Same as fix, but will not print out informative messages. This may be useful in a large script where the user does not want excessive harmless messages. If the violation is not fixable, it will still throw an exception. -In addition, as of Astropy version 0.4.0 the following 'combined' options are -available: +In addition the following combined options are available: * **fix+ignore** * **fix+warn** @@ -126,37 +125,38 @@ available: * **silentfix+warn** * **silentfix+exception** -These options combine the semantics of the basic options. For example +These options combine the semantics of the basic options. For example, ``silentfix+exception`` is actually equivalent to just ``silentfix`` in that fixable errors will be fixed silently, but any unfixable errors will raise an -exception. On the other hand ``silentfix+warn`` will issue warnings for +exception. On the other hand, ``silentfix+warn`` will issue warnings for unfixable errors, but will stay silent about any fixed errors. Verifications at Different Data Object Levels -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +============================================= -We'll examine what Astropy's verification does at the three different levels: +We will examine what ``astropy``'s verification does at the three different +levels: Verification at HDUList -""""""""""""""""""""""" +----------------------- At the HDU List level, the verification is only for two simple cases: -1. Verify that the first HDU in the HDU list is a Primary HDU. This is a - fixable case. The fix is to insert a minimal Primary HDU into the HDU list. +1. Verify that the first HDU in the HDU list is a primary HDU. This is a + fixable case. The fix is to insert a minimal primary HDU into the HDU list. -2. Verify second or later HDU in the HDU list is not a Primary HDU. Violation - will not be fixable. +2. Verify the second or later HDU in the HDU list is not a primary HDU. + Violation will not be fixable. Verification at Each HDU -"""""""""""""""""""""""" +------------------------ For each HDU, the mandatory keywords, their locations in the header, and their values will be verified. Each FITS HDU has a fixed set of required keywords in -a fixed order. For example, the Primary HDU's header must at least have the +a fixed order. For example, the primary HDU's header must at least have the following keywords: .. parsed-literal:: @@ -168,75 +168,97 @@ following keywords: If any of the mandatory keywords are missing or in the wrong order, the fix option will fix them:: - >>> hdu.header # has a 'bad' header - SIMPLE = T / - NAXIS = 0 - BITPIX = 8 / - >>> hdu.verify('fix') # fix it - Output verification result: - 'BITPIX' card at the wrong place (card 2). Fixed by moving it to the right - place (card 1). - >>> h.header # voila! - SIMPLE = T / conforms to FITS standard - BITPIX = 8 / array data type - NAXIS = 0 - + >>> from astropy.io import fits + >>> filename = fits.util.get_testdata_filepath('verify.fits') + >>> hdul = fits.open(filename) + >>> hdul[0].header + SIMPLE = T / conforms to FITS standard + NAXIS = 0 / NUMBER OF AXES + BITPIX = 8 / BITS PER PIXEL + >>> hdul[0].verify('fix') # doctest: +SHOW_WARNINGS + VerifyWarning: Verification reported errors: + VerifyWarning: 'BITPIX' card at the wrong place (card 2). + Fixed by moving it to the right place (card 1). + VerifyWarning: Note: astropy.io.fits uses zero-based indexing. + >>> hdul[0].header # voila! + SIMPLE = T / conforms to FITS standard + BITPIX = 8 / BITS PER PIXEL + NAXIS = 0 / NUMBER OF AXES + >>> hdul.close() Verification at Each Card -""""""""""""""""""""""""" +------------------------- The lowest level, the Card, also has the most complicated verification -possibilities. Here is a lit of fixable and not fixable Cards: +possibilities. + +Examples +^^^^^^^^ + +.. + EXAMPLE START + Verification at Each Card in astropy.io.fits + +Here is a list of fixable and not fixable Cards: Fixable Cards: -1. floating point numbers with lower case 'e' or 'd' +1. Floating point numbers with lower case 'e' or 'd':: + + >>> from astropy.io import fits + >>> c = fits.Card.fromstring('FIX1 = 2.1e23') + >>> c.verify('silentfix') + >>> print(c) + FIX1 = 2.1E23 + +2. The equal sign is before column nine in the card image:: -2. the equal sign is before column 9 in the card image + >>> c = fits.Card.fromstring('FIX2= 2') + >>> c.verify('silentfix') + >>> print(c) + FIX2 = 2 -3. string value without enclosing quotes +3. String value without enclosing quotes:: -4. missing equal sign before column 9 in the card image + >>> c = fits.Card.fromstring('FIX3 = string value without quotes') + >>> c.verify('silentfix') + >>> print(c) + FIX3 = 'string value without quotes' -5. space between numbers and E or D in floating point values +4. Missing equal sign before column nine in the card image. -6. unparsable values will be "fixed" as a string +5. Space between numbers and E or D in floating point values:: -Here are some examples of fixable cards: + >>> c = fits.Card.fromstring('FIX5 = 2.4 e 03') + >>> c.verify('silentfix') + >>> print(c) + FIX5 = 2.4E03 - >>> hdu.header[4:] # has a bunch of fixable cards - FIX1 = 2.1e23 - FIX2= 2 - FIX3 = string value without quotes - FIX4 2 - FIX5 = 2.4 e 03 - FIX6 = '2 10 ' - >>> hdu.header[5] # can still access the values before the fix - 2 - >>> hdu.header['fix4'] - 2 - >>> hdu.header['fix5'] - 2400.0 - >>> hdu.verify('silentfix') - >>> hdu.header[4:] - FIX1 = 2.1E23 - FIX2 = 2 - FIX3 = 'string value without quotes' - FIX4 = 2 - FIX5 = 2.4E03 - FIX6 = '2 10 ' +6. Unparsable values will be "fixed" as a string:: + + >>> c = fits.Card.fromstring('FIX6 = 2 10 ') + >>> c.verify('fix+warn') # doctest: +SHOW_WARNINGS + VerifyWarning: Verification reported errors: + VerifyWarning: Card 'FIX6' is not FITS standard + (invalid value string: '2 10'). + Fixed 'FIX6' card to meet the FITS standard. + VerifyWarning: Note: astropy.io.fits uses zero-based indexing. + >>> print(c) + FIX6 = '2 10 ' Unfixable Cards: -1. illegal characters in keyword name +1. Illegal characters in keyword name. -We'll summarize the verification with a "life-cycle" example:: +We will summarize the verification with a "life-cycle" example:: >>> h = fits.PrimaryHDU() # create a PrimaryHDU >>> # Try to add an non-standard FITS keyword 'P.I.' (FITS does no allow >>> # '.' in the keyword), if using the update() method - doesn't work! - >>> h['P.I.'] = 'Hubble' - ValueError: Illegal keyword name 'P.I.' + >>> h.header['P.I.'] = 'Hubble' # doctest: +SHOW_WARNINGS + VerifyWarning: Keyword name 'P.I.' is greater than 8 characters or + contains characters not allowed by the FITS standard; + a HIERARCH card will be created. >>> # Have to do it the hard way (so a user will not do this by accident) >>> # First, create a card image and give verbatim card content (including >>> # the proper spacing, but no need to add the trailing blanks) @@ -244,105 +266,122 @@ We'll summarize the verification with a "life-cycle" example:: >>> h.header.append(c) # then append it to the header >>> # Now if we try to write to a FITS file, the default output >>> # verification will not take it. - >>> h.writeto('pi.fits') - Output verification result: - HDU 0: - Card 4: - Unfixable error: Illegal keyword name 'P.I.' - ...... - raise VerifyError - VerifyError + >>> h.writeto('pi.fits') # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + VerifyError: HDU 0: + Card 5: + Card 'P.I. ' is not FITS standard (equal sign not at column 8). + Illegal keyword name 'P.I. ' >>> # Must set the output_verify argument to 'ignore', to force writing a >>> # non-standard FITS file >>> h.writeto('pi.fits', output_verify='ignore') >>> # Now reading a non-standard FITS file >>> # astropy.io.fits is magnanimous in reading non-standard FITS files - >>> hdus = fits.open('pi.fits') - >>> hdus[0].header - SIMPLE = T / conforms to FITS standard - BITPIX = 8 / array data type - NAXIS = 0 / number of array dimensions - EXTEND = T - P.I. = 'Hubble' + >>> hdul = fits.open('pi.fits') + >>> hdul[0].header # doctest: +SHOW_WARNINGS + SIMPLE = T / conforms to FITS standard + BITPIX = 8 / array data type + NAXIS = 0 / number of array dimensions + EXTEND = T + HIERARCH P.I. = 'Hubble ' + P.I. = 'Hubble ' + VerifyWarning: Verification reported errors: + VerifyWarning: Card 'P.I. ' is not FITS standard (equal sign + not at column 8). Fixed 'P.I. ' card to meet the FITS standard. + VerifyWarning: Unfixable error: Illegal keyword name 'P.I. ' + VerifyWarning: Note: astropy.io.fits uses zero-based indexing. >>> # even when you try to access the offending keyword, it does NOT >>> # complain - >>> hdus[0].header['p.i.'] + >>> hdul[0].header['p.i.'] 'Hubble' >>> # But if you want to make sure if there is anything wrong/non-standard, >>> # use the verify() method - >>> hdus.verify() - Output verification result: - HDU 0: - Card 4: - Unfixable error: Illegal keyword name 'P.I.' + >>> hdul.verify() # doctest: +SHOW_WARNINGS + VerifyWarning: Verification reported errors: + VerifyWarning: HDU 0: + VerifyWarning: Card 5: + VerifyWarning: Illegal keyword name 'P.I. ' + VerifyWarning: Note: astropy.io.fits uses zero-based indexing. + >>> hdul.close() +.. + EXAMPLE END -Verification using the FITS Checksum Keyword Convention -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Verification Using the FITS Checksum Keyword Convention +======================================================= The North American FITS committee has reviewed the FITS Checksum Keyword -Convention for possible adoption as a FITS Standard. This convention provides -an integrity check on information contained in FITS HDUs. The convention -consists of two header keyword cards: CHECKSUM and DATASUM. The CHECKSUM +Convention for possible adoption as a FITS Standard. This convention provides +an integrity check on information contained in FITS HDUs. The convention +consists of two header keyword cards: CHECKSUM and DATASUM. The CHECKSUM keyword is defined as an ASCII character string whose value forces the 32-bit 1's complement checksum accumulated over all the 2880-byte FITS logical records -in the HDU to equal negative zero. The DATASUM keyword is defined as a +in the HDU to equal negative zero. The DATASUM keyword is defined as a character string containing the unsigned integer value of the 32-bit 1's -complement checksum of the data records in the HDU. Verifying the the +complement checksum of the data records in the HDU. Verifying the accumulated checksum is still equal to negative zero provides a fairly reliable way to determine that the HDU has not been modified by subsequent data processing operations or corrupted while copying or storing the file on physical media. -In order to avoid any impact on performance, by default Astropy will not verify -HDU checksums when a file is opened or generate checksum values when a file is -written. In fact, CHECKSUM and DATASUM cards are automatically removed from -HDU headers when a file is opened, and any CHECKSUM or DATASUM cards are -stripped from headers when a HDU is written to a file. In order to verify the +In order to avoid any impact on performance, by default ``astropy`` will not +verify HDU checksums when a file is opened or generate checksum values when a +file is written. In fact, CHECKSUM and DATASUM cards are automatically removed +from HDU headers when a file is opened, and any CHECKSUM or DATASUM cards are +stripped from headers when an HDU is written to a file. In order to verify the checksum values for HDUs when opening a file, the user must supply the checksum keyword argument in the call to the open convenience function with a value of -True. When this is done, any checksum verification failure will cause a -warning to be issued (via the warnings module). If checksum verification is +True. When this is done, any checksum verification failure will cause a +warning to be issued (via the warnings module). If checksum verification is requested in the open, and no CHECKSUM or DATASUM cards exist in the HDU -header, the file will open without comment. Similarly, in order to output the +header, the file will open without comment. Similarly, in order to output the CHECKSUM and DATASUM cards in an HDU header when writing to a file, the user must supply the checksum keyword argument with a value of True in the call to -the writeto function. It is possible to write only the DATASUM card to the +the ``writeto()`` function. It is possible to write only the DATASUM card to the header by supplying the checksum keyword argument with a value of 'datasum'. -Here are some examples:: - - >>> # Open the file pix.fits verifying the checksum values for all HDUs - >>> hdul = fits.open('pix.fits', checksum=True) - -:: - - >>> # Open the file in.fits where checksum verification fails for the - >>> # primary HDU - >>> hdul = fits.open('in.fits', checksum=True) - Warning: Checksum verification failed for HDU #0. - -:: - - >>> # Create file out.fits containing an HDU constructed from data and - >>> # header containing both CHECKSUM and DATASUM cards. - >>> fits.writeto('out.fits', data, header, checksum=True) - -:: - - >>> # Create file out.fits containing all the HDUs in the HDULIST - >>> # hdul with each HDU header containing only the DATASUM card - >>> hdul.writeto('out.fits', checksum='datasum') - -:: - - >>> # Create file out.fits containing the HDU hdu with both CHECKSUM - >>> # and DATASUM cards in the header - >>> hdu.writeto('out.fits', checksum=True) - -:: - - >>> # Append a new HDU constructed from array data to the end of - >>> # the file existingfile.fits with only the appended HDU - >>> # containing both CHECKSUM and DATASUM cards. - >>> fits.append('existingfile.fits', data, checksum=True) +Examples +-------- + +.. + EXAMPLE START + Verification Using the FITS Checksum Keyword Convention + +To verify the checksum values for HDUs when opening a file:: + + >>> # Open the file checksum.fits verifying the checksum values for all HDUs + >>> filename = fits.util.get_testdata_filepath('checksum.fits') + >>> hdul = fits.open(filename, checksum=True) + >>> hdul.close() + >>> # Open the file in.fits where checksum verification fails + >>> filename = fits.util.get_testdata_filepath('checksum_false.fits') + >>> hdul = fits.open(filename, checksum=True) # doctest: +SHOW_WARNINGS + AstropyUserWarning: Checksum verification failed for HDU ('PRIMARY', 1). + AstropyUserWarning: Datasum verification failed for HDU ('PRIMARY', 1). + AstropyUserWarning: Checksum verification failed for HDU ('RATE', 1). + AstropyUserWarning: Datasum verification failed for HDU ('RATE', 1). + >>> # Create file out.fits containing an HDU constructed from data + >>> # containing both CHECKSUM and DATASUM cards. + >>> data = hdul[0].data + >>> fits.writeto('out.fits', data=data, checksum=True) + >>> hdun = fits.open('out.fits', checksum=True) + >>> hdun.close() + + >>> # Create file out.fits containing all the HDUs in the HDULIST + >>> # hdul with each HDU header containing only the DATASUM card + >>> hdul.writeto('out2.fits', checksum='datasum') + + >>> # Create file out.fits containing the HDU hdu with both CHECKSUM + >>> # and DATASUM cards in the header + >>> hdu = hdul[1] + >>> hdu.writeto('out3.fits', checksum=True) + + >>> # Append a new HDU constructed from array data to the end of + >>> # the file existingfile.fits with only the appended HDU + >>> # containing both CHECKSUM and DATASUM cards. + >>> fits.append('out3.fits', data, checksum=True) + >>> hdul.close() + +.. + EXAMPLE END diff --git a/docs/io/misc.rst b/docs/io/misc.rst index a03ed47960bb..587aa3ea2f3a 100644 --- a/docs/io/misc.rst +++ b/docs/io/misc.rst @@ -1,9 +1,9 @@ -********************************************** -Miscellaneous Input/Output (`astropy.io.misc`) -********************************************** +************************************************************** +ECVS, HDF5, Parquet, PyArrow CSV, YAML (`astropy.io.misc`) +************************************************************** The `astropy.io.misc` module contains miscellaneous input/output routines that -do not fit elsewhere, and are often used by other Astropy sub-packages. For +do not fit elsewhere, and are often used by other ``astropy`` sub-packages. For example, `astropy.io.misc.hdf5` contains functions to read/write :class:`~astropy.table.Table` objects from/to HDF5 files, but these should not be imported directly by users. Instead, users can access this @@ -11,6 +11,10 @@ functionality via the :class:`~astropy.table.Table` class itself (see :ref:`table_io`). Routines that are intended to be used directly by users are listed in the `astropy.io.misc` section. -.. automodapi:: astropy.io.misc +Reference/API +============= -.. automodapi:: astropy.io.misc.hdf5 +.. toctree:: + :maxdepth: 2 + + misc_ref_api diff --git a/docs/io/misc_ref_api.rst b/docs/io/misc_ref_api.rst new file mode 100644 index 000000000000..c9c78752c58c --- /dev/null +++ b/docs/io/misc_ref_api.rst @@ -0,0 +1,15 @@ +Reference/API +************* + +.. automodapi:: astropy.io.misc + +.. automodapi:: astropy.io.misc.ecsv + +.. automodapi:: astropy.io.misc.hdf5 + + +.. automodapi:: astropy.io.misc.parquet + +.. automodapi:: astropy.io.misc.pyarrow.csv + +.. automodapi:: astropy.io.misc.yaml diff --git a/docs/io/overview.rst b/docs/io/overview.rst new file mode 100644 index 000000000000..63fda68c451f --- /dev/null +++ b/docs/io/overview.rst @@ -0,0 +1,34 @@ +Overview of Astropy File I/O +**************************** + +Astropy provides two main interfaces for reading and writing data: + +- :ref:`High-level Unified I/O ` interface that is designed to be consistent + and easy to use. This allows working with different types of data such as + :ref:`tables `, :ref:`images `, and even + :ref:`cosmologies `. +- Low-level I/O sub-packages that are directly responsible for + reading and writing data in specific formats such as :ref:`FITS ` + or :ref:`VOTable `. + +In general we recommend starting with the high-level interface unless you have a +specific need for the low-level interface. + +.. list-table:: Comparison of high-level and low-level interfaces + :widths: 50 50 + :header-rows: 1 + + * - High-level Unified I/O + - Low-level I/O + * - Use ``read()`` and ``write()`` class methods of + output data class, e.g., ``data = QTable.read("data.fits")`` returns a + `~astropy.table.QTable`. + - Interfaces are specific to format, e.g., ``hdus = fits.open("data.fits")`` + returns an `~astropy.io.fits.HDUList`. + * - Read and write entire file at once. + - Support varies, e.g., :ref:`FITS ` has memory-mapped + read access. + * - Automatically determine file format in common cases. + - Specify format explicitly. + * - Help documentation via class method, e.g., ``QTable.read.help("fits")``. + - Help documentation varies, e.g., ``help(fits.open)`` or API docs. diff --git a/docs/io/registry.rst b/docs/io/registry.rst index 90ccb3c82cd4..f8ba21a22044 100644 --- a/docs/io/registry.rst +++ b/docs/io/registry.rst @@ -6,100 +6,197 @@ I/O Registry (`astropy.io.registry`) .. note:: - The I/O registry is only meant to be used directly by users who want - to define their own custom readers/writers. Users who want to find - out more about what built-in formats are supported by - :class:`~astropy.table.Table` by default should see - :ref:`table_io`. No built-in formats are currently defined for - :class:`~astropy.nddata.NDData`, but this will be added in - future). + The I/O registry is only meant to be used directly by users who want to + define their own custom readers/writers. Users who want to find out more + about what built-in formats are supported by :class:`~astropy.table.Table` + by default should see :ref:`table_io`. + Likewise :ref:`cosmology_io` for built-in formats supported by + :class:`~astropy.cosmology.Cosmology`. + No built-in formats are currently defined for + :class:`~astropy.nddata.NDData`, but this will be added in future. Introduction ============ -The I/O registry is a sub-module used to define the readers/writers available -for the :class:`~astropy.table.Table` and -:class:`~astropy.nddata.NDData` classes. +The I/O registry is a submodule used to define the readers/writers available +for the :class:`~astropy.table.Table`, :class:`~astropy.nddata.NDData`, +and :class:`~astropy.cosmology.Cosmology` classes. -Using `astropy.io.registry` + +Custom Read/Write Functions =========================== -The following example demonstrates how to create a reader for the -:class:`~astropy.table.Table` class. First, we can create a highly -simplistic FITS reader which just reads the data as a structured array:: +This section demonstrates how to create a custom reader/writer. A reader is +written as a function that can take any arguments except ``format`` (which is +needed when manually specifying the format — see below) and returns an +instance of the :class:`~astropy.table.Table` or +:class:`~astropy.nddata.NDData` classes (or subclasses). + + +Examples +-------- + +.. + EXAMPLE START + Using astropy.io.registry to Create a Custom Reader/Writer - from astropy.table import Table +Here we assume that we are trying to write a reader/writer for the +:class:`~astropy.table.Table` class:: - def fits_table_reader(filename, hdu=1): - from astropy.io import fits - data = fits.open(filename)[hdu].data - return Table(data) + >>> from astropy.table import Table -and then register it:: + >>> def my_table_reader(filename, some_option=1): + ... # Read in the table by any means necessary + ... return table # should be an instance of Table - from astropy.io import registry - registry.register_reader('fits', Table, fits_table_reader) +Such a function can then be registered with the I/O registry:: -Reader functions can take any arguments except ``format`` (since this -is reserved for :func:`~astropy.io.registry.read`) and should return an instance of the class specified as the second argument of ``register_reader`` (:class:`~astropy.table.Table` in the above case.) + >>> from astropy.io import registry + >>> registry.register_reader('my-table-format', Table, my_table_reader) -We can then read in a FITS table with:: +where the first argument is the name of the format, the second argument is the +class that the function returns an instance for, and the third argument is the +reader itself. - t = Table.read('catalog.fits', format='fits') +We can then read in a table with:: + + >>> d = Table.read('my_table_file.mtf', format='my-table-format') # doctest: +SKIP In practice, it would be nice to have the ``read`` method automatically -identify that this file was a FITS file, so we can construct a function that -can recognize FITS files, which we refer to here as an *identifier* function. -An identifier function should take a first argument that should be a string +identify that this file is in the ``my-table-format`` format, so we can +construct a function that can recognize these files, which we refer to here as +an *identifier* function. + +An identifier function should take a first argument that is a string which indicates whether the identifier is being called from ``read`` or -``write``, and should then accept arbitrary number of positional and keyword +``write``, and should then accept an arbitrary number of positional and keyword arguments via ``*args`` and ``**kwargs``, which are the arguments passed to -``Table.read``. We can write a simplistic function that only looks at +the ``read`` method. + +In the above case, we can write a function that only looks at filenames (but in practice, this function could even look at the first few -bytes of the file for example). The only requirement is that it return a -boolean indicating whether the input matches that expected for the format:: +bytes of the file, for example). The only requirement for the identifier +function is that it return a boolean indicating whether the input matches that +expected for the format. In our example, we want to automatically recognize +files with filenames ending in ``.mtf`` as being in the ``my-table-format`` +format:: + + >>> import os + >>> + >>> def identify_mtf(origin, *args, **kwargs): + ... return (isinstance(args[0], str) and + ... os.path.splitext(args[0].lower())[1] == '.mtf') - def fits_identify(origin, *args, **kwargs): - return isinstance(args[0], basestring) and \ - args[0].lower().split('.')[-1] in ['fits', 'fit'] +.. note:: -.. note:: Identifier functions should be prepared for arbitrary input - in - particular, the first argument may not be a filename or file - object, so it should not assume that this is the case. + Identifier functions should be prepared for arbitrary input — in + particular, the first argument may not be a filename or file object, so it + should not assume that this is the case. -We then register this identifier function:: +We then register this identifier function, similarly to the reader function:: - registry.register_identifier('fits', Table, fits_identify) + >>> registry.register_identifier('my-table-format', Table, identify_mtf) -And we can then do:: +Having registered this function, we can then do:: - t = Table.read('catalog.fits') + >>> t = Table.read('catalog.mtf') # doctest: +SKIP If multiple formats match the current input, then an exception is raised, and similarly if no format matches the current input. In that case, the format should be explicitly given with the ``format=`` keyword argument. -Similarly, it is possible to create custom writers. To go with our simplistic FITS reader above, we can write a simplistic FITS writer:: +It is also possible to create custom writers. To go with our custom reader +above, we can write a custom writer:: + + >>> def my_table_writer(table, filename, overwrite=False): + ... # Write the table out to a file + ... return None # generally None, but other values are not forbidden. - def fits_table_writer(table, filename, clobber=False): - import numpy as np - from astropy.io import fits - fits.writeto(filename, np.array(table), clobber=clobber) +Writer functions should take a dataset object (either an instance of the +:class:`~astropy.table.Table` or :class:`~astropy.nddata.NDData` +classes or subclasses), and any number of subsequent positional and keyword +arguments — although as for the reader, the ``format`` keyword argument cannot +be used. We then register the writer:: - io_registry.register_writer('fits', Table, fits_table_writer) + >>> registry.register_writer('my-table-format', Table, my_table_writer) + +We can write the table out to a file: + +.. testsetup:: + >>> t = Table() + +>>> t.write('catalog_new.mtf', format='my-table-format') + +Since we have already registered the identifier function, we can also do:: + + >>> t.write('catalog_new.mtf') -And we can then write the file out to a FITS file:: +.. testcleanup:: + >>> registry.unregister_reader('my-table-format', Table) + >>> registry.unregister_writer('my-table-format', Table) + >>> registry.unregister_identifier('my-table-format', Table) - t.write('catalog_new.fits', format='fits') -If we have registered the identifier as above, we can simply do:: +.. + EXAMPLE END + + +Registries, local and default +============================= + +.. versionchanged:: 5.0 + +As of Astropy 5.0 the I/O registry submodule has switched to a class-based +architecture, allowing for the creation of custom registries. +The three supported registry types are read-only -- +:class:`~astropy.io.registry.UnifiedInputRegistry` -- +write-only -- :class:`~astropy.io.registry.UnifiedOutputRegistry` -- +and read/write -- :class:`~astropy.io.registry.UnifiedIORegistry`. + + >>> from astropy.io.registry import UnifiedIORegistry + >>> example_reg = UnifiedIORegistry() + >>> print([m for m in dir(example_reg) if not m.startswith("_")]) + ['available_registries', 'delay_doc_updates', 'get_formats', 'get_reader', + 'get_writer', 'identify_format', 'read', 'register_identifier', + 'register_reader', 'register_writer', 'unregister_identifier', + 'unregister_reader', 'unregister_writer', 'write'] + +For backward compatibility all the methods on this registry have corresponding +module-level functions, which work with the default global read/write registry. +These functions were used in the previous examples. This new registry is empty. + + >>> example_reg.get_formats() +
+ Data class Format Read Write Auto-identify + float64 float64 float64 float64 float64 + ---------- ------- ------- ------- ------------- + +We can register read / write / identify methods with this registry object: + + >>> example_reg.register_reader('my-table-format', Table, my_table_reader) + >>> example_reg.get_formats() +
+ Data class Format Read Write Auto-identify + str5 str15 str3 str2 str2 + ---------- --------------- ---- ----- ------------- + Table my-table-format Yes No No + + +What is the use of a custom registries? + + 1. To make read-only or write-only registries. + 2. To allow for different readers for the same format. + 3. To allow for an object to have different *kinds* of readers and writers. + E.g. |Cosmology| which supports both file I/O and object conversion. - t.write('catalog_new.fits') Reference/API ============= -.. automodapi:: astropy.io.registry +.. toctree:: + :maxdepth: 2 + + registry_ref_api diff --git a/docs/io/registry_ref_api.rst b/docs/io/registry_ref_api.rst new file mode 100644 index 000000000000..6ef75d74d0e1 --- /dev/null +++ b/docs/io/registry_ref_api.rst @@ -0,0 +1,4 @@ +Reference/API +************* + +.. automodapi:: astropy.io.registry diff --git a/docs/io/typing.rst b/docs/io/typing.rst new file mode 100644 index 000000000000..f8d1f07049c4 --- /dev/null +++ b/docs/io/typing.rst @@ -0,0 +1,48 @@ +******************************** +I/O Typing (`astropy.io.typing`) +******************************** + + +``astropy.io`` provides type annotations through the :mod:`astropy.io.typing` module. +These type annotations allow users to specify the expected types of variables, function +parameters, and return values when working with I/O. By using type annotations, +developers can improve code readability, catch potential type-related errors early, and +enable better code documentation and tooling support. + +For example, the following function uses type annotations to specify that the +``filename`` parameter can be any type of path-like object (e.g. a string, byte-string, +or pathlib.Path object). + +.. code-block:: python + + from astropy.io import fits + from astropy.io.typing import PathLike + + def read_fits_file(filename: PathLike) -> fits.HDUList: + return fits.open(filename) + + +The :mod:`astropy.io.typing` module also provides type aliases for file-like objects +that support reading and writing. The following example uses the +:class:`~astropy.io.typing.ReadableFileLike` type alias to specify that the ``fileobj`` +parameter can be any file-like object that supports reading. Using a +:class:`~typing.TypeVar`, the return type of the function is specified to be the same +type as the file-like object can read. + + +.. code-block:: python + + from typing import TypeVar + from astropy.io.typing import ReadableFileLike + + R = TypeVar('R') # type of object returned by fileobj.read() + + def read_from_file(fileobj: ReadableFileLike[R]) -> R: + """Reads from a file-like object and returns the result.""" + return fileobj.read() + + +Reference/API +============= + +.. automodapi:: astropy.io.typing diff --git a/docs/io/unified.rst b/docs/io/unified.rst index b317d1de56e1..b9702d269532 100644 --- a/docs/io/unified.rst +++ b/docs/io/unified.rst @@ -1,285 +1,70 @@ -.. doctest-skip-all - .. _table_io: +.. _io-unified: -Unified file read/write interface -=================================== - -Astropy provides a unified interface for reading and writing data in different formats. -For many common cases this will simplify the process of file I/O and reduce the need to -master the separate details of all the I/O packages within Astropy. This functionality is -still in active development and the number of supported formats will be increasing. For -details on the implementation see :ref:`io_registry`. - -Getting started with Table I/O ------------------------------- - -The :class:`~astropy.table.Table` class includes two methods, -:meth:`~astropy.table.Table.read` and -:meth:`~astropy.table.Table.write`, that make it possible to read from -and write to files. A number of formats are automatically supported (see -`Built-in table readers/writers`_) and new file formats and extensions can be -registered with the :class:`~astropy.table.Table` class (see -:ref:`io_registry`). - -To use this interface, first import the :class:`~astropy.table.Table` class, then -simply call the :class:`~astropy.table.Table` -:meth:`~astropy.table.Table.read` method with the name of the file and -the file format, for instance ``'ascii.daophot'``:: - - >>> from astropy.table import Table - >>> t = Table.read('photometry.dat', format='ascii.daophot') - -It is possible to load tables directly from the Internet using URLs. For example, -download tables from Vizier catalogues in CDS format (``'ascii.cds'``):: - - >>> t = Table.read("ftp://cdsarc.u-strasbg.fr/pub/cats/VII/253/snrs.dat", - ... readme="ftp://cdsarc.u-strasbg.fr/pub/cats/VII/253/ReadMe", - ... format="ascii.cds") - -For certain file formats, the format can be automatically detected, for -example from the filename extension:: - - >>> t = Table.read('table.tex') - -Similarly, for writing, the format can be explicitly specified:: - - >>> t.write(filename, format='latex') - -As for the :meth:`~astropy.table.Table.read` method, the format may -be automatically identified in some cases. - -Any additional arguments specified will depend on the format. For examples of this see the -section `Built-in table readers/writers`_. This section also provides the full list of -choices for the ``format`` argument. - -.. _built_in_readers_writers: - -Built-in table readers/writers ------------------------------- - -The full list of built-in readers and writers is shown in the table below: - -=========================== ==== ===== ============= ========== - Format Read Write Auto-identify Deprecated -=========================== ==== ===== ============= ========== - aastex Yes Yes No Yes - ascii Yes Yes No - ascii.aastex Yes Yes No - ascii.basic Yes Yes No - ascii.cds Yes No No - ascii.commented_header Yes Yes No - ascii.daophot Yes No No - ascii.ecsv Yes Yes No - ascii.fixed_width Yes Yes No -ascii.fixed_width_no_header Yes Yes No - ascii.fixed_width_two_line Yes Yes No - ascii.html Yes Yes Yes - ascii.ipac Yes Yes No - ascii.latex Yes Yes Yes - ascii.no_header Yes Yes No - ascii.rdb Yes Yes Yes - ascii.sextractor Yes No No - ascii.tab Yes Yes No - ascii.csv Yes Yes Yes - cds Yes No No Yes - daophot Yes No No Yes - fits Yes Yes Yes - hdf5 Yes Yes Yes - html Yes Yes No Yes - ipac Yes Yes No Yes - latex Yes Yes No Yes - rdb Yes Yes No Yes - votable Yes Yes Yes -=========================== ==== ===== ============= ========== - -Deprecated format names like ``aastex`` will be removed in a future version. -Use the full name (e.g. ``ascii.aastex``) instead. - -.. _table_io_ascii: - -ASCII formats -^^^^^^^^^^^^^^ - -The :meth:`~astropy.table.Table.read` and -:meth:`~astropy.table.Table.write` methods can be used to read and write formats -supported by `astropy.io.ascii`. - -Use ``format='ascii'`` in order to interface to the generic -:func:`~astropy.io.ascii.read` and :func:`~astropy.io.ascii.write` -functions from `astropy.io.ascii`. When reading a table this means -that all supported ASCII table formats will be tried in order to successfully -parse the input. For example:: - - >>> t = Table.read('astropy/io/ascii/tests/t/latex1.tex', format='ascii') - >>> print t - cola colb colc - ---- ---- ---- - a 1 2 - b 3 4 - -When writing a table with ``format='ascii'`` the output is a basic -character-delimited file with a single header line containing the -column names. - -All additional arguments are passed to the `astropy.io.ascii` -:func:`~astropy.io.ascii.read` and :func:`~astropy.io.ascii.write` -functions. Further details are available in the sections on -:ref:`io_ascii_read_parameters` and :ref:`io_ascii_write_parameters`. For example, to change -column delimiter and the output format for the ``colc`` column use:: - - >>> t.write(sys.stdout, format='ascii', delimiter='|', formats={'colc': '%0.2f'}) - cola|colb|colc - a|1|2.00 - b|3|4.00 - -A full list of the supported ``format`` values and corresponding format types -for ASCII tables is given below. The ``Suffix`` column indicates the filename -suffix where the format will be auto-detected, while the ``Write`` column -indicates which support write functionality. +High-level Unified File I/O +*************************** -=============================== ====== ===== ============================================================================================ - Format Suffix Write Description -=============================== ====== ===== ============================================================================================ -``ascii`` Yes ASCII table in any supported format (uses guessing) -``ascii.aastex`` Yes :class:`~astropy.io.ascii.AASTex`: AASTeX deluxetable used for AAS journals -``ascii.basic`` Yes :class:`~astropy.io.ascii.Basic`: Basic table with custom delimiters -``ascii.cds`` :class:`~astropy.io.ascii.Cds`: CDS format table -``ascii.commented_header`` Yes :class:`~astropy.io.ascii.CommentedHeader`: Column names in a commented line -``ascii.daophot`` :class:`~astropy.io.ascii.Daophot`: IRAF DAOphot format table -``ascii.fixed_width`` Yes :class:`~astropy.io.ascii.FixedWidth`: Fixed width -``ascii.fixed_width_no_header`` Yes :class:`~astropy.io.ascii.FixedWidthNoHeader`: Fixed width with no header -``ascii.fixed_width_two_line`` Yes :class:`~astropy.io.ascii.FixedWidthTwoLine`: Fixed width with second header line -``ascii.ipac`` Yes :class:`~astropy.io.ascii.Ipac`: IPAC format table -``ascii.html`` .html Yes :class:`~astropy.io.ascii.HTML`: HTML table -``ascii.latex`` .tex Yes :class:`~astropy.io.ascii.Latex`: LaTeX table -``ascii.no_header`` Yes :class:`~astropy.io.ascii.NoHeader`: Basic table with no headers -``ascii.rdb`` .rdb Yes :class:`~astropy.io.ascii.Rdb`: Tab-separated with a type definition header line -``ascii.sextractor`` :class:`~astropy.io.ascii.SExtractor`: SExtractor format table -``ascii.tab`` Yes :class:`~astropy.io.ascii.Tab`: Basic table with tab-separated values -``ascii.csv`` .csv Yes :class:`~astropy.io.ascii.Csv`: Basic table with comma-separated values -=============================== ====== ===== ============================================================================================ +``astropy`` uses the :ref:`I/O Registry ` to provide a unified interface +for reading and writing data in different formats. For many common cases this will +streamline the process of file I/O and reduce the need to learn the separate details of +all of the low-level I/O packages within ``astropy``. -.. note:: +.. toctree:: + :maxdepth: 2 - When specifying a specific ASCII table format using the unified interface, the format name is - prefixed with ``ascii.`` in order to identify the format as ASCII-based. Compare the - table above to the `astropy.io.ascii` list of :ref:`supported_formats`. Therefore the following - are equivalent:: + unified_image + unified_table - >>> dat = ascii.read('file.dat', format='daophot') - >>> dat = Table.read('file.dat', format='ascii.daophot') +**Overview** - For compatibility with astropy version 0.2 and earlier, the following format - values are also allowed in ``Table.read()``: ``daophot``, ``ipac``, ``html``, ``latex``, and ``rdb``. +The fundamental idea for the unified interface is that each data container class such as +`~astropy.table.Table` or `~astropy.nddata.CCDData` has class methods ``read()`` +and ``write()`` that can be used to read and write data. -.. _table_io_fits: +The first positional argument to these methods specifies the input or output. In +general the input can be a file name, a file-like object or a URL. For some formats, +most notably the :ref:`table_io_ascii` formats, the input can also be a string or +list of strings representing the data. The output can be a file name or a file-like +object. -FITS -^^^^ +The file format is specified using the ``format`` keyword argument. This is required +unless the format can be uniquely determined from the file name or file content. -Reading/writing from/to `FITS `_ -files is supported with ``format='fits'``. In most cases, existing FITS -files should be automatically identified as such based on the header of the -file, but if not, or if writing to disk, then the format should be explicitly -specified. +**Example** -If a FITS table file contains only a single table, then it can be read in -with:: +The example below shows how to read a table in the specialized DAOphot format and write +it back to FITS format. Notice that FITS is a format where the interface recognizes the +format automatically from the file name, so the ``format`` argument is not needed. +In this example we use a file that is installed with astropy:: - >>> t = Table.read('data.fits') - -If more than one table is present in the file, the first table found will be -read in and a warning will be emitted:: - - >>> t = Table.read('data.fits') - WARNING: hdu= was not specified but multiple tables are present, reading in first available table (hdu=1) [astropy.io.fits.connect] - -To write to a new file:: - - >>> t.write('new_table.fits') - -At this time, the ``meta`` attribute of the -:class:`~astropy.table.Table` class is simply an ordered -dictionary and does not fully represent the structure of a FITS -header (for example, keyword comments are dropped). This is likely -to change in a future release. - -.. _table_io_hdf5: - -HDF5 -^^^^^^^^ - -Reading/writing from/to `HDF5 `_ files is -supported with ``format='hdf5'`` (this requires `h5py -`_ to be installed). However, the ``.hdf5`` -file extension is automatically recognized when writing files, and HDF5 files -are automatically identified (even with a different extension) when reading -in (using the first few bytes of the file to identify the format), so in most -cases you will not need to explicitly specify ``format='hdf5'``. - -Since HDF5 files can contain multiple tables, the full path to the table -should be specified via the ``path=`` argument when reading and writing. -For example, to read a table called ``data`` from an HDF5 file named -``observations.hdf5``, you can do:: - - >>> t = Table.read('observations.hdf5', path='data') - -To read a table nested in a group in the HDF5 file, you can do:: - - >>> t = Table.read('observations.hdf5', path='group/data') - -To write a table to a new file, the path should also be specified:: - - >>> t.write('new_file.hdf5', path='updated_data') - -It is also possible to write a table to an existing file using ``append=True``:: - - >>> t.write('observations.hdf5', path='updated_data', append=True) - -As with other formats, the ``overwrite=True`` argument is supported for -overwriting existing files. To overwrite only a single table within an HDF5 -file that has multiple datasets, use *both* the ``overwrite=True`` and -``append=True`` arguments. - -Finally, when writing to HDF5 files, the ``compression=`` argument can be -used to ensure that the data is compressed on disk:: - - >>> t.write('new_file.hdf5', path='updated_data', compression=True) - - - - -.. _table_io_votable: - -VO Tables -^^^^^^^^^^^ - -Reading/writing from/to `VO table `_ -files is supported with ``format='votable'``. In most cases, existing VO -tables should be automatically identified as such based on the header of the -file, but if not, or if writing to disk, then the format should be explicitly -specified. - -If a VO table file contains only a single table, then it can be read in with:: + >>> from astropy.table import Table + >>> from astropy.utils.data import get_pkg_data_filename + >>> photometry_file = get_pkg_data_filename('data/daophot.dat', + ... package='astropy.io.ascii.tests') + >>> t = Table.read(photometry_file, format='ascii.daophot') + >>> t.write('photometry.fits') # doctest: +IGNORE_WARNINGS - >>> t = Table.read('aj285677t3_votable.xml') +The FITS writer will issue a few warnings because the units read from the DAOphot +file do not match the FITS conventions, but the data in the file is perfectly fine. -If more than one table is present in the file, an error will be raised, -unless the table ID is specified via the ``table_id=`` argument:: +.. testcleanup:: - >>> t = Table.read('catalog.xml') - Traceback (most recent call last): - ... - ValueError: Multiple tables found: table id should be set via the table_id= argument. The available tables are twomass, spitzer + >>> import os + >>> os.remove('photometry.fits') - >>> t = Table.read('catalog.xml', table_id='twomass') +Each file format is handled by a specific reader or writer, and each of those +functions will have its own set of arguments. -To write to a new file, the ID of the table should also be specified (unless -``t.meta['ID']`` is defined):: +**Getting Help** - >>> t.write('new_catalog.xml', table_id='updated_table', format='votable') +To get help on the available arguments for each format, use the ``help()`` method of the +appropriate ``read()`` or ``write()`` class method, e.g., `astropy.table.Table.read`. +In the examples below we do not show the long output: -When writing, the ``compression=True`` argument can be used to force -compression of the data on disk, and the ``overwrite=True`` argument can be -used to overwrite an existing file. + >>> from astropy.table import Table + >>> from astropy.nddata import CCDData + >>> CCDData.read.help('fits') # doctest: +IGNORE_OUTPUT + >>> Table.read.help('ascii') # doctest: +IGNORE_OUTPUT + >>> Table.read.help('ascii.latex') # doctest: +IGNORE_OUTPUT + >>> Table.write.help('hdf5') # doctest: +IGNORE_OUTPUT + >>> Table.write.help('csv') # doctest: +IGNORE_OUTPUT diff --git a/docs/io/unified_image.rst b/docs/io/unified_image.rst new file mode 100644 index 000000000000..54cea76a8fcc --- /dev/null +++ b/docs/io/unified_image.rst @@ -0,0 +1,44 @@ +.. _io_unified_image: + +Image Data +========== + +Reading and writing CCD image data in the unified I/O interface is supported +though the :ref:`CCDData class ` using FITS file format: + +.. testsetup:: + >>> from astropy.nddata import CCDData + >>> ccd = CCDData([1, 2, 3], unit='adu') + >>> ccd.write('image.fits') + +:: + + >>> # Read CCD image + >>> from astropy.nddata import CCDData + >>> ccd = CCDData.read('image.fits') + + >>> # Write back CCD image + >>> ccd.write('new_image.fits') + +.. testcleanup:: + >>> del ccd + >>> import os + >>> os.remove('new_image.fits') + +Note that the unit is stored in the ``BUNIT`` keyword in the header on saving, +and is read from the header if it is present. + +Detailed help on the available keyword arguments for reading and writing +can be obtained via the ``help()`` method as follows:: + + >>> from astropy.nddata import CCDData + >>> CCDData.read.help('fits') # Get help on the CCDData FITS reader # doctest: +ELLIPSIS + ========================================= + CCDData.read(format='fits') documentation + ========================================= + ... + >>> CCDData.write.help('fits') # Get help on the CCDData FITS writer # doctest: +ELLIPSIS + ========================================== + CCDData.write(format='fits') documentation + ========================================== + ... diff --git a/docs/io/unified_table.rst b/docs/io/unified_table.rst new file mode 100644 index 000000000000..73ba04bd4472 --- /dev/null +++ b/docs/io/unified_table.rst @@ -0,0 +1,321 @@ +.. _read_write_tables: + +Table Data +========== + +The :class:`~astropy.table.QTable` and :class:`~astropy.table.Table` classes includes two +methods, :meth:`~astropy.table.Table.read` and :meth:`~astropy.table.Table.write`, that +make it possible to read from and write to files. A number of formats are supported (see +`Built-in table readers/writers`_) and new file formats and extensions can be registered +with the :class:`~astropy.table.Table` class (see :ref:`I/O Registry `). + +.. toctree:: + :maxdepth: 1 + :caption: Table Formats + + unified_table_text + unified_table_fits + unified_table_hdf5 + unified_table_parquet + unified_table_votable + +.. + EXAMPLE START + Reading a DAOPhot Table + +To use this interface, first import the :class:`~astropy.table.Table` class, +then call the :class:`~astropy.table.Table` +:meth:`~astropy.table.Table.read` method with the name of the file and +the file format, for instance ``'ascii.daophot'``: + +.. testsetup:: + >>> import os + >>> with open('photometry.dat', 'w') as f: # doctest: +IGNORE_OUTPUT + ... f.write("#N ID XCENTER YCENTER\n") + ... f.write("#U ## pixel pixel \n") + ... f.write("#F %-9d %-10.3f %-10.3f\n") + ... f.write("#\n") + ... f.write("14 138.538 256.405\n") + ... f.write("18 18.114 280.170\n") + >>> from astropy.table import Table + >>> t = Table.read('photometry.dat', format='ascii.daophot') + >>> t.write('table.tex', format='latex') + +>>> from astropy.table import Table +>>> t = Table.read('photometry.dat', format='ascii.daophot') + + +.. testcleanup:: + + >>> os.remove('photometry.dat') + +.. + EXAMPLE END + +.. + EXAMPLE START + Reading a Table directly from the Internet + +It is possible to load tables directly from the Internet using URLs. For +example, download an example Spitzer catalog (a :ref:`VOTable `): + +.. doctest-remote-data:: + + >>> from astropy.table import Table + >>> t = Table.read("http://www.astropy.org/astropy-data/photometry/spitzer_example_catalog.xml", format="votable") + +For certain file formats the format can be automatically detected, for +example, from the filename extension:: + + >>> t = Table.read('table.tex') + +.. + EXAMPLE END + +.. + EXAMPLE START + Writing a LaTeX Table + +For writing a table, the format can be explicitly specified:: + + >>> t.write('some_filename', format='latex') + +.. testcleanup:: + + >>> import pathlib + >>> pathlib.Path.unlink('table.tex') + >>> pathlib.Path.unlink('some_filename') + +As for the :meth:`~astropy.table.Table.read` method, the format may +be automatically identified in some cases. + +The underlying file handler will also automatically detect various +compressed data formats and uncompress them as far as +supported by the Python installation (see +:meth:`~astropy.utils.data.get_readable_fileobj`). + +For writing, you can also specify details about the `Table serialization +methods`_ via the ``serialize_method`` keyword argument. This allows +fine control of the way to write out certain columns, for instance +writing an ISO format Time column as a pair of JD1/JD2 floating +point values (for full resolution) or as a formatted ISO date string. + +.. + EXAMPLE END + +.. _built_in_readers_writers: + +Built-In Table Readers/Writers +------------------------------ + +The :class:`~astropy.table.Table` class has built-in support for various input +and output formats including :ref:`table_io_ascii`, +:ref:`table_io_fits`, :ref:`table_io_hdf5`, :ref:`table_io_pandas`, +:ref:`table_io_parquet`, and :ref:`table_io_votable`. + +A full list of the supported formats and corresponding classes is shown in the +table below. The ``Write`` column indicates those formats that support write +functionality, and the ``Suffix`` column indicates the filename suffix +indicating a particular format. If the value of ``Suffix`` is ``auto``, the +format is auto-detected from the file itself. Not all formats support +auto-detection. + +=========================== ===== ====== ============================================================================================ + Format Write Suffix Description +=========================== ===== ====== ============================================================================================ + ascii Yes Text table in most supported formats (uses guessing) + ascii.aastex Yes :class:`~astropy.io.ascii.AASTex`: AASTeX deluxetable used for AAS journals + ascii.basic Yes :class:`~astropy.io.ascii.Basic`: Basic table with custom delimiters + ascii.cds No :class:`~astropy.io.ascii.Cds`: CDS format table + ascii.commented_header Yes :class:`~astropy.io.ascii.CommentedHeader`: Column names in a commented line + ascii.csv Yes .csv :class:`~astropy.io.ascii.Csv`: Basic table with comma-separated values + ascii.daophot No :class:`~astropy.io.ascii.Daophot`: IRAF DAOphot format table + ascii.ecsv Yes .ecsv :class:`~astropy.io.ascii.Ecsv`: Basic table with Enhanced CSV (supporting metadata) + ascii.fixed_width Yes :class:`~astropy.io.ascii.FixedWidth`: Fixed width +ascii.fixed_width_no_header Yes :class:`~astropy.io.ascii.FixedWidthNoHeader`: Fixed width with no header + ascii.fixed_width_two_line Yes :class:`~astropy.io.ascii.FixedWidthTwoLine`: Fixed width with second header line + ascii.html Yes .html :class:`~astropy.io.ascii.HTML`: HTML table + ascii.ipac Yes :class:`~astropy.io.ascii.Ipac`: IPAC format table + ascii.latex Yes .tex :class:`~astropy.io.ascii.Latex`: LaTeX table + ascii.mesa No .data :class:`~astropy.io.ascii.Mesa`: MESA stellar evolution code output format + ascii.mrt Yes :class:`~astropy.io.ascii.Mrt`: AAS Machine-Readable Table format + ascii.no_header Yes :class:`~astropy.io.ascii.NoHeader`: Basic table with no headers + ascii.qdp Yes .qdp :class:`~astropy.io.ascii.QDP`: Quick and Dandy Plotter files + ascii.rdb Yes .rdb :class:`~astropy.io.ascii.Rdb`: Tab-separated with a type definition header line + ascii.rst Yes .rst :class:`~astropy.io.ascii.RST`: reStructuredText simple format table + ascii.sextractor No :class:`~astropy.io.ascii.SExtractor`: SExtractor format table + ascii.tab Yes :class:`~astropy.io.ascii.Tab`: Basic table with tab-separated values + ascii.tdat Yes .tdat :class:`~astropy.io.ascii.Tdat`: Transportable Database Aggregate Table format + fits Yes auto :mod:`~astropy.io.fits`: Flexible Image Transport System file + hdf5 Yes auto |HDF5|: Hierarchical Data Format binary file + jsviewer Yes JavaScript viewer format (write-only) + pandas.csv Yes :func:`pandas.read_csv` and :meth:`pandas.DataFrame.to_csv` + pandas.fwf No :func:`pandas.read_fwf` (fixed width format) + pandas.html Yes :func:`pandas.read_html` and :meth:`pandas.DataFrame.to_html` + pandas.json Yes :func:`pandas.read_json` and :meth:`pandas.DataFrame.to_json` + parquet Yes auto |Parquet|: Apache Parquet binary file + parquet.votable Yes Parquet file(s) with VOTable metadata + pyarrow.csv No :func:`~astropy.io.misc.pyarrow.csv.read_csv`: Performant CSV reader + votable Yes auto :mod:`~astropy.io.votable`: Table format used by Virtual Observatory (VO) initiative + votable.parquet Yes Parquet serialization of VOTables. Specify this format for writing, reading is automatic. +=========================== ===== ====== ============================================================================================ + +Details +------- + +.. _table_serialization_methods: + +Table Serialization Methods +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``astropy`` supports fine-grained control of the way to write out (serialize) +the columns in a Table. For instance, if you are writing an ISO format +Time column to an ECSV text table file, you may want to write this as a pair +of JD1/JD2 floating point values for full resolution (perfect "round-trip"), +or as a formatted ISO date string so that the values are easily readable by +your other applications. + +The default method for serialization depends on the format (FITS, ECSV, HDF5). +For instance HDF5 is a binary format and so it would make sense to store a Time +object as JD1/JD2, while ECSV is a flat text format and commonly you +would want to see the date in the same format as the Time object. The defaults +also reflect an attempt to minimize compatibility issues between ``astropy`` +versions. For instance, it was possible to write Time columns to ECSV as +formatted strings in a version prior to the ability to write as JD1/JD2 +pairs, so the current default for ECSV is to write as formatted strings. + +The two classes which have configurable serialization methods are `~astropy.time.Time` +and `~astropy.table.MaskedColumn`. The defaults for each format are listed below: + +====== ==================== =============== +Format Time MaskedColumn +====== ==================== =============== +FITS ``jd1_jd2`` ``null_value`` +ECSV ``formatted_value`` ``null_value`` +HDF5 ``jd1_jd2`` ``data_mask`` +YAML ``jd2_jd2`` --- +====== ==================== =============== + +Examples +"""""""" + +.. + EXAMPLE START + Table Serialization Methods in astropy.io + +Start by making a table with a Time column and masked column:: + + >>> import sys + >>> from astropy.time import Time + >>> from astropy.table import Table, MaskedColumn + + >>> t = Table(masked=True) + >>> t['tm'] = Time(['2000-01-01', '2000-01-02']) + >>> t['mc1'] = MaskedColumn([1.0, 2.0], mask=[True, False]) + >>> t['mc2'] = MaskedColumn([3.0, 4.0], mask=[False, True]) + >>> t +
+ tm mc1 mc2 + Time float64 float64 + ----------------------- ------- ------- + 2000-01-01 00:00:00.000 -- 3.0 + 2000-01-02 00:00:00.000 2.0 -- + +Now specify that you want all `~astropy.time.Time` columns written as JD1/JD2 +and the ``mc1`` column written as a data/mask pair and write to ECSV:: + + >>> serialize_method = {Time: 'jd1_jd2', 'mc1': 'data_mask'} + >>> t.write(sys.stdout, format='ascii.ecsv', serialize_method=serialize_method) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE + # %ECSV 1.0 + ... + # schema: astropy-2.0 + tm.jd1 tm.jd2 mc1 mc1.mask mc2 + 2451544.0 0.5 1.0 True 3.0 + 2451546.0 -0.5 2.0 False "" + +(Spaces added for clarity) + +Notice that the ``tm`` column has been replaced by the ``tm.jd1`` and ``tm.jd2`` +columns, and likewise a new column ``mc1.mask`` has appeared and it explicitly +contains the mask values. When this table is read back with the ``ascii.ecsv`` +reader then the original columns are reconstructed. + +The ``serialize_method`` argument can be set in two different ways: + +- As a single string like ``data_mask``. This value then applies to every + column, and is a convenient strategy for a masked table with no Time columns. +- As a `dict`, where the key can be either a single column name or a class (as + shown in the example above), and the value is the corresponding serialization + method. + +.. + EXAMPLE END + +Reading and Writing Column Objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. EXAMPLE START: Reading and Writing Column Objects + +Individual table columns do not have their own functions for reading and writing +but it is easy to select just a single column (here "obstime") from a table for writing:: + + >>> from astropy.time import Time + >>> tab = Table({'name': ['AB Aur', 'SU Aur'], + ... 'obstime': Time(['2013-05-23T14:23:12', '2011-11-11T11:11:11'])}) + >>> tab[['obstime']].write('obstime.fits') + +Note the notation ``[['obstime']]`` in the last line - indexing a table with a list of strings +gives us a new table with the columns given by the strings. Since the inner list has only +one element, the resulting table has only one column. + +Then, we can read back that single-column table and extract the column from it:: + + >>> col = Table.read('obstime.fits').columns[0] + >>> type(col) + + +.. testcleanup:: + + >>> import pathlib + >>> pathlib.Path.unlink('obstime.fits') + +.. EXAMPLE END + +Note on Filenames +^^^^^^^^^^^^^^^^^ +Both the :meth:`~astropy.table.Table.read` and +:meth:`~astropy.table.Table.write` methods can accept file paths of the form +``~/data/file.csv`` or ``~username/data/file.csv``. These tilde-prefixed paths +are expanded in the same way as is done by many command-line utilities, to +represent the home directory of the current or specified user, respectively. + +Command-Line Utility +^^^^^^^^^^^^^^^^^^^^ + +.. note:: + + In v7.1, the ``showtable`` command is now deprecated to avoid a name clash on Debian; + use ``showtable-astropy`` instead. The deprecated command will be removed in a future + release. + +For convenience, the command-line tool ``showtable-astropy`` can be used to print the +content of tables for the formats supported by the unified I/O interface. + +Example +""""""" + +.. + EXAMPLE START + Viewing the Contents of a Table on the Command Line + +To view the contents of a table on the command line:: + + $ showtable-astropy astropy/io/fits/tests/data/table.fits + + target V_mag + ------- ----- + NGC1001 11.1 + NGC1002 12.3 + NGC1003 15.2 + +To get full documentation on the usage and available options, do ``showtable-astropy --help``. diff --git a/docs/io/unified_table_fits.rst b/docs/io/unified_table_fits.rst new file mode 100644 index 000000000000..99df4b9236ac --- /dev/null +++ b/docs/io/unified_table_fits.rst @@ -0,0 +1,626 @@ +.. _table_io_fits: + + +FITS +---- + +Reading and writing tables in `FITS `_ format is +supported with ``format='fits'``. In most cases, existing FITS files should be +automatically identified as such based on the header of the file, but if not, +or if writing to disk, then the format should be explicitly specified. + +Reading +^^^^^^^ + +If a FITS table file contains only a single table, then it can be read in +as shown below. In this example we use a file that is installed with astropy:: + + >>> from astropy.table import Table + >>> from astropy.utils.data import get_pkg_data_filename + >>> chandra_events = get_pkg_data_filename('data/chandra_time.fits', + ... package='astropy.io.fits.tests') + >>> t = Table.read(chandra_events) + +A benefit of using the unified interface to read the table is that it will reconstruct +any :ref:`mixin_columns` that were written to that HDU. + +If more than one table is present in the file, you can select the HDU +by index or by name:: + + >>> t = Table.read(chandra_events, hdu="EVENTS") + +In this case if the ``hdu`` argument is omitted, then the first table found +will be read in and a warning will be emitted. + +You can also read a table from the HDUs of an in-memory FITS file. :: + + >>> from astropy.io import fits + >>> with fits.open(chandra_events) as hdul: + ... t = Table.read(hdul["EVENTS"]) + +If a column contains unit information, it will have an associated +`astropy.units` object:: + + >>> t["energy"].unit + Unit("eV") + +It is also possible to get directly a table with columns as +`~astropy.units.Quantity` objects by using the `~astropy.table.QTable` class:: + + >>> from astropy.table import QTable + >>> t2 = QTable.read(chandra_events, hdu=1) + >>> t2['energy'] + + +Writing +^^^^^^^ + +To write a table ``t`` to a new file:: + + >>> t.write('new_table.fits') + +If the file already exists and you want to overwrite it, then set the +``overwrite`` keyword:: + + >>> t.write('existing_table.fits', overwrite=True) + +If you want to append a table to an existing file, set the ``append`` +keyword:: + + >>> t.write('existing_table.fits', append=True) + + +.. testcleanup:: + + >>> import pathlib + >>> pathlib.Path.unlink('new_table.fits') + >>> pathlib.Path.unlink('existing_table.fits') + +Alternatively, you can use the convenience function +:func:`~astropy.io.fits.table_to_hdu` to create a single +binary table HDU and insert or append that to an existing +:class:`~astropy.io.fits.HDUList`. + +There is support for writing a table which contains :ref:`mixin_columns` such +as `~astropy.time.Time` or `~astropy.coordinates.SkyCoord`. This uses FITS +``COMMENT`` cards to capture additional information needed order to fully +reconstruct the mixin columns when reading back from FITS. The information is a +Python `dict` structure which is serialized using YAML. + +Keywords +^^^^^^^^ + +The FITS keywords associated with an HDU table are represented in the ``meta`` +ordered dictionary attribute of a :ref:`Table `. After reading +a table you can view the available keywords in a readable format using: + + >>> for key, value in t.meta.items(): + ... print(f'{key} = {value}') + EXTNAME = EVENTS + HDUNAME = EVENTS + TLMIN2 = 0 + ... + +This does not include the "internal" FITS keywords that are required to specify +the FITS table properties (e.g., ``NAXIS``, ``TTYPE1``). ``HISTORY`` and +``COMMENT`` keywords are treated specially and are returned as a list of + + >>> t.meta['MY_KEYWD'] = 'my value' + >>> t.meta['COMMENT'] = ['First comment', 'Second comment', 'etc'] + >>> t.write('my_table.fits', overwrite=True) + +.. testcleanup:: + >>> pathlib.Path.unlink('my_table.fits') + +The keyword names (e.g., ``MY_KEYWD``) will be automatically capitalized prior +to writing. + +.. _fits_astropy_native: + + +TDISPn Keyword +^^^^^^^^^^^^^^ + +TDISPn FITS keywords will map to and from the `~astropy.table.Column` ``format`` +attribute if the display format is convertible to and from a Python display +format. Below are the rules used for both conversion directions. + +TDISPn to Python format string +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +TDISPn format characters are defined in the table below. + +============ ================================================================ + Format Description +============ ================================================================ +Aw Character +Lw Logical +Iw.m Integer +Bw.m Binary, integers only +Ow.m Octal, integers only +Zw.m Hexadecimal, integers only +Fw.d Floating-point, fixed decimal notation +Ew.dEe Floating-point, exponential notation +ENw.d Engineering; E format with exponent multiple of three +ESw.d Scientific; same as EN but non-zero leading digit if not zero +Gw.dEe General; appears as F if significance not lost, also E +Dw.dEe Floating-point, exponential notation, double precision +============ ================================================================ + +Where w is the width in characters of displayed values, m is the minimum number +of digits displayed, d is the number of digits to the right of decimal, and e +is the number of digits in the exponent. The .m and Ee fields are optional. + +The A (character), L (logical), F (floating point), and G (general) display +formats can be directly translated to Python format strings. The other formats +need to be modified to match Python display formats. + +For the integer formats (I, B, O, and Z), the width (w) value is used to add +space padding to the left of the column value. The minimum number (m) value is +not used. For the E, G, D, EN, and ES formats (floating point exponential) the +width (w) and precision (d) are both used, but the exponential (e) is not used. + +Python format string to TDISPn +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The conversion from Python format strings back to TDISPn is slightly more +complicated. + +Python strings map to the TDISP format A if the Python formatting string does +not contain right space padding. It will accept left space padding. The same +applies to the logical format L. + +The integer formats (decimal integer, binary, octal, hexadecimal) map to the +I, B, O, and Z TDISP formats respectively. Integer formats do not accept a +zero padded format string or a format string with no left padding defined (a +width is required in the TDISP format standard for the Integer formats). + +For all float and exponential values, zero padding is not accepted. There +must be at least a width or precision defined. If only a width is defined, +there is no precision set for the TDISPn format. If only a precision is +defined, the width is set to the precision plus an extra padding value +depending on format type, and both are set in the TDISPn format. Otherwise, +if both a width and precision are present they are both set in the TDISPn +format. A Python ``f`` or ``F`` map to TDISP F format. The Python ``g`` or +``G`` map to TDISP G format. The Python ``e`` and ``E`` map to TDISP E format. + +.. _unified_table_fits_masked_columns: + +Masked Columns +^^^^^^^^^^^^^^ + +Tables that contain `~astropy.table.MaskedColumn` columns can be written to +FITS. By default this will replace the masked data elements with certain +sentinel values according to the FITS standard: + +- ``NaN`` for float columns. +- Value of ``TNULLn`` for integer columns, as defined by the column + ``fill_value`` attribute. +- Null string for string columns (not currently implemented). + +When the file is read back those elements are marked as masked in the returned +table, but see `issue #4708 `_ +for problems in all three cases. It is possible to deactivate the masking with +``mask_invalid=False``. + +The FITS standard has a few limitations: + +- Not all data types are supported (e.g., logical / boolean). +- Integer columns require picking one value as the NULL indicator. If + all possible values are represented in valid data (e.g., an unsigned + int columns with all 256 possible values in valid data), then there + is no way to represent missing data. +- The masked data values are permanently lost, precluding the possibility + of later unmasking the values. + +``astropy`` provides a work-around for this limitation that users can choose to +use. The key part is to use the ``serialize_method='data_mask'`` keyword +argument when writing the table. This tells the FITS writer to split each masked +column into two separate columns, one for the data and one for the mask. +When it gets read back that process is reversed and the two columns are +merged back into one masked column. + +:: + + >>> from astropy.table.table_helpers import simple_table + >>> t = simple_table(masked=True) + >>> t['d'] = [False, False, True] + >>> t['d'].mask = [True, False, False] + >>> t +
+ a b c d + int64 float64 str1 bool + ----- ------- ---- ----- + -- 1.0 c -- + 2 2.0 -- False + 3 -- e True + +:: + + >>> t.write('data.fits', serialize_method='data_mask', overwrite=True) + >>> Table.read('data.fits') +
+ a b c d + int64 float64 bytes1 bool + ----- ------- ------ ----- + -- 1.0 c -- + 2 2.0 -- False + 3 -- e True + +.. warning:: This option goes outside of the established FITS standard for + representing missing data, so users should be careful about choosing this + option, especially if other (non-``astropy``) users will be reading the + file(s). Behind the scenes, ``astropy`` is converting the masked columns + into two distinct data and mask columns, then writing metadata into + ``COMMENT`` cards to allow reconstruction of the original data. + +``astropy`` Native Objects (Mixin Columns) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It is possible to store not only standard `~astropy.table.Column` objects to a +FITS table HDU, but also any ``astropy`` native objects +(:ref:`mixin_columns`) within a `~astropy.table.Table` or +`~astropy.table.QTable`. This includes `~astropy.time.Time`, +`~astropy.units.Quantity`, `~astropy.coordinates.SkyCoord`, and many others. + +In general, a mixin column may contain multiple data components as well as +object attributes beyond the standard Column attributes like ``format`` or +``description``. Abiding by the rules set by the FITS standard requires the +mapping of these data components and object attributes to the appropriate FITS +table columns and keywords. Thus, a well defined protocol has been developed +to allow the storage of these mixin columns in FITS while allowing the object to +"round-trip" through the file with no loss of data or attributes. + +Quantity +~~~~~~~~ + +A `~astropy.units.Quantity` mixin column in a `~astropy.table.QTable` is +represented in a FITS table using the ``TUNITn`` FITS column keyword to +incorporate the unit attribute of Quantity. For example:: + + >>> from astropy.table import QTable + >>> import astropy.units as u + >>> t = QTable([[1, 2] * u.angstrom]) + >>> t.write('my_table.fits', overwrite=True) + >>> qt = QTable.read('my_table.fits') + >>> qt + + col0 + Angstrom + float64 + -------- + 1.0 + 2.0 + +.. testcleanup:: + >>> pathlib.Path.unlink('my_table.fits') + +Time +~~~~ + +``astropy`` provides the following features for reading and writing ``Time``: + +- Writing and reading `~astropy.time.Time` Table columns to and from FITS + tables. +- Reading time coordinate columns in FITS tables (compliant with the time + standard) as `~astropy.time.Time` Table columns. + +Writing and reading ``astropy`` Time columns +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, a `~astropy.time.Time` mixin column within a `~astropy.table.Table` +or `~astropy.table.QTable` will be written to FITS in full precision. This will +be done using the FITS time standard by setting the necessary FITS header +keywords. + +The default behavior for reading a FITS table into a `~astropy.table.Table` +has historically been to convert all FITS columns to `~astropy.table.Column` +objects, which have closely matching properties. For some columns, however, +closer native ``astropy`` representations are possible, and you can indicate +these should be used by passing ``astropy_native=True`` (for backwards +compatibility, this is not done by default). This will convert columns +conforming to the FITS time standard to `~astropy.time.Time` instances, +avoiding any loss of precision and preserving information about the time +system if set in the fits header. + +Example +~~~~~~~ + +.. + EXAMPLE START + Writing and Reading Time Columns to/from FITS Tables + +To read a FITS table into `~astropy.table.Table`: + + >>> from astropy.time import Time + >>> from astropy.table import Table + >>> from astropy.coordinates import EarthLocation + >>> t = Table() + >>> t['a'] = Time([100.0, 200.0], scale='tt', format='mjd', + ... location=EarthLocation(-2446354, 4237210, 4077985, unit='m')) + >>> t.write('my_table.fits', overwrite=True) + >>> tm = Table.read('my_table.fits', astropy_native=True) + >>> tm['a'] +
+ ... + ... Unique source identifier (unique within a particular Data Release) (source_id) + ... + ... Right ascension (ICRS) at Ep=2016.0 (ra) + ... + ... Declination (ICRS) at Ep=2016.0 (dec) + ... + ... + ... + ... + ... + ...
368078797593645.019123737330.16323692972
557057342208044.853345087860.12810861107
794139533734444.974115339120.22751845955
"""))) + >>> table = votable.get_first_table() + >>> for field in votable.iter_fields_and_params(): + ... print(field) + + + + +We can now generate a ``SkyCoord`` object from the VOTable information:: + + >>> from astropy.coordinates import SkyCoord + >>> SkyCoord(table.array.data["RA_ICRS"], + ... table.array.data["DE_ICRS"], + ... frame=votable.get_coosys_by_id( + ... table.get_field_by_id("RA_ICRS").ref + ... ).to_astropy_frame(), + ... unit="deg") + diff --git a/docs/io/votable/dataorigin.rst b/docs/io/votable/dataorigin.rst new file mode 100644 index 000000000000..b0f5afaa9659 --- /dev/null +++ b/docs/io/votable/dataorigin.rst @@ -0,0 +1,129 @@ +Introduction +------------ + +Extract basic provenance information from VOTable header. The information is described in +DataOrigin IVOA note: https://www.ivoa.net/documents/DataOrigin/. + +DataOrigin includes both the query information (such as publisher, contact, versions, etc.) +and the Dataset origin (such as Creator, bibliographic links, URL, etc.) + +This API retrieves Metadata from INFO in VOTable. + + +Getting Started +--------------- + +For the following example, we would first reconstruct a VOTable DataOrigin based on a query to +VizieR catalogue J/AJ/167/18. In practice, you would obtain this table directly from +the VO service of interest:: + + >>> from astropy.io.votable.dataorigin import add_data_origin_info + >>> from astropy.io.votable.tree import VOTableFile + >>> from astropy.table import Column, Table + >>> # For this example, the table data itself is irrelevant. + >>> table = Table([ + ... Column(name="id", data=[1, 2, 3, 4]), + ... Column(name="bmag", unit="mag", data=[5.6, 7.9, 12.4, 11.3])]) + >>> votable = VOTableFile().from_table(table) + >>> votable.description = "Period variations of 32 contact binaries (Hong+, 2024)" + >>> # Order is important here for the example. + >>> add_data_origin_info(votable, "data_ivoid", "ivo://cds.vizier/j/aj/167/18", + ... content="IVOID of underlying data collection") + >>> add_data_origin_info(votable, "creator", "Hong K.", + ... content="First author or institution") + >>> add_data_origin_info(votable, "cites", "bibcode:2024AJ....167...18H", + ... content="Article or Data origin sources") + >>> add_data_origin_info(votable, "journal", "Astronomical Journal (AAS)", + ... content="Editor name (article)") + >>> add_data_origin_info(votable, "original_date", "2024", + ... content="Year of the article publication") + >>> # The rest in alphabetical order. + >>> add_data_origin_info(votable, "citation", "doi:10.26093/cds/vizier.51670018") + >>> add_data_origin_info(votable, "contact", "cds-question@unistra.fr") + >>> add_data_origin_info(votable, "publication_date", "2024-11-06") + >>> add_data_origin_info(votable, "publisher", "CDS") + >>> add_data_origin_info(votable, "reference_url", "https://cdsarc.cds.unistra.fr/viz-bin/cat/J/AJ/167/18") + >>> add_data_origin_info(votable, "request_date", "2025-03-05T14:18:05") + >>> add_data_origin_info(votable, "rights_uri", "https://cds.unistra.fr/vizier-org/licences_vizier.html") + >>> add_data_origin_info(votable, "server_software", "7.4.5") + >>> add_data_origin_info(votable, "service_protocol", "ivo://ivoa.net/std/ConeSearch/v1.03") + + +To extract DataOrigin from VOTable:: + + >>> from astropy.io.votable.dataorigin import extract_data_origin + >>> data_origin = extract_data_origin(votable) + >>> print(data_origin) + publisher: CDS + server_software: 7.4.5 + service_protocol: ivo://ivoa.net/std/ConeSearch/v1.03 + request_date: 2025-03-05T14:18:05 + contact: cds-question@unistra.fr + + data_ivoid: ivo://cds.vizier/j/aj/167/18 + citation: doi:10.26093/cds/vizier.51670018 + reference_url: https://cdsarc.cds.unistra.fr/viz-bin/cat/J/AJ/167/18 + rights_uri: https://cds.unistra.fr/vizier-org/licences_vizier.html + creator: Hong K. + journal: Astronomical Journal (AAS) + cites: bibcode:2024AJ....167...18H + original_date: 2024 + publication_date: 2024-11-06 + + +Contents and metadata +--------------------- + +`astropy.io.votable.dataorigin.extract_data_origin` returns a `astropy.io.votable.dataorigin.DataOrigin` (class) container which is made of: + +* a `astropy.io.votable.dataorigin.QueryOrigin` (class) container describing the request. + ``QueryOrigin`` is considered to be unique for the whole VOTable. + It includes metadata like the publisher, the contact, date of execution, query, etc. + +* a list of `astropy.io.votable.dataorigin.DatasetOrigin` (class) container for each Element having DataOrigin information. + ``DataSetOrigin`` is a basic provenance of the datasets queried. Each attribute is a list. + It includes metadata like authors, ivoid, landing pages, .... + +Examples +-------- + +Get the (Data Center) publisher and the Creator of the dataset:: + + >>> print(data_origin.query.publisher) + CDS + >>> print(data_origin.origin[0].creator) + ['Hong K.'] + +Other capabilities +------------------ + +DataOrigin container includes VO Elements: + +* Extract list of `astropy.io.votable.tree.Info`: + + >>> # get DataOrigin with the description of each INFO + >>> for dataset_origin in data_origin.origin: + ... for info in dataset_origin.infos: + ... print(f"{info.name}: {info.value} ({info.content})") + data_ivoid: ivo://cds.vizier/j/aj/167/18 (IVOID of underlying data collection) + creator: Hong K. (First author or institution) + cites: bibcode:2024AJ....167...18H (Article or Data origin sources) + journal: Astronomical Journal (AAS) (Editor name (article)) + original_date: 2024 (Year of the article publication) + ... + +* Extract tree node `astropy.io.votable.tree.Element`; + The following example extracts the citation from the header (in APA style): + + >>> # get the Title retrieved in Element + >>> origin = data_origin.origin[0] + >>> vo_elt = origin.get_votable_element() + >>> title = vo_elt.description if vo_elt else "" + >>> print(f"APA: {','.join(origin.creator)} ({origin.publication_date[0]}). {title} [Dataset]. {data_origin.query.publisher}. {origin.citation[0]}") + APA: Hong K. (2024-11-06). Period variations of 32 contact binaries (Hong+, 2024) [Dataset]. CDS. doi:10.26093/cds/vizier.51670018 + +* Add Data Origin INFO into VOTable: + + >>> from astropy.io.votable import dataorigin + >>> dataorigin.add_data_origin_info(votable, "query", "Data center name") + >>> dataorigin.add_data_origin_info(votable.resources[0], "creator", "Author name") diff --git a/docs/io/votable/index.rst b/docs/io/votable/index.rst index 18c567cab206..786be659baf4 100644 --- a/docs/io/votable/index.rst +++ b/docs/io/votable/index.rst @@ -4,212 +4,132 @@ .. _astropy-io-votable: -******************************************* -VOTable XML handling (`astropy.io.votable`) -******************************************* +*************************************** +VOTable Handling (`astropy.io.votable`) +*************************************** Introduction ============ -The `astropy.io.votable` subpackage converts VOTable XML files to and -from Numpy record arrays. +The `astropy.io.votable` sub-package converts VOTable XML files to and +from ``numpy`` record arrays. + +.. note:: + + If you want to read or write a single table in VOTable format, the + recommended method is via the :ref:`table_io` interface. In particular, + see the :ref:`Unified I/O VO Tables ` section. Getting Started =============== -Reading a VOTable file +Reading a VOTable File ---------------------- -To read in a VOTable file, pass a file path to +To read a VOTable file, pass a file path to `~astropy.io.votable.parse`:: - from astropy.io.votable import parse - votable = parse("votable.xml") + from astropy.io.votable import parse + votable = parse("votable.xml") ``votable`` is a `~astropy.io.votable.tree.VOTableFile` object, which can be used to retrieve and manipulate the data and save it back out to disk. -VOTable files are made up of nested ``RESOURCE`` elements, each of -which may contain one or more ``TABLE`` elements. The ``TABLE`` -elements contain the arrays of data. - -To get at the ``TABLE`` elements, one can write a loop over the -resources in the ``VOTABLE`` file:: - - for resource in votable.resources: - for table in resource.tables: - # ... do something with the table ... - pass - -However, if the nested structure of the resources is not important, -one can use `~astropy.io.votable.tree.VOTableFile.iter_tables` to -return a flat list of all tables:: - - for table in votable.iter_tables(): - # ... do something with the table ... - pass - -Finally, if there is expected to be only one table in the file, it -might be simplest to just use -`~astropy.io.votable.tree.VOTableFile.get_first_table`:: - - table = votable.get_first_table() - -Even easier, there is a convenience method to parse a VOTable file and -return the first table all in one step:: - - from astropy.io.votable import parse_single_table - table = parse_single_table("votable.xml") - -From a `~astropy.io.votable.tree.Table` object, one can get the data itself -in the ``array`` member variable:: - - data = table.array +Writing a VOTable File +---------------------- -This data is a Numpy record array. +This section describes writing table data in the VOTable format using the +`~astropy.io.votable` package directly. For some cases, however, the high-level +:ref:`table_io` will often suffice and is somewhat more convenient to use. See +the :ref:`Unified I/O VOTable ` section for details. -The columns get their names from both the ``ID`` and ``name`` -attributes of the ``FIELD`` elements in the ``VOTABLE`` file. For -example, suppose we had a ``FIELD`` specified as follows: +To save a VOTable file, call the +`~astropy.io.votable.tree.VOTableFile.to_xml` method, or equivalently +the ``write`` convenience method. Both accept either a string or +Unicode path, or a Python file-like object:: -.. code-block:: xml + votable.to_xml('output.xml') - - - representing the ICRS declination of the center of the image. - - + # Equivalently: + votable.write('output.xml') .. note:: - The mapping from VOTable ``name`` and ``ID`` attributes to Numpy - dtype ``names`` and ``titles`` is highly confusing. + ``VOTableFile.write()`` serialises the existing ``VOTableFile`` object + as-is, preserving all metadata including PARAMs and INFOs. This is + different from ``Table.write(format='votable')``, which converts an + `~astropy.table.Table` into a new VOTable and does not preserve + VOTable-specific metadata such as PARAMs and INFOs. - In VOTable, ``ID`` is guaranteed to be unique, but is not - required. ``name`` is not guaranteed to be unique, but is - required. - - In Numpy record dtypes, ``names`` are required to be unique and - are required. ``titles`` are not required, and are not required - to be unique. - - Therefore, VOTable's ``ID`` most closely maps to Numpy's - ``names``, and VOTable's ``name`` most closely maps to Numpy's - ``titles``. However, in some cases where a VOTable ``ID`` is not - provided, a Numpy ``name`` will be generated based on the VOTable - ``name``. Unfortunately, VOTable fields do not have an attribute - that is both unique and required, which would be the most - convenient mechanism to uniquely identify a column. - - When converting from a `astropy.io.votable.tree.Table` object to - an `astropy.table.Table` object, one can specify whether to give - preference to ``name`` or ``ID`` attributes when naming the - columns. By default, ``ID`` is given preference. To give - ``name`` preference, pass the keyword argument - ``use_names_over_ids=True``:: - - >>> votable.get_first_table().to_table(use_names_over_ids=True) - -This column of data can be extracted from the record array using:: - - >>> table.array['dec_targ'] - array([17.15153360566, 17.15153360566, 17.15153360566, 17.1516686826, - 17.1516686826, 17.1516686826, 17.1536197136, 17.1536197136, - 17.1536197136, 17.15375479055, 17.15375479055, 17.15375479055, - 17.1553884541, 17.15539736932, 17.15539752176, - 17.25736014763, - # ... - 17.2765703], dtype=object) - -or equivalently:: - - >>> table.array['Dec'] - array([17.15153360566, 17.15153360566, 17.15153360566, 17.1516686826, - 17.1516686826, 17.1516686826, 17.1536197136, 17.1536197136, - 17.1536197136, 17.15375479055, 17.15375479055, 17.15375479055, - 17.1553884541, 17.15539736932, 17.15539752176, - 17.25736014763, - # ... - 17.2765703], dtype=object) - -Building a new table from scratch ---------------------------------- - -It is also possible to build a new table, define some field datatypes -and populate it with data:: - - from astropy.io.votable.tree import VOTableFile, Resource, Table, Field - - # Create a new VOTable file... - votable = VOTableFile() - - # ...with one resource... - resource = Resource() - votable.resources.append(resource) - - # ... with one table - table = Table(votable) - resource.tables.append(table) +There are a number of data storage formats supported by +`astropy.io.votable`. The ``TABLEDATA`` format is XML-based and +stores values as strings representing numbers. The ``BINARY`` format +is more compact, and stores numbers in base64-encoded binary. VOTable +version 1.3 adds the ``BINARY2`` format, which allows for masking of +any data type, including integers and bit fields which cannot be +masked in the older ``BINARY`` format. The storage format can be set +on a per-table basis using the `~astropy.io.votable.tree.TableElement.format` +attribute, or globally using the +`~astropy.io.votable.tree.VOTableFile.set_all_tables_format` method:: - # Define some fields - table.fields.extend([ - Field(votable, name="filename", datatype="char", arraysize="*"), - Field(votable, name="matrix", datatype="double", arraysize="2x2")]) + votable.get_first_table().format = 'binary' + votable.set_all_tables_format('binary') + votable.to_xml('binary.xml') - # Now, use those field definitions to create the numpy record arrays, with - # the given number of rows - table.create_arrays(2) +The VOTable elements +==================== - # Now table.array can be filled with data - table.array[0] = ('test1.xml', [[1, 0], [0, 1]]) - table.array[1] = ('test2.xml', [[0.5, 0.3], [0.2, 0.1]]) +VOTables are built from nested elements. Let's for example build a +votable containing an ``INFO`` element:: - # Now write the whole thing to a file. - # Note, we have to use the top-level votable file object - votable.to_xml("new_votable.xml") + >>> from astropy.io.votable.tree import VOTableFile, Info + >>> vot = VOTableFile() + >>> vot.infos.append(Info(name="date_obs", value="2025-01-01")) -Outputting a VOTable file -------------------------- +These elements can be: -To save a VOTable file, simply call the -`~astropy.io.votable.tree.VOTableFile.to_xml` method. It accepts -either a string or Unicode path, or a Python file-like object:: +- `~astropy.io.votable.tree.CooSys` +- `~astropy.io.votable.tree.TimeSys` +- `~astropy.io.votable.tree.Info` +- `~astropy.io.votable.tree.Param` +- `~astropy.io.votable.tree.Group` +- `~astropy.io.votable.tree.Resource` +- `~astropy.io.votable.tree.Link` +- `~astropy.io.votable.tree.TableElement` +- `~astropy.io.votable.tree.Field` +- `~astropy.io.votable.tree.Values` +- `~astropy.io.votable.tree.MivotBlock` - votable.to_xml('output.xml') +Here are some detailed explanations on some of these elements: -There are a number of data storage formats supported by -`astropy.io.votable`. The ``TABLEDATA`` format is XML-based and -stores values as strings representing numbers. The ``BINARY`` format -is more compact, and stores numbers in base64-encoded binary. VOTable -version 1.3 adds the ``BINARY2`` format, which allows for masking of -any data type, including integers and bit fields which can not be -masked in the older ``BINARY`` format. The storage format can be set -on a per-table basis using the `~astropy.io.votable.tree.Table.format` -attribute, or globally using the -`~astropy.io.votable.tree.VOTableFile.set_all_tables_format` method:: +.. toctree:: + :maxdepth: 1 - votable.get_first_table().format = 'binary' - votable.set_all_tables_format('binary') - votable.to_xml('binary.xml') + table_element + coosys_element + mivot_blocks Using `astropy.io.votable` ========================== -Standard compliance +Standard Compliance ------------------- -`astropy.io.votable.tree.Table` supports the `VOTable Format Definition +`astropy.io.votable.tree.TableElement` supports the `VOTable Format Definition Version 1.1 -`_, +`_, `Version 1.2 -`_, -and the `Version 1.3 proposed recommendation -`_. +`_, +`Version 1.3 +`_, +`Version 1.4 +`_, +and `Version 1.5 +`_, Some flexibility is provided to support the 1.0 draft version and -other non-standard usage in the wild. To support these cases, set the -keyword argument ``pedantic`` to ``False`` when parsing. +other nonstandard usage in the wild, see :ref:`verifying-votables` for more +details. .. note:: @@ -217,112 +137,49 @@ keyword argument ``pedantic`` to ``False`` when parsing. is documented in more detail in :ref:`warnings` and :ref:`exceptions`. -Output always conforms to the 1.1, 1.2 or 1.3 spec, depending on the +Output always conforms to the 1.1, 1.2, 1.3, 1.4, 1.5 spec, depending on the input. -.. _pedantic-mode: +.. _verifying-votables: + +Verifying VOTables +------------------ -Pedantic mode -^^^^^^^^^^^^^ +Many VOTable files in the wild do not conform to the VOTable specification. You +can set what should happen when a violation is encountered with the ``verify`` +keyword, which can take three values: -Many VOTABLE files in the wild do not conform to the VOTABLE -specification. If reading one of these files causes exceptions, you -may turn off pedantic mode in `astropy.io.votable` by passing -``pedantic=False`` to the `~astropy.io.votable.parse` or -`~astropy.io.votable.parse_single_table` functions:: + * ``'ignore'`` - Attempt to parse the VOTable silently. This is the default + setting. + * ``'warn'`` - Attempt to parse the VOTable, but raise appropriate + :ref:`warnings`. It is possible to limit the number of warnings of the + same type to a maximum value using the + `astropy.io.votable.exceptions.conf.max_warnings + ` item in the + :ref:`astropy_config`. + * ``'exception'`` - Do not parse the VOTable and raise an exception. + +The ``verify`` keyword can be used with the :func:`~astropy.io.votable.parse` +or :func:`~astropy.io.votable.parse_single_table` functions:: from astropy.io.votable import parse - votable = parse("votable.xml", pedantic=False) - -Note, however, that it is good practice to report these errors to the -author of the application that generated the VOTABLE file to bring the -file into compliance with the specification. - -Even with ``pedantic`` turned off, many warnings may still be omitted. -These warnings are all of the type -`~astropy.io.votable.exceptions.VOTableSpecWarning` and can be turned -off using the standard Python `warnings` module. - -Missing values --------------- - -Any value in the table may be "missing". `astropy.io.votable` stores -a Numpy masked array in each `~astropy.io.votable.tree.Table` -instance. This behaves like an ordinary Numpy masked array, except -for variable-length fields. For those fields, the datatype of the -column is "object" and another Numpy masked array is stored there. -Therefore, operations on variable length columns will not work -- this -is simply because variable length columns are not directly supported -by Numpy masked arrays. - -Datatype mappings ------------------ - -The datatype specified by a ``FIELD`` element is mapped to a Numpy -type according to the following table: - - ================================ ======================================================================== - VOTABLE type Numpy type - ================================ ======================================================================== - boolean b1 - -------------------------------- ------------------------------------------------------------------------ - bit b1 - -------------------------------- ------------------------------------------------------------------------ - unsignedByte u1 - -------------------------------- ------------------------------------------------------------------------ - char (*variable length*) O - In Python 2.x, a `str` object; in 3.x, a ``bytes()`` object. - -------------------------------- ------------------------------------------------------------------------ - char (*fixed length*) S - -------------------------------- ------------------------------------------------------------------------ - unicodeChar (*variable length*) O - In Python 2.x, a `unicode` object, in utf-16; in 3.x a `str` object - -------------------------------- ------------------------------------------------------------------------ - unicodeChar (*fixed length*) U - -------------------------------- ------------------------------------------------------------------------ - short i2 - -------------------------------- ------------------------------------------------------------------------ - int i4 - -------------------------------- ------------------------------------------------------------------------ - long i8 - -------------------------------- ------------------------------------------------------------------------ - float f4 - -------------------------------- ------------------------------------------------------------------------ - double f8 - -------------------------------- ------------------------------------------------------------------------ - floatComplex c8 - -------------------------------- ------------------------------------------------------------------------ - doubleComplex c16 - ================================ ======================================================================== - -If the field is a fixed size array, the data is stored as a Numpy -fixed-size array. - -If the field is a variable size array (that is ``arraysize`` contains -a '*'), the cell will contain a Python list of Numpy values. Each -value may be either an array or scalar depending on the ``arraysize`` -specifier. - -Examining field types ---------------------- - -To look up more information about a field in a table, one can use the -`~astropy.io.votable.tree.Table.get_field_by_id` method, which returns -the `~astropy.io.votable.tree.Field` object with the given ID. For -example:: - - >>> field = table.get_field_by_id('Dec') - >>> field.datatype - 'char' - >>> field.unit - 'deg' + votable = parse("votable.xml", verify='warn') -.. note:: - Field descriptors should not be mutated. To change the set of - columns, convert the Table to an `astropy.table.Table`, make the - changes, and then convert it back. +It is possible to change the default ``verify`` value through the +`astropy.io.votable.conf.verify ` item in the +:ref:`astropy_config`. + +Note that ``'ignore'`` or ``'warn'`` mean that ``astropy`` will attempt to +parse the VOTable, but if the specification has been violated then success +cannot be guaranteed. + +It is good practice to report any errors to the author of the application that +generated the VOTable file to bring the file into compliance with the +specification. .. _votable-serialization: -Data serialization formats +Data Serialization Formats -------------------------- VOTable supports a number of different serialization formats. @@ -345,14 +202,22 @@ VOTable supports a number of different serialization formats. - `FITS `__ - stores the data in an external FITS file. This serialization is not + stores the data in an external FITS file. This serialization is not supported by the `astropy.io.votable` writer, since it requires writing multiple files. +- ``PARQUET`` + stores the data in an external PARQUET file, similar to FITS serialization. + Reading and writing is fully supported by the `astropy.io.votable` writer and + the `astropy.io.votable.parse` reader. The parquet file can be + referenced with either absolute and relative paths. The parquet + serialization can be used as part of the unified Table I/O (see next + section), by setting the ``format`` argument to ``'votable.parquet'``. + The serialization format can be selected in two ways: 1) By setting the ``format`` attribute of a - `astropy.io.votable.tree.Table` object:: + `astropy.io.votable.tree.TableElement` object:: votable.get_first_table().format = "binary" votable.to_xml("new_votable.xml") @@ -367,7 +232,7 @@ Converting to/from an `astropy.table.Table` ------------------------------------------- The VOTable standard does not map conceptually to an -`astropy.table.Table`. However, a single table within the ``VOTable`` +`astropy.table.Table`. However, a single table within the ``VOTable`` file may be converted to and from an `astropy.table.Table`:: from astropy.io.votable import parse_single_table @@ -383,71 +248,63 @@ file with just a single table:: .. note:: By default, ``to_table`` will use the ``ID`` attribute from the files to - create the column names for the `~astropy.table.Table` object. However, - it may be that you want to use the ``name`` attributes instead. For this, - set the ``use_names_over_ids`` keyword to `True`. Note that since field + create the column names for the `~astropy.table.Table` object. However, + it may be that you want to use the ``name`` attributes instead. For this, + set the ``use_names_over_ids`` keyword to `True`. Note that since field ``names`` are not guaranteed to be unique in the VOTable specification, - but column names are required to be unique in Numpy structured arrays (and + but column names are required to be unique in ``numpy`` structured arrays (and thus `astropy.table.Table` objects), the names may be renamed by appending numbers to the end in some cases. -Performance considerations +Performance Considerations -------------------------- File reads will be moderately faster if the ``TABLE`` element includes -an nrows_ attribute. If the number of rows is not specified, the +an nrows_ attribute. If the number of rows is not specified, the record array must be resized repeatedly during load. -.. _nrows: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC10 +.. _nrows: http://www.ivoa.net/documents/REC/VOTable/VOTable-20040811.html#ToC10 -See Also -======== -- `VOTable Format Definition Version 1.1 - `_ +Data Origin +=========== -- `VOTable Format Definition Version 1.2 - `_ +.. _astropy-io-votable-dataorigin: -- `VOTable Format Definition Version 1.3, Proposed Recommendatation - `_ +.. include:: dataorigin.rst -Reference/API -============= - -.. automodapi:: astropy.io.votable - :no-inheritance-diagram: - :skip: VOWarning - :skip: VOTableChangeWarning - :skip: VOTableSpecWarning - :skip: UnimplementedWarning - :skip: IOWarning - :skip: VOTableSpecError +See Also +======== -.. automodapi:: astropy.io.votable.tree - :no-inheritance-diagram: +- `VOTable Format Definition Version 1.1 + `_ -.. automodapi:: astropy.io.votable.converters - :no-inheritance-diagram: +- `VOTable Format Definition Version 1.2 + `_ -.. automodapi:: astropy.io.votable.ucd - :no-inheritance-diagram: +- `VOTable Format Definition Version 1.3 + `_ -.. automodapi:: astropy.io.votable.util - :no-inheritance-diagram: +- `VOTable Format Definition Version 1.4 + `_ -.. automodapi:: astropy.io.votable.validator - :no-inheritance-diagram: +- `VOTable Format Definition Version 1.5 + `_ -.. automodapi:: astropy.io.votable.xmlutil - :no-inheritance-diagram: +- `MIVOT Recommendation Version 1.0 + `_ +.. note that if this section gets too long, it should be moved to a separate + doc page - see the top of performance.inc.rst for the instructions on how to do + that +.. include:: performance.inc.rst -astropy.io.votable.exceptions Module ------------------------------------- +Reference/API +============= .. toctree:: - :maxdepth: 1 + :maxdepth: 2 - api_exceptions.rst + ref_api + api_exceptions diff --git a/docs/io/votable/mivot_blocks.rst b/docs/io/votable/mivot_blocks.rst new file mode 100644 index 000000000000..d81e6891d231 --- /dev/null +++ b/docs/io/votable/mivot_blocks.rst @@ -0,0 +1,129 @@ +.. doctest-skip-all + +Reading and writing VO model annotations (MIVOT) +------------------------------------------------ + +A ``RESOURCE`` element can be a ``MIVOT`` block since VOTable version 1.5. + +Introduction +^^^^^^^^^^^^ +Model Instances in VOTables (`MIVOT `_) +defines a syntax to map VOTable data to any model serialised in VO-DML (Virtual Observatory Data Modeling Language). +This annotation schema operates as a bridge between data and the models. It associates both column/param metadata and data +from the VOTable to the data model elements (class, attributes, types, etc.). It also brings up VOTable data or +metadata that were possibly missing in the table, e.g., coordinate system description, or curation tracing. +The data model elements are grouped in an independent annotation block complying with the MIVOT XML schema which +is added as an extra resource above the table element. +The MIVOT syntax allows to describe a data structure as a hierarchy of classes. +It is also able to represent relations and compositions between them. It can moreover build up data model objects by +aggregating instances from different tables of the VOTable. + +Astropy implementation +^^^^^^^^^^^^^^^^^^^^^^ +The purpose of Astropy is not to process VO annotations. +It is just to allow related packages to get and set MIVOT blocks from/into VOTables. +For this reason, in this implementation MIVOT annotations are both imported and exported as strings. +The current implementation prevents client code from injecting into VOTables strings +that are not MIVOT serializations. + +MivotBlock implementation: + +- MIVOT blocks are handled by the :class:`astropy.io.votable.tree.MivotBlock` class. +- A MivotBlock instance can only be carried by a resource with "type=meta". +- This instance holds the XML mapping block as a string. +- MivotBlock objects are instanced by the Resource parser. +- The MivotBlock class has its own logic that operates both parsing and IO functionalities. + +Example +""""""" + +.. code-block:: xml + + + + + + ... + + + + .... +
+ + + +Reading a VOTable containing a MIVOT block +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To read in a VOTable file containing or not a MIVOT Resource, pass a file path to `~astropy.io.votable.parse`: + +.. code-block:: python + + >>> from astropy.io.votable import parse + >>> from astropy.utils.data import get_pkg_data_filename + >>> votable = parse(get_pkg_data_filename("data/test.order.xml", package="astropy.io.votable.tests")) + + + + + + +The parse function will call the MIVOT parser if it detects a MIVOT block. + +Building a Resource containing a MIVOT block +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Construct the MIVOT block by passing the XML block as a parameter: + +.. code-block:: python + + >>> from astropy.io.votable import tree + >>> from astropy.io.votable.tree import MivotBlock, Resource, VOTableFile + >>> mivot_block = MivotBlock(""" + + Unit test mapping block + + + """) + +Build a new resource: + +.. code-block:: python + + >>> mivot_resource = Resource() + +Give it the type meta: + +.. code-block:: python + + >>> mivot_resource.type = "meta" + +Then add it the MIVOT block: + +.. code-block:: python + + >>> mivot_resource.mivot_block = mivot_block + +Now you have a MIVOT resource that you can add to an object Resource creating a new Resource: + +.. code-block:: python + + >>> votable = VOTableFile() + >>> r1 = Resource() + >>> r1.type = "results" + >>> r1.resources.append(mivot_resource) + +You can add an `astropy.io.votable.tree.TableElement` to the resource: + +.. code-block:: python + + >>> table = tree.TableElement(votable) + >>> r1.tables.append(t1) + >>> votable.resources.append(r1) + >>> for resource in votable.resources: + ... print(resource.mivot_block.content) + + Unit test mapping block + + diff --git a/docs/io/votable/performance.inc.rst b/docs/io/votable/performance.inc.rst new file mode 100644 index 000000000000..8a54ff093fed --- /dev/null +++ b/docs/io/votable/performance.inc.rst @@ -0,0 +1,12 @@ +.. note that if this is changed from the default approach of using an *include* + (in index.rst) to a separate performance page, the header needs to be changed + from === to ***, the filename extension needs to be changed from .inc.rst to + .rst, and a link needs to be added in the subpackage toctree + +.. _astropy-io-votable-performance: + +.. Performance Tips +.. ================ +.. +.. Here we provide some tips and tricks for how to optimize performance of code +.. using `astropy.io.votable`. diff --git a/docs/io/votable/ref_api.rst b/docs/io/votable/ref_api.rst new file mode 100644 index 000000000000..908a33a9d853 --- /dev/null +++ b/docs/io/votable/ref_api.rst @@ -0,0 +1,36 @@ +.. doctest-skip-all + +.. include:: references.txt + +Reference/API +************* + +.. automodapi:: astropy.io.votable + :no-inheritance-diagram: + :skip: VOWarning + :skip: VOTableChangeWarning + :skip: VOTableSpecWarning + :skip: UnimplementedWarning + :skip: IOWarning + :skip: VOTableSpecError + +.. automodapi:: astropy.io.votable.tree + :no-inheritance-diagram: + +.. automodapi:: astropy.io.votable.converters + :no-inheritance-diagram: + +.. automodapi:: astropy.io.votable.ucd + :no-inheritance-diagram: + +.. automodapi:: astropy.io.votable.util + :no-inheritance-diagram: + +.. automodapi:: astropy.io.votable.validator + :no-inheritance-diagram: + +.. automodapi:: astropy.io.votable.xmlutil + :no-inheritance-diagram: + +.. automodapi:: astropy.io.votable.dataorigin + :no-inheritance-diagram: diff --git a/docs/io/votable/references.txt b/docs/io/votable/references.txt index f3ff4c8be309..e4975684c303 100644 --- a/docs/io/votable/references.txt +++ b/docs/io/votable/references.txt @@ -1,23 +1,24 @@ -.. _BINARY: http://www.ivoa.net/Documents/PR/VOTable/VOTable-20040322.html#ToC27 +.. _BINARY: http://www.ivoa.net/documents/PR/VOTable/VOTable-20040322.html#ToC27 .. _BINARY2: http://www.ivoa.net/documents/VOTable/20130315/PR-VOTable-1.3-20130315.html#sec:BIN2 -.. _COOSYS: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC19 -.. _DESCRIPTION: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC19 -.. _FIELD: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC24 -.. _FIELDref: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC31 -.. _FITS: http://fits.gsfc.nasa.gov/fits_documentation.html -.. _GROUP: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC31 -.. _ID: http://www.w3.org/TR/REC-xml/#id -.. _INFO: http://www.ivoa.net/Documents/VOTable/20040811/REC-VOTable-1.1-20040811.html#ToC19 -.. _LINK: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC22 -.. _multidimensional arrays: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC12 -.. _numerical accuracy: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC26 -.. _PARAM: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC24 -.. _PARAMref: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC31 -.. _RESOURCE: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC21 -.. _TABLE: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC23 -.. _TABLEDATA: http://www.ivoa.net/Documents/PR/VOTable/VOTable-20040322.html#ToC25 -.. _unified content descriptor: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC28 -.. _unique type: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC29 -.. _units: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC27 -.. _VALUES: http://www.ivoa.net/Documents/REC/VOTable/VOTable-20040811.html#ToC30 -.. _VOTABLE: http://www.ivoa.net/Documents/PR/VOTable/VOTable-20040322.html#ToC9 +.. _COOSYS: http://www.ivoa.net/documents/VOTable/20191021/REC-VOTable-1.4-20191021.html#ToC20 +.. _DESCRIPTION: http://www.ivoa.net/documents/REC/VOTable/VOTable-20040811.html#ToC19 +.. _FIELD: http://www.ivoa.net/documents/REC/VOTable/VOTable-20040811.html#ToC24 +.. _FIELDref: http://www.ivoa.net/documents/REC/VOTable/VOTable-20040811.html#ToC31 +.. _FITS: https://fits.gsfc.nasa.gov/fits_documentation.html +.. _GROUP: http://www.ivoa.net/documents/REC/VOTable/VOTable-20040811.html#ToC31 +.. _ID: https://www.w3.org/TR/xml-id/ +.. _INFO: http://www.ivoa.net/documents/VOTable/20040811/REC-VOTable-1.1-20040811.html#ToC19 +.. _LINK: http://www.ivoa.net/documents/REC/VOTable/VOTable-20040811.html#ToC22 +.. _multidimensional arrays: http://www.ivoa.net/documents/REC/VOTable/VOTable-20040811.html#ToC12 +.. _numerical accuracy: http://www.ivoa.net/documents/REC/VOTable/VOTable-20040811.html#ToC26 +.. _PARAM: http://www.ivoa.net/documents/REC/VOTable/VOTable-20040811.html#ToC24 +.. _PARAMref: http://www.ivoa.net/documents/REC/VOTable/VOTable-20040811.html#ToC31 +.. _RESOURCE: http://www.ivoa.net/documents/REC/VOTable/VOTable-20040811.html#ToC21 +.. _TABLE: http://www.ivoa.net/documents/REC/VOTable/VOTable-20040811.html#ToC23 +.. _TABLEDATA: http://www.ivoa.net/documents/PR/VOTable/VOTable-20040322.html#ToC25 +.. _TIMESYS: http://www.ivoa.net/documents/VOTable/20191021/REC-VOTable-1.4-20191021.html#ToC21 +.. _unified content descriptor: http://www.ivoa.net/documents/REC/VOTable/VOTable-20040811.html#ToC28 +.. _unique type: http://www.ivoa.net/documents/REC/VOTable/VOTable-20040811.html#ToC29 +.. _units: http://www.ivoa.net/documents/REC/VOTable/VOTable-20040811.html#ToC27 +.. _VALUES: http://www.ivoa.net/documents/REC/VOTable/VOTable-20040811.html#ToC30 +.. _VOTABLE: http://www.ivoa.net/documents/PR/VOTable/VOTable-20040322.html#ToC9 diff --git a/docs/io/votable/table_element.rst b/docs/io/votable/table_element.rst new file mode 100644 index 000000000000..3f8034735eb9 --- /dev/null +++ b/docs/io/votable/table_element.rst @@ -0,0 +1,245 @@ +.. doctest-skip-all + +Table element +------------- + +VOTable files can contain ``RESOURCE`` elements, each of +which may contain one or more ``TABLE`` elements. The ``TABLE`` +elements contain the arrays of data. + +To get at the ``TABLE`` elements, you can write a loop over the +resources in the ``VOTABLE`` file:: + + for resource in votable.resources: + for table in resource.tables: + # ... do something with the table ... + pass + +However, if the nested structure of the resources is not important, +you can use `~astropy.io.votable.tree.VOTableFile.iter_tables` to +return a flat list of all tables:: + + for table in votable.iter_tables(): + # ... do something with the table ... + pass + +Finally, if you expect only one table in the file, it might be most convenient +to use `~astropy.io.votable.tree.VOTableFile.get_first_table`:: + + table = votable.get_first_table() + +Alternatively, there is a convenience method to parse a VOTable file and +return the first table all in one step:: + + from astropy.io.votable import parse_single_table + table = parse_single_table("votable.xml") + +From a `~astropy.io.votable.tree.TableElement` object, you can get the data itself +in the ``array`` member variable:: + + data = table.array + +This data is a ``numpy`` record array. + +The columns get their names from both the ``ID`` and ``name`` +attributes of the ``FIELD`` elements in the ``VOTABLE`` file. + +.. + EXAMPLE START + Reading a VOTable File with astropy.io.votable + +Suppose we had a ``FIELD`` specified as follows: + +.. code-block:: xml + + + + representing the ICRS declination of the center of the image. + + + +.. note:: + + The mapping from VOTable ``name`` and ``ID`` attributes to ``numpy`` + dtype ``names`` and ``titles`` is highly confusing. + + In VOTable, ``ID`` is guaranteed to be unique, but is not + required. ``name`` is not guaranteed to be unique, but is + required. + + In ``numpy`` record dtypes, ``names`` are required to be unique and + are required. ``titles`` are not required, and are not required + to be unique. + + Therefore, VOTable's ``ID`` most closely maps to ``numpy``'s + ``names``, and VOTable's ``name`` most closely maps to ``numpy``'s + ``titles``. However, in some cases where a VOTable ``ID`` is not + provided, a ``numpy`` ``name`` will be generated based on the VOTable + ``name``. Unfortunately, VOTable fields do not have an attribute + that is both unique and required, which would be the most + convenient mechanism to uniquely identify a column. + + When converting from an `astropy.io.votable.tree.TableElement` object to + an `astropy.table.Table` object, you can specify whether to give + preference to ``name`` or ``ID`` attributes when naming the + columns. By default, ``ID`` is given preference. To give + ``name`` preference, pass the keyword argument + ``use_names_over_ids=True``:: + + >>> votable.get_first_table().to_table(use_names_over_ids=True) + +This column of data can be extracted from the record array using:: + + >>> table.array['dec_targ'] + array([17.15153360566, 17.15153360566, 17.15153360566, 17.1516686826, + 17.1516686826, 17.1516686826, 17.1536197136, 17.1536197136, + 17.1536197136, 17.15375479055, 17.15375479055, 17.15375479055, + 17.1553884541, 17.15539736932, 17.15539752176, + 17.25736014763, + # ... + 17.2765703], dtype=object) + +or equivalently:: + + >>> table.array['Dec'] + array([17.15153360566, 17.15153360566, 17.15153360566, 17.1516686826, + 17.1516686826, 17.1516686826, 17.1536197136, 17.1536197136, + 17.1536197136, 17.15375479055, 17.15375479055, 17.15375479055, + 17.1553884541, 17.15539736932, 17.15539752176, + 17.25736014763, + # ... + 17.2765703], dtype=object) + +.. + EXAMPLE END + +Datatype Mappings +^^^^^^^^^^^^^^^^^ + +The datatype specified by a ``FIELD`` element is mapped to a ``numpy`` +type according to the following table: + + ================================ ========================= + VOTABLE type NumPy type + ================================ ========================= + boolean b1 + -------------------------------- ------------------------- + bit b1 + -------------------------------- ------------------------- + unsignedByte u1 + -------------------------------- ------------------------- + char (*variable length*) O - A ``bytes()`` object. + -------------------------------- ------------------------- + char (*fixed length*) S + -------------------------------- ------------------------- + unicodeChar (*variable length*) O - A `str` object + -------------------------------- ------------------------- + unicodeChar (*fixed length*) U + -------------------------------- ------------------------- + short i2 + -------------------------------- ------------------------- + int i4 + -------------------------------- ------------------------- + long i8 + -------------------------------- ------------------------- + float f4 + -------------------------------- ------------------------- + double f8 + -------------------------------- ------------------------- + floatComplex c8 + -------------------------------- ------------------------- + doubleComplex c16 + ================================ ========================= + +If the field is a fixed-size array, the data is stored as a ``numpy`` +fixed-size array. + +If the field is a variable-size array (that is, ``arraysize`` contains +a '*'), the cell will contain a Python list of ``numpy`` values. Each +value may be either an array or scalar depending on the ``arraysize`` +specifier. + +Examining Field Types +^^^^^^^^^^^^^^^^^^^^^ + +To look up more information about a field in a table, you can use the +`~astropy.io.votable.tree.TableElement.get_field_by_id` method, which returns +the `~astropy.io.votable.tree.Field` object with the given ID. + +.. + EXAMPLE START + Examining Field Types in VOTables with astropy.io.votable + +To look up more information about a field:: + + >>> field = table.get_field_by_id('Dec') + >>> field.datatype + 'char' + >>> field.unit + 'deg' + +.. note:: + Field descriptors should not be mutated. To change the set of + columns, convert the Table to an `astropy.table.Table`, make the + changes, and then convert it back. + +.. + EXAMPLE END + +Building a New Table from Scratch +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It is also possible to build a new table, define some field datatypes, +and populate it with data. + +.. + EXAMPLE START + Building a New Table from a VOTable File + +To build a new table from a VOTable file:: + + from astropy.io.votable.tree import VOTableFile, Resource, TableElement, Field + + # Create a new VOTable file... + votable = VOTableFile() + + # ...with one resource... + resource = Resource() + votable.resources.append(resource) + + # ... with one table + table = TableElement(votable) + resource.tables.append(table) + + # Define some fields + table.fields.extend([ + Field(votable, name="filename", datatype="char", arraysize="*"), + Field(votable, name="matrix", datatype="double", arraysize="2x2")]) + + # Now, use those field definitions to create the numpy record arrays, with + # the given number of rows + table.create_arrays(2) + + # Now table.array can be filled with data + table.array[0] = ('test1.xml', [[1, 0], [0, 1]]) + table.array[1] = ('test2.xml', [[0.5, 0.3], [0.2, 0.1]]) + + # Now write the whole thing to a file. + # Note, we have to use the top-level votable file object + votable.to_xml("new_votable.xml") + +.. + EXAMPLE END + +Missing Values +^^^^^^^^^^^^^^ + +Any value in the table may be "missing". `astropy.io.votable` stores +a ``numpy`` masked array in each `~astropy.io.votable.tree.TableElement` +instance. This behaves like an ordinary ``numpy`` masked array, except +for variable-length fields. For those fields, the datatype of the +column is "object" and another ``numpy`` masked array is stored there. +Therefore, operations on variable-length columns will not work — this +is because variable-length columns are not directly supported +by ``numpy`` masked arrays. diff --git a/docs/known_issues.rst b/docs/known_issues.rst index fce68f7bcb35..382aa576c985 100644 --- a/docs/known_issues.rst +++ b/docs/known_issues.rst @@ -1,8 +1,6 @@ -.. doctest-skip-all - -============ +************ Known Issues -============ +************ .. contents:: :local: @@ -11,199 +9,294 @@ Known Issues While most bugs and issues are managed using the `astropy issue tracker `_, this document lists issues that are too difficult to fix, may require some -intervention from the user to workaround, or are due to bugs in other +intervention from the user to work around, or are caused by bugs in other projects or packages. -Issues listed on this page are grouped into two categories: The first is known +Issues listed on this page are grouped into two categories: The first is known issues and shortcomings in actual algorithms and interfaces that currently do not have fixes or workarounds, and that users should be aware of when writing -code that uses Astropy. Some of those issues are still platform-specific, -while others are very general. The second category is common issues that come -up when configuring, building, or installing Astropy. This also includes +code that uses ``astropy``. Some of those issues are still platform-specific, +while others are very general. The second category is of common issues that come +up when configuring, building, or installing ``astropy``. This also includes cases where the test suite can report false negatives depending on the context/ platform on which it was run. -Known deficiencies ------------------- +Known Deficiencies +================== .. _quantity_issues: -Quantities lose their units with some operations -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Quantities Lose Their Units with Some Operations +------------------------------------------------ -Quantities are subclassed from numpy's `~numpy.ndarray` and in some numpy operations -(and in scipy operations using numpy internally) the subclass is ignored, which -means that either a plain array is returned, or a `~astropy.units.quantity.Quantity` without units. -E.g.:: +Quantities are subclassed from ``numpy``'s `~numpy.ndarray` and while we have +ensured that ``numpy`` functions will work well with them, they do not always +work in functions from ``scipy`` or other packages that use ``numpy`` +internally, but ignore the subclass. Furthermore, at a few places in ``numpy`` +itself we cannot control the behaviour. For instance, care must be taken when +setting array slices using Quantities:: >>> import astropy.units as u >>> import numpy as np - >>> q = u.Quantity(np.arange(10.), u.m) - >>> np.dot(q,q) - 285.0 - >>> np.hstack((q,q)) - + >>> a = np.ones(4) + >>> a[2:3] = 2*u.kg + >>> a # doctest: +FLOAT_CMP + array([1., 1., 2., 1.]) -Also in-place operations where the output is a normal `~numpy.ndarray` -will drop the unit silently (at least in numpy <= 1.9):: +:: - >>> a = np.arange(10.) - >>> a *= 1.*u.kg - >>> a - array([ 0., 1., 2., 3., 4., 5., 6., 7., 8., 9.]) + >>> a = np.ones(4) + >>> a[2:3] = 1*u.cm/u.m + >>> a # doctest: +FLOAT_CMP + array([1., 1., 1., 1.]) -Work-arounds are available for some cases. For the above:: +Either set single array entries or use lists of Quantities:: - >>> q.dot(q) - + >>> a = np.ones(4) + >>> a[2] = 1*u.cm/u.m + >>> a # doctest: +FLOAT_CMP + array([1. , 1. , 0.01, 1. ]) - >>> u.Quantity([q, q]).flatten() - +:: -An incomplete list of specific functions which are known to exhibit this behavior follows. + >>> a = np.ones(4) + >>> a[2:3] = [1*u.cm/u.m] + >>> a # doctest: +FLOAT_CMP + array([1. , 1. , 0.01, 1. ]) -* `numpy.dot` -* `numpy.hstack`, `numpy.vstack`, ``numpy.c_``, ``numpy.r_``, `numpy.append` -* `numpy.where` -* `numpy.choose` -* `numpy.vectorize` -* pandas DataFrame(s) +Both will throw an exception if units do not cancel, e.g.:: + >>> a = np.ones(4) + >>> a[2] = 1*u.cm + Traceback (most recent call last): + ... + TypeError: only dimensionless scalar quantities can be converted to Python scalars -See: https://github.com/astropy/astropy/issues/1274 +See: https://github.com/astropy/astropy/issues/7582 -Quantities float comparison with np.isclose fails -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Multiplying a `pandas.Series` with an `~astropy.units.Unit` does not produce a |Quantity| +----------------------------------------------------------------------------------------- -Comparing Quantities floats using the numpy function `~numpy.isclose` fails on -numpy 1.9 as the comparison between ``a`` and ``b`` is made using the formula +Quantities may work with certain operations on `~pandas.Series` but +this behaviour is not tested. +For example, multiplying a `~pandas.Series` instance +with a unit will *not* return a |Quantity|. It will return a `~pandas.Series` +object without any unit: -.. math:: +.. doctest-requires:: pandas>=2.0 - |a - b| \le (a_\textrm{tol} + r_\textrm{tol} \times |b|) + >>> import pandas as pd + >>> import astropy.units as u + >>> a = pd.Series([1., 2., 3.]) + >>> a * u.m + 0 1.0 + 1 2.0 + 2 3.0 + dtype: float64 -This will result in the following traceback when using this with Quantities:: +To avoid this, it is best to initialize the |Quantity| directly: - >>> from astropy import units as u, constants as const - >>> import numpy as np - >>> np.isclose(500* u.km/u.s, 300 * u.km / u.s) - UnitsError: Can only apply 'add' function to dimensionless quantities when - other argument is not a quantity (unless the latter is all zero/infinity/nan) +.. doctest-requires:: pandas>=2.0 -An easy solution is:: + >>> u.Quantity(a, u.m) + - >>> np.isclose(500* u.km/u.s, 300 * u.km / u.s, atol=1e-8 * u.mm / u.s) - array([False], dtype=bool) +Note that the overrides pandas provides are not complete, and +as a consequence, using the (in-place) shift operator does work: +.. doctest-requires:: pandas>=2.0 -Table sorting can silently fail on MacOS X or Windows with Python 3 and Numpy < 1.6.2 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + >>> b = a << u.m + >>> b + + >>> a <<= u.m + >>> a + -In Python 3, prior to Numpy 1.6.2, there was a bug (in Numpy) that caused -sorting of structured arrays to silently fail under certain circumstances (for -example if the Table contains string columns) on MacOS X, Windows, and possibly -other platforms other than Linux. Since ``Table.sort`` relies on Numpy to -internally sort the data, it is also affected by this bug. If you are using -Python 3, and need the sorting functionality for tables, we recommend updating -to a more recent version of Numpy. +But this is fragile as this may stop working in future versions of +pandas if they decide to override the dunder methods. +See: https://github.com/astropy/astropy/issues/11247 -Remote data utilities in `astropy.utils.data` fail on some Python distributions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Using Numpy array creation functions to initialize Quantity +----------------------------------------------------------- +Trying the following example will ignore the unit: -The remote data utilities in `astropy.utils.data` depend on the Python -standard library `shelve` module, which in some cases depends on the -standard library `bsddb` module. Some Python distributions, including but -not limited to + >>> np.full(10, 1 * u.m) + array([1., 1., 1., 1., 1., 1., 1., 1., 1., 1.]) -* OS X, Python 2.7.5 via homebrew -* Linux, Python 2.7.6 via conda [#]_ -* Linux, Python 2.6.9 via conda +However, the following works as one would expect -are built without support for the ``bsddb`` module, resulting in an error -such as:: + >>> np.full(10, 1.0, like=u.Quantity([], u.m)) + - ImportError: No module named _bsddb +and is equivalent to:: -One workaround is to install the ``bsddb3`` module. + >>> np.full(10, 1) << u.m + +`~numpy.zeros`, `~numpy.ones`, and `~numpy.empty` behave similarly. -mmap support for ``astropy.io.fits`` on GNU Hurd -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +`~numpy.arange` also supports the ``like`` keyword argument -On Hurd and possibly other platforms ``flush()`` on memory-mapped files is not -implemented, so writing changes to a mmap'd FITS file may not be reliable and is -thus disabled. Attempting to open a FITS file in writeable mode with mmap will -result in a warning (and mmap will be disabled on the file automatically). + >>> np.arange(0 * u.cm, 1 * u.cm, 1 * u.mm, like=u.Quantity([], u.cm)) + -See: https://github.com/astropy/astropy/issues/968 +Also note that the unit of the output array is dictated by that of the ``stop`` +argument, and that, like for quantities generally, the data has a floating-point +dtype. If ``stop`` is a pure number, the unit of the output will default to that +of the ``like`` argument. + +As with ``~numpy.full`` and similar functions, one may alternatively move the +units outside of the call to `~numpy.arange`:: + + >>> np.arange(0, 10, 1) << u.mm + +Or use `~numpy.linspace`: -Bug with unicode endianness in ``io.fits`` for big-endian processors -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + >>> np.linspace(0 * u.cm, 9 * u.mm, 10) + -On big-endian processors (e.g. SPARC, PowerPC, MIPS), string columnn in FITS -files may not be correctly read when using the ``Table.read`` interface. This -will be fixed in a subsequent bug fix release of Astropy (see `bug report here -`_) +Quantities Lose Their Units When Broadcasted +-------------------------------------------- -Error *'buffer' does not have the buffer interface* in ``io.fits`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +When broadcasting Quantities, it is necessary to pass ``subok=True`` to +`~numpy.broadcast_to`, or else a bare `~numpy.ndarray` will be returned:: -For Python 2.7.x versions prior to 2.7.4, the `astropy.io.fits` may under -certain circumstances output the following error:: + >>> q = u.Quantity(np.arange(10.), u.m) + >>> b = np.broadcast_to(q, (2, len(q))) + >>> b # doctest: +FLOAT_CMP + array([[0., 1., 2., 3., 4., 5., 6., 7., 8., 9.], + [0., 1., 2., 3., 4., 5., 6., 7., 8., 9.]]) + >>> b2 = np.broadcast_to(q, (2, len(q)), subok=True) + >>> b2 # doctest: +FLOAT_CMP + - TypeError: 'buffer' does not have the buffer interface +This is analogous to the case of passing a Quantity to `~numpy.array`:: -This can be resolved by upgrading to Python 2.7.4 or later (at the time of -writing, the latest Python 2.7.x version is 2.7.9). + >>> a = np.array(q) + >>> a # doctest: +FLOAT_CMP + array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.]) + >>> a2 = np.array(q, subok=True) + >>> a2 # doctest: +FLOAT_CMP + +See: https://github.com/astropy/astropy/issues/7832 -Floating point precision issues on Python 2.6 on Microsoft Windows -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Chained Quantity comparisons to dimensionless zero can be misleading +-------------------------------------------------------------------- -When converting floating point numbers to strings on Python 2.6 on a -Microsoft Windows platform, some of the requested precision may be -lost. +When chaining comparisons using Quantities and dimensionless zero, +the result may be misleading:: -The easiest workaround is to install Python 2.7. + >>> 0 * u.Celsius == 0 * u.m # Correct + False + >>> 0 * u.Celsius == 0 == 0 * u.m # Misleading + np.True_ -The Python issue: http://bugs.python.org/issue7117 +What the second comparison is really doing is this:: + >>> (0 * u.Celsius == 0) and (0 == 0 * u.m) + np.True_ -Color printing on Windows -^^^^^^^^^^^^^^^^^^^^^^^^^ +See: https://github.com/astropy/astropy/issues/15103 -Colored printing of log messages and other colored text does work in Windows -but only when running in the IPython console. Colors are not currently +numpy.prod cannot be applied to Quantity +---------------------------------------- + +Using ``numpy.prod`` function on a Quantity would result in error. +This is because correctly implementing it for Quantity is fairly +difficult, since, unlike for most numpy functions, the result unit +depends on the shape of the input (rather than only on the units +of the inputs). + + >>> np.prod([1, 2, 3] * u.m) + Traceback (most recent call last): + ... + astropy.units.errors.UnitsError: Cannot use 'reduce' method on ufunc multiply with a Quantity instance as it would change the unit. + +See: https://github.com/astropy/astropy/issues/18429 + +def_unit should not be used for logarithmic unit +------------------------------------------------ + +When defining custom unit involving logarithmic unit, ``def_unit`` usage +should be avoided because it might result in surprising behavior:: + + >>> dBW = u.def_unit('dBW', u.dB(u.W)) + >>> 1 * dBW + Traceback (most recent call last): + ... + TypeError: unsupported operand type(s) for *: 'int' and 'Unit' + +Instead, it could be defined directly as such:: + + >>> dBW = u.dB(u.W) + >>> 1 * dBW + + +See: https://github.com/astropy/astropy/issues/5945 + +mmap Support for ``astropy.io.fits`` on GNU Hurd +------------------------------------------------ + +On Hurd and possibly other platforms, ``flush()`` on memory-mapped files are not +implemented, so writing changes to a mmap'd FITS file may not be reliable and is +thus disabled. Attempting to open a FITS file in writeable mode with mmap will +result in a warning (and mmap will be disabled on the file automatically). + +See: https://github.com/astropy/astropy/issues/968 + + +Color Printing on Windows +------------------------- + +Colored printing of log messages and other colored text does work in Windows, +but only when running in the IPython console. Colors are not currently supported in the basic Python command-line interpreter on Windows. +``numpy.int64`` does not decompose input ``Quantity`` objects +------------------------------------------------------------- +Python's ``int()`` goes through ``__index__`` +while ``numpy.int64`` or ``numpy.int_`` do not go through ``__index__``. This +means that an upstream fix in NumPy is required in order for +``astropy.units`` to control decomposing the input in these functions:: -Build/installation/test issues ------------------------------- + >>> np.int64((15 * u.km) / (15 * u.imperial.foot)) + np.int64(1) + >>> np.int_((15 * u.km) / (15 * u.imperial.foot)) + np.int64(1) + >>> int((15 * u.km) / (15 * u.imperial.foot)) + 3280 -Anaconda users should upgrade with ``conda``, not ``pip`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +To convert a dimensionless `~astropy.units.Quantity` to an integer, it is +therefore recommended to use ``int(...)``. -Upgrading Astropy in the anaconda python distribution using ``pip`` can result +Build/Installation/Test Issues +============================== + +Anaconda Users Should Upgrade with ``conda``, Not ``pip`` +--------------------------------------------------------- + +Upgrading ``astropy`` in the Anaconda Python distribution using ``pip`` can result in a corrupted install with a mix of files from the old version and the new version. Anaconda users should update with ``conda update astropy``. There -may be a brief delay between the release of Astropy on PyPI and its release +may be a brief delay between the release of ``astropy`` on PyPI and its release via the ``conda`` package manager; users can check the availability of new versions with ``conda search astropy``. -Locale errors in MacOS X and Linux -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Locale Errors in MacOS X and Linux +---------------------------------- -On MacOS X, you may see the following error when running ``setup.py``:: +On MacOS X, you may see the following error when running ``pip``:: - ... + ... ValueError: unknown locale: UTF-8 This is due to the ``LC_CTYPE`` environment variable being incorrectly set to @@ -212,7 +305,7 @@ This is due to the ``LC_CTYPE`` environment variable being incorrectly set to On MacOS X or Linux (or other platforms) you may also encounter the following error:: - ... + ... stderr = stderr.decode(stdio_encoding) TypeError: decode() argument 1 must be str, not None @@ -242,117 +335,5 @@ see something like:: LC_TIME="en_US.UTF-8" LC_ALL="en_US.UTF-8" -If so, you can go ahead and try running ``setup.py`` again (in the new +If so, you can go ahead and try running ``pip`` again (in the new terminal). - - -Creating a Time object fails with ValueError after upgrading Astropy -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In some cases, have users have upgraded Astropy from an older version to v1.0 -or greater they have run into the following crash when trying to create a -`~astropy.time.Time` object:: - - >>> datetime = Time('2012-03-01T13:08:00', scale='utc') - Traceback (most recent call last): - ... - ValueError: Input values did not match any of the formats where - the format keyword is optional [u'astropy_time', u'datetime', - u'jyear_str', u'iso', u'isot', u'yday', u'byear_str'] - -This problem can occur when there is a version mismatch between the compiled -ERFA library (this is included as part of Astropy in most distributions), and -the version of the Astropy Python source. - -This can have a number of causes. The most likely is that when installing the -new Astropy version, your previous Astropy version was not fully uninstalled -first, resulting in a mishmash of versions. Your best bet is to fully remove -Astropy from its installation path, and reinstall from scratch using your -preferred installation method. How to remove the old version may be a simple -matter if removing the entire ``astropy/`` directory from within the -``site-packages`` directory it is installed in. However, if in doubt, ask -how best to uninstall packages from your preferred Python distribution. - -Another possible cause of this, in particular for people developing on Astropy -and installing from a source checkout, is simply that your Astropy build -directory is unclean. To fix this, run ``git clean -dfx``. This removes -*all* build artifacts from the repository that aren't normally tracked by git. -Make sure before running this that there are no untracked files in the -repository you intend to save. Then rebuild/reinstall from the clean repo. - - -Failing logging tests when running the tests in IPython -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When running the Astropy tests using ``astropy.test()`` in an IPython -interpreter some of the tests in the ``astropy/tests/test_logger.py`` *might* -fail, depending on the version of IPython or other factors. -This is due to mutually incompatible behaviors in IPython and py.test, and is -not due to a problem with the test itself or the feature being tested. - -See: https://github.com/astropy/astropy/issues/717 - - -Some docstrings can not be displayed in IPython < 0.13.2 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Displaying long docstrings that contain Unicode characters may fail on -some platforms in the IPython console (prior to IPython version -0.13.2):: - - >>> import astropy.units as u - - >>> u.Angstrom? - ERROR: UnicodeEncodeError: 'ascii' codec can't encode character u'\xe5' in - position 184: ordinal not in range(128) [IPython.core.page] - -This can be worked around by changing the default encoding to ``utf-8`` -by adding the following to your ``sitecustomize.py`` file:: - - import sys - sys.setdefaultencoding('utf-8') - -Note that in general, `this is not recommended -`_, -because it can hide other Unicode encoding bugs in your application. -However, in general if your application does not deal with text -processing and you just want docstrings to work, this may be -acceptable. - -The IPython issue: https://github.com/ipython/ipython/pull/2738 - - -Installation fails on Mageia-2 or Mageia-3 distributions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Building may fail with warning messages such as:: - - unable to find 'pow' or 'sincos' - -at the linking phase. Upgrading the OS packages for Python should -fix the issue, though an immediate workaround is to edit the file:: - - /usr/lib/python2.7/config/Makefile - -and search for the line that adds the option ``-Wl,--no-undefined`` to the -``LDFLAGS`` variable and remove that option. - - -Crash on upgrading from Astropy 0.2 to a newer version -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It is possible for installation of a new version of Astropy, or upgrading of an -existing installation to crash due to not having permissions on the -``~/.astropy/`` directory (in your home directory) or some file or subdirectory -in that directory. In particular this can occur if you installed Astropy as -the root user (such as with ``sudo``) at any point. This can manifest in -several ways, but the most common is a traceback ending with ``ImportError: -cannot import name config``. To resolve this issue either run ``sudo chown -R - ~/.astropy`` or, if you don't need anything in it you can blow -it away with ``sudo rm -rf ~/.astropy``. - -See for example: https://github.com/astropy/astropy/issues/987 - -.. [#] Continuum `says - `_ - this will be fixed in their next Python build. diff --git a/docs/license.rst b/docs/license.rst index a7982682f354..2e9c2f58b55e 100644 --- a/docs/license.rst +++ b/docs/license.rst @@ -7,7 +7,7 @@ Astropy License Astropy is licensed under a 3-clause BSD style license: -.. include:: ../licenses/LICENSE.rst +.. include:: ../LICENSE.rst Other Licenses ============== diff --git a/docs/logging.rst b/docs/logging.rst index a0a4a649b12d..1c782c95ae01 100644 --- a/docs/logging.rst +++ b/docs/logging.rst @@ -2,6 +2,11 @@ Logging system ************** +.. note:: + + The Astropy logging system is meant for internal ``astropy`` usage. For use + in other packages, we recommend implementing your own logger instead. + Overview ======== @@ -24,7 +29,7 @@ levels: * ERROR: indicates a more serious issue, including exceptions -By default, only WARNING and ERROR messages are displayed, and are sent to a +By default, INFO, WARNING and ERROR messages are displayed, and are sent to a log file located at ``~/.astropy/astropy.log`` (if the file is writeable). Configuring the logging system @@ -36,11 +41,15 @@ First, import the logger:: The threshold level (defined above) for messages can be set with e.g.:: - log.setLevel('INFO') + log.setLevel('DEBUG') Color (enabled by default) can be disabled with:: - log.setColor(False) + log.disable_color() + +and enabled with:: + + log.enable_color() Warnings from ``warnings.warn`` can be logged with:: @@ -52,7 +61,7 @@ which can be disabled with:: and exceptions can be included in the log with:: - log.set_exception_logging() + log.enable_exception_logging() which can be disabled with:: @@ -111,10 +120,10 @@ emitted in the ``astropy.wcs`` sub-package. Using the configuration file ============================ -Options for the logger can be set in the ``[config.logging_helper]`` section +Options for the logger can be set in the ``[logger]`` section of the Astropy configuration file:: - [config.logging_helper] + [logger] # Threshold for the logging messages. Logging messages that are less severe # than this level will be ignored. The levels are 'DEBUG', 'INFO', 'WARNING', @@ -133,7 +142,8 @@ of the Astropy configuration file:: # Whether to always log messages to a log file log_to_file = True - # The file to log messages to + # The file to log messages to. If empty string is given, it defaults to a + # file `astropy.log` in the astropy config directory. log_file_path = '~/.astropy/astropy.log' # Threshold for logging messages to log_file_path @@ -142,10 +152,13 @@ of the Astropy configuration file:: # Format for log file entries log_file_format = '%(asctime)s, %(origin)s, %(levelname)s, %(message)s' + # The encoding (e.g., UTF-8) to use for the log file. If empty string is + # given, it defaults to the platform-preferred encoding. + log_file_encoding = "" + Reference/API ============= .. automodapi:: astropy.logger :no-inheritance-diagram: - diff --git a/docs/lts_policy.rst b/docs/lts_policy.rst new file mode 100644 index 000000000000..45b6803fb1e9 --- /dev/null +++ b/docs/lts_policy.rst @@ -0,0 +1,7 @@ +******************* +LTS Backport Policy +******************* + +Starting with astropy v6.0.0, there will be no more designated Long-Term Stable +(LTS) releases of astropy - see `APE 21 +`_ for more details. diff --git a/docs/make.bat b/docs/make.bat index 93dfe92b9c98..aa930134b5de 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -37,6 +37,8 @@ if "%1" == "help" ( if "%1" == "clean" ( for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i del /q /s %BUILDDIR%\* + del /q /s api + del /q /s generated goto end ) diff --git a/docs/modeling/add-units.rst b/docs/modeling/add-units.rst new file mode 100644 index 000000000000..9183e74405a8 --- /dev/null +++ b/docs/modeling/add-units.rst @@ -0,0 +1,124 @@ +.. _add_units: + +Adding support for units in a model (Advanced) +============================================== + +Evaluation +---------- + +To make it so that your models can accept parameters with units and be evaluated +using inputs with units, you need to make sure that the +:meth:`~astropy.modeling.Model.evaluate` method works correctly with +input values and parameters with units. For simple arithmetic, this may work +out of the box since :class:`~astropy.units.Quantity` objects are understood by +a number of Numpy functions. + +If users of your models provide input during evaluation that is not compatible +with the parameter units, they may get cryptic errors such as:: + + UnitsError : Can only apply 'subtract' function to dimensionless quantities + when other argument is not a quantity (unless the latter is all + zero/infinity/nan) + +There are several attributes or properties that can be set on models that adjust +the behavior of models with units. These attributes can be changed from the +defaults in the class definition, e.g.:: + + class MyModel(Model): + input_units = {'x': u.deg} + ... + +Note that these are all optional. + +.. _models_input_units: + +``input_units`` +^^^^^^^^^^^^^^^ + +You can easily add checking of the input units by adding an ``input_units`` +property or attribute on your model class. This should return either `None` (to +indicate no constraints) or a dictionary where the keys are the input names +(e.g. ``x`` for many 1D models) and the values are the units expected, which can +be a function of the parameter units:: + + @property + def input_units(self): + if self.mean.unit is None: + return None + else: + return {'x': self.mean.unit} + +If the user then gives values with incorrect input units, a clear error will be +displayed:: + + UnitsError: Units of input 'x', (dimensionless), could not be converted to + required input units of m (length) + +Note that the input units don't have to match exactly those returned by +``input_units``, but be convertible to them. In addition, ``input_units`` can +also be specified as an attribute rather than a property in simple cases:: + + input_units = {'x': u.deg} + +.. _models_return_units: + +``return_units`` +^^^^^^^^^^^^^^^^ + +Similarly to :ref:`models_input_units`, this should be dictionary that maps the return +values of a model to units. If :meth:`~astropy.modeling.Model.evaluate` was called +with quantities but returns unitless values, the units are added to the output. +If the return values are quantities in different units, they are converted to +``return_units``. + +``input_units_strict`` +^^^^^^^^^^^^^^^^^^^^^^ + +If set to `True`, values that are passed in compatible units will be converted +to the exact units specified in ``input_units``. + +This attribute can also be a +dictionary that maps input names to a Boolean to enable converting of that input +to the specified unit. + +``input_units_equivalencies`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This can be set to a dictionary that maps the input names to a list of +equivalencies, for example:: + + input_units_equivalencies = {'nu': u.spectral()} + +``_input_units_allow_dimensionless`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If set to `True`, values that are plain scalars or Numpy arrays can be passed to +evaluate even if ``input_units`` specifies that the input should have units. It +is up to the :meth:`~astropy.modeling.Model.evaluate` to then decide how to +handle these dimensionless values. This attribute can also be a dictionary that +maps input names to a Boolean to enable passing dimensionless values to +:meth:`~astropy.modeling.Model.evaluate` for that input. + + +Fitting +------- + +To allow models with parameters that have units to be fitted to data with units, +you will need to add a method called ``_parameter_units_for_data_units`` to your +model class. This should take two arguments ``input_units`` and +``output_units`` - ``input_units`` will be set to a dictionary with +the units of the independent variables in the data, while ``output_units`` will +be set to a dictionary with the units the dependent variables in the data (for +example, for a simple 1D model, ``input_units`` will have one key, ``x``, and +``output_units`` will have one key, ``y``). This method should then return +a dictionary giving for each parameter the units the parameter should be +converted to so that the model could be used on the data if units were removed +from both the models and the data. The following example shows the +implementation for the 1D Gaussian:: + + def _parameter_units_for_data_units(self, inputs_unit, outputs_unit): + return {'mean': inputs_unit['x'], + 'stddev': inputs_unit['x'], + 'amplitude': outputs_unit['y']} + +With this method in place, the model can then be fit to data that has units. diff --git a/docs/modeling/algorithms.rst b/docs/modeling/algorithms.rst deleted file mode 100644 index 5d2dff792905..000000000000 --- a/docs/modeling/algorithms.rst +++ /dev/null @@ -1,61 +0,0 @@ -********** -Algorithms -********** - -Univariate polynomial evaluation -================================ - -* The evaluation of 1-D polynomials uses Horner's algorithm. - -* The evaluation of 1-D Chebyshev and Legendre polynomials uses Clenshaw's - algorithm. - - -Multivariate polynomial evaluation -================================== - -* Multivariate Polynomials are evaluated following the algorithm in [1]_ . The - algorithm uses the following notation: - - - **multiindex** is a tuple of non-negative integers for which the length is - defined in the following way: - - .. math:: \alpha = (\alpha1, \alpha2, \alpha3), |\alpha| = \alpha1+\alpha2+\alpha3 - - - - **inverse lexical order** is the ordering of monomials in such a way that - :math:`{x^a < x^b}` if and only if there exists :math:`{1 \le i \le n}` - such that :math:`{a_n = b_n, \dots, a_{i+1} = b_{i+1}, a_i < b_i}`. - - In this ordering :math:`y^2 > x^2*y` and :math:`x*y > y` - - - **Multivariate Horner scheme** uses d+1 variables :math:`r_0, ...,r_d` to - store intermediate results, where *d* denotes the number of variables. - - Algorithm: - - 1. Set *di* to the max number of variables (2 for a 2-D polynomials). - - 2. Set :math:`r_0` to :math:`c_{\alpha(0)}`, where c is a list of - coefficients for each multiindex in inverse lexical order. - - 3. For each monomial, n, in the polynomial: - - - determine :math:`k = max \{1 \leq j \leq di: \alpha(n)_j \neq \alpha(n-1)_j\}` - - - Set :math:`r_k := l_k(x)* (r_0 + r_1 + \dots + r_k)` - - - Set :math:`r_0 = c_{\alpha(n)}, r_1 = \dots r_{k-1} = 0.` - - 4. return :math:`r_0 + \dots + r_{di}` - -* The evaluation of multivariate Chebyshev and Legendre polynomials uses a - variation of the above Horner's scheme, in which every Legendre or Chebyshev - function is considered a separate variable. In this case the length of the - :math:`\alpha` indices tuple is equal to the number of functions in x plus - the number of functions in y. In addition the Chebyshev and Legendre - functions are cached for efficiency. - - - -.. [1] J. M. Pena, Thomas Sauer, "On the Multivariate Horner Scheme", SIAM Journal on Numerical Analysis, Vol 37, No. 4 diff --git a/docs/modeling/compound-models.rst b/docs/modeling/compound-models.rst index fe6f55559ca8..55574d7fe45b 100644 --- a/docs/modeling/compound-models.rst +++ b/docs/modeling/compound-models.rst @@ -1,20 +1,129 @@ -.. _compound-models: +.. include:: links.inc -Compound Models -=============== +.. _compound-models-intro: -.. versionadded:: 1.0 +Combining Models +**************** -As noted in the :ref:`introduction to the modeling package -`, it is now possible to create new models just by -combining existing models using the arithmetic operators ``+``, ``-``, ``*``, -``/``, and ``**``, as well as by model composition using ``|`` and -concatenation (explained below) with ``&``. +Basics +====== + +While the Astropy modeling package makes it very easy to define :doc:`new +models ` either from existing functions, or by writing a +`~astropy.modeling.Model` subclass, an additional way to create new models is +by combining them using arithmetic expressions. This works with models built +into Astropy, and most user-defined models as well. For example, it is +possible to create a superposition of two Gaussians like so:: + + >>> from astropy.modeling import models + >>> g1 = models.Gaussian1D(1, 0, 0.2) + >>> g2 = models.Gaussian1D(2.5, 0.5, 0.1) + >>> g1_plus_2 = g1 + g2 + +The resulting object ``g1_plus_2`` is itself a new model. + +.. note:: + The model ``g1_plus_2`` is a `~astropy.modeling.CompoundModel` which contains + the models ``g1`` and ``g2`` without any parameter duplication. Meaning changes + to the parameters of ``g1_plus_2`` will affect the parameters of ``g1`` or ``g2`` + and vice versa; if one does not want this to occur one can copy the models prior + to adding them using the ``.copy()`` method ``g1.copy() + g2.copy()``. In + general this applies to any `~astropy.modeling.CompoundModel` constructed using a + binary operation, so that `~astropy.modeling.CompoundModel` follows the Python + convention for construction of container objects. For more information on this + please see the `API Changes in astropy.modeling `__ + +Evaluating, say, ``g1_plus_2(0.25)`` is the same as evaluating ``g1(0.25) + g2(0.25)``:: + + >>> g1_plus_2(0.25) # doctest: +FLOAT_CMP + 0.5676756958301329 + >>> g1_plus_2(0.25) == g1(0.25) + g2(0.25) + True + +This model can be further combined with other models in new expressions. + +These new compound models can also be fitted to data, like most other models +(though this currently requires one of the non-linear fitters): + +.. plot:: + :include-source: + import warnings + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling import models, fitting + + # Generate fake data + rng = np.random.default_rng(seed=42) + g1 = models.Gaussian1D(1, 0, 0.2) + g2 = models.Gaussian1D(2.5, 0.5, 0.1) + x = np.linspace(-1, 1, 200) + y = g1(x) + g2(x) + rng.normal(0., 0.2, x.shape) + + # Now to fit the data create a new superposition with initial + # guesses for the parameters: + gg_init = models.Gaussian1D(1, 0, 0.1) + models.Gaussian1D(2, 0.5, 0.1) + fitter = fitting.SLSQPLSQFitter() + + with warnings.catch_warnings(): + # Ignore a warning on clipping to bounds from the fitter + warnings.filterwarnings('ignore', message='Values in x were outside bounds', + category=RuntimeWarning) + gg_fit = fitter(gg_init, x, y) + + # Plot the data with the best-fit model + fig, ax = plt.subplots(figsize=(8, 5)) + ax.plot(x, y, 'ko') + ax.plot(x, gg_fit(x)) + ax.set(xlabel='Position', ylabel='Flux') + +This works for 1-D models, 2-D models, and combinations thereof, though there +are some complexities involved in correctly matching up the inputs and outputs +of all models used to build a compound model. You can learn more details in +the :doc:`compound-models` documentation. + +Astropy models also support convolution through the function +`~astropy.convolution.convolve_models`, which returns a compound model. + +For instance, the convolution of two Gaussian functions is also a Gaussian +function in which the resulting mean position is the average of the +mean positions and the variance is the sum of the variances. + +.. plot:: + :include-source: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling import models + from astropy.convolution import convolve_models + + g1 = models.Gaussian1D(1, -1, 1) + g2 = models.Gaussian1D(1, 1, 1) + g3 = convolve_models(g1, g2) + + x = np.linspace(-3, 3, 50) + fig, ax = plt.subplots(figsize=(8, 5)) + ax.plot(x, g1(x), label='g1') + ax.plot(x, g2(x), label='g2') + ax.plot(x, g3(x), label='g3 (Convolution)') + ax.legend() + + +.. _compound-models: + +A comprehensive description +=========================== Some terminology ---------------- +It is possible to create new models just by +combining existing models using the arithmetic operators ``+``, ``-``, ``*``, +``/``, and ``**``, or by model composition using ``|`` and +concatenation (explained below) with ``&``, as well as using :func:`~astropy.modeling.fix_inputs` +for :ref:`reducing the number of inputs to a model `. + + In discussing the compound model feature, it is useful to be clear about a few terms where there have been points of confusion: @@ -23,14 +132,14 @@ few terms where there have been points of confusion: - All models in `astropy.modeling`, whether it represents some `function `, a `rotation `, etc., are represented in the - abstract by a model *class*--specifically a subclass of - `~astropy.modeling.Model`--that encapsulates the routine for evaluating the - model, a list of its required parameters, and other metadata about the - model. + abstract by a model *class* --- specifically a subclass of + `~astropy.modeling.Model` --- that encapsulates the routine for + evaluating the model, a list of its required parameters, and other + metadata about the model. - Per typical object-oriented parlance, a model *instance* is the object - created when when calling a model class with some arguments--in most cases - values for the model's parameters. + created when calling a model class with some arguments --- in most + cases values for the model's parameters. A model class, by itself, cannot be used to perform any computation because most models, at least, have one or more parameters that must be specified @@ -42,8 +151,8 @@ few terms where there have been points of confusion: >>> Gaussian1D Name: Gaussian1D - Inputs: ('x',) - Outputs: ('y',) + N_inputs: 1 + N_outputs: 1 Fittable parameters: ('amplitude', 'mean', 'stddev') We can then create a model *instance* by passing in values for the three @@ -65,16 +174,13 @@ few terms where there have been points of confusion: distinction is either irrelevant or clear from context. But a distinction will be made where necessary. -- A *compound model* can be created by combining two or more existing models-- - be they model *instances* or *classes*, and can be models that come with - Astropy, :doc:`user defined models `, or other compound models--using - Python expressions consisting of one or more of the supported binary - operators. +- A *compound model* can be created by combining two or more existing model instances + which can be models that come with Astropy, :doc:`user defined models `, or + other compound models --- using Python expressions consisting of one or more of the + supported binary operators. - In some places the term *composite model* is used interchangeably with - *compound model*. This can be seen in the cases of the now deprecated - `~astropy.modeling.SerialCompositeModel` and - `~astropy.modeling.SummedCompositeModel`. However, this document uses the + *compound model*. However, this document uses the term *composite model* to refer *only* to the case of a compound model created from the functional composition of two or more models using the pipe operator ``|`` as explained below. This distinction is used consistently @@ -84,252 +190,80 @@ few terms where there have been points of confusion: Creating compound models ------------------------ -As discussed in the :ref:`introduction to compound models -`, the only way, currently, to create compound models is +The only way to create compound models is to combine existing single models and/or compound models using expressions in Python with the binary operators ``+``, ``-``, ``*``, ``/``, ``**``, ``|``, -and ``&``, each of which is discussed in the following sections. The operands -used in these expressions may be model *classes*, or model *instances*. In -other words, any object for which either ``isinstance(obj, Model)`` or -``issubclass(obj, Model)`` is `True`. +and ``&``, each of which is discussed in the following sections. +The result of combining two models is a model instance:: -.. _compound-model-classes: + >>> two_gaussians = Gaussian1D(1.1, 0.1, 0.2) + Gaussian1D(2.5, 0.5, 0.1) + >>> two_gaussians # doctest: +FLOAT_CMP + -Compound model classes -^^^^^^^^^^^^^^^^^^^^^^ +This expression creates a new model instance that is ready to be used for evaluation:: -We start by demonstrating how new compound model *classes* can be created -by combining other classes. This is more advanced usage, but it's useful to -understand that this is what's going on under the hood in the more basic usage -of :ref:`compound model instances `. + >>> two_gaussians(0.2) # doctest: +FLOAT_CMP + 0.9985190841886609 -When all models involved in the expression are classes, the result of the -expression is, itself, a class (remember, classes in Python are themselves also -objects just like strings and integers or model instances):: - - >>> TwoGaussians = Gaussian1D + Gaussian1D - >>> from astropy.modeling import Model - >>> isinstance(TwoGaussians, Model) - False - >>> issubclass(TwoGaussians, Model) - True +The ``print`` function provides more information about this object:: -When we inspect the variable ``TwoGaussians`` by printing its representation at -the command prompt we can get some more information about it:: - - >>> TwoGaussians - - Name: CompoundModel... + >>> print(two_gaussians) + Model: CompoundModel... Inputs: ('x',) Outputs: ('y',) - Fittable parameters: ('amplitude_0', 'mean_0', 'stddev_0', 'amplitude_1', 'mean_1', 'stddev_1') + Model set size: 1 Expression: [0] + [1] Components: - [0]: - Name: Gaussian1D - Inputs: ('x',) - Outputs: ('y',) - Fittable parameters: ('amplitude', 'mean', 'stddev') + [0]: - [1]: - Name: Gaussian1D - Inputs: ('x',) - Outputs: ('y',) - Fittable parameters: ('amplitude', 'mean', 'stddev') - -There are a number of things to point out here: This model class has six -fittable parameters. How parameters are handled is discussed further in the + [1]: + Parameters: + amplitude_0 mean_0 stddev_0 amplitude_1 mean_1 stddev_1 + ----------- ------ -------- ----------- ------ -------- + 1.1 0.1 0.2 2.5 0.5 0.1 + +There are a number of things to point out here: This model has six +fittable parameters. How parameters are handled is discussed further in the section on :ref:`compound-model-parameters`. We also see that there is a listing of the *expression* that was used to create this compound model, which in this case is summarized as ``[0] + [1]``. The ``[0]`` and ``[1]`` refer to the first and second components of the model listed next (in this case both -components are the `~astropy.modeling.functional_models.Gaussian1D` class). +components are the `~astropy.modeling.functional_models.Gaussian1D` objects). Each component of a compound model is a single, non-compound model. This is the case even when including an existing compound model in a new expression. -The existing compound model is not treated as a single model--instead the +The existing compound model is not treated as a single model --- instead the expression represented by that compound model is extended. An expression involving two or more compound models results in a new expression that is the concatenation of all involved models' expressions:: - >>> FourGaussians = TwoGaussians + TwoGaussians - >>> FourGaussians - - Name: CompoundModel... + >>> four_gaussians = two_gaussians + two_gaussians + >>> print(four_gaussians) + Model: CompoundModel... Inputs: ('x',) Outputs: ('y',) - Fittable parameters: ('amplitude_0', 'mean_0', 'stddev_0', ..., 'amplitude_3', 'mean_3', 'stddev_3') + Model set size: 1 Expression: [0] + [1] + [2] + [3] Components: - [0]: - Name: Gaussian1D - Inputs: ('x',) - Outputs: ('y',) - Fittable parameters: ('amplitude', 'mean', 'stddev') - ... - [3]: - Name: Gaussian1D - Inputs: ('x',) - Outputs: ('y',) - Fittable parameters: ('amplitude', 'mean', 'stddev') - -In a future version it may be possible to "freeze" a compound model, so that -from the user's perspective it is treated as a single model. However, as this -is the default behavior it is good to be aware of. - - -Model names -^^^^^^^^^^^ - -In the last two examples another notable feature of the generated compound -model classes is that the class name, as displayed when printing the class at -the command prompt, is not "TwoGaussians", "FourGaussians", etc. Instead it is -a generated name consisting of "CompoundModel" followed by an essentially -arbitrary integer that is chosen simply so that every compound model has a -unique default name. This is a limitation at present, due to the limitation -that it is not generally possible in Python when an object is created by an -expression for it to "know" the name of the variable it will be assigned to, if -any. It may be possible in the future to work around this in limited cases, -but for now there are a couple workarounds for creating compound model classes -with friendlier names. The first is to use the -`Model.rename ` class method on the result of -the model expression:: - - >>> TwoGaussians = (Gaussian1D + Gaussian1D).rename('TwoGaussians') - >>> TwoGaussians - - Name: TwoGaussians (CompoundModel...) - ... - -This actually takes the generated compound model and creates a light subclass -of it with the desired name. This does not impose any additional overhead. An -alternative syntax, which is equivalent to what -`~astropy.modeling.Model.rename` is doing, is to directly use the model -expression as the base class of a new class:: - - >>> class TwoGaussians(Gaussian1D + Gaussian1D): - ... """A superposition of two Gaussians.""" - ... - >>> TwoGaussians - - Name: TwoGaussians (CompoundModel...) - ... - -Because the result of the expression ``Gaussian1D + Gaussian1D`` *is* a class, -it can be used directly in the standard class declaration syntax -``class ClassName(Base):`` as the base. This syntax also has the advantage of -allowing a docstring to be assigned to the new class. In future versions it -may be possible to customize other aspects of compound model classes in this -way. Single model classes can also be given custom names by using -`~astropy.modeling.Model.rename`, and model instances can be given names as -well. This can be used to good effect, for example as shown in the section on -:ref:`compound-model-indexing`. - - -.. _compound-model-instances: - -Compound models with model instances -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -So far we have seen how to create compound model *classes* from expressions -involving other model classes. This is the most "generic" way to create new -models from existing models. However, many may find it more useful most of the -time, especially when providing an initial guess to a fitter, to create a new -model from a combination of model *instances* with already defined parameter -values. This can also be done and works mostly the same way:: - - >>> both_gaussians = Gaussian1D(1, 0, 0.2) + Gaussian1D(2.5, 0.5, 0.1) - >>> both_gaussians # doctest: +FLOAT_CMP - - -Unlike when a model was created from model classes, this expression does not -directly return a new class; instead it creates a model instance that is ready -to be used for evaluation:: - - >>> both_gaussians(0.2) # doctest: +FLOAT_CMP - 0.6343031510582392 - -This was found to be much more convenient and natural, in this case, than -returning a class. It is worth understanding that the way this works under the -hood is to create the compound class, and then immediately instantiate it with -the already known parameter values. We can see this by checking the type of -``both_gaussians``:: - - >>> type(both_gaussians) # doctest: +FLOAT_CMP - - Name: CompoundModel... - Inputs: ('x',) - Outputs: ('y',) - Fittable parameters: ('amplitude_0', 'mean_0', 'stddev_0', 'amplitude_1', 'mean_1', 'stddev_1') - Expression: [0] + [1] - Components: - [0]: + [0]: [1]: - -It is also possible, and sometimes useful, to make a compound model from a -combination of classes *and* instances in the same expression:: - - >>> from astropy.modeling.models import Linear1D, Sine1D - >>> MyModel = Linear1D + Sine1D(amplitude=1, frequency=1) - >>> MyModel - - Name: CompoundModel... - Inputs: ('x',) - Outputs: ('y',) - Fittable parameters: ('slope_0', 'intercept_0', 'amplitude_1', 'frequency_1') - Expression: [0] + [1] - Components: - [0]: - Name: Linear1D - Inputs: ('x',) - Outputs: ('y',) - Fittable parameters: ('slope', 'intercept') - [1]: - -In this case the result is always a class. However (and this is not -immediately obvious by the representation) the difference is that the -``amplitude`` and ``frequency`` parameters for the -`~astropy.modeling.functional_models.Sine1D` part of the model are -"baked into" the class as default values for those parameters. So it is -possible to instantiate one of these models by specifying just the ``slope`` -and ``intercept`` parameters for the -`~astropy.modeling.functional_models.Linear1D` part of the model:: - - >>> my_model = MyModel(1, 0) - >>> my_model(0.25) # doctest +FLOAT_CMP - 1.25 - -This does not prevent the other parameters from being overridden, however:: - - >>> my_model = MyModel(slope_0=1, intercept_0=0, frequency_1=2) - >>> my_model(0.125) # doctest +FLOAT_CMP - 1.125 - -In fact, this is currently the only way to use a `polynomial -` model in a compound model, because the design of -the polynomial models is currently such that they must be instantiated in order -to specify their polynomial degree. Because the polynomials are already -designed so that their coefficients all default to zero, this "limitation" -should not have any practical drawbacks. - -.. note:: - - There is currently a caveat in the example of combining model classes and - instances, which is that the parameter values of model *instances* are only - treated as defaults if the expression is written in such a way that all - model instances are to the right of all model classes. This limitation - will be lifted in a later version--in particular, Python 3 offers a lot - more flexibility with respect to how function arguments are handled. + [2]: + + [3]: + Parameters: + amplitude_0 mean_0 stddev_0 amplitude_1 ... stddev_2 amplitude_3 mean_3 stddev_3 + ----------- ------ -------- ----------- ... -------- ----------- ------ -------- + 1.1 0.1 0.2 2.5 ... 0.2 2.5 0.5 0.1 Operators --------- Arithmetic operators -^^^^^^^^^^^^^^^^^^^^ +-------------------- Compound models can be created from expressions that include any number of the arithmetic operators ``+``, ``-``, ``*``, ``/``, and @@ -338,8 +272,9 @@ objects in Python. .. note:: - In the case of division ``/`` always means floating point division--integer - division and the ``//`` operator is not supported for models). + In the case of division ``/`` always means floating point division + --- integer division and the ``//`` operator is not supported for + models. As demonstrated in previous examples, for models that have a single output the result of evaluating a model like ``A + B`` is to evaluate ``A`` and @@ -357,7 +292,7 @@ arrays. .. _compound-model-composition: Model composition -^^^^^^^^^^^^^^^^^ +----------------- The sixth binary operator that can be used to create compound models is the composition operator, also known as the "pipe" operator ``|`` (not to be @@ -372,7 +307,7 @@ when evaluated, is equivalent to evaluating :math:`g \circ f = g(f(x))`. This is in part because there is no operator symbol supported in Python that corresponds well to this. The ``|`` operator should instead be read like the `pipe operator - `_ of UNIX shell syntax: + `_ of UNIX shell syntax: It chains together models by piping the output of the left-hand operand to the input of the right-hand operand, forming a "pipeline" of models, or transformations. @@ -384,58 +319,108 @@ inputs. For simple functional models this is exactly the same as functional composition, except for the aforementioned caveat about ordering. For -example: +example, to create the following compound model: + +.. graphviz:: + + digraph { + in0 [shape="none", label="input 0"]; + out0 [shape="none", label="output 0"]; + redshift0 [shape="box", label="RedshiftScaleFactor"]; + gaussian0 [shape="box", label="Gaussian1D(1, 0.75, 0.1)"]; + + in0 -> redshift0; + redshift0 -> gaussian0; + gaussian0 -> out0; + } .. plot:: :include-source: import numpy as np - from astropy.modeling.models import Redshift, Gaussian1D - - class RedshiftedGaussian(Redshift | Gaussian1D(1, 0.75, 0.1)): - """Evaluates a Gaussian with optional redshift applied to the input.""" + import matplotlib.pyplot as plt + from astropy.modeling.models import RedshiftScaleFactor, Gaussian1D x = np.linspace(0, 1.2, 100) - g0 = RedshiftedGaussian(z_0=0) + g0 = RedshiftScaleFactor(0) | Gaussian1D(1, 0.75, 0.1) - plt.figure(figsize=(8, 3)) - plt.plot(x, g0(x), 'g--', lw=2, label='$z=0$') + fig, ax = plt.subplots(figsize=(8, 5)) + ax.plot(x, g0(x), 'g--', label='$z=0$') for z in (0.2, 0.4, 0.6): - g = RedshiftedGaussian(z_0=z) - plt.plot(x, g(x), color=plt.cm.OrRd(z), lw=2, - label='$z={0}$'.format(z)) + g = RedshiftScaleFactor(z) | Gaussian1D(1, 0.75, 0.1) + ax.plot(x, g(x), color=plt.cm.OrRd(z), + label=f'$z={z}$') + + ax.set(xlabel='Energy', ylabel='Flux') + ax.legend() + +If you wish to perform redshifting in the wavelength space instead of energy, +and would also like to conserve flux, here is another way to do it using +model *instances*: + +.. plot:: + :include-source: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling.models import RedshiftScaleFactor, Gaussian1D, Scale - plt.xlabel('Energy') - plt.ylabel('Flux') - plt.legend() + x = np.linspace(1000, 5000, 1000) + g0 = Gaussian1D(1, 2000, 200) # No redshift is same as redshift with z=0 -When working with models with multiple inputs and outputs the same idea + fig, ax = plt.subplots(figsize=(8, 5)) + ax.plot(x, g0(x), 'g--', label='$z=0$') + + for z in (0.2, 0.4, 0.6): + rs = RedshiftScaleFactor(z).inverse # Redshift in wavelength space + sc = Scale(1. / (1 + z)) # Rescale the flux to conserve energy + g = rs | g0 | sc + ax.plot(x, g(x), color=plt.cm.OrRd(z), + label=f'$z={z}$') + + ax.set(xlabel='Wavelength', ylabel='Flux') + ax.legend() + +When working with models with multiple inputs and outputs, the same idea applies. If each input is thought of as a coordinate axis, then this defines a pipeline of transformations for the coordinates on each axis (though it does not necessarily guarantee that these transformations are separable). For example: +.. graphviz:: + + digraph { + in0 [shape="none", label="input 0"]; + in1 [shape="none", label="input 1"]; + out0 [shape="none", label="output 0"]; + out1 [shape="none", label="output 1"]; + rot0 [shape="box", label="Rotation2D"]; + gaussian0 [shape="box", label="Gaussian2D(1, 0, 0, 0.1, 0.3)"]; + + in0 -> rot0; + in1 -> rot0; + rot0 -> gaussian0; + rot0 -> gaussian0; + gaussian0 -> out0; + gaussian0 -> out1; + } + .. plot:: :include-source: import numpy as np + import matplotlib.pyplot as plt from astropy.modeling.models import Rotation2D, Gaussian2D - class RotatedGaussian(Rotation2D | Gaussian2D(1, 0, 0, 0.1, 0.3)): - """A Gaussian2D composed with a coordinate rotation.""" - x, y = np.mgrid[-1:1:0.01, -1:1:0.01] - plt.figure(figsize=(8, 2.5)) + fig, axs = plt.subplots(figsize=(8, 2.5), ncols=3) - for idx, theta in enumerate((0, 45, 90)): - g = RotatedGaussian(theta) - plt.subplot(1, 3, idx + 1) - plt.imshow(g(x, y), origin='lower') - plt.xticks([]) - plt.yticks([]) - plt.title('Rotated $ {0}^\circ $'.format(theta)) + for idx, (theta, ax) in enumerate(zip((0, 45, 90), axs)): + g = Rotation2D(theta) | Gaussian2D(1, 0, 0, 0.1, 0.3) + ax.imshow(g(x, y), origin='lower') + ax.set(xticks=[], yticks=[], title=rf'Rotated $ {theta}^\circ $') .. note:: @@ -448,7 +433,7 @@ Normally it is not possible to compose, say, a model with two outputs and a function of only one input:: >>> from astropy.modeling.models import Rotation2D - >>> Rotation2D | Gaussian1D + >>> Rotation2D() | Gaussian1D() # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... ModelDefinitionError: Unsupported operands for |: Rotation2D (n_inputs=2, n_outputs=2) and Gaussian1D (n_inputs=1, n_outputs=1); n_outputs for the left-hand model must match n_inputs for the right-hand model. @@ -462,17 +447,38 @@ especially when used in concert with :ref:`mappings `. .. _compound-model-concatenation: Model concatenation -^^^^^^^^^^^^^^^^^^^ +------------------- The concatenation operator ``&``, sometimes also referred to as a "join", combines two models into a single, fully separable transformation. That is, it makes a new model that takes the inputs to the left-hand model, concatenated with the inputs to the right-hand model, and returns a tuple consisting of the two models' outputs concatenated together, without mixing in any way. In other -words, it simply evaluates the two models in parallel--it can be thought of as -something like a tuple of models. For example, given two coordinate axes, we -can scale each coordinate by a different factor by concatenating two -`~astropy.modeling.functional_models.Scale` models:: +words, it simply evaluates the two models in parallel --- it can be thought of as +something like a tuple of models. + +For example, given two coordinate axes, we can scale each coordinate +by a different factor by concatenating two +`~astropy.modeling.functional_models.Scale` models. + +.. graphviz:: + + digraph { + in0 [shape="none", label="input 0"]; + in1 [shape="none", label="input 1"]; + out0 [shape="none", label="output 0"]; + out1 [shape="none", label="output 1"]; + scale0 [shape="box", label="Scale(factor=1.2)"]; + scale1 [shape="box", label="Scale(factor=3.4)"]; + + in0 -> scale0; + scale0 -> out0; + + in1 -> scale1; + scale1 -> out1; + } + +:: >>> from astropy.modeling.models import Scale >>> separate_scales = Scale(factor=1.2) & Scale(factor=3.4) @@ -481,7 +487,30 @@ can scale each coordinate by a different factor by concatenating two We can also combine concatenation with composition to build chains of transformations that use both "1D" and "2D" models on two (or more) coordinate -axes:: +axes: + +.. graphviz:: + + digraph { + in0 [shape="none", label="input 0"]; + in1 [shape="none", label="input 1"]; + out0 [shape="none", label="output 0"]; + out1 [shape="none", label="output 1"]; + scale0 [shape="box", label="Scale(factor=1.2)"]; + scale1 [shape="box", label="Scale(factor=3.4)"]; + rot0 [shape="box", label="Rotation2D(90)"]; + + in0 -> scale0; + scale0 -> rot0; + + in1 -> scale1; + scale1 -> rot0; + + rot0 -> out0; + rot0 -> out1; + } + +:: >>> scale_and_rotate = ((Scale(factor=1.2) & Scale(factor=3.4)) | ... Rotation2D(90)) @@ -503,6 +532,40 @@ transformation matrix:: >>> allclose(scale_and_rotate(1, 2), affine(1, 2)) True +Other Topics +============ + +Model names +----------- + +In the above two examples another notable feature of the generated compound +model classes is that the class name, as displayed when printing the class at +the command prompt, is not "TwoGaussians", "FourGaussians", etc. Instead it is +a generated name consisting of "CompoundModel" followed by an essentially +arbitrary integer that is chosen simply so that every compound model has a +unique default name. This is a limitation at present, due to the limitation +that it is not generally possible in Python when an object is created by an +expression for it to "know" the name of the variable it will be assigned to, if +any. +It is possible to directly assign a name to the compound model instance +by using the `Model.name ` attribute:: + + >>> two_gaussians.name = "TwoGaussians" + >>> print(two_gaussians) # doctest: +SKIP + Model: CompoundModel... + Name: TwoGaussians + Inputs: ('x',) + Outputs: ('y',) + Model set size: 1 + Expression: [0] + [1] + Components: + [0]: + + [1]: + Parameters: + amplitude_0 mean_0 stddev_0 amplitude_1 mean_1 stddev_1 + ----------- ------ -------- ----------- ------ -------- + 1.1 0.1 0.2 2.5 0.5 0.1 .. _compound-model-indexing: @@ -516,35 +579,35 @@ expression that defined the model, from left to right, regardless of the order of operations. For example:: >>> from astropy.modeling.models import Const1D - >>> A = Const1D.rename('A') - >>> B = Const1D.rename('B') - >>> C = Const1D.rename('C') + >>> A = Const1D(1.1, name='A') + >>> B = Const1D(2.1, name='B') + >>> C = Const1D(3.1, name='C') >>> M = A + B * C - >>> M - - Name: CompoundModel... - ... + >>> print(M) + Model: CompoundModel... + Inputs: ('x',) + Outputs: ('y',) + Model set size: 1 Expression: [0] + [1] * [2] Components: - [0]: - Name: A (Const1D) - ... + [0]: - [1]: - Name: B (Const1D) - ... + [1]: - [2]: - Name: C (Const1D) - ... + [2]: + Parameters: + amplitude_0 amplitude_1 amplitude_2 + ----------- ----------- ----------- + 1.1 2.1 3.1 -In this example the expression is evaluated ``(B * C) + A``--that is, the + +In this example the expression is evaluated ``(B * C) + A`` --- that is, the multiplication is evaluated before the addition per usual arithmetic rules. However, the components of this model are simply read off left to right from the expression ``A + B * C``, with ``A -> 0``, ``B -> 1``, ``C -> 2``. If we had instead defined ``M = C * B + A`` then the indices would be reversed (though the expression is mathematically equivalent). This convention is -chosen for simplicity--given the list of components it is not necessary to +chosen for simplicity --- given the list of components it is not necessary to jump around when mentally mapping them to the expression. We can pull out each individual component of the compound model ``M`` by using @@ -552,11 +615,7 @@ indexing notation on it. Following from the above example, ``M[1]`` should return the model ``B``:: >>> M[1] - - Name: B (Const1D) - Inputs: ('x',) - Outputs: ('y',) - Fittable parameters: ('amplitude',) + We can also take a *slice* of the compound model. This returns a new compound model that evaluates the *subexpression* involving the models selected by the @@ -566,91 +625,116 @@ The start point is inclusive and the end point is exclusive. So a slice like *operators* between them). So the resulting model evaluates just the subexpression ``B * C``:: - >>> M[1:] - - Name: CompoundModel... + >>> print(M[1:]) + Model: CompoundModel Inputs: ('x',) Outputs: ('y',) - Fittable parameters: ('amplitude_1', 'amplitude_2') + Model set size: 1 Expression: [0] * [1] Components: - [0]: - Name: B (Const1D) - ... + [0]: - [1]: - Name: C (Const1D) - ... + [1]: + Parameters: + amplitude_0 amplitude_1 + ----------- ----------- + 2.1 3.1 + +.. note:: + + There is a change in the parameter names of a slice from versions + prior to 4.0. Previously, the parameter names were identical to that + of the model being sliced. Now, they are what is expected for a + compound model of this type apart from the model sliced. That is, + the sliced model always starts with its own relative index for its + components, thus the parameter names start with a 0 suffix. + +.. note:: + + Starting with 4.0, the behavior of slicing is more restrictive than + previously. For example if:: + + m = m1 * m2 + m3 + + and one sliced by + using ``m[1:3]``, previously that would return the model: ``m2 + m3`` + even though there was never any such submodel of m. Starting with 4.0 + a slice must correspond to a submodel (something that corresponds + to an intermediate result of the computational chain of evaluating + the compound model). So:: + + m1 * m2 + + is a submodel (i.e.,``m[:2]``) but + ``m[1:3]`` is not. Currently this also means that in simpler expressions + such as:: -The new compound model for the subexpression can be instantiated and evaluated + m = m1 + m2 + m3 + m4 + + where any slice should be valid in + principle, only slices that include m1 are valid since it is part of + all submodules. The order of evaluation is:: + + ((m1 + m2) + m3) + m4 + + Anyone creating compound models that wishes submodels to be available + is advised to use parentheses explicitly or define intermediate + models to be used in subsequent expressions so that they can be + extracted with a slice or simple index depending on the context. + For example, to make ``m2 + m3`` accessible by slice, define ``m`` as:: + + m = m1 + (m2 + m3) + m4. In this case ``m[1:3]`` will work. + +The new compound model for the subexpression can be evaluated like any other:: - >>> m = M[1:](2, 3) - >>> m - - >>> m(0) - 6.0 + >>> M[1:](0) # doctest: +FLOAT_CMP + 6.51 Although the model ``M`` was composed entirely of ``Const1D`` models in this example, it was useful to give each component a unique name (``A``, ``B``, ``C``) in order to differentiate between them. This can also be used for indexing and slicing:: - >>> M['B'] - - Name: B (Const1D) + >>> print(M['B']) + Model: Const1D + Name: B Inputs: ('x',) Outputs: ('y',) - Fittable parameters: ('amplitude',) + Model set size: 1 + Parameters: + amplitude + --------- + 2.1 + In this case ``M['B']`` is equivalent to ``M[1]``. But by using the name we do not have to worry about what index that component is in (this becomes especially useful when combining multiple compound models). A current limitation, however, is that each component of a compound model must have a -unique name--if some components have duplicate names then they can only be -accessed by their integer index. This may improve in a future release. +unique name --- if some components have duplicate names then they can only be +accessed by their integer index. Slicing also works with names. When using names the start and end points are *both inclusive*:: - >>> M['B':'C'] - - ... + >>> print(M['B':'C']) + Model: CompoundModel... + Inputs: ('x',) + Outputs: ('y',) + Model set size: 1 Expression: [0] * [1] Components: - [0]: - Name: B (Const1D) - ... + [0]: - [1]: - Name: C (Const1D) - ... + [1]: + Parameters: + amplitude_0 amplitude_1 + ----------- ----------- + 2.1 3.1 So in this case ``M['B':'C']`` is equivalent to ``M[1:3]``. -All of the above applies equally well to compound models composed of model -instances. Individual model instances can be given a name by passing in the -``name=`` argument when instantiating them. These names are used in the same was -as class names were in the class-based examples:: - - >>> a = Const1D(amplitude=1, name='A') - >>> b = Const1D(amplitude=2, name='B') - >>> c = Const1D(amplitude=3, name='C') - >>> m = a + b * c - -Because this model is composed entirely of constants it doesn't matter what -input we pass in, so 0 is used without loss of generality:: - - >>> m(0) - 7.0 - >>> m[1:](0) # b * c - 6.0 - >>> m['A':'B'](0) # a + b - 3.0 - >>> m['B':'C'](0) # b * c, again - 6.0 - - .. _compound-model-parameters: Parameters @@ -659,7 +743,7 @@ Parameters A question that frequently comes up when first encountering compound models is how exactly all the parameters are dealt with. By now we've seen a few examples that give some hints, but a more detailed explanation is in order. -This is also one of the biggest areas for possible improvements--the current +This is also one of the biggest areas for possible improvements --- the current behavior is meant to be practical, but is not ideal. (Some possible improvements include being able to rename parameters, and providing a means of narrowing down the number of parameters in a compound model.) @@ -680,14 +764,14 @@ belongs to. For example:: >>> Gaussian1D.param_names ('amplitude', 'mean', 'stddev') - >>> (Gaussian1D + Gaussian1D).param_names + >>> (Gaussian1D() + Gaussian1D()).param_names ('amplitude_0', 'mean_0', 'stddev_0', 'amplitude_1', 'mean_1', 'stddev_1') For consistency's sake, this scheme is followed even if not all of the components have overlapping parameter names:: - >>> from astropy.modeling.models import Redshift - >>> (Redshift | (Gaussian1D + Gaussian1D)).param_names + >>> from astropy.modeling.models import RedshiftScaleFactor + >>> (RedshiftScaleFactor() | (Gaussian1D() + Gaussian1D())).param_names ('z_0', 'amplitude_1', 'mean_1', 'stddev_1', 'amplitude_2', 'mean_2', 'stddev_2') @@ -701,8 +785,9 @@ are still tied back to the compound model:: >>> a = Gaussian1D(1, 0, 0.2, name='A') >>> b = Gaussian1D(2.5, 0.5, 0.1, name='B') + >>> m = a + b >>> m.amplitude_0 - Parameter('amplitude_0', value=1.0) + Parameter('amplitude', value=1.0) is equivalent to:: @@ -717,17 +802,17 @@ Updating one updates the other:: Parameter('amplitude', value=42.0) >>> m['A'].amplitude = 99 >>> m.amplitude_0 - Parameter('amplitude_0', value=99.0) + Parameter('amplitude', value=99.0) Note, however, that the original -`~astropy.modeling.functional_models.Gaussian1D` instance ``a`` has not been +`~astropy.modeling.functional_models.Gaussian1D` instance ``a`` has been updated:: >>> a.amplitude - Parameter('amplitude', value=1.0) + Parameter('amplitude', value=99.0) -This is because currently, when a compound model is created, copies are made of -the original models. +This is different than the behavior in versions prior to 4.0. Now compound model +parameters share the same Parameter instance as the original model. .. _compound-model-mappings: @@ -762,23 +847,92 @@ an example like:: >>> m = (Scale(1.2) & Scale(3.4)) | Rotation2D(90) + +.. graphviz:: + + digraph { + in0 [shape="none", label="input 0"]; + in1 [shape="none", label="input 1"]; + out0 [shape="none", label="output 0"]; + out1 [shape="none", label="output 1"]; + scale0 [shape="box", label="Scale(factor=1.2)"]; + scale1 [shape="box", label="Scale(factor=3.4)"]; + rot0 [shape="box", label="Rotation2D(90)"]; + + in0 -> scale0; + scale0 -> rot0; + + in1 -> scale1; + scale1 -> rot0; + + rot0 -> out0; + rot0 -> out1; + } + where two coordinate inputs are scaled individually and then rotated into each other. However, say we wanted to scale only one of those coordinates. It would be fine to simply use ``Scale(1)`` for one them, or any other model that is effectively a no-op. But that also adds unnecessary computational overhead, so we might as well simply specify that that coordinate is not to be scaled or transformed in any way. This is a good use case for -`~astropy.modeling.mappings.Identity`:: +`~astropy.modeling.mappings.Identity`: + +.. graphviz:: + + digraph { + in0 [shape="none", label="input 0"]; + in1 [shape="none", label="input 1"]; + out0 [shape="none", label="output 0"]; + out1 [shape="none", label="output 1"]; + scale0 [shape="box", label="Scale(factor=1.2)"]; + identity0 [shape="box", label="Identity(1)"]; + rot0 [shape="box", label="Rotation2D(90)"]; + + in0 -> scale0; + scale0 -> rot0; + + in1 -> identity0; + identity0 -> rot0; + + rot0 -> out0; + rot0 -> out1; + } + +:: >>> from astropy.modeling.models import Identity >>> m = Scale(1.2) & Identity(1) >>> m(1, 2) # doctest: +FLOAT_CMP (1.2, 2.0) + This scales the first input, and passes the second one through unchanged. We can use this to build up more complicated steps in a many-axis WCS transformation. If for example we had 3 axes and only wanted to scale the -first one:: +first one: + +.. graphviz:: + + digraph { + in0 [shape="none", label="input 0"]; + in1 [shape="none", label="input 1"]; + in2 [shape="none", label="input 2"]; + out0 [shape="none", label="output 0"]; + out1 [shape="none", label="output 1"]; + out2 [shape="none", label="output 2"]; + scale0 [shape="box", label="Scale(1.2)"]; + identity0 [shape="box", label="Identity(2)"]; + + in0 -> scale0; + scale0 -> out0; + + in1 -> identity0; + in2 -> identity0; + identity0 -> out1; + identity0 -> out2; + } + +:: >>> m = Scale(1.2) & Identity(2) >>> m(1, 2, 3) # doctest: +FLOAT_CMP @@ -795,10 +949,31 @@ number of outputs the `~astropy.modeling.mappings.Mapping` should produce. A 1-tuple means that whatever inputs come in to the `~astropy.modeling.mappings.Mapping`, only one will be output. And so on for 2-tuple or higher (though the length of the tuple cannot be greater than the -number of inputs--it will not pull values out of thin air). The elements of +number of inputs --- it will not pull values out of thin air). The elements of this mapping are integers corresponding to the indices of the inputs. For -example, a mapping of ``Mapping((0,))`` is equivalent to ``Identity(1)``--it -simply takes the first (0-th) input and returns it:: +example, a mapping of ``Mapping((0,))`` is equivalent to ``Identity(1)`` --- it +simply takes the first (0-th) input and returns it: + +.. graphviz:: + + digraph G { + in0 [shape="none", label="input 0"]; + + subgraph cluster_A { + shape=rect; + color=black; + label="(0,)"; + + a [shape=point, label=""]; + } + + out0 [shape="none", label="output 0"]; + + in0 -> a; + a -> out0; + } + +:: >>> from astropy.modeling.models import Mapping >>> m = Mapping((0,)) @@ -807,36 +982,381 @@ simply takes the first (0-th) input and returns it:: Likewise ``Mapping((0, 1))`` is equivalent to ``Identity(2)``, and so on. However, `~astropy.modeling.mappings.Mapping` also allows outputs to be -reordered arbitrarily:: +reordered arbitrarily: + +.. graphviz:: + + digraph G { + { + rank=same; + in0 [shape="none", label="input 0"]; + in1 [shape="none", label="input 1"]; + } + + subgraph cluster_A { + shape=rect; + color=black; + label="(1, 0)"; + + { + rank=same; + a [shape=point, label=""]; + b [shape=point, label=""]; + } + + { + rank=same; + c [shape=point, label=""]; + d [shape=point, label=""]; + } + + a -> c [style=invis]; + a -> d [constraint=false]; + b -> c [constraint=false]; + } + + { + rank=same; + out0 [shape="none", label="output 0"]; + out1 [shape="none", label="output 1"]; + } + + in0 -> a; + in1 -> b; + c -> out0; + d -> out1; + } + +:: >>> m = Mapping((1, 0)) >>> m(1.0, 2.0) (2.0, 1.0) + +.. graphviz:: + + digraph G { + { + rank=same; + in0 [shape="none", label="input 0"]; + in1 [shape="none", label="input 1"]; + in2 [shape="none", label="input 2"]; + } + + subgraph cluster_A { + shape=rect; + color=black; + label="(1, 0, 2)"; + + { + rank=same; + a [shape=point, label=""]; + b [shape=point, label=""]; + c [shape=point, label=""]; + } + + { + rank=same; + d [shape=point, label=""]; + e [shape=point, label=""]; + f [shape=point, label=""]; + } + + a -> d [style=invis]; + a -> e [constraint=false]; + b -> d [constraint=false]; + c -> f [constraint=false]; + } + + { + rank=same; + out0 [shape="none", label="output 0"]; + out1 [shape="none", label="output 1"]; + out2 [shape="none", label="output 2"]; + } + + in0 -> a; + in1 -> b; + in2 -> c; + d -> out0; + e -> out1; + f -> out2; + } + +:: + >>> m = Mapping((1, 0, 2)) >>> m(1.0, 2.0, 3.0) (2.0, 1.0, 3.0) -Outputs may also be dropped:: +Outputs may also be dropped: + +.. graphviz:: + + digraph G { + { + rank=same; + in0 [shape="none", label="input 0"]; + in1 [shape="none", label="input 1"]; + } + + subgraph cluster_A { + shape=rect; + color=black; + label="(1,)"; + + { + rank=same; + a [shape=point, label=""]; + b [shape=point, label=""]; + } + + { + rank=same; + c [shape=point, label=""]; + } + + a -> c [style=invis]; + b -> c [constraint=false]; + } + + out0 [shape="none", label="output 0"]; + + in0 -> a; + in1 -> b; + c -> out0; + } + +:: >>> m = Mapping((1,)) >>> m(1.0, 2.0) 2.0 + +.. graphviz:: + + digraph G { + { + rank=same; + in0 [shape="none", label="input 0"]; + in1 [shape="none", label="input 1"]; + in2 [shape="none", label="input 2"]; + } + + subgraph cluster_A { + shape=rect; + color=black; + label="(0, 2)"; + + { + rank=same; + a [shape=point, label=""]; + b [shape=point, label=""]; + c [shape=point, label=""]; + } + + { + rank=same; + d [shape=point, label=""]; + e [shape=point, label=""]; + } + + a -> d [style=invis]; + a -> d [constraint=false]; + c -> e [constraint=false]; + } + + { + rank=same; + out0 [shape="none", label="output 0"]; + out1 [shape="none", label="output 1"]; + } + + in0 -> a; + in1 -> b; + in2 -> c; + d -> out0; + e -> out1; + } + +:: + >>> m = Mapping((0, 2)) >>> m(1.0, 2.0, 3.0) (1.0, 3.0) -Or duplicated:: +Or duplicated: + +.. graphviz:: + + digraph G { + in0 [shape="none", label="input 0"]; + + subgraph cluster_A { + shape=rect; + color=black; + label="(0, 0)"; + + a [shape=point, label=""]; + + { + rank=same; + b [shape=point, label=""]; + c [shape=point, label=""]; + } + + a -> b [style=invis]; + a -> b [constraint=false]; + a -> c [constraint=false]; + } + + { + rank=same; + out0 [shape="none", label="output 0"]; + out1 [shape="none", label="output 1"]; + } + + in0 -> a; + b -> out0; + c -> out1; + } + +:: >>> m = Mapping((0, 0)) >>> m(1.0) (1.0, 1.0) + +.. graphviz:: + + digraph G { + { + rank=same; + in0 [shape="none", label="input 0"]; + in1 [shape="none", label="input 1"]; + in2 [shape="none", label="input 2"]; + } + + subgraph cluster_A { + shape=rect; + color=black; + label="(0, 1, 1, 2)"; + + { + rank=same; + a [shape=point, label=""]; + b [shape=point, label=""]; + c [shape=point, label=""]; + } + + { + rank=same; + d [shape=point, label=""]; + e [shape=point, label=""]; + f [shape=point, label=""]; + g [shape=point, label=""]; + } + + a -> d [style=invis]; + a -> d [constraint=false]; + b -> e [constraint=false]; + b -> f [constraint=false]; + c -> g [constraint=false]; + } + + { + rank=same; + out0 [shape="none", label="output 0"]; + out1 [shape="none", label="output 1"]; + out2 [shape="none", label="output 2"]; + out3 [shape="none", label="output 3"]; + } + + in0 -> a; + in1 -> b; + in2 -> c; + d -> out0; + e -> out1; + f -> out2; + g -> out3; + } + +:: + >>> m = Mapping((0, 1, 1, 2)) >>> m(1.0, 2.0, 3.0) (1.0, 2.0, 2.0, 3.0) A complicated example that performs multiple transformations, some separable, -some not, on three coordinate axes might look something like:: +some not, on three coordinate axes might look something like: + +.. graphviz:: + + digraph G { + { + rank=same; + in0 [shape="none", label="input 0"]; + in1 [shape="none", label="input 1"]; + in2 [shape="none", label="input 2"]; + } + + { + rank=same; + poly0 [shape=rect, label="Poly1D(3, c0=1, c3=1)"]; + identity0 [shape=rect, label="Identity(1)"]; + poly1 [shape=rect, label="Poly1D(2, c2=1)"]; + } + + subgraph cluster_A { + shape=rect; + color=black; + label="(0, 2, 1)"; + + { + rank=same; + a [shape=point, label=""]; + b [shape=point, label=""]; + c [shape=point, label=""]; + } + + { + rank=same; + d [shape=point, label=""]; + e [shape=point, label=""]; + f [shape=point, label=""]; + } + + a -> d [style=invis]; + d -> e [style=invis]; + a -> d [constraint=false]; + c -> e [constraint=false]; + b -> f [constraint=false]; + } + + poly2 [shape="rect", label="Poly2D(4, c0_0=1, c1_1=1, c2_2=2)"]; + gaussian0 [shape="rect", label="Gaussian1D(1, 0, 4)"]; + + { + rank=same; + out0 [shape="none", label="output 0"]; + out1 [shape="none", label="output 1"]; + } + + in0 -> poly0; + in1 -> identity0; + in2 -> poly1; + poly0 -> a; + identity0 -> b; + poly1 -> c; + d -> poly2; + e -> poly2; + f -> gaussian0; + poly2 -> out0; + gaussian0 -> out1; + } + +:: >>> from astropy.modeling.models import Polynomial1D as Poly1D >>> from astropy.modeling.models import Polynomial2D as Poly2D @@ -861,3 +1381,58 @@ This opens up the possibility of essentially arbitrarily complex transformation graphs. Currently the tools do not exist to make it easy to navigate and reason about highly complex compound models that use these mappings, but that is a possible enhancement for future versions. + +.. _model-reduction: + +Model Reduction +--------------- + +In order to save much duplication in the construction of complex models, it is +possible to define one complex model that covers all cases where the +variables that distinguish the models are made part of the model's input +variables. The ``fix_inputs`` function allows defining models derived from +the more complex one by setting one or more of the inputs to a constant +value. Examples of this sort of situation arise when working out +the transformations from detector pixel to RA, Dec, and lambda for +spectrographs when the slit locations may be moved (e.g., fiber fed or +commandable slit masks), or different orders may be selected (e.g., Eschelle). +In the case of order, one may have a function of pixel ``x``, ``y``, ``spectral_order`` +that map into ``RA``, ``Dec`` and ``lambda``. Without specifying ``spectral_order``, it is +ambiguous what ``RA``, ``Dec`` and ``Lambda`` corresponds to a pixel location. It +is usually possible to define a function of all three inputs. Presuming +this model is ``general_transform`` then ``fix_inputs`` may be used to define +the transform for a specific order as follows: + +:: + >>> order1_transform = fix_inputs(general_transform, {'order': 1}) # doctest: +SKIP + +creates a new compound model that takes only pixel position and generates +``RA``, ``Dec``, and ``lambda``. The ``fix_inputs`` function can be used to set input +values by position (0 is the first) or by input variable name, and more +than one can be set in the dictionary supplied. + +If the input model has a bounding_box, the generated model will have the +bounding for the input coordinate removed. + + +.. test_replace_submodel + +Replace submodels +----------------- + + +:meth:`~astropy.modeling.core.CompoundModel.replace_submodel` creates a new model by +replacing a submodel with a matching name with another submodel. The number of +inputs and outputs of the old and new submodels should match. +:: + + >>> from astropy.modeling import models + >>> shift = models.Shift(-1) & models.Shift(-1) + >>> scale = models.Scale(2) & models.Scale(3) + >>> scale.name = "Scale" + >>> model = shift | scale + >>> model(2, 1) # doctest: +FLOAT_CMP + (2.0, 0.0) + >>> new_model = model.replace_submodel('Scale', models.Rotation2D(90, name='Rotation')) + >>> new_model(2, 1) # doctest: +FLOAT_CMP + (6.12e-17, 1.0) diff --git a/docs/modeling/example-fitting-constraints.rst b/docs/modeling/example-fitting-constraints.rst new file mode 100644 index 000000000000..83a839d909da --- /dev/null +++ b/docs/modeling/example-fitting-constraints.rst @@ -0,0 +1,161 @@ +Fitting with constraints +======================== + +`~astropy.modeling.fitting` support constraints, however, different fitters support +different types of constraints. The `~astropy.modeling.fitting.Fitter.supported_constraints` +attribute shows the type of constraints supported by a specific fitter:: + + >>> from astropy.modeling import fitting + >>> fitting.LinearLSQFitter.supported_constraints + ['fixed'] + >>> fitting.TRFLSQFitter.supported_constraints + ['fixed', 'tied', 'bounds'] + >>> fitting.SLSQPLSQFitter.supported_constraints + ['bounds', 'eqcons', 'ineqcons', 'fixed', 'tied'] + +Fixed Parameter Constraint +-------------------------- + +All fitters support fixed (frozen) parameters through the ``fixed`` argument +to models or setting the `~astropy.modeling.Parameter.fixed` +attribute directly on a parameter. + +For linear fitters, freezing a polynomial coefficient means that the +corresponding term will be subtracted from the data before fitting a +polynomial without that term to the result. For example, fixing ``c0`` in a +polynomial model will fit a polynomial with the zero-th order term missing +to the data minus that constant. The fixed coefficients and corresponding terms +are restored to the fit polynomial and this is the polynomial returned from the fitter:: + + >>> import numpy as np + >>> rng = np.random.default_rng(seed=12345) + >>> from astropy.modeling import models, fitting + >>> x = np.arange(1, 10, .1) + >>> p1 = models.Polynomial1D(2, c0=[1, 1], c1=[2, 2], c2=[3, 3], + ... n_models=2) + >>> p1 # doctest: +FLOAT_CMP + + >>> y = p1(x, model_set_axis=False) + >>> n = (rng.standard_normal(y.size)).reshape(y.shape) + >>> p1.c0.fixed = True + >>> pfit = fitting.LinearLSQFitter() + >>> new_model = pfit(p1, x, y + n) # doctest: +IGNORE_WARNINGS + >>> print(new_model) # doctest: +SKIP + Model: Polynomial1D + Inputs: ('x',) + Outputs: ('y',) + Model set size: 2 + Degree: 2 + Parameters: + c0 c1 c2 + --- ------------------ ------------------ + 1.0 2.072116176718454 2.99115839177437 + 1.0 1.9818866652726403 3.0024208951927585 + +The syntax to fix the same parameter ``c0`` using an argument to the model +instead of ``p1.c0.fixed = True`` would be:: + + >>> p1 = models.Polynomial1D(2, c0=[1, 1], c1=[2, 2], c2=[3, 3], + ... n_models=2, fixed={'c0': True}) + + +Bounded Constraints +------------------- + +Bounded fitting is supported through the ``bounds`` arguments to models or by +setting `~astropy.modeling.Parameter.min` and `~astropy.modeling.Parameter.max` +attributes on a parameter. The following fitters support bounds internally: + +* `~astropy.modeling.fitting.TRFLSQFitter` +* `~astropy.modeling.fitting.DogBoxLSQFitter` +* `~astropy.modeling.fitting.SLSQPLSQFitter` + +The `~astropy.modeling.fitting.LevMarLSQFitter` algorithm uses an unsophisticated +method of handling bounds and is no longer recommended (see +:ref:`modeling-getting-started-nonlinear-notes` for more details). + +.. _tied: + +Tied Constraints +---------------- + +The `~astropy.modeling.Parameter.tied` constraint is often useful with +:ref:`Compound models `. In this example we will +read a spectrum from a file called ``spec.txt`` and simultaneously fit +Gaussians to the emission lines while linking their wavelengths and +linking the flux of the [OIII] Îģ4959 line to the [OIII] Îģ5007 line. + +.. plot:: + :include-source: + + import numpy as np + from astropy.io import ascii + from astropy.modeling import fitting, models + from astropy.utils.data import get_pkg_data_filename + from matplotlib import pyplot as plt + + fname = get_pkg_data_filename("data/spec.txt", package="astropy.modeling.tests") + spec = ascii.read(fname) + wave = spec["lambda"] + flux = spec["flux"] + + # Use the (vacuum) rest wavelengths of known lines as initial values + # for the fit. + Hbeta = 4862.721 + O3_4959 = 4960.295 + O3_5007 = 5008.239 + + # Create Gaussian1D models for each of the H-beta and [OIII] lines. + hbeta_broad = models.Gaussian1D(amplitude=15, mean=Hbeta, stddev=20) + hbeta_narrow = models.Gaussian1D(amplitude=20, mean=Hbeta, stddev=2) + o3_4959 = models.Gaussian1D(amplitude=70, mean=O3_4959, stddev=2) + o3_5007 = models.Gaussian1D(amplitude=180, mean=O3_5007, stddev=2) + + # Create a polynomial model to fit the continuum. + mean_flux = flux.mean() + cont = np.where(flux > mean_flux, mean_flux, flux) + linfitter = fitting.LinearLSQFitter() + poly_cont = linfitter(models.Polynomial1D(1), wave, cont) + + # Create a compound model for the four emission lines and the continuum. + model = hbeta_broad + hbeta_narrow + o3_4959 + o3_5007 + poly_cont + + # Tie the ratio of the intensity of the two [OIII] lines. + def tie_o3_ampl(model): + return model.amplitude_3 / 2.98 + + o3_4959.amplitude.tied = tie_o3_ampl + + # Tie the wavelengths of the two [OIII] lines + def tie_o3_wave(model): + return model.mean_3 * O3_4959 / O3_5007 + + o3_4959.mean.tied = tie_o3_wave + + # Tie the wavelengths of the two (narrow and broad) H-beta lines + def tie_hbeta_wave1(model): + return model.mean_1 + + hbeta_broad.mean.tied = tie_hbeta_wave1 + + # Tie the wavelengths of the H-beta lines to the [OIII] 5007 line + def tie_hbeta_wave2(model): + return model.mean_3 * Hbeta / O3_5007 + + hbeta_narrow.mean.tied = tie_hbeta_wave2 + + # Simultaneously fit all the emission lines and continuum. + fitter = fitting.TRFLSQFitter() + fitted_model = fitter(model, wave, flux) + fitted_lines = fitted_model(wave) + + # Plot the data and the fitted model + fig, ax = plt.subplots(figsize=(9, 6)) + ax.plot(wave, flux, label="Data") + ax.plot(wave, fitted_lines, color="C1", label="Fitted Model") + ax.legend(loc="upper left") + ax.text(4860, 45, r"$H\beta$ (broad + narrow)", rotation=90) + ax.text(4958, 68, r"[OIII] $\lambda 4959$", rotation=90) + ax.text(4995, 140, r"[OIII] $\lambda 5007$", rotation=90) + ax.set(xlim=(4700, 5100), xlabel="Wavelength (Angstrom)", ylabel="Flux") + plt.show() diff --git a/docs/modeling/example-fitting-line.rst b/docs/modeling/example-fitting-line.rst new file mode 100644 index 000000000000..315123c25fab --- /dev/null +++ b/docs/modeling/example-fitting-line.rst @@ -0,0 +1,149 @@ +.. _example_fitting_line: + +Fitting a Line +============== + +Fitting a line to (x,y) data points is a common case in many areas. +Examples fits are given for fitting, fitting using the uncertainties +as weights, and fitting using iterative sigma clipping. + +Simple Fit +---------- + +Here the (x,y) data points are fit with a line. The (x,y) data +points are simulated and have a range of uncertainties to give +a realistic example. + +.. plot:: + :include-source: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling import models, fitting + + # define a model for a line + line_orig = models.Linear1D(slope=1.0, intercept=0.5) + + # generate x, y data non-uniformly spaced in x + # add noise to y measurements + npts = 30 + rng = np.random.default_rng(10) + x = rng.uniform(0.0, 10.0, npts) + y = line_orig(x) + yunc = np.absolute(rng.normal(0.5, 2.5, npts)) + y += rng.normal(0.0, yunc, npts) + + # initialize a linear fitter + fit = fitting.LinearLSQFitter() + + # initialize a linear model + line_init = models.Linear1D() + + # fit the data with the fitter + fitted_line = fit(line_init, x, y) + + # plot + fig, ax = plt.subplots() + ax.plot(x, y, 'ko', label='Data') + ax.plot(x, line_orig(x), 'b-', label='Simulation Model') + ax.plot(x, fitted_line(x), 'k-', label='Fitted Model') + ax.set(xlabel='x', ylabel='y') + ax.legend() + +Fit using uncertainties +----------------------- + +Fitting can be done using the uncertainties as weights. +To get the standard weighting of 1/unc^2 for the case of +Gaussian errors, the weights to pass to the fitting are 1/unc. + +.. plot:: + :include-source: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling import models, fitting + + # define a model for a line + line_orig = models.Linear1D(slope=1.0, intercept=0.5) + + # generate x, y data non-uniformly spaced in x + # add noise to y measurements + npts = 30 + rng = np.random.default_rng(10) + x = rng.uniform(0.0, 10.0, npts) + y = line_orig(x) + yunc = np.absolute(rng.normal(0.5, 2.5, npts)) + y += rng.normal(0.0, yunc, npts) + + # initialize a linear fitter + fit = fitting.LinearLSQFitter() + + # initialize a linear model + line_init = models.Linear1D() + + # fit the data with the fitter + fitted_line = fit(line_init, x, y, weights=1.0/yunc) + + # plot + fig, ax = plt.subplots() + ax.errorbar(x, y, yerr=yunc, fmt='ko', label='Data') + ax.plot(x, line_orig(x), 'b-', label='Simulation Model') + ax.plot(x, fitted_line(x), 'k-', label='Fitted Model') + ax.set(xlabel='x', ylabel='y') + ax.legend() + +Iterative fitting using sigma clipping +-------------------------------------- + +When fitting, there may be data that are outliers from the fit +that can significantly bias the fitting. These outliers can +be identified and removed from the fitting iteratively. +Note that the iterative sigma clipping assumes all the data +have the same uncertainties for the sigma clipping decision. + +.. plot:: + :include-source: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.stats import sigma_clip + from astropy.modeling import models, fitting + + # define a model for a line + line_orig = models.Linear1D(slope=1.0, intercept=0.5) + + # generate x, y data non-uniformly spaced in x + # add noise to y measurements + npts = 30 + rng = np.random.default_rng(10) + x = rng.uniform(0.0, 10.0, npts) + y = line_orig(x) + yunc = np.absolute(rng.normal(0.5, 2.5, npts)) + y += rng.normal(0.0, yunc, npts) + + # make true outliers + y[3] = line_orig(x[3]) + 6 * yunc[3] + y[10] = line_orig(x[10]) - 4 * yunc[10] + + # initialize a linear fitter + fit = fitting.LinearLSQFitter() + + # initialize the outlier removal fitter + or_fit = fitting.FittingWithOutlierRemoval(fit, sigma_clip, niter=3, sigma=3.0) + + # initialize a linear model + line_init = models.Linear1D() + + # fit the data with the fitter + fitted_line, mask = or_fit(line_init, x, y, weights=1.0/yunc) + filtered_data = np.ma.masked_array(y, mask=mask) + + # plot + fig, ax = plt.subplots() + ax.errorbar(x, y, yerr=yunc, fmt="ko", fillstyle="none", label="Clipped Data") + ax.plot(x, filtered_data, "ko", label="Fitted Data") + ax.plot(x, line_orig(x), 'b-', label='Simulation Model') + ax.plot(x, fitted_line(x), 'k-', label='Fitted Model') + ax.set(xlabel='x', ylabel='y') + ax.legend() diff --git a/docs/modeling/example-fitting-model-sets.rst b/docs/modeling/example-fitting-model-sets.rst new file mode 100644 index 000000000000..c59c6831e22c --- /dev/null +++ b/docs/modeling/example-fitting-model-sets.rst @@ -0,0 +1,171 @@ +.. _example-fitting-model-sets: + +Fitting Model Sets +================== + +Astropy model sets let you fit the same (linear) model to lots of independent +data sets. It solves the linear equations simultaneously, so can avoid looping. +But getting the data into the right shape can be a bit tricky. + +The time savings could be worth the effort. In the example below, if we change +the width*height of the data cube to 500*500 it takes 140 ms on a 2015 MacBook Pro +to fit the models using model sets. Doing the same fit by looping over the 500*500 models +takes 1.5 minutes, more than 600 times slower. + +In the example below, we create a 3D data cube where the first dimension is a ramp -- +for example as from non-destructive readouts of an IR detector. So each pixel has a +depth along a time axis, and flux that results a total number of counts that is +increasing with time. We will be fitting a 1D polynomial vs. time to estimate the +flux in counts/second (the slope of the fit). We will use just a small image +of 3 rows by 4 columns, with a depth of 10 non-destructive reads. + +First, import the necessary libraries: + + >>> import numpy as np + >>> rng = np.random.default_rng(seed=12345) + >>> from astropy.modeling import models, fitting + + >>> depth, width, height = 10, 3, 4 # Time is along the depth axis + >>> t = np.arange(depth, dtype=np.float64)*10. # e.g. readouts every 10 seconds + +The number of counts in neach pixel is flux*time with the addition of some Gaussian noise:: + + >>> fluxes = np.arange(1. * width * height).reshape(width, height) + >>> image = fluxes[np.newaxis, :, :] * t[:, np.newaxis, np.newaxis] + >>> image += rng.normal(0., image*0.05, size=image.shape) # Add noise + >>> image.shape + (10, 3, 4) + +Create the models and the fitter. We need N=width*height instances of the same linear, +parametric model (model sets currently only work with linear models and fitters):: + + >>> N = width * height + >>> line = models.Polynomial1D(degree=1, n_models=N) + >>> fit = fitting.LinearLSQFitter() + >>> print(f"We created {len(line)} models") + We created 12 models + +We need to get the data to be fit into the right shape. It's not possible to just feed +the 3D data cube. In this case, the time axis can be one dimensional. +The fluxes have to be organized into an array that is of shape ``width*height,depth`` -- in +other words, we are reshaping to flatten last two axes and transposing to put them first:: + + >>> pixels = image.reshape((depth, width*height)) + >>> y = pixels.T + >>> print("x axis is one dimensional: ",t.shape) + x axis is one dimensional: (10,) + >>> print("y axis is two dimensional, N by len(x): ", y.shape) + y axis is two dimensional, N by len(x): (12, 10) + +Fit the model. It fits the N models simultaneously:: + + >>> new_model = fit(line, x=t, y=y) + >>> print(f"We fit {len(new_model)} models") + We fit 12 models + +Fill an array with values computed from the best fit and reshape it to match the original:: + + >>> best_fit = new_model(t, model_set_axis=False).T.reshape((depth, height, width)) + >>> print("We reshaped the best fit to dimensions: ", best_fit.shape) + We reshaped the best fit to dimensions: (10, 4, 3) + +Now inspect the model:: + + >>> print(new_model) # doctest: +FLOAT_CMP + Model: Polynomial1D + Inputs: ('x',) + Outputs: ('y',) + Model set size: 12 + Degree: 1 + Parameters: + c0 c1 + ------------------- ------------------ + 0.0 0.0 + 0.7435257251672668 0.9788645710692938 + -2.9342067207465647 2.038294797728997 + -4.258776494573452 3.1951399579785678 + 2.364390501364263 3.9973270072631104 + 2.161531512810536 4.939542306192216 + 3.9930177540418823 5.967786182181591 + -6.825657765397985 7.2680615507233215 + -6.675677073701012 8.321048309260679 + -11.91115500400788 9.025794163936956 + -4.123655771677581 9.938564642105128 + -0.7256700167533869 10.989896974949136 + + >>> print("The new_model has a param_sets attribute with shape: ",new_model.param_sets.shape) + The new_model has a param_sets attribute with shape: (2, 12) + + >>> print(f"And values that are the best-fit parameters for each pixel:\n{new_model.param_sets}") # doctest: +FLOAT_CMP + And values that are the best-fit parameters for each pixel: + [[ 0. 0.74352573 -2.93420672 -4.25877649 2.3643905 + 2.16153151 3.99301775 -6.82565777 -6.67567707 -11.911155 + -4.12365577 -0.72567002] + [ 0. 0.97886457 2.0382948 3.19513996 3.99732701 + 4.93954231 5.96778618 7.26806155 8.32104831 9.02579416 + 9.93856464 10.98989697]] + +Plot the fit along a couple of pixels: + + >>> def plotramp(ax, t, image, best_fit, row, col): + ... ax.plot(t, image[:, row, col], '.', label=f'data pixel {row},{col}') + ... ax.plot(t, best_fit[:, row, col], '-', label=f'fit to pixel {row},{col}') + ... ax.set(xlabel='Time', ylabel='Counts') + ... ax.legend(loc='upper left') + >>> fig, ax = plt.subplots(figsize=(10, 5)) # doctest: +SKIP + >>> plotramp(ax, t, image, best_fit, 1, 1) # doctest: +SKIP + >>> plotramp(ax, t, image, best_fit, 2, 1) # doctest: +SKIP + +The data and the best fit model are shown together on one plot. + +.. plot:: + + import numpy as np + import matplotlib.pyplot as plt + from scipy import stats + from astropy.modeling import models, fitting + + # Set up the shape of the image and create the time axis + depth,width,height=10,3,4 # Time is along the depth axis + t = np.arange(depth, dtype=np.float64)*10. # e.g. readouts every 10 seconds + + # Make up a flux in each pixel + fluxes = np.arange(1.*width*height).reshape(height, width) + # Create the ramps by integrating the fluxes along the time steps + image = fluxes[np.newaxis, :, :] * t[:, np.newaxis, np.newaxis] + # Add some Gaussian noise to each sample + image += stats.norm.rvs(0., image*0.05, size=image.shape) # Add noise + + # Create the models and the fitter + N = width * height # This is how many instances we need + line = models.Polynomial1D(degree=1, n_models=N) + fit = fitting.LinearLSQFitter() + + # We need to get the data to be fit into the right shape + # In this case, the time axis can be one dimensional. + # The fluxes have to be organized into an array + # that is of shape `(width*height, depth)` + # i.e we are reshaping to flatten last two axes and + # transposing to put them first. + pixels = image.reshape((depth, width*height)) + y = pixels.T + + # Fit the model. It does the looping over the N models implicitly + new_model = fit(line, x=t, y=y) + + # Fill an array with values computed from the best fit and reshape it to match the original + best_fit = new_model(t, model_set_axis=False).T.reshape((depth, height, width)) + + + # Plot the fit along a couple of pixels + def plotramp(ax, t, image, best_fit, row, col): + ax.plot(t, image[:, row, col], '.', label=f'data pixel {row},{col}') + ax.plot(t, best_fit[:, row, col], '-', label=f'fit to pixel {row},{col}') + ax.set(xlabel='Time', ylabel='Counts') + ax.legend(loc='upper left') + + + fig, ax = plt.subplots(figsize=(10, 5)) + plotramp(ax, t, image, best_fit, 1, 1) + plotramp(ax, t, image, best_fit, 3, 2) + plt.show() diff --git a/docs/modeling/fitting.rst b/docs/modeling/fitting.rst index 6de39f165351..1fc7c7b01a47 100644 --- a/docs/modeling/fitting.rst +++ b/docs/modeling/fitting.rst @@ -9,8 +9,8 @@ instance of `~astropy.modeling.FittableModel` as input and modify its users to easily add other fitters. Linear fitting is done using Numpy's `numpy.linalg.lstsq` function. There are -currently two non-linear fitters which use `scipy.optimize.leastsq` and -`scipy.optimize.fmin_slsqp`. +currently non-linear fitters which use `scipy.optimize.leastsq`, +`scipy.optimize.least_squares`, and `scipy.optimize.fmin_slsqp`. The rules for passing input to fitters are: @@ -21,105 +21,173 @@ The rules for passing input to fitters are: argument just as used when evaluating models; this may be required for the fitter to know how to broadcast the input data. - -Fitting examples ----------------- - -- Fitting a polynomial model to multiple data sets simultaneously:: - - >>> from astropy.modeling import models, fitting - >>> import numpy as np - >>> p1 = models.Polynomial1D(3) - >>> p1.c0 = 1 - >>> p1.c1 = 2 - >>> print(p1) - Model: Polynomial1D - Inputs: ('x',) - Outputs: ('y',) - Model set size: 1 - Degree: 3 - Parameters: - c0 c1 c2 c3 - --- --- --- --- - 1.0 2.0 0.0 0.0 - >>> x = np.arange(10) - >>> y = p1(x) - >>> yy = np.array([y, y]) - >>> p2 = models.Polynomial1D(3, n_models=2) - >>> pfit = fitting.LinearLSQFitter() - >>> new_model = pfit(p2, x, yy) - >>> print(new_model) # doctest: +SKIP - Model: Polynomial1D - Inputs: 1 - Outputs: 1 - Model set size: 2 - Degree: 3 - Parameters: - c0 c1 c2 c3 - --- --- ------------------ ----------------- - 1.0 2.0 -5.86673908219e-16 3.61636197841e-17 - 1.0 2.0 -5.86673908219e-16 3.61636197841e-17 - -Fitters support constrained fitting. - -- All fitters support fixed (frozen) parameters through the ``fixed`` argument - to models or setting the `~astropy.modeling.Parameter.fixed` - attribute directly on a parameter. - - For linear fitters, freezing a polynomial coefficient means that a polynomial - without that term will be fitted to the data. For example, fixing ``c0`` in a - polynomial model will fit a polynomial with the zero-th order term missing. - However, the fixed value of the coefficient is used when evaluating the - model:: - - >>> x = np.arange(1, 10, .1) - >>> p1 = models.Polynomial1D(2, c0=[1, 1], c1=[2, 2], c2=[3, 3], - ... n_models=2) - >>> p1 - - >>> y = p1(x, model_set_axis=False) - >>> p1.c0.fixed = True - >>> pfit = fitting.LinearLSQFitter() - >>> new_model = pfit(p1, x, y) - >>> print(new_model) # doctest: +SKIP - Model: Polynomial1D - Inputs: 1 - Outputs: 1 - Model set size: 2 - Degree: 2 - Parameters: - c0 c1 c2 - --- ------------- ------------- - 1.0 2.38641216243 2.96827885742 - 1.0 2.38641216243 2.96827885742 - -- A parameter can be `~astropy.modeling.Parameter.tied` (linked to - another parameter). This can be done in two ways:: - - >>> def tiedfunc(g1): - ... mean = 3 * g1.stddev - ... return mean - >>> g1 = models.Gaussian1D(amplitude=10., mean=3, stddev=.5, - ... tied={'mean': tiedfunc}) - - or:: - - >>> g1 = models.Gaussian1D(amplitude=10., mean=3, stddev=.5) - >>> g1.mean.tied = tiedfunc - -Bounded fitting is supported through the ``bounds`` arguments to models or by -setting `~astropy.modeling.Parameter.min` and `~astropy.modeling.Parameter.max` -attributes on a parameter. Bounds for the -`~astropy.modeling.fitting.LevMarLSQFitter` are always exactly satisfied--if -the value of the parameter is outside the fitting interval, it will be reset to -the value at the bounds. The `~astropy.modeling.fitting.SLSQPLSQFitter` handles -bounds internally. - -- Different fitters support different types of constraints:: - - >>> fitting.LinearLSQFitter.supported_constraints - ['fixed'] - >>> fitting.LevMarLSQFitter.supported_constraints - ['fixed', 'tied', 'bounds'] - >>> fitting.SLSQPLSQFitter.supported_constraints - ['bounds', 'eqcons', 'ineqcons', 'fixed', 'tied'] +* The `~astropy.modeling.fitting.LinearLSQFitter` currently works only with + simple (not compound) models. + +* The current fitters work only with models that have a single output + (including bivariate functions such as + `~astropy.modeling.polynomial.Chebyshev2D` but not compound models that map + ``x, y -> x', y'``). + +* The units of the fitting data and the model parameters are stripped before fitting + so that the underlying ``scipy`` methods can handle this data. One should be aware + of this when fitting data with units as unit conversions will only be performed + initially. These conversions will be performed using the ``equivalencies`` + argument to the fitter combined with the ``model.input_units_equivalencies`` attribute + of the model being fit. + +.. note:: + In general, non-linear fitters do not support fitting to data which contains + non-finite values: ``NaN``, ``Inf``, or ``-Inf``. This is a limitation of the + underlying scipy library. As a consequence, an error will be raised whenever + any non-finite value is present in the data to be fitted. To avoid this error + users should "filter" the non-finite values from their data, for example + when fitting a ``model``, with a ``fitter`` using ``data`` containing non-finite + values one can "filter" these problems as follows for the 1D case:: + + # Filter non-finite values from data + mask = np.isfinite(data) + # Fit model to filtered data + model = fitter(model, x[mask], data[mask]) + + or for the 2D case:: + + # Filter non-finite values from data + mask = np.isfinite(data) + # Fit model to filtered data + model = fitter(model, x[mask], y[mask], data[mask]) + +.. _modeling-getting-started-nonlinear-notes: + +Notes on non-linear fitting +--------------------------- + +There are several non-linear fitters, which rely on several different +optimization algorithms. Which one you should choose will depend on the problem +you are trying to solve. The main recommended non-linear fitters are: + +* :class:`~astropy.modeling.fitting.TRFLSQFitter`, which uses the Trust Region Reflective + (TRF) algorithm that is particularly suitable for large sparse problems with bounds, see + `scipy.optimize.least_squares` for more details. + +* :class:`~astropy.modeling.fitting.DogBoxLSQFitter`, which uses the dogleg algorithm + with rectangular trust regions, typical use case is small problems with bounds. Not + recommended for problems with rank-deficient Jacobian, see `scipy.optimize.least_squares` + for more details. + +* :class:`~astropy.modeling.fitting.LMLSQFitter`, which uses the Levenberg-Marquardt (LM) + algorithm as implemented by `scipy.optimize.least_squares`. Does not handle bounds and/or + sparse Jacobians. Usually the most efficient method for small unconstrained problems. + If a Levenberg-Marquardt algorithm is desired for your problem, it is now recommended that + you use this fitter instead of :class:`~astropy.modeling.fitting.LevMarLSQFitter` as it + makes use of the recommended version of this algorithm in scipy. However, if your problem + makes use of bounds, you should use another non-linear fitter instead such as + :class:`~astropy.modeling.fitting.TRFLSQFitter` or :class:`~astropy.modeling.fitting.DogBoxLSQFitter` + +Note that the :class:`~astropy.modeling.fitting.LevMarLSQFitter` fitter, which +uses the Levenberg-Marquardt algorithm via the scipy legacy function +`scipy.optimize.leastsq`, is no longer recommended. This fitter supports +parameter bounds via an unsophisticated min/max condition whereby during each +step of the fitting, parameters that are out of bounds are simply reset to the +min or max of the bounds. This can cause parameters to "stick" to one of the +bounds if during the fitting process the parameter gets close to the bound. If +the models you are fitting make use of bounds, you should make use of one of the +other fitters such as :class:`~astropy.modeling.fitting.TRFLSQFitter` or +:class:`~astropy.modeling.fitting.DogBoxLSQFitter`, and if you do not need +bounds and specifically want to use the Levenberg-Marquardt algorithm, you +should use :class:`~astropy.modeling.fitting.LMLSQFitter`. + +.. _modeling-getting-started-1d-fitting: + +Simple 1-D model fitting +------------------------ + +In this section, we look at a simple example of fitting a Gaussian to a +simulated dataset. We use the `~astropy.modeling.functional_models.Gaussian1D` +and `~astropy.modeling.functional_models.Trapezoid1D` models and the +`~astropy.modeling.fitting.TRFLSQFitter` fitter to fit the data: + +.. plot:: + :include-source: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling import models, fitting + + # Generate fake data + rng = np.random.default_rng(0) + x = np.linspace(-5., 5., 200) + y = 3 * np.exp(-0.5 * (x - 1.3)**2 / 0.8**2) + y += rng.normal(0., 0.2, x.shape) + + # Fit the data using a box model. + # Bounds are not really needed but included here to demonstrate usage. + t_init = models.Trapezoid1D(amplitude=1., x_0=0., width=1., slope=0.5, + bounds={"x_0": (-5., 5.)}) + fit_t = fitting.TRFLSQFitter() + t = fit_t(t_init, x, y, maxiter=200) + + # Fit the data using a Gaussian + g_init = models.Gaussian1D(amplitude=1., mean=0, stddev=1.) + fit_g = fitting.TRFLSQFitter() + g = fit_g(g_init, x, y) + + # Plot the data with the best-fit model + fig, ax = plt.subplots(figsize=(8, 5)) + ax.plot(x, y, 'ko') + ax.plot(x, t(x), label='Trapezoid') + ax.plot(x, g(x), label='Gaussian') + ax.set(xlabel='Position', ylabel='Flux') + ax.legend(loc=2) + +As shown above, once instantiated, the fitter class can be used as a function +that takes the initial model (``t_init`` or ``g_init``) and the data values +(``x`` and ``y``), and returns a fitted model (``t`` or ``g``). + +.. _modeling-getting-started-2d-fitting: + +Simple 2-D model fitting +------------------------ + +Similarly to the 1-D example, we can create a simulated 2-D data dataset, and +fit a polynomial model to it. This could be used for example to fit the +background in an image. + +.. plot:: + :include-source: + + import warnings + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling import models, fitting + from astropy.utils.exceptions import AstropyUserWarning + + # Generate fake data + rng = np.random.default_rng(0) + y, x = np.mgrid[:128, :128] + z = 2. * x ** 2 - 0.5 * x ** 2 + 1.5 * x * y - 1. + z += rng.normal(0., 0.1, z.shape) * 50000. + + # Fit the data using astropy.modeling + p_init = models.Polynomial2D(degree=2) + fit_p = fitting.LMLSQFitter() + + with warnings.catch_warnings(): + # Ignore model linearity warning from the fitter + warnings.filterwarnings('ignore', message='Model is linear in parameters', + category=AstropyUserWarning) + p = fit_p(p_init, x, y, z) + + # Plot the data with the best-fit model + fig, axs = plt.subplots(figsize=(8, 2.5), ncols=3) + ax1 = axs[0] + ax1.imshow(z, origin='lower', interpolation='nearest', vmin=-1e4, vmax=5e4) + ax1.set_title("Data") + ax2 = axs[1] + ax2.imshow(p(x, y), origin='lower', interpolation='nearest', vmin=-1e4, + vmax=5e4) + ax2.set_title("Model") + ax3 = axs[2] + ax3.imshow(z - p(x, y), origin='lower', interpolation='nearest', vmin=-1e4, + vmax=5e4) + ax3.set_title("Residual") diff --git a/docs/modeling/index.rst b/docs/modeling/index.rst index 3bdc667a1600..77b5aaec24d9 100644 --- a/docs/modeling/index.rst +++ b/docs/modeling/index.rst @@ -10,337 +10,129 @@ Introduction ============ `astropy.modeling` provides a framework for representing models and performing -model evaluation and fitting. It currently supports 1-D and 2-D models and -fitting with parameter constraints. +model evaluation and fitting. A number of predefined 1-D and 2-D models are +provided and the capability for custom, user defined models is supported. +Different fitting algorithms can be used with any model. For those fitters +with the capabilities fitting can be done using uncertainties, parameters with +bounds, and priors. -It is designed to be easily extensible and flexible. Models do not reference -fitting algorithms explicitly and new fitting algorithms may be added without -changing the existing models (though not all models can be used with all -fitting algorithms due to constraints such as model linearity). +.. _modeling-using: -The goal is to eventually provide a rich toolset of models and fitters such -that most users will not need to define new model classes, nor special purpose -fitting routines (while making it reasonably easy to do when necessary). +Using Modeling +============== -.. note:: +.. toctree:: + :maxdepth: 2 - `astropy.modeling` is currently a work-in-progress, and thus it is likely - there will still be API changes in later versions of Astropy. Backwards - compatibility support between versions will still be maintained as much as - possible, but new features and enhancements are coming in future versions. - If you have specific ideas for how it might be improved, feel free to let - us know on the `astropy-dev mailing list`_ or at - http://feedback.astropy.org + Models + Compound Models + Model Parameters + Fitting + Using Units with Models and Fitting -Getting started -=============== +.. _getting-started-example: + +A Simple Example +================ + +This simple example illustrates defining a model, +calculating values based on input x values, and using fitting data with a model. + + .. plot:: + :include-source: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling import models, fitting + + # define a model for a line + line_orig = models.Linear1D(slope=1.0, intercept=0.5) + + # generate x, y data non-uniformly spaced in x + # add noise to y measurements + npts = 30 + rng = np.random.default_rng(10) + x = rng.uniform(0.0, 10.0, npts) + y = line_orig(x) + y += rng.normal(0.0, 1.5, npts) -The examples here use the predefined models and assume the following modules -have been imported:: + # initialize a linear fitter + fit = fitting.LinearLSQFitter() - >>> import numpy as np - >>> from astropy.modeling import models, fitting + # initialize a linear model + line_init = models.Linear1D() + # fit the data with the fitter + fitted_line = fit(line_init, x, y) -Using Models ------------- + # plot the model + fig, ax = plt.subplots() + ax.plot(x, y, 'ko', label='Data') + ax.plot(x, fitted_line(x), 'k-', label='Fitted Model') + ax.set(xlabel='x', ylabel='y') + ax.legend() -The `astropy.modeling` package defines a number of models that are collected -under a single namespace as ``astropy.modeling.models``. Models behave like -parametrized functions:: +.. _advanced_topics: - >>> from astropy.modeling import models - >>> g = models.Gaussian1D(amplitude=1.2, mean=0.9, stddev=0.5) - >>> print(g) - Model: Gaussian1D - Inputs: ('x',) - Outputs: ('y',) - Model set size: 1 - Parameters: - amplitude mean stddev - --------- ---- ------ - 1.2 0.9 0.5 +Advanced Topics +=============== + +.. toctree:: + :maxdepth: 2 + + Performance Tips + Extending Models + Extending Fitters + Adding support for units to models + Joint Fitting + Parallel Fitting -Model parameters can be accessed as attributes:: - >>> g.amplitude - Parameter('amplitude', value=1.2) - >>> g.mean - Parameter('mean', value=0.9) - >>> g.stddev - Parameter('stddev', value=0.5) - -and can also be updated via those attributes:: - - >>> g.amplitude = 0.8 - >>> g.amplitude - Parameter('amplitude', value=0.8) - -Models can be evaluated by calling them as functions:: - - >>> g(0.1) - 0.22242984036255528 - >>> g(np.linspace(0.5, 1.5, 7)) - array([ 0.58091923, 0.71746405, 0.7929204 , 0.78415894, 0.69394278, - 0.54952605, 0.3894018 ]) - -As the above example demonstrates, in general most models evaluate array-like -inputs according to the standard `Numpy broadcasting rules`_ for arrays. - -Models can therefore already be useful to evaluate common functions, -independently of the fitting features of the package. - - -Simple 1-D model fitting ------------------------- - -In this section, we look at a simple example of fitting a Gaussian to a -simulated dataset. We use the `~astropy.modeling.functional_models.Gaussian1D` -and `~astropy.modeling.functional_models.Trapezoid1D` models and the -`~astropy.modeling.fitting.LevMarLSQFitter` fitter to fit the data: - -.. plot:: - :include-source: - - import numpy as np - from astropy.modeling import models, fitting - - # Generate fake data - np.random.seed(0) - x = np.linspace(-5., 5., 200) - y = 3 * np.exp(-0.5 * (x - 1.3)**2 / 0.8**2) - y += np.random.normal(0., 0.2, x.shape) - - # Fit the data using a box model - t_init = models.Trapezoid1D(amplitude=1., x_0=0., width=1., slope=0.5) - fit_t = fitting.LevMarLSQFitter() - t = fit_t(t_init, x, y) - - # Fit the data using a Gaussian - g_init = models.Gaussian1D(amplitude=1., mean=0, stddev=1.) - fit_g = fitting.LevMarLSQFitter() - g = fit_g(g_init, x, y) - - # Plot the data with the best-fit model - plt.figure(figsize=(8,5)) - plt.plot(x, y, 'ko') - plt.plot(x, t(x), 'b-', lw=2, label='Trapezoid') - plt.plot(x, g(x), 'r-', lw=2, label='Gaussian') - plt.xlabel('Position') - plt.ylabel('Flux') - plt.legend(loc=2) - -As shown above, once instantiated, the fitter class can be used as a function -that takes the initial model (``t_init`` or ``g_init``) and the data values -(``x`` and ``y``), and returns a fitted model (``t`` or ``g``). - - -Simple 2-D model fitting ------------------------- - -Similarly to the 1-D example, we can create a simulated 2-D data dataset, and -fit a polynomial model to it. This could be used for example to fit the -background in an image. - -.. plot:: - :include-source: - - import warnings - import numpy as np - from astropy.modeling import models, fitting - - # Generate fake data - np.random.seed(0) - y, x = np.mgrid[:128, :128] - z = 2. * x ** 2 - 0.5 * x ** 2 + 1.5 * x * y - 1. - z += np.random.normal(0., 0.1, z.shape) * 50000. - - # Fit the data using astropy.modeling - p_init = models.Polynomial2D(degree=2) - fit_p = fitting.LevMarLSQFitter() - - with warnings.catch_warnings(): - # Ignore model linearity warning from the fitter - warnings.simplefilter('ignore') - p = fit_p(p_init, x, y, z) - - # Plot the data with the best-fit model - plt.figure(figsize=(8,2.5)) - plt.subplot(1,3,1) - plt.imshow(z, origin='lower', interpolation='nearest', vmin=-1e4, vmax=5e4) - plt.title("Data") - plt.subplot(1,3,2) - plt.imshow(p(x, y), origin='lower', interpolation='nearest', vmin=-1e4, - vmax=5e4) - plt.title("Model") - plt.subplot(1,3,3) - plt.imshow(z - p(x, y), origin='lower', interpolation='nearest', vmin=-1e4, - vmax=5e4) - plt.title("Residual") - -A list of models is provided in the `Reference/API`_ section. The fitting -framework includes many useful features that are not demonstrated here, such as -weighting of datapoints, fixing or linking parameters, and placing lower or -upper limits on parameters. For more information on these, take a look at the -:doc:`fitting` documentation. - - -Model sets ----------- - -In some cases it is necessary to describe many models of the same type but with -different sets of parameter values. This could be done simply by instantiating -as many instances of a `~astropy.modeling.Model` as are needed. But that can -be inefficient for a large number of models. To that end, all model classes in -`astropy.modeling` can also be used to represent a model *set* which is a -collection of models of the same type, but with different values for their -parameters. - -To instantiate a model set, use argument ``n_models=N`` where ``N`` is the -number of models in the set when constructing the model. The value of each -parameter must be a list or array of length ``N``, such that each item in -the array corresponds to one model in the set:: - - >>> g = models.Gaussian1D(amplitude=[1, 2], mean=[0, 0], - ... stddev=[0.1, 0.2], n_models=2) - >>> print(g) - Model: Gaussian1D - Inputs: ('x',) - Outputs: ('y',) - Model set size: 2 - Parameters: - amplitude mean stddev - --------- ---- ------ - 1.0 0.0 0.1 - 2.0 0.0 0.2 - -This is equivalent to two Gaussians with the parameters ``amplitude=1, mean=0, -stddev=0.1`` and ``amplitude=2, mean=0, stddev=0.2`` respectively. When -printing the model the parameter values are displayed as a table, with each row -corresponding to a single model in the set. - -The number of models in a model set can be determined using the `len` builtin:: - - >>> len(g) - 2 - -Single models have a length of 1, and are not considered a model set as such. - -When evaluating a model set, by default the input must be the same length as -the number of models, with one input per model:: - - >>> g([0, 0.1]) - array([ 1. , 1.76499381]) - -The result is an array with one result per model in the set. It is also -possible to broadcast a single value to all models in the set:: - - >>> g(0) - array([ 1., 2.]) - -Model sets are used primarily for fitting, allowing a large number of models of -the same type to be fitted simultaneously (and independently from each other) -to some large set of inputs. For example, fitting a polynomial to the time -response of each pixel in a data cube. This can greatly speed up the fitting -process, especially for linear models. - - -.. _compound-models-intro: - -Compound models ---------------- -.. versionadded:: 1.0 - - This feature is experimental and expected to see significant further - development, but the basic usage is stable and expected to see wide use. - -While the Astropy modeling package makes it very easy to define :doc:`new -models ` either from existing functions, or by writing a -`~astropy.modeling.Model` subclass, an additional way to create new models is -by combining them using arithmetic expressions. This works with models built -into Astropy, and most user-defined models as well. For example, it is -possible to create a superposition of two Gaussians like so:: - - >>> from astropy.modeling import models - >>> g1 = models.Gaussian1D(1, 0, 0.2) - >>> g2 = models.Gaussian1D(2.5, 0.5, 0.1) - >>> g1_plus_2 = g1 + g2 - -The resulting object ``g1_plus_2`` is itself a new model. Evaluating, say, -``g1_plus_2(0.25)`` is the same as evaluating ``g1(0.25) + g2(0.25)``:: - - >>> g1_plus_2(0.25) # doctest: +FLOAT_CMP - 0.5676756958301329 - >>> g1_plus_2(0.25) == g1(0.25) + g2(0.25) - True - -This model can be further combined with other models in new expressions. It is -also possible to define entire new model *classes* using arithmetic expressions -of other model classes. This allows general compound models to be created -without specifying any parameter values up front. This more advanced usage is -explained in more detail in the :ref:`compound model documentation -`. - -These new compound models can also be fitted to data, like most other models: - -.. plot:: - :include-source: - - import numpy as np - from astropy.modeling import models, fitting - - # Generate fake data - np.random.seed(42) - g1 = models.Gaussian1D(1, 0, 0.2) - g2 = models.Gaussian1D(2.5, 0.5, 0.1) - x = np.linspace(-1, 1, 200) - y = g1(x) + g2(x) + np.random.normal(0., 0.2, x.shape) +Pre-Defined Models +================== - # Now to fit the data create a new superposition with initial - # guesses for the parameters: - gg_init = models.Gaussian1D(1, 0, 0.1) + models.Gaussian1D(2, 0.5, 0.1) - fitter = fitting.SLSQPLSQFitter() - gg_fit = fitter(gg_init, x, y) - - # Plot the data with the best-fit model - plt.figure(figsize=(8,5)) - plt.plot(x, y, 'ko') - plt.plot(x, gg_fit(x), 'r-', lw=2) - plt.xlabel('Position') - plt.ylabel('Flux') +.. To be expanded to include all pre-defined models -This works for 1-D models, 2-D models, and combinations thereof, though there -are some complexities involved in correctly matching up the inputs and outputs -of all models used to build a compound model. You can learn more details in -the :doc:`compound-models` documentation. +Some of the pre-defined models are listed and illustrated. +.. toctree:: + :maxdepth: 2 + + 1D Models + 2D Models + Physical Models + Polynomial Models + Powerlaw Models + Spline Models -Using `astropy.modeling` -======================== +Examples +======== .. toctree:: - :maxdepth: 1 + :maxdepth: 2 - models - parameters - fitting - compound-models - new - algorithms + Fitting a line + example-fitting-constraints + example-fitting-model-sets + +.. TODO list + fitting with masks + fitting with priors + fitting with units + defining 1d model + defining 2d model + fitting 2d model + defining and using a WCS/gWCS model + defining and using a Tabular1D model + statistics functions and how to make your own + compound models Reference/API ============= -.. automodapi:: astropy.modeling -.. automodapi:: astropy.modeling.mappings -.. automodapi:: astropy.modeling.functional_models -.. automodapi:: astropy.modeling.powerlaws -.. automodapi:: astropy.modeling.polynomial -.. automodapi:: astropy.modeling.projections -.. automodapi:: astropy.modeling.rotations -.. automodapi:: astropy.modeling.fitting -.. automodapi:: astropy.modeling.optimizers -.. automodapi:: astropy.modeling.statistic +.. toctree:: + :maxdepth: 2 + reference_api diff --git a/docs/modeling/jointfitter.rst b/docs/modeling/jointfitter.rst new file mode 100644 index 000000000000..12ca0e857d11 --- /dev/null +++ b/docs/modeling/jointfitter.rst @@ -0,0 +1,231 @@ +.. _jointfitter: + +JointFitter +=========== + +There are cases where one may wish to fit multiple datasets with models that +share parameters. This is possible with the +`astropy.modeling.fitting.JointFitter`. Basically, this fitter is +setup with a list of defined models, the parameters in common between the +different models, and the initial values for those parameters. Then the fitter +is called supplying as many x and y arrays, one for each model to be fit. The +fit parameters are the result of the jointly fitting the models to the +combined datasets. + +.. note:: + The JointFitter uses the scipy.optimize.leastsq. In addition, it + does not support fixed, bounded, or tied parameters at this time. + +Example: Spectral Line +====================== + +This example is for two spectral segments with different spectral resolutions +that have the same spectral line in the wavelength region that is overlapping +between both segments. + +We will need to define a Gaussian function that has mean wavelength, area, and +width parameters. This is needed as the `astropy.modeling.functional_models.Gaussian1D` +function has mean wavelength, central intensity, and width parameters, but the +central intensity of a line will be different at different spectral resolutions, +but the area will be the same. + +First, imports needed for this example + + >>> # imports + >>> import math + >>> import numpy as np + >>> from astropy.modeling import fitting, Fittable1DModel + >>> from astropy.modeling.parameters import Parameter + >>> from astropy.modeling.functional_models import FLOAT_EPSILON + +Now define AreaGaussian1D with area instead of intensity as a parameter. +This new is modified and trimmed version of the standard Gaussian1D model. + + >>> class AreaGaussian1D(Fittable1DModel): + ... """ + ... One dimensional Gaussian model with area as a parameter. + ... + ... Parameters + ... ---------- + ... area : float or `~astropy.units.Quantity`. + ... Integrated area + ... Note: amplitude = area / (stddev * np.sqrt(2 * np.pi)) + ... mean : float or `~astropy.units.Quantity`. + ... Mean of the Gaussian. + ... stddev : float or `~astropy.units.Quantity`. + ... Standard deviation of the Gaussian with FWHM = 2 * stddev * np.sqrt(2 * np.log(2)). + ... """ + ... area = Parameter(default=1) + ... mean = Parameter(default=0) + ... + ... # Ensure stddev makes sense if its bounds are not explicitly set. + ... # stddev must be non-zero and positive. + ... stddev = Parameter(default=1, bounds=(FLOAT_EPSILON, None)) + ... + ... @staticmethod + ... def evaluate(x, area, mean, stddev): + ... """ + ... AreaGaussian1D model function. + ... """ + ... return (area / (stddev * np.sqrt(2 * np.pi))) * np.exp( + ... -0.5 * (x - mean) ** 2 / stddev ** 2 + ... ) + +Data to be fit is simulated. The 1st spectral segment will have a spectral +resolution that is a factor of 2 higher than the second segment. The first +segment will have wavelengths from 1 to 6 and the second from 4 to 10 giving +an overlapping wavelength region from 4 to 6. + + >>> # Generate fake data + >>> mean = 5.1 + >>> sigma1 = 0.2 + >>> sigma2 = 0.4 + >>> noise = 0.10 + + >>> # compute the central amplitudes so the lines in each segment have the + >>> # same area + >>> area = 1.5 + >>> amp1 = area / np.sqrt(2.0 * math.pi * sigma1 ** 2) + >>> amp2 = area / np.sqrt(2.0 * math.pi * sigma2 ** 2) + + >>> # segment 1 + >>> rng = np.random.default_rng(147) + >>> x1 = np.linspace(1.0, 6.0, 200) + >>> y1 = amp1 * np.exp(-0.5 * (x1 - mean) ** 2 / sigma1 ** 2) + >>> y1 += rng.normal(0.0, noise, x1.shape) + + >>> # segment 2 + >>> x2 = np.linspace(4.0, 10.0, 200) + >>> y2 = amp2 * np.exp(-0.5 * (x2 - mean) ** 2 / sigma2 ** 2) + >>> y2 += rng.normal(0.0, noise, x2.shape) + +Now define the models to be fit and fitter to use. Then fit the two simulated +datasets. + + >>> # define the two models to be fit + >>> gjf1 = AreaGaussian1D(area=1.0, mean=5.0, stddev=1.0) + >>> gjf2 = AreaGaussian1D(area=1.0, mean=5.0, stddev=1.0) + +.. doctest-requires:: scipy + + >>> # define the jointfitter specifying the parameters in common and their initial values + >>> fit_joint = fitting.JointFitter( + ... [gjf1, gjf2], {gjf1: ["area", "mean"], gjf2: ["area", "mean"]}, [1.0, 5.0] + ... ) + >>> + >>> # perform the fit + >>> g12 = fit_joint(x1, y1, x2, y2) + + +The resulting fit parameters show that the area and mean wavelength of the +two AreaGaussian1D models are exactly the same while the width (stddev) is +different reflecting the different spectral resolutions of the two segments. + +AreaGaussian1 parameters + +.. doctest-requires:: scipy + + >>> print(gjf1.param_names) + ('area', 'mean', 'stddev') + >>> print(gjf1.parameters) + [1.49823951 5.10494811 0.19918164] + +AreaGaussian2 parameters + +.. doctest-requires:: scipy + + >>> print(gjf1.param_names) + ('area', 'mean', 'stddev') + >>> print(gjf2.parameters) + [1.49823951 5.10494811 0.39860539] + + +The simulated data and best fit models can be plotted showing good agreement +between the two AreaGaussian1D models and the two spectral segments. + +.. plot:: + + # imports + import numpy as np + import math + import matplotlib.pyplot as plt + from astropy.modeling import fitting, Fittable1DModel + from astropy.modeling.parameters import Parameter + from astropy.modeling.functional_models import FLOAT_EPSILON + + + class AreaGaussian1D(Fittable1DModel): + """ + One dimensional Gaussian model with area as a parameter. + + Parameters + ---------- + area : float or `~astropy.units.Quantity`. + Integrated area + Note: amplitude = area / (stddev * np.sqrt(2 * np.pi)) + mean : float or `~astropy.units.Quantity`. + Mean of the Gaussian. + stddev : float or `~astropy.units.Quantity`. + Standard deviation of the Gaussian with FWHM = 2 * stddev * np.sqrt(2 * np.log(2)). + """ + + area = Parameter(default=1) + mean = Parameter(default=0) + + # Ensure stddev makes sense if its bounds are not explicitly set. + # stddev must be non-zero and positive. + stddev = Parameter(default=1, bounds=(FLOAT_EPSILON, None)) + + @staticmethod + def evaluate(x, area, mean, stddev): + """ + AreaGaussian1D model function. + """ + return (area / (stddev * np.sqrt(2 * np.pi))) * np.exp( + -0.5 * (x - mean) ** 2 / stddev ** 2 + ) + + + # Generate fake data + mean = 5.1 + sigma1 = 0.2 + sigma2 = 0.4 + noise = 0.10 + + # compute the central amplitudes so the lines in each segment have the + # same area + area = 1.5 + amp1 = area / np.sqrt(2.0 * math.pi * sigma1 ** 2) + amp2 = area / np.sqrt(2.0 * math.pi * sigma2 ** 2) + + # segment 1 + rng = np.random.default_rng(147) + x1 = np.linspace(1.0, 6.0, 200) + y1 = amp1 * np.exp(-0.5 * (x1 - mean) ** 2 / sigma1 ** 2) + y1 += rng.normal(0.0, noise, x1.shape) + + # segment 2 + x2 = np.linspace(4.0, 10.0, 200) + y2 = amp2 * np.exp(-0.5 * (x2 - mean) ** 2 / sigma2 ** 2) + y2 += rng.normal(0.0, noise, x2.shape) + + # define the two models to be fit + gjf1 = AreaGaussian1D(area=1.0, mean=5.0, stddev=1.0) + gjf2 = AreaGaussian1D(area=1.0, mean=5.0, stddev=1.0) + + # define the jointfitter specifying the parameters in common and their initial values + fit_joint = fitting.JointFitter( + [gjf1, gjf2], {gjf1: ["area", "mean"], gjf2: ["area", "mean"]}, [1.0, 5.0] + ) + + # perform the fit + g12 = fit_joint(x1, y1, x2, y2) + + # Plot the data with the best-fit models + fig, ax = plt.subplots(figsize=(8, 5)) + ax.plot(x1, y1, "bo", alpha=0.25) + ax.plot(x2, y2, "go", alpha=0.25) + ax.plot(x1, gjf1(x1), "b--", label="AreaGaussian1") + ax.plot(x2, gjf2(x2), "g--", label="AreaGaussian2") + ax.set(xlabel="Wavelength", ylabel="Flux") + ax.legend(loc=2) diff --git a/docs/modeling/links.inc b/docs/modeling/links.inc index a4ce3d42d3f0..93a2cb076e8b 100644 --- a/docs/modeling/links.inc +++ b/docs/modeling/links.inc @@ -1 +1,4 @@ -.. _Numpy broadcasting rules: http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html +.. _Numpy broadcasting rules: https://numpy.org/doc/stable/user/basics.broadcasting.html +.. _Generalized World Coordinate System (GWCS): https://gwcs.readthedocs.io/en/latest/ +.. _ASDF: https://asdf-standard.readthedocs.io/en/latest/ +.. _SIP: https://fits.gsfc.nasa.gov/registry/sip.html diff --git a/docs/modeling/models.rst b/docs/modeling/models.rst index a33955082406..5cb2466c0813 100644 --- a/docs/modeling/models.rst +++ b/docs/modeling/models.rst @@ -1,318 +1,969 @@ .. include:: links.inc + +.. _models: + +****** +Models +****** + +.. _basics-models: + +Basics +====== + +The `astropy.modeling` package defines a number of models that are collected +under a single namespace as ``astropy.modeling.models``. Models behave like +parametrized functions:: + + >>> import numpy as np + >>> from astropy.modeling import models + >>> g = models.Gaussian1D(amplitude=1.2, mean=0.9, stddev=0.5) + >>> print(g) + Model: Gaussian1D + Inputs: ('x',) + Outputs: ('y',) + Model set size: 1 + Parameters: + amplitude mean stddev + --------- ---- ------ + 1.2 0.9 0.5 + +Model parameters can be accessed as attributes:: + + >>> g.amplitude + Parameter('amplitude', value=1.2) + >>> g.mean + Parameter('mean', value=0.9) + >>> g.stddev # doctest: +FLOAT_CMP + Parameter('stddev', value=0.5, bounds=(1.1754943508222875e-38, None)) + +and can also be updated via those attributes:: + + >>> g.amplitude = 0.8 + >>> g.amplitude + Parameter('amplitude', value=0.8) + +Models can be evaluated by calling them as functions:: + + >>> g(0.1) + 0.22242984036255528 + >>> g(np.linspace(0.5, 1.5, 7)) # doctest: +FLOAT_CMP + array([0.58091923, 0.71746405, 0.7929204 , 0.78415894, 0.69394278, + 0.54952605, 0.3894018 ]) + +As the above example demonstrates, in general most models evaluate array-like +inputs according to the standard `Numpy broadcasting rules`_ for arrays. +Models can therefore already be useful to evaluate common functions, +independently of the fitting features of the package. + .. _modeling-instantiating: -*********************************** -Instantiating and Evaluating Models -*********************************** -The base class of all models is `~astropy.modeling.Model`, however -fittable models should subclass `~astropy.modeling.FittableModel`. -Fittable models can be linear or nonlinear in a regression analysis sense. +Instantiating and Evaluating Models +----------------------------------- -In general models are instantiated by providing the parameter values that +In general, models are instantiated by supplying the parameter values that define that instance of the model to the constructor, as demonstrated in the section on :ref:`modeling-parameters`. Additionally, a `~astropy.modeling.Model` instance may represent a single model -with one set of parameters, or a model *set* consisting of a set of parameters -each representing a different parameterization of the same parametric model. -For example one may instantiate a single Gaussian model with one mean, standard -deviation, and amplitude. Or one may create a set of N Gaussians, each one of -which would be fitted to, for example, a different plane in an image cube. +with one set of parameters, or a :ref:`Model set ` consisting +of a set of parameters each representing a different parameterization of the same +parametric model. For example, you may instantiate a single Gaussian model +with one mean, standard deviation, and amplitude. Or you may create a set +of N Gaussians, each one of which would be evaluated on, for example, a +different plane in an image cube. -Regardless of whether using a single model, or a model set, parameter values -may be scalar values, or arrays of any size and shape, so long as they are -compatible according to the standard `Numpy broadcasting rules`_. For example, -a model may be instantiated with all scalar parameters:: +For example, a single Gaussian model may be instantiated with all scalar parameters:: >>> from astropy.modeling.models import Gaussian1D >>> g = Gaussian1D(amplitude=1, mean=0, stddev=1) - >>> g - + >>> g # doctest: +FLOAT_CMP + -Or it may use all array parameters. For example if all parameters are 2x2 -arrays the model is computed element-wise using all elements in the arrays:: +The newly created model instance ``g`` now works like a Gaussian function +with the specific parameters. It takes a single input:: - >>> g = Gaussian1D(amplitude=[[1, 2], [3, 4]], mean=[[0, 1], [1, 0]], - ... stddev=[[0.1, 0.2], [0.3, 0.4]]) - >>> g - - >>> g(0) - array([[ 1.00000000e+00, 7.45330634e-06], - [ 1.15977604e-02, 4.00000000e+00]]) + >>> g.inputs + ('x',) + >>> g(x=0) + 1.0 -Or it may even use a mix of scalar values and arrays of different sizes and -dimensions so long as they are compatible:: +The model can also be called without explicitly using keyword arguments:: - >>> g = Gaussian1D(amplitude=[[1, 2], [3, 4]], mean=0.1, stddev=[0.1, 0.2]) >>> g(0) - array([[ 0.60653066, 1.76499381], - [ 1.81959198, 3.52998761]]) + 1.0 -In this case, four values are computed--one using each element of the amplitude -array. Each model uses a mean of 0.1, and a standard deviation of 0.1 is -used with the amplitudes of 1 and 3, and 0.2 is used with amplitudes 2 and 4. +Or a set of Gaussians may be instantiated by passing multiple parameter values:: -If any of the parameters have incompatible values this will result in an -error:: - - >>> g = Gaussian1D(amplitude=1, mean=[1, 2], stddev=[1, 2, 3]) + >>> from astropy.modeling.models import Gaussian1D + >>> gset = Gaussian1D(amplitude=[1, 1.5, 2], + ... mean=[0, 1, 2], + ... stddev=[1., 1., 1.], + ... n_models=3) + >>> print(gset) # doctest: +FLOAT_CMP + Model: Gaussian1D + Inputs: ('x',) + Outputs: ('y',) + Model set size: 3 + Parameters: + amplitude mean stddev + --------- ---- ------ + 1.0 0.0 1.0 + 1.5 1.0 1.0 + 2.0 2.0 1.0 + +This model also works like a Gaussian function. The three models in +the model set can be evaluated on the same input:: + + >>> gset(1.) + array([0.60653066, 1.5 , 1.21306132]) + +or on ``N=3`` inputs:: + + >>> gset([1, 2, 3]) + array([0.60653066, 0.90979599, 1.21306132]) + +For a comprehensive example of fitting a model set see :ref:`example-fitting-model-sets`. + +Model inverses +-------------- + +All models have a `Model.inverse ` property +which may, for some models, return a new model that is the analytic inverse of +the model it is attached to. For example:: + + >>> from astropy.modeling.models import Linear1D + >>> linear = Linear1D(slope=0.8, intercept=1.0) + >>> linear.inverse + + +The inverse of a model will always be a fully instantiated model in its own +right, and so can be evaluated directly like:: + + >>> linear.inverse(2.0) + 1.25 + +It is also possible to assign a *custom* inverse to a model. This may be +useful, for example, in cases where a model does not have an analytic inverse, +but may have an approximate inverse that was computed numerically and is +represented by another model. This works even if the target model has a +default analytic inverse--in this case the default is overridden with the +custom inverse:: + + >>> from astropy.modeling.models import Polynomial1D + >>> linear.inverse = Polynomial1D(degree=1, c0=-1.25, c1=1.25) + >>> linear.inverse + + +If a custom inverse has been assigned to a model, it can be deleted with +``del model.inverse``. This resets the inverse to its default (if one exists). +If a default does not exist, accessing ``model.inverse`` raises a +`NotImplementedError`. For example polynomial models do not have a default +inverse:: + + >>> del linear.inverse + >>> linear.inverse + + >>> p = Polynomial1D(degree=2, c0=1.0, c1=2.0, c2=3.0) + >>> p.inverse Traceback (most recent call last): + File "", line 1, in + File "astropy\modeling\core.py", line 796, in inverse + raise NotImplementedError("An analytical inverse transform has not " + NotImplementedError: No analytical or user-supplied inverse transform + has been implemented for this model. + +One may certainly compute an inverse and assign it to a polynomial model +though. + +.. note:: + + When assigning a custom inverse to a model no validation is performed to + ensure that it is actually an inverse or even approximate inverse. So + assign custom inverses at your own risk. + +Bounding Boxes +-------------- + +.. _bounding-boxes: + +Efficient Model Rendering with Bounding Boxes ++++++++++++++++++++++++++++++++++++++++++++++ + + +All `Model ` subclasses have a +`bounding_box ` attribute that +can be used to set the limits over which the model is significant. This greatly +improves the efficiency of evaluation when the input range is much larger than +the characteristic width of the model itself. For example, to create a sky model +image from a large survey catalog, each source should only be evaluated over the +pixels to which it contributes a significant amount of flux. This task can +otherwise be computationally prohibitive on an average CPU. + +The :func:`Model.render ` method can be used to +evaluate a model on an output array, or input coordinate arrays, limiting the +evaluation to the `bounding_box ` region if +it is set. This function will also produce postage stamp images of the model if +no other input array is passed. To instead extract postage stamps from the data +array itself, see :ref:`cutout_images`. + +Using the standard Bounding Box ++++++++++++++++++++++++++++++++ + +For basic usage, see `Model.bounding_box `. +By default no `~astropy.modeling.Model.bounding_box` is set, except on model +subclasses where a ``bounding_box`` property or method is explicitly defined. +The default is then the minimum rectangular region symmetric about the position +that fully contains the model. If the model does not have a finite extent, +the containment criteria are noted in the documentation. For example, see +``Gaussian2D.bounding_box``. + +.. warning:: + + Accessing the `Model.bounding_box ` + property when it has not been set, or does not have a default will + result in a ``NotImplementedError``. If this behavior is undesirable, + then one can instead use the `Model.get_bounding_box ` + method instead. This method will return the bounding box if one exists + (by setting or default) otherwise it will return ``None`` instead + of raising an error. + +A `Model.bounding_box ` default can be +set by the user to any callable. This is particularly useful for models created +with `~astropy.modeling.custom_model` or as a `~astropy.modeling.core.CompoundModel`:: + + >>> from astropy.modeling import custom_model + >>> def ellipsoid(x, y, z, x0=0, y0=0, z0=0, a=2, b=3, c=4, amp=1): + ... rsq = ((x - x0) / a) ** 2 + ((y - y0) / b) ** 2 + ((z - z0) / c) ** 2 + ... val = (rsq < 1) * amp + ... return val ... - InputParameterError: Parameter 'mean' of shape (2,) cannot be broadcast - with parameter 'stddev' of shape (3,). All parameter arrays must have - shapes that are mutually compatible according to the broadcasting rules. - -.. _modeling-model-sets: - -Model Sets -========== - -By default, `~astropy.modeling.Model` instances represent a single model. -There are two ways, when instantiating a `~astropy.modeling.Model` instance, to -create a model set instead. The first is to specify the ``n_models`` argument -when instantiating the model:: - - >>> g = Gaussian1D(amplitude=[1, 2], mean=[0, 0], stddev=[0.1, 0.2], - ... n_models=2) - >>> g - - -When specifying some ``n_models=N`` this requires that the parameter values be -arrays of some kind, the first *axis* of which has as length of ``N``. This -axis is referred to as the ``model_set_axis``, and by default is is the ``0th`` -axis of parameter arrays. In this case the parameters were given as 1-D arrays -of length 2. The values ``amplitude=1, mean=0, stddev=0.1`` are the parameters -for the first model in the set. The values ``amplitude=2, mean=0, stddev=0.2`` -are the parameters defining the second model in the set. - -This has different semantics from simply using array values for the parameters, -in that ensures that parameter values and input values are matched up according -to the model_set_axis before any other array broadcasting rules are applied. - -For example, in the previous section we created a model with array values -like:: - - >>> g = Gaussian1D(amplitude=[[1, 2], [3, 4]], mean=0.1, stddev=[0.1, 0.2]) - -If instead we treat the rows as values for two different model sets, this -particular instantiation will fail, since only one value is given for mean:: - - >>> g = Gaussian1D(amplitude=[[1, 2], [3, 4]], mean=0.1, stddev=[0.1, 0.2], - ... n_models=2) - Traceback (most recent call last): + >>> class Ellipsoid3D(custom_model(ellipsoid)): + ... # A 3D ellipsoid model + ... def bounding_box(self): + ... return ((self.z0 - self.c, self.z0 + self.c), + ... (self.y0 - self.b, self.y0 + self.b), + ... (self.x0 - self.a, self.x0 + self.a)) ... - InputParameterError: All parameter values must be arrays of dimension at - least 1 for model_set_axis=0 (the value given for 'mean' is only - 0-dimensional) - -To get around this for now, provide two values for mean:: - - >>> g = Gaussian1D(amplitude=[[1, 2], [3, 4]], mean=[0.1, 0.1], - ... stddev=[0.1, 0.2], n_models=2) - -This is different from the case without ``n_models=2``. It does not mean that -the value of amplitude is a 2x2 array. Rather, it means there are *two* values -for amplitude (one for each model in the set), each of which is 1-D array of -length 2. The value for the first model is ``[1, 2]``, and the value for the -second model is ``[3, 4]``. Likewise, scalar values are given for the mean and -standard deviation of each model in the set. - -When evaluating this model on a single input we get a different result from the -single-model case:: - - >>> g(0) - array([[ 0.60653066, 1.21306132], - [ 2.64749071, 3.52998761]]) + >>> model1 = Ellipsoid3D() + >>> model1.bounding_box + ModelBoundingBox( + intervals={ + x0: Interval(lower=-2.0, upper=2.0) + x1: Interval(lower=-3.0, upper=3.0) + x2: Interval(lower=-4.0, upper=4.0) + } + model=Ellipsoid3D(inputs=('x0', 'x1', 'x2')) + order='C' + ) + +By default models are evaluated on any inputs. By passing a flag they can be evaluated +only on inputs within the bounding box. For inputs outside of the bounding_box a ``fill_value`` is +returned (``np.nan`` by default):: + + >>> model1(-5, 1, 1) + 0.0 + >>> model1(-5, 1, 1, with_bounding_box=True) + nan + >>> model1(-5, 1, 1, with_bounding_box=True, fill_value=-1) + -1.0 + +`Model.bounding_box ` can be set on any +model instance via the usage of the property setter. For example for a single +input model one needs to only set a tuple of the lower and upper bounds :: + + >>> from astropy.modeling.models import Polynomial1D + >>> model2 = Polynomial1D(2) + >>> model2.bounding_box = (-1, 1) + >>> model2.bounding_box + ModelBoundingBox( + intervals={ + x: Interval(lower=-1, upper=1) + } + model=Polynomial1D(inputs=('x',)) + order='C' + ) + >>> model2(-2) + 0.0 + >>> model2(-2, with_bounding_box=True) + nan + >>> model2(-2, with_bounding_box=True, fill_value=47) + 47.0 + +For multi-input models, `Model.bounding_box ` +can be set on any model instance by specifying a tuple of lower/upper bound tuples :: + + >>> from astropy.modeling.models import Polynomial2D + >>> model3 = Polynomial2D(2) + >>> model3.bounding_box = ((-2, 2), (-1, 1)) + >>> model3.bounding_box + ModelBoundingBox( + intervals={ + x: Interval(lower=-1, upper=1) + y: Interval(lower=-2, upper=2) + } + model=Polynomial2D(inputs=('x', 'y')) + order='C' + ) + >>> model3(-2, 0) + 0.0 + >>> model3(-2, 0, with_bounding_box=True) + nan + >>> model3(-2, 0, with_bounding_box=True, fill_value=7) + 7.0 + +Note that if one wants to directly recover the tuple used to formulate +a bounding box, then one can use the +`ModelBoundingBox.bounding_box() ` +method :: + + >>> model1.bounding_box.bounding_box() + ((np.float64(-4.0), np.float64(4.0)), (np.float64(-3.0), np.float64(3.0)), (np.float64(-2.0), np.float64(2.0))) + >>> model2.bounding_box.bounding_box() + (-1, 1) + >>> model3.bounding_box.bounding_box() + ((-2, 2), (-1, 1)) + +.. warning:: + + When setting multi-dimensional bounding boxes it is important to + remember that by default the tuple of tuples is assumed to be ``'C'`` ordered, + which means that the bound tuples will be ordered in the reverse order + to their respective input order. That is if the inputs are in the order + ``('x', 'y', 'z')`` then the bounds will need to be listed in ``('z', 'y', 'x')`` + order. + +The if one does not want to work directly with the default ``'C'`` ordered +bounding boxes. It is possible to use the alternate ``'F'`` ordering, which +orders the bounding box tuple in the same order as the inputs. To do this +one can use the `bind_bounding_box ` +function, and passing the ``order='F'`` keyword argument :: + + >>> from astropy.modeling import bind_bounding_box + >>> model4 = Polynomial2D(3) + >>> bind_bounding_box(model4, ((-1, 1), (-2, 2)), order='F') + >>> model4.bounding_box + ModelBoundingBox( + intervals={ + x: Interval(lower=-1, upper=1) + y: Interval(lower=-2, upper=2) + } + model=Polynomial2D(inputs=('x', 'y')) + order='F' + ) + >>> model4(-2, 0) + 0.0 + >>> model4(-2, 0, with_bounding_box=True) + nan + >>> model4(-2, 0, with_bounding_box=True, fill_value=12) + 12.0 + >>> model4.bounding_box.bounding_box() + ((-1, 1), (-2, 2)) + >>> model4.bounding_box.bounding_box(order='C') + ((-2, 2), (-1, 1)) + +.. warning:: + + Currently when combining models the bounding boxes of components are + combined only when joining models with the ``&`` operator. + For the other operators bounding boxes for compound models must be assigned + explicitly. A future release will determine the appropriate bounding box + for a compound model where possible. + +Using the Compound Bounding Box ++++++++++++++++++++++++++++++++ + +Sometimes it is useful to have multiple bounding boxes for the same model, +which are selectable when the model is evaluated. In this case, one should +consider using a `CompoundBoundingBox `. + +A common use case for this may be if the model has a single "discrete" +selector input (for example ``'slit_id'``), which among other things, +determines what bounding box should be applied to the other inputs. To +do this one needs to first define a dictionary of bounding box tuples, +with dictionary keys being the specific values of the selector input +corresponding to that specific bounding box :: + + >>> from astropy.modeling.models import Shift, Identity + >>> model1 = Shift(1) & Shift(2) & Identity(1) + >>> model1.inputs = ('x', 'y', 'slit_id') + >>> bboxes = { + ... 0: ((0, 1), (1, 2)), + ... 1: ((2, 3), (3, 4)) + ... } + +In order for the compound bounding box to function one must specify a list +of selector arguments, where the elements of this list are tuples of the input's +name and whether or not the bounding box should be applied to the selector argument +or not. In this case, it makes sense for the selector argument to be ignored :: + + >>> from astropy.modeling.core import bind_compound_bounding_box + >>> selector_args = [('slit_id', True)] + >>> bind_compound_bounding_box(model1, bboxes, selector_args, order='F') + >>> model1.bounding_box + CompoundBoundingBox( + bounding_boxes={ + (0,) = ModelBoundingBox( + intervals={ + x: Interval(lower=0, upper=1) + y: Interval(lower=1, upper=2) + } + ignored=['slit_id'] + model=CompoundModel(inputs=('x', 'y', 'slit_id')) + order='F' + ) + (1,) = ModelBoundingBox( + intervals={ + x: Interval(lower=2, upper=3) + y: Interval(lower=3, upper=4) + } + ignored=['slit_id'] + model=CompoundModel(inputs=('x', 'y', 'slit_id')) + order='F' + ) + } + selector_args = SelectorArguments( + Argument(name='slit_id', ignore=True) + ) + ) + >>> model1(0.5, 1.5, 0, with_bounding_box=True) + (1.5, 3.5, 0.0) + >>> model1(0.5, 1.5, 1, with_bounding_box=True) + (np.float64(nan), np.float64(nan), np.float64(nan)) + +Multiple selector arguments can also be used, in this case the keys of the +dictionary of bounding boxes need to be specified as tuples of values :: + + >>> model2 = Shift(1) & Shift(2) & Identity(2) + >>> model2.inputs = ('x', 'y', 'slit_x', 'slit_y') + >>> bboxes = { + ... (0, 0): ((0, 1), (1, 2)), + ... (0, 1): ((2, 3), (3, 4)), + ... (1, 0): ((4, 5), (5, 6)), + ... (1, 1): ((6, 7), (7, 8)), + ... } + >>> selector_args = [('slit_x', True), ('slit_y', True)] + >>> bind_compound_bounding_box(model2, bboxes, selector_args, order='F') + >>> model2.bounding_box + CompoundBoundingBox( + bounding_boxes={ + (0, 0) = ModelBoundingBox( + intervals={ + x: Interval(lower=0, upper=1) + y: Interval(lower=1, upper=2) + } + ignored=['slit_x', 'slit_y'] + model=CompoundModel(inputs=('x', 'y', 'slit_x', 'slit_y')) + order='F' + ) + (0, 1) = ModelBoundingBox( + intervals={ + x: Interval(lower=2, upper=3) + y: Interval(lower=3, upper=4) + } + ignored=['slit_x', 'slit_y'] + model=CompoundModel(inputs=('x', 'y', 'slit_x', 'slit_y')) + order='F' + ) + (1, 0) = ModelBoundingBox( + intervals={ + x: Interval(lower=4, upper=5) + y: Interval(lower=5, upper=6) + } + ignored=['slit_x', 'slit_y'] + model=CompoundModel(inputs=('x', 'y', 'slit_x', 'slit_y')) + order='F' + ) + (1, 1) = ModelBoundingBox( + intervals={ + x: Interval(lower=6, upper=7) + y: Interval(lower=7, upper=8) + } + ignored=['slit_x', 'slit_y'] + model=CompoundModel(inputs=('x', 'y', 'slit_x', 'slit_y')) + order='F' + ) + } + selector_args = SelectorArguments( + Argument(name='slit_x', ignore=True) + Argument(name='slit_y', ignore=True) + ) + ) + >>> model2(0.5, 1.5, 0, 0, with_bounding_box=True) + (1.5, 3.5, 0.0, 0.0) + >>> model2(0.5, 1.5, 1, 1, with_bounding_box=True) + (np.float64(nan), np.float64(nan), np.float64(nan), np.float64(nan)) + +Note that one can also specify the ordering for all the bounding boxes +comprising the compound bounding using the ``order`` keyword argument. + +Another use case for this maybe a if one wants to use multiple bounding +boxes for the same model, where the user chooses the bounding box when +evaluating the model. In this case, one must still choose a selector +argument as a fall back default for bounding box selection; however, this +argument should not be ignored by the bounding box:: + + >>> from astropy.modeling.models import Polynomial2D + >>> from astropy.modeling import bind_compound_bounding_box + >>> model = Polynomial2D(3) + >>> bboxes = { + ... 0: ((0, 1), (1, 2)), + ... 1: ((2, 3), (3, 4)) + ... } + >>> selector_args = [('x', False)] + >>> bind_compound_bounding_box(model, bboxes, selector_args, order='F') + >>> model.bounding_box + CompoundBoundingBox( + bounding_boxes={ + (0,) = ModelBoundingBox( + intervals={ + x: Interval(lower=0, upper=1) + y: Interval(lower=1, upper=2) + } + model=Polynomial2D(inputs=('x', 'y')) + order='F' + ) + (1,) = ModelBoundingBox( + intervals={ + x: Interval(lower=2, upper=3) + y: Interval(lower=3, upper=4) + } + model=Polynomial2D(inputs=('x', 'y')) + order='F' + ) + } + selector_args = SelectorArguments( + Argument(name='x', ignore=False) + ) + ) + +For the user to select the bounding box on evaluation, instead of +specifying, ``with_bounding_box=True`` as the keyword argument; the user +instead specifies ``with_bounding_box=`` :: + + >>> model(0.5, 1.5, with_bounding_box=0) + 0.0 + >>> model(0.5, 1.5, with_bounding_box=1) + nan + + +Ignoring Inputs in Bounding Boxes ++++++++++++++++++++++++++++++++++ + +Both `standard bounding box ` +and `CompoundBoundingBox ` +support ignoring specific inputs from enforcement by the bounding box. Effectively, +for multi-dimensional models one can define bounding boxes so that bounds are +only applied to a subset of the model's inputs rather than the default of enforcing +a bound of some kind on every input. Note that use of this feature is equivalent +to defining the bounds for an input to be ``[-np.inf, np.inf]``. + +.. warning:: + The ``ignored`` input feature is not available when constructing/adding bounding + boxes to models using tuples and the property interface. That is one cannot + ignore inputs when setting bounding boxes using ``model.bounding_box = (-1, 1)``. + This feature is only available via the methods + `bind_bounding_box ` and + `bind_compound_bounding_box `. + +Ignoring inputs for a bounding box can be achieved via passing a list of the input +name strings to be ignored to the ``ignored`` keyword argument in any of the main +bounding box interfaces. :: + + >>> from astropy.modeling.models import Polynomial1D + >>> from astropy.modeling import bind_bounding_box + >>> model1 = Polynomial2D(3) + >>> bind_bounding_box(model1, {'x': (-1, 1)}, ignored=['y']) + >>> model1.bounding_box + ModelBoundingBox( + intervals={ + x: Interval(lower=-1, upper=1) + } + ignored=['y'] + model=Polynomial2D(inputs=('x', 'y')) + order='C' + ) + >>> model1(-2, 0, with_bounding_box=True) + nan + >>> model1(0, 300, with_bounding_box=True) + 0.0 + +Similarly, the ignored inputs will be applied to all of the bounding boxes +contained within a compound bounding box. :: + + >>> from astropy.modeling import bind_compound_bounding_box + >>> model2 = Polynomial2D(3) + >>> bboxes = { + ... 0: {'x': (0, 1)}, + ... 1: {'x': (1, 2)} + ... } + >>> selector_args = [('x', False)] + >>> bind_compound_bounding_box(model2, bboxes, selector_args, ignored=['y'], order='F') + >>> model2.bounding_box + CompoundBoundingBox( + bounding_boxes={ + (0,) = ModelBoundingBox( + intervals={ + x: Interval(lower=0, upper=1) + } + ignored=['y'] + model=Polynomial2D(inputs=('x', 'y')) + order='F' + ) + (1,) = ModelBoundingBox( + intervals={ + x: Interval(lower=1, upper=2) + } + ignored=['y'] + model=Polynomial2D(inputs=('x', 'y')) + order='F' + ) + } + selector_args = SelectorArguments( + Argument(name='x', ignore=False) + ) + ) + >>> model2(0.5, 300, with_bounding_box=0) + 0.0 + >>> model2(0.5, 300, with_bounding_box=1) + nan + + +Efficient evaluation with `Model.render() ` +-------------------------------------------------------------------------- + +When a model is evaluated over a range much larger than the model itself, it +may be prudent to use the :func:`Model.render ` +method if efficiency is a concern. The :func:`render ` +method can be used to evaluate the model on an +array of the same dimensions. ``model.render()`` can be called with no +arguments to return a "postage stamp" of the bounding box region. + +In this example, we generate a 300x400 pixel image of 100 2D Gaussian sources. +For comparison, the models are evaluated both with and without using bounding +boxes. By using bounding boxes, the evaluation speed increases by approximately +a factor of 10 with negligible loss of information. -Each row in this output is the output for each model in the set. The first is -the value of the Gaussian with ``amplitude=[1, 2], mean=0.1, stddev=0.1``, and -the second is the value of the Gaussian with ``amplitude=[3, 4], mean=0.1, -stddev=0.2``. - -We can also pass a different input to each model in a model set by passing in -an array input:: - - >>> g([0, 1]) - array([[ 6.06530660e-01, 1.21306132e+00], - [ 1.20195892e-04, 1.60261190e-04]]) +.. plot:: + :include-source: + + import numpy as np + from time import time + from astropy.modeling import models + import matplotlib.pyplot as plt + from matplotlib.patches import Rectangle + + imshape = (300, 400) + y, x = np.indices(imshape) + + # Generate random source model list + rng = np.random.default_rng(0) + nsrc = 100 + model_params = [ + dict(amplitude=rng.uniform(.5, 1), + x_mean=rng.uniform(0, imshape[1] - 1), + y_mean=rng.uniform(0, imshape[0] - 1), + x_stddev=rng.uniform(2, 6), + y_stddev=rng.uniform(2, 6), + theta=rng.uniform(0, 2 * np.pi)) + for _ in range(nsrc)] + + model_list = [models.Gaussian2D(**kwargs) for kwargs in model_params] + + # Render models to image using bounding boxes + bb_image = np.zeros(imshape) + t_bb = time() + for model in model_list: + model.render(bb_image) + t_bb = time() - t_bb + + # Render models to image using full evaluation + full_image = np.zeros(imshape) + t_full = time() + for model in model_list: + model.bounding_box = None + model.render(full_image) + t_full = time() - t_full + + flux = full_image.sum() + diff = (full_image - bb_image) + max_err = diff.max() + + # Plots + fig, axs = plt.subplots(figsize=(16, 7), ncols=2) + fig.subplots_adjust(left=.05, right=.97, bottom=.03, top=.97, wspace=0.15) + + # Full model image + ax1 = axs[0] + ax1.imshow(full_image, origin='lower') + ax1.set_title(f'Full Models\nTiming: {t_full:.2f} seconds', fontsize=16) + ax1.set(xlabel='x', ylabel='y') + + # Bounded model image with boxes overplotted + ax2 = axs[1] + ax2.imshow(bb_image, origin='lower') + for model in model_list: + del model.bounding_box # Reset bounding_box to its default + dy, dx = np.diff(model.bounding_box).flatten() + pos = (model.x_mean.value - dx / 2, model.y_mean.value - dy / 2) + r = Rectangle(pos, dx, dy, edgecolor='w', facecolor='none', alpha=.25) + ax2.add_patch(r) + ax2.set_title(f'Bounded Models\nTiming: {t_bb:.2f} seconds', fontsize=16) + ax2.set(xlabel='x', ylabel='y') + + # Difference image + fig2, ax = plt.subplots(figsize=(16, 8)) + im = ax.imshow(diff, vmin=-max_err, vmax=max_err) + fig2.colorbar(im, format='%.1e') + ax.set_title(f'Difference Image\nTotal Flux Err = {((flux - np.sum(bb_image)) / flux):.0e}') + ax.set(xlabel='x', ylabel='y') + plt.show() + + + +.. _separability: + +Model Separability +------------------ + +Simple models have a boolean `Model.separable ` property. +It indicates whether the outputs are independent and is essential for computing the +separability of compound models using the :func:`~astropy.modeling.is_separable` function. +Having a separable compound model means that it can be decomposed into independent models, +which in turn is useful in many applications. +For example, it may be easier to define inverses using the independent parts of a model +than the entire model. +In other cases, tools using `Generalized World Coordinate System (GWCS)`_, +can be more flexible and take advantage of separable spectral and spatial transforms. + +If a custom subclass of `~astropy.modeling.Model` needs to override the +computation of its separability it can implement the +``_calculate_separability_matrix`` method which should return the separability +matrix for that model. -By default this uses the same concept of a ``model_set_axis``. The first -dimension of the input array is used to map inputs to corresponding models in -the model set. We can use this, for example, to evaluate the model on 1-D -array inputs with a different input to each model set:: - >>> g([[0, 1], [2, 3]]) - array([[ 6.06530660e-01, 5.15351422e-18], - [ 7.57849134e-20, 8.84815213e-46]]) +.. _modeling-model-sets: -In this case the first model is evaluated on ``[0, 1]``, and the second model -is evaluated on ``[2, 3]``. If the input has length greater than the number of -models in the set then this is in error:: +Model Sets +========== - >>> g([0, 1, 2]) +In some cases it is useful to describe many models of the same type but with +different sets of parameter values. This could be done simply by instantiating +as many instances of a `~astropy.modeling.Model` as are needed. But that can +be inefficient for a large number of models. To that end, all model classes in +`astropy.modeling` can also be used to represent a model **set** which is a +collection of models of the same type, but with different values for their +parameters. + +To instantiate a model set, use argument ``n_models=N`` where ``N`` is the +number of models in the set when constructing the model. The value of each +parameter must be a list or array of length ``N``, such that each item in +the array corresponds to one model in the set:: + + >>> from astropy.modeling import models + >>> g = models.Gaussian1D(amplitude=[1, 2], mean=[0, 0], + ... stddev=[0.1, 0.2], n_models=2) + >>> print(g) + Model: Gaussian1D + Inputs: ('x',) + Outputs: ('y',) + Model set size: 2 + Parameters: + amplitude mean stddev + --------- ---- ------ + 1.0 0.0 0.1 + 2.0 0.0 0.2 + +This is equivalent to two Gaussians with the parameters ``amplitude=1, mean=0, +stddev=0.1`` and ``amplitude=2, mean=0, stddev=0.2`` respectively. When +printing the model the parameter values are displayed as a table, with each row +corresponding to a single model in the set. + +The number of models in a model set can be determined using the `len` builtin:: + + >>> len(g) + 2 + +Single models have a length of 1, and are not considered a model set as such. + +When evaluating a model set, by default the input must be the same length as +the number of models, with one input per model:: + + >>> g([0, 0.1]) # doctest: +FLOAT_CMP + array([1. , 1.76499381]) + +The result is an array with one result per model in the set. It is also +possible to broadcast a single input value to all models in the set:: + + >>> g(0) # doctest: +FLOAT_CMP + array([1., 2.]) + +Or when the input is an array:: + + >>> x = np.array([[0, 0, 0], [0.1, 0.1, 0.1]]) + >>> print(x) + [[0. 0. 0. ] + [0.1 0.1 0.1]] + >>> g(x) + array([[1. , 1. , 1. ], + [1.76499381, 1.76499381, 1.76499381]]) + +Internally the shape of the inputs, outputs, and parameter values is controlled +by an attribute - ``model_set_axis``. In the above case ``model_set_axis=0``:: + + >>> g.model_set_axis + 0 + +This indicates that elements along the 0-th axis will be passed as inputs to individual models. +Sometimes it may be useful to pass inputs along a different axis, for example the 1st axis:: + + >>> x = np.array([[0, 0, 0], [0.1, 0.1, 0.1]]).T + >>> print(x) + [[0. 0.1] + [0. 0.1] + [0. 0.1]] + +Because there are two models in this model set and we are passing three inputs +along the 0th axis, evaluation will fail:: + + >>> g(x) Traceback (most recent call last): ... ValueError: Input argument 'x' does not have the correct dimensions in model_set_axis=0 for a model set with n_models=2. -And input like ``[0, 1, 2]`` wouldn't work anyways because it is not compatible -with the array dimensions of the parameter values. However, what if we wanted -to evaluate all models in the set on the input ``[0, 1]``? We could do this -by simply repeating:: - - >>> g([[0, 1], [0, 1]]) - array([[ 6.06530660e-01, 5.15351422e-18], - [ 2.64749071e+00, 1.60261190e-04]]) - -But there is a workaround for this use case that does not necessitate -duplication. This is to include the argument ``model_set_axis=False``:: - - >>> g([0, 1], model_set_axis=False) - array([[ 6.06530660e-01, 5.15351422e-18], - [ 2.64749071e+00, 1.60261190e-04]]) - -What ``model_set_axis=False`` implies is that an array-like input should not be -treated as though any of its dimensions map to models in a model set. And -rather, the given input should be used to evaluate all the models in the model -set. For scalar inputs like ``g(0)``, ``model_set_axis=False`` is implied -automatically. But for array inputs it is necessary to avoid ambiguity. - - -Inputs and Outputs -================== - -Models have an `~astropy.modeling.Model.n_inputs` attribute, which shows how -many coordinates the model expects as an input. All models expect coordinates -as separate arguments. For example a 2-D model expects x and y coordinate -values to be passed separately, i.e. as two scalars or array-like values. - -Models also have an attribute `~astropy.modeling.Model.n_outputs`, which shows -the number of output coordinates. The `~astropy.modeling.Model.n_inputs` and -`~astropy.modeling.Model.n_outputs` attributes can be used when chaining -transforms by adding models in :class:`series -` or in :class:`parallel -`. Because composite models can be -nested within other composite models, creating theoretically infinitely complex -models, a mechanism to map input data to models is needed. In this case the -input may be wrapped in a `~astropy.modeling.LabeledInput` object-- a dict-like -object whose items are ``{label: data}`` pairs. - - -Further examples -================ - -The examples here assume this import statement was executed:: +There are two ways to get around this. ``model_set_axis`` can be passed in +when the model is evaluated:: + + >>> g(x, model_set_axis=1) + array([[1. , 1.76499381], + [1. , 1.76499381], + [1. , 1.76499381]]) + +Or when the model is initialized:: + + >>> g = models.Gaussian1D(amplitude=[[1, 2]], mean=[[0, 0]], + ... stddev=[[0.1, 0.2]], n_models=2, + ... model_set_axis=1) + >>> g(x) + array([[1. , 1.76499381], + [1. , 1.76499381], + [1. , 1.76499381]]) + +Note that in the latter case, the shape of the individual parameters has changed to 2D +because now the parameters are defined along the 1st axis. + +The value of ``model_set_axis`` is either an integer number, representing the axis along which +the different parameter sets and inputs are defined, or a boolean of value ``False``, +in which case it indicates all model sets should use the same inputs on evaluation. +For example, the above model has a value of 1 for ``model_set_axis``. +If ``model_set_axis=False`` is passed the two models will be evaluated on the same input:: + + >>> g.model_set_axis + 1 + >>> result = g(x, model_set_axis=False) + >>> result + array([[[1. , 0.60653066], + [2. , 1.76499381]], + + [[1. , 0.60653066], + [2. , 1.76499381]], + + [[1. , 0.60653066], + [2. , 1.76499381]]]) + >>> result[: , 0] + array([[1. , 0.60653066], + [1. , 0.60653066], + [1. , 0.60653066]]) + >>> result[: , 1] + array([[2. , 1.76499381], + [2. , 1.76499381], + [2. , 1.76499381]]) + +Currently model sets are most useful for fitting a set of **linear** models +(:ref:`example `) +allowing a large number of models of the same type to be fitted simultaneously +(and independently from each other) to some large set of inputs, such as +fitting a polynomial to the time response of each pixel in a data cube. +This can greatly speed up the fitting process. The speed-up is due to solving +the set of equations to find the exact solution. Nonlinear models, which require +an iterative algorithm, cannot be currently fit using model sets. Model sets of nonlinear +models can only be evaluated. + +When fitting model sets it is important that data arrays are passed to the fitter +in the correct shape. The shape depends on the ``model_set_axis`` attribute of the +model to be fit. The rule is that the index of the dependent variable that corresponds +to a model set should be along the ``model_set_axis`` dimension. For example, for a +1D model set with 3 models with ``model_set_axis == 1`` the shape of ``y`` should be (x, 3):: - >>> from astropy.modeling.models import Gaussian1D, Polynomial1D >>> import numpy as np + >>> from astropy.modeling.models import Polynomial1D + >>> from astropy.modeling.fitting import LinearLSQFitter + >>> fitter = LinearLSQFitter() + >>> x = np.arange(4) + >>> y = np.array([2*x+1, x+4, x]).T + >>> print(y) + [[1 4 0] + [3 5 1] + [5 6 2] + [7 7 3]] + >>> print(y.shape) + (4, 3) + >>> m = Polynomial1D(1, n_models=3, model_set_axis=1) + >>> mfit = fitter(m, x, y) + +For 2D models with 3 models and ``model_set_axis = 0`` the shape of ``z`` should be (3, x, y):: -- Create a model set of two 1-D Gaussians:: - - >>> x = np.arange(1, 10, .1) - >>> g1 = Gaussian1D(amplitude=[10, 9], mean=[2, 3], - ... stddev=[0.15, .1], n_models=2) - >>> print g1 - Model: Gaussian1D - Inputs: ('x',) - Outputs: ('y',) - Model set size: 2 - Parameters: - amplitude mean stddev - --------- ---- ------ - 10.0 2.0 0.15 - 9.0 3.0 0.1 - - Evaluate all models in the set on one set of input coordinates:: - - >>> y = g1(x, model_set_axis=False) # broadcast the array to all models - >>> print(y.shape) - (2, 90) - - or different inputs for each model in the set:: - - >>> y = g1([x, x + 3]) - >>> print(y.shape) - (2, 90) - -.. plot:: - - import matplotlib.pyplot as plt - import numpy as np - from astropy.modeling import models, fitting - x = np.arange(1, 10, .1) - g1 = models.Gaussian1D(amplitude=[10, 9], mean=[2,3], stddev=[.15,.1], - n_models=2) - y = g1(x, model_set_axis=False) - plt.figure(figsize=(8, 4)) - plt.plot(x, y.T) - plt.title('Evaluate two Gaussian1D models on 1 set of input data') - plt.show() - -.. plot:: - - import matplotlib.pyplot as plt - import numpy as np - from astropy.modeling import models, fitting - x = np.arange(1, 10, .1) - g1 = models.Gaussian1D(amplitude=[10, 9], mean=[2,3], stddev=[.15,.1], - n_models=2) - y = g1([x, x - 3]) - plt.figure(figsize=(8, 4)) - plt.plot(x, y[0]) - plt.plot(x - 3, y[1]) - plt.title('Evaluate two Gaussian1D models with 2 sets of input data') - plt.show() - - -- Evaluating a set of multiple polynomial models with one input data set - creates multiple output data sets:: - - >>> p1 = Polynomial1D(degree=1, n_models=5) - >>> p1.c1 = [0, 1, 2, 3, 4] - >>> print p1 - Model: Polynomial1D - Inputs: ('x',) - Outputs: ('y',) - Model set size: 5 - Degree: 1 - Parameters: - c0 c1 - --- --- - 0.0 0.0 - 0.0 1.0 - 0.0 2.0 - 0.0 3.0 - 0.0 4.0 - >>> y = p1(x, model_set_axis=False) - - -.. plot:: - - import matplotlib.pyplot as plt - import numpy as np - from astropy.modeling import models, fitting - x = np.arange(1, 10, .1) - p1 = models.Polynomial1D(1, n_models=5) - p1.c1 = [0, 1, 2, 3, 4] - y = p1(x, model_set_axis=False) - plt.figure(figsize=(8, 4)) - plt.plot(x, y.T) - plt.title("Polynomial1D with a 5 model set on the same input") - plt.show() - -- When passed a 2-D array, the same polynomial will map each row of the array - to one model in the set, one for one:: - - >>> x = np.arange(30).reshape(5, 6) - >>> y = p1(x) - >>> y - array([[ 0., 0., 0., 0., 0., 0.], - [ 6., 7., 8., 9., 10., 11.], - [ 24., 26., 28., 30., 32., 34.], - [ 54., 57., 60., 63., 66., 69.], - [ 96., 100., 104., 108., 112., 116.]]) - >>> print(y.shape) - (5, 6) + >>> import numpy as np + >>> from astropy.modeling.models import Polynomial2D + >>> from astropy.modeling.fitting import LinearLSQFitter + >>> fitter = LinearLSQFitter() + >>> x = np.arange(8).reshape(2, 4) + >>> y = x + >>> z = np.asarray([2 * x + 1, x + 4, x + 3]) + >>> print(z.shape) + (3, 2, 4) + >>> m = Polynomial2D(1, n_models=3, model_set_axis=0) + >>> mfit = fitter(m, x, y, z) + +.. _modeling-asdf: + +Model Serialization (Writing a Model to a File) +=============================================== + +Models are serializable using the `ASDF`_ +format. This can be useful in many contexts, one of which is the implementation of a +`Generalized World Coordinate System (GWCS)`_. + +Serializing a model to disk is possible by assigning the object to ``AsdfFile.tree``: + +.. doctest-requires:: asdf-astropy + + >>> from asdf import AsdfFile + >>> from astropy.modeling import models + >>> rotation = models.Rotation2D(angle=23.7) + >>> f = AsdfFile() + >>> f.tree['model'] = rotation + >>> f.write_to('rotation.asdf') + +To read the file and create the model: + +.. doctest-requires:: asdf-astropy + + >>> import asdf + >>> with asdf.open('rotation.asdf') as f: + ... model = f.tree['model'] + >>> print(model) + Model: Rotation2D + Inputs: ('x', 'y') + Outputs: ('x', 'y') + Model set size: 1 + Parameters: + angle + ----- + 23.7 + +Compound models can also be serialized. Please note that some model attributes (e.g ``meta``, +``tied`` parameter constraints used in fitting), as well as model sets are not yet serializable. +For more information on serialization of models, see :ref:`asdf-astropy:asdf-astropy`. diff --git a/docs/modeling/new-fitter.rst b/docs/modeling/new-fitter.rst new file mode 100644 index 000000000000..9e26b95234c0 --- /dev/null +++ b/docs/modeling/new-fitter.rst @@ -0,0 +1,187 @@ +.. _new_fitter: + +Defining New Fitter Classes +*************************** + +This section describes how to add a new nonlinear fitting algorithm to this +package or write a user-defined fitter. In short, one needs to define an error +function and a ``__call__`` method and define the types of constraints which +work with this fitter (if any). + +The details are described below using scipy's SLSQP algorithm as an example. +The base class for all fitters is `~astropy.modeling.fitting.Fitter`:: + + class SLSQPFitter(Fitter): + supported_constraints = ['bounds', 'eqcons', 'ineqcons', 'fixed', + 'tied'] + + def __init__(self): + # Most currently defined fitters take no arguments in their + # __init__, but the option certainly exists for custom fitters + super().__init__() + +All fitters take a model (their ``__call__`` method modifies the model's +parameters) as their first argument. + +Next, the error function takes a list of parameters returned by an iteration of +the fitting algorithm and input coordinates, evaluates the model with them and +returns some type of a measure for the fit. In the example the sum of the +squared residuals is used as a measure of fitting.:: + + def objective_function(self, fps, *args): + model = args[0] + meas = args[-1] + model.fitparams(fps) + res = self.model(*args[1:-1]) - meas + return np.sum(res**2) + +The ``__call__`` method performs the fitting. As a minimum it takes all +coordinates as separate arguments. Additional arguments are passed as +necessary:: + + def __call__(self, model, x, y , maxiter=MAXITER, epsilon=EPS): + if model.linear: + raise ModelLinearityException( + 'Model is linear in parameters; ' + 'non-linear fitting methods should not be used.') + model_copy = model.copy() + init_values, _ = model_to_fit_params(model_copy) + self.fitparams = optimize.fmin_slsqp(self.errorfunc, p0=init_values, + args=(y, x), + bounds=self.bounds, + eqcons=self.eqcons, + ineqcons=self.ineqcons) + return model_copy + +Defining a Plugin Fitter +======================== + +`astropy.modeling` includes a plugin mechanism which allows fitters +defined outside of astropy's core to be inserted into the +`astropy.modeling.fitting` namespace through the use of entry points. +Entry points are references to importable objects. A tutorial on defining +entry points can be found in `setuptools' documentation `_. +Plugin fitters must to extend from the `~astropy.modeling.fitting.Fitter` +base class. For the fitter to be discovered and inserted into +`astropy.modeling.fitting` the entry points must be inserted into +the `astropy.modeling` entry point group:: + + setup( + # ... + entry_points = {'astropy.modeling': 'PluginFitterName = fitter_module:PlugFitterClass'} + ) + +This would allow users to import the ``PlugFitterName`` through `astropy.modeling.fitting` by:: + + from astropy.modeling.fitting import PlugFitterName + +One project which uses this functionality is `Saba `_ +and be can be used as a reference. + +Using a Custom Statistic Function +================================= + +This section describes how to write a new fitter with a user-defined statistic +function. The example below shows a specialized class which fits a straight +line with uncertainties in both variables. + +The following import statements are needed:: + + import numpy as np + from astropy.modeling.fitting import (_validate_model, + fitter_to_model_params, + model_to_fit_params, Fitter, + _convert_input) + from astropy.modeling.optimizers import Simplex + +First one needs to define a statistic. This can be a function or a callable +class.:: + + def chi_line(measured_vals, updated_model, x_sigma, y_sigma, x): + """ + Chi^2 statistic for fitting a straight line with uncertainties in x and + y. + + Parameters + ---------- + measured_vals : array + updated_model : `~astropy.modeling.ParametricModel` + model with parameters set by the current iteration of the optimizer + x_sigma : array + uncertainties in x + y_sigma : array + uncertainties in y + + """ + model_vals = updated_model(x) + if x_sigma is None and y_sigma is None: + return np.sum((model_vals - measured_vals) ** 2) + elif x_sigma is not None and y_sigma is not None: + weights = 1 / (y_sigma ** 2 + updated_model.parameters[1] ** 2 * + x_sigma ** 2) + return np.sum((weights * (model_vals - measured_vals)) ** 2) + else: + if x_sigma is not None: + weights = 1 / x_sigma ** 2 + else: + weights = 1 / y_sigma ** 2 + return np.sum((weights * (model_vals - measured_vals)) ** 2) + +In general, to define a new fitter, all one needs to do is provide a statistic +function and an optimizer. In this example we will let the optimizer be an +optional argument to the fitter and will set the statistic to ``chi_line`` +above:: + + class LineFitter(Fitter): + """ + Fit a straight line with uncertainties in both variables + + Parameters + ---------- + optimizer : class or callable + one of the classes in optimizers.py (default: Simplex) + """ + + def __init__(self, optimizer=Simplex): + self.statistic = chi_line + super().__init__(optimizer, statistic=self.statistic) + +The last thing to define is the ``__call__`` method:: + + def __call__(self, model, x, y, x_sigma=None, y_sigma=None, **kwargs): + """ + Fit data to this model. + + Parameters + ---------- + model : `~astropy.modeling.core.ParametricModel` + model to fit to x, y + x : array + input coordinates + y : array + input coordinates + x_sigma : array + uncertainties in x + y_sigma : array + uncertainties in y + kwargs : dict + optional keyword arguments to be passed to the optimizer + + Returns + ------ + model_copy : `~astropy.modeling.core.ParametricModel` + a copy of the input model with parameters set by the fitter + + """ + model_copy = _validate_model(model, + self._opt_method.supported_constraints) + + farg = _convert_input(x, y) + farg = (model_copy, x_sigma, y_sigma) + farg + p0, _, _ = model_to_fit_params(model_copy) + + fitparams, self.fit_info = self._opt_method( + self.objective_function, p0, farg, **kwargs) + fitter_to_model_params(model_copy, fitparams) + + return model_copy diff --git a/docs/modeling/new-model.rst b/docs/modeling/new-model.rst new file mode 100644 index 000000000000..24e252f6fe3c --- /dev/null +++ b/docs/modeling/new-model.rst @@ -0,0 +1,285 @@ +.. _modeling-new-classes: + +************************** +Defining New Model Classes +************************** + +This document describes how to add a model to the package or to create a +user-defined model. In short, one needs to define all model parameters and +write a function which evaluates the model, that is, computes the mathematical +function that implements the model. If the model is fittable, a function to +compute the derivatives with respect to parameters is required if a linear +fitting algorithm is to be used and optional if a non-linear fitter is to be +used. + + +Basic custom models +=================== + +For most cases, the `~astropy.modeling.custom_model` decorator provides an +easy way to make a new `~astropy.modeling.Model` class from an existing Python +callable. The following example demonstrates how to set up a model consisting +of two Gaussians: + +.. plot:: + :include-source: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling.models import custom_model + from astropy.modeling.fitting import TRFLSQFitter + + # Define model + @custom_model + def sum_of_gaussians(x, amplitude1=1.0, mean1=-1.0, sigma1=1.0, + amplitude2=1.0, mean2=1.5, sigma2=1.0): + return (amplitude1 * np.exp(-0.5 * ((x - mean1) / sigma1)**2) + + amplitude2 * np.exp(-0.5 * ((x - mean2) / sigma2)**2)) + + # Generate fake data with some noise + rng = np.random.default_rng(0) + x = np.linspace(-5., 5., 200) + m_ref = sum_of_gaussians(amplitude1=2., mean1=-0.5, sigma1=0.4, + amplitude2=0.5, mean2=2., sigma2=1.0) + y = m_ref(x) + rng.normal(0., 0.05, x.shape) + + # Fit model to data + m_init = sum_of_gaussians() + fit = TRFLSQFitter() + m = fit(m_init, x, y) + + # Plot the data and the best fit + fig, ax = plt.subplots() + ax.plot(x, y, 'o', color='k') + ax.plot(x, m(x)) + + +This decorator also supports setting a model's +`~astropy.modeling.FittableModel.fit_deriv` as well as creating models with +more than one inputs. Note that when creating a model from a function with +multiple outputs, the keyword argument ``n_outputs`` must be set to the +number of outputs of the function. It can also be used as a normal factory +function (for example ``SumOfGaussians = custom_model(sum_of_gaussians)``) +rather than as a decorator. See the `~astropy.modeling.custom_model` +documentation for more examples. + + +A step by step definition of a 1-D Gaussian model +================================================= + +The example described in `Basic custom models`_ can be used for most simple +cases, but the following section describes how to construct model classes in +general. Defining a full model class may be desirable, for example, to +provide more specialized parameters, or to implement special functionality not +supported by the basic `~astropy.modeling.custom_model` factory function. + +The details are explained below with a 1-D Gaussian model as an example. There +are two base classes for models. If the model is fittable, it should inherit +from `~astropy.modeling.FittableModel`; if not it should subclass +`~astropy.modeling.Model`. + +If the model takes parameters they should be specified as class attributes in +the model's class definition using the `~astropy.modeling.Parameter` +descriptor. All arguments to the Parameter constructor are optional, and may +include a default value for that parameter, a text description of the parameter +(useful for `help` and documentation generation), as well default constraints +and custom getters/setters for the parameter value. It is also possible to +define a "validator" method for each parameter, enabling custom code to check +whether that parameter's value is valid according to the model definition (for +example if it must be non-negative). See the example in +`Parameter.validator ` for more details. +Note, that if pickling the model is important the validator function should be +assigned directly to the instance ``Parameter._validator`` instead of using +the decorator. + +:: + + from astropy.modeling import Fittable1DModel, Parameter + + class Gaussian1D(Fittable1DModel): + n_inputs = 1 + n_outputs = 1 + + amplitude = Parameter() + mean = Parameter() + stddev = Parameter() + +The ``n_inputs`` and ``n_outputs`` class attributes must be integers +indicating the number of independent variables that are input to evaluate the +model, and the number of outputs it returns. The labels of the inputs and +outputs, ``inputs`` and ``outputs``, are generated automatically. It is possible +to overwrite the default ones by assigning the desired values in the class ``__init__`` +method, after calling ``super``. ``outputs`` and ``inputs`` must be tuples of +strings with length ``n_outputs`` and ``n_inputs`` respectively. +Outputs may have the same labels as inputs (eg. ``inputs = ('x', 'y')`` and ``outputs = ('x', 'y')``). +However, inputs must not conflict with each other (eg. ``inputs = ('x', 'x')`` is +incorrect) and likewise for outputs. + +There are two helpful base classes in the modeling package that can be used to +avoid specifying ``n_inputs`` and ``n_outputs`` for most common models. These are +`~astropy.modeling.Fittable1DModel` and `~astropy.modeling.Fittable2DModel`. +For example, the actual `~astropy.modeling.functional_models.Gaussian1D` model is +a subclass of `~astropy.modeling.Fittable1DModel`. This helps cut +down on boilerplate by not having to specify ``n_inputs``, ``n_outputs``, ``inputs`` +and ``outputs`` for many models (follow the link to Gaussian1D to see its source code, for +example). + +Fittable models can be linear or nonlinear in a regression sense. The default +value of the `~astropy.modeling.Model.linear` attribute is ``False``. Linear +models should define the ``linear`` class attribute as ``True``. Because this +model is non-linear we can stick with the default. + +Models which inherit from `~astropy.modeling.Fittable1DModel` have the +``Model._separable`` property already set to ``True``. +All other models should define this property to indicate the +:ref:`separability`. + +Next, provide methods called ``evaluate`` to evaluate the model and +``fit_deriv``, to compute its derivatives with respect to parameters. These +may be normal methods, `classmethod`, or `staticmethod`, though the convention +is to use `staticmethod` when the function does not depend on any of the +object's other attributes (i.e., it does not reference ``self``) or any of the +class's other attributes as in the case of `classmethod`. The evaluation +method takes all input coordinates as separate arguments and all of the model's +parameters in the same order they would be listed by +`~astropy.modeling.Model.param_names`. + +For this example:: + + @staticmethod + def evaluate(x, amplitude, mean, stddev): + return amplitude * np.exp((-(1 / (2. * stddev**2)) * (x - mean)**2)) + +It should be made clear that the ``evaluate`` method must be designed to take +the model's parameter values as arguments. This may seem at odds with the fact +that the parameter values are already available via attribute of the model +(eg. ``model.amplitude``). However, passing the parameter values directly to +``evaluate`` is a more efficient way to use it in many cases, such as fitting. + +Users of your model would not generally use ``evaluate`` directly. Instead +they create an instance of the model and call it on some input. The +``__call__`` method of models uses ``evaluate`` internally, but users do not +need to be aware of it. The default ``__call__`` implementation also handles +details such as checking that the inputs are correctly formatted and follow +Numpy's broadcasting rules before attempting to evaluate the model. + +Like ``evaluate``, the ``fit_deriv`` method takes as input all coordinates and +all parameter values as arguments. There is an option to compute numerical +derivatives for nonlinear models in which case the ``fit_deriv`` method should +be ``None``:: + + @staticmethod + def fit_deriv(x, amplitude, mean, stddev): + d_amplitude = np.exp(- 0.5 / stddev**2 * (x - mean)**2) + d_mean = (amplitude * + np.exp(- 0.5 / stddev**2 * (x - mean)**2) * + (x - mean) / stddev**2) + d_stddev = (2 * amplitude * + np.exp(- 0.5 / stddev**2 * (x - mean)**2) * + (x - mean)**2 / stddev**3) + return [d_amplitude, d_mean, d_stddev] + + +Note that we did *not* have to define an ``__init__`` method or a ``__call__`` +method for our model. For most models the ``__init__`` follows the same pattern, +taking the parameter values as positional arguments, followed by several optional +keyword arguments (constraints, etc.). The modeling framework automatically generates an +``__init__`` for your class that has the correct calling signature (see for +yourself by calling ``help(Gaussian1D.__init__)`` on the example model we just +defined). + +There are cases where it might be desirable to define a custom ``__init__``. +For example, the `~astropy.modeling.functional_models.Gaussian2D` model takes +an optional ``cov_matrix`` argument which can be used as an alternative way to +specify the x/y_stddev and theta parameters. This is perfectly valid so long +as the ``__init__`` determines appropriate values for the actual parameters and +then calls the super ``__init__`` with the standard arguments. Schematically +this looks something like: + +.. code-block:: python + + def __init__(self, amplitude, x_mean, y_mean, x_stddev=None, + y_stddev=None, theta=None, cov_matrix=None, **kwargs): + # The **kwargs here should be understood as other keyword arguments + # accepted by the basic Model.__init__ (such as constraints) + if cov_matrix is not None: + # Set x/y_stddev and theta from the covariance matrix + x_stddev = ... + y_stddev = ... + theta = ... + + # Don't pass on cov_matrix since it doesn't mean anything to the base + # class + super().__init__(amplitude, x_mean, y_mean, x_stddev, y_stddev, theta, + **kwargs) + + +Full example +------------ + +.. code-block:: python + + import numpy as np + from astropy.modeling import Fittable1DModel, Parameter + + class Gaussian1D(Fittable1DModel): + amplitude = Parameter() + mean = Parameter() + stddev = Parameter() + + @staticmethod + def evaluate(x, amplitude, mean, stddev): + return amplitude * np.exp((-(1 / (2. * stddev**2)) * (x - mean)**2)) + + @staticmethod + def fit_deriv(x, amplitude, mean, stddev): + d_amplitude = np.exp((-(1 / (stddev**2)) * (x - mean)**2)) + d_mean = (2 * amplitude * + np.exp((-(1 / (stddev**2)) * (x - mean)**2)) * + (x - mean) / (stddev**2)) + d_stddev = (2 * amplitude * + np.exp((-(1 / (stddev**2)) * (x - mean)**2)) * + ((x - mean)**2) / (stddev**3)) + return [d_amplitude, d_mean, d_stddev] + + +A full example of a LineModel +============================= + +This example demonstrates one other optional feature for model classes, which +is an *inverse*. An `~astropy.modeling.Model.inverse` implementation should be +a `property` that returns a new model instance (not necessarily of the same +class as the model being inverted) that computes the inverse of that model, so +that for some model instance with an inverse, ``model.inverse(model(*input)) == +input``. + +.. code-block:: python + + import numpy as np + from astropy.modeling import Fittable1DModel, Parameter + + class LineModel(Fittable1DModel): + slope = Parameter() + intercept = Parameter() + linear = True + + @staticmethod + def evaluate(x, slope, intercept): + return slope * x + intercept + + @staticmethod + def fit_deriv(x, slope, intercept): + d_slope = x + d_intercept = np.ones_like(x) + return [d_slope, d_intercept] + + @property + def inverse(self): + new_slope = self.slope ** -1 + new_intercept = -self.intercept / self.slope + return LineModel(slope=new_slope, intercept=new_intercept) + +.. note:: + + The above example is essentially equivalent to the built-in + `~astropy.modeling.functional_models.Linear1D` model. diff --git a/docs/modeling/new.rst b/docs/modeling/new.rst deleted file mode 100644 index 2e2dcbdc1ec4..000000000000 --- a/docs/modeling/new.rst +++ /dev/null @@ -1,429 +0,0 @@ -.. _modeling-new-classes: - -Defining New Model Classes -========================== - -This document describes how to add a model to the package or to create a -user-defined model. In short, one needs to define all model parameters and -write a function which evaluates the model, that is, computes the mathematical -function that implements the model. If the model is fittable, a function to -compute the derivatives with respect to parameters is required if a linear -fitting algorithm is to be used and optional if a non-linear fitter is to be -used. - - -Basic custom models -------------------- - -For most cases, the `~astropy.modeling.custom_model` decorator provides an -easy way to make a new `~astropy.modeling.Model` class from an existing Python -callable. The following example demonstrates how to set up a model consisting -of two Gaussians: - -.. plot:: - :include-source: - - import numpy as np - from astropy.modeling.models import custom_model - from astropy.modeling.fitting import LevMarLSQFitter - - # Define model - @custom_model - def sum_of_gaussians(x, amplitude1=1., mean1=-1., sigma1=1., - amplitude2=1., mean2=1., sigma2=1.): - return (amplitude1 * np.exp(-0.5 * ((x - mean1) / sigma1)**2) + - amplitude2 * np.exp(-0.5 * ((x - mean2) / sigma2)**2)) - - # Generate fake data - np.random.seed(0) - x = np.linspace(-5., 5., 200) - m_ref = sum_of_gaussians(amplitude1=2., mean1=-0.5, sigma1=0.4, - amplitude2=0.5, mean2=2., sigma2=1.0) - y = m_ref(x) + np.random.normal(0., 0.1, x.shape) - - # Fit model to data - m_init = sum_of_gaussians() - fit = LevMarLSQFitter() - m = fit(m_init, x, y) - - # Plot the data and the best fit - plt.plot(x, y, 'o', color='k') - plt.plot(x, m(x), color='r', lw=2) - - -This decorator also supports setting a model's -`~astropy.modeling.FittableModel.fit_deriv` as well as creating models with -more than one inputs. It can also be used as a normal factory function (for -example ``SumOfGaussians = custom_model(sum_of_gaussians)``) rather than as a -decorator. See the `~astropy.modeling.custom_model` documentation for more -examples. - - -A step by step definition of a 1-D Gaussian model -------------------------------------------------- - -The example described in `Basic custom models`_ can be used for most simple -cases, but the following section describes how to construct model classes in -general. Defining a full model class may be desirable, for example, to -provide more specialized parameters, or to implement special functionality not -supported by the basic `~astropy.modeling.custom_model` factory function. - -The details are explained below with a 1-D Gaussian model as an example. There -are two base classes for models. If the model is fittable, it should inherit -from `~astropy.modeling.FittableModel`; if not it should subclass -`~astropy.modeling.Model`. - -If the model takes parameters they should be specified as class attributes in -the model's class definition using the `~astropy.modeling.Parameter` -descriptor. All arguments to the Parameter constructor are optional, and may -include a default value for that parameter, a text description of the parameter -(useful for `help` and documentation generation), as well default constraints -and custom getters/setters for the parameter value. - -:: - - from astropy.modeling import FittableModel, Parameter - - class Gaussian1D(FittableModel): - inputs = ('x',) - outputs = ('y',) - - amplitude = Parameter() - mean = Parameter() - stddev = Parameter() - -The ``inputs`` and ``outputs`` class attributes must be tuples of strings -indicating the number of independent variables that are input to evaluate the -model, and the number of outputs it returns. The labels of the inputs and -outputs (in this case ``'x'`` and ``'y'`` respectively) are currently used for -informational purposes only and have no requirements on them other than that -they do not conflict with parameter names. Outputs may have the same labels as -inputs (eg. ``inputs = ('x', 'y')`` and ``outputs = ('x', 'y')``). However, -inputs must not conflict with each other (eg. ``inputs = ('x', 'x')`` is -incorrect) and likewise for outputs. The lengths of these tuples are -important for specifying the correct number of inputs and outputs. These -attributes supersede the ``n_inputs`` and ``n_outputs`` attributes in older -versions of this package. - -There are two helpful base classes in the modeling package that can be used to -avoid specifying ``inputs`` and ``outputs`` for most common models. These are -`~astropy.modeling.Fittable1DModel` and `~astropy.modeling.Fittable2DModel`. -For example, the real `~astropy.modeling.functional_models.Gaussian1D` model is -actually a subclass of `~astropy.modeling.Fittable1DModel`. This helps cut -down on boilerplate by not having to specify ``inputs`` and ``outputs`` for -many models (follow the link to Gaussian1D to see its source code, for -example). - -Fittable models can be linear or nonlinear in a regression sense. The default -value of the `~astropy.modeling.Model.linear` attribute is ``False``. Linear -models should define the ``linear`` class attribute as ``True``. Because this -model is non-linear we can stick with the default. - -Next, provide methods called ``evaluate`` to evaluate the model and -``fit_deriv``, to compute its derivatives with respect to parameters. These -may be normal methods, `classmethod`, or `staticmethod`, though the convention -is to use `staticmethod` when the function does not depend on any of the -object's other attributes (i.e., it does not reference ``self``) or any of the -class's other attributes as in the case of `classmethod`. The evaluation -method takes all input coordinates as separate arguments and all of the model's -parameters in the same order they would be listed by -`~astropy.modeling.Model.param_names`. - -For this example:: - - @staticmethod - def evaluate(x, amplitude, mean, stddev): - return amplitude * np.exp((-(1 / (2. * stddev**2)) * (x - mean)**2)) - -It should be made clear that the ``evaluate`` method must be designed to take -the model's parameter values as arguments. This may seem at odds with the fact -that the parameter values are already available via attribute of the model -(eg. ``model.amplitude``). However, passing the parameter values directly to -``evaluate`` is a more efficient way to use it in many cases, such as fitting. - -Users of your model would not generally use ``evaluate`` directly. Instead -they create an instance of the model and call it on some input. The -``__call__`` method of models uses ``evaluate`` internally, but users do not -need to be aware of it. The default ``__call__`` implementation also handles -details such as checking that the inputs are correctly formatted and follow -Numpy's broadcasting rules before attempting to evaluate the model. - -Like ``evaluate``, the ``fit_deriv`` method takes as input all coordinates and -all parameter values as arguments. There is an option to compute numerical -derivatives for nonlinear models in which case the ``fit_deriv`` method should -be ``None``:: - - @staticmethod - def fit_deriv(x, amplitude, mean, stddev): - d_amplitude = np.exp((-(1 / (stddev**2)) * (x - mean)**2)) - d_mean = (2 * amplitude * - np.exp((-(1 / (stddev**2)) * (x - mean)**2)) * - (x - mean) / (stddev**2)) - d_stddev = (2 * amplitude * - np.exp((-(1 / (stddev**2)) * (x - mean)**2)) * - ((x - mean)**2) / (stddev**3)) - return [d_amplitude, d_mean, d_stddev] - - -Note that we did *not* have to define an ``__init__`` method or a ``__call__`` -method for our model (this contrasts with Astropy versions 0.4.x and earlier). -For most models the ``__init__`` follows the same pattern, taking the parameter -values as positional arguments, followed by several optional keyword arguments -(constraints, etc.). The modeling framework automatically generates an -``__init__`` for your class that has the correct calling signature (see for -yourself by calling ``help(Gaussian1D.__init__)`` on the example model we just -defined). - -There are cases where it might be desirable to define a custom ``__init__``. -For example, the `~astropy.modeling.functional_models.Gaussian2D` model takes -an optional ``cov_matrix`` argument which can be used as an alternative way to -specify the x/y_stddev and theta parameters. This is perfectly valid so long -as the ``__init__`` determines appropriate values for the actual parameters and -then calls the super ``__init__`` with the standard arguments. Schematically -this looks something like: - -.. code-block:: python - - def __init__(self, amplitude, x_mean, y_mean, x_stddev=None, - y_stddev=None, theta=None, cov_matrix=None, **kwargs): - # The **kwargs here should be understood as other keyword arguments - # accepted by the basic Model.__init__ (such as constraints) - if cov_matrix is not None: - # Set x/y_stddev and theta from the covariance matrix - x_stddev = ... - y_stddev = ... - theta = ... - - # Don't pass on cov_matrix since it doesn't mean anything to the base - # class - super(Gaussian2D, self).__init__(amplitude, x_mean, y_mean, x_stddev, - y_stddev, theta, **kwargs) - - -Full example -^^^^^^^^^^^^ - -.. code-block:: python - - from astropy.modeling import FittableModel, Parameter - - class Gaussian1D(FittableModel): - amplitude = Parameter() - mean = Parameter() - stddev = Parameter() - - @staticmethod - def evaluate(x, amplitude, mean, stddev): - return amplitude * np.exp((-(1 / (2. * stddev**2)) * (x - mean)**2)) - - @staticmethod - def fit_deriv(x, amplitude, mean, stddev): - d_amplitude = np.exp((-(1 / (stddev**2)) * (x - mean)**2)) - d_mean = (2 * amplitude * - np.exp((-(1 / (stddev**2)) * (x - mean)**2)) * - (x - mean) / (stddev**2)) - d_stddev = (2 * amplitude * - np.exp((-(1 / (stddev**2)) * (x - mean)**2)) * - ((x - mean)**2) / (stddev**3)) - return [d_amplitude, d_mean, d_stddev] - - -A full example of a LineModel ------------------------------ - -This example demonstrates one other optional feature for model classes, which -is an *inverse*. An `~astropy.modeling.Model.inverse` implementation should be -a `property` that returns a new model instance (not necessarily of the same -class as the model being inverted) that computes the inverse of that model, so -that for some model instance with an inverse, ``model.inverse(model(*input)) == -input``. - -.. code-block:: python - - from astropy.modeling import FittableModel, Parameter - import numpy as np - - class LineModel(FittableModel): - slope = Parameter() - intercept = Parameter() - linear = True - - @staticmethod - def evaluate(x, slope, intercept): - return slope * x + intercept - - @staticmethod - def fit_deriv(x, slope, intercept): - d_slope = x - d_intercept = np.ones_like(x) - return [d_slope, d_intercept] - - @property - def inverse(self): - new_slope = self.slope ** -1 - new_intercept = -self.intercept / self.slope - return LineModel(slope=new_slope, intercept=new_intercept) - - -Defining New Fitter Classes -=========================== - -This section describes how to add a new nonlinear fitting algorithm to this -package or write a user-defined fitter. In short, one needs to define an error -function and a ``__call__`` method and define the types of constraints which -work with this fitter (if any). - -The details are described below using scipy's SLSQP algorithm as an example. -The base class for all fitters is `~astropy.modeling.fitting.Fitter`:: - - class SLSQPFitter(Fitter): - supported_constraints = ['bounds', 'eqcons', 'ineqcons', 'fixed', - 'tied'] - - def __init__(self): - # Most currently defined fitters take no arguments in their - # __init__, but the option certainly exists for custom fitters - super(SLSQPFitter, self).__init__() - -All fitters take a model (their ``__call__`` method modifies the model's -parameters) as their first argument. - -Next, the error function takes a list of parameters returned by an iteration of -the fitting algorithm and input coordinates, evaluates the model with them and -returns some type of a measure for the fit. In the example the sum of the -squared residuals is used as a measure of fitting.:: - - def objective_function(self, fps, *args): - model = args[0] - meas = args[-1] - model.fitparams(fps) - res = self.model(*args[1:-1]) - meas - return np.sum(res**2) - -The ``__call__`` method performs the fitting. As a minimum it takes all -coordinates as separate arguments. Additional arguments are passed as -necessary:: - - def __call__(self, model, x, y , maxiter=MAXITER, epsilon=EPS): - if model.linear: - raise ModelLinearityException( - 'Model is linear in parameters; ' - 'non-linear fitting methods should not be used.') - model_copy = model.copy() - init_values, _ = _model_to_fit_params(model_copy) - self.fitparams = optimize.fmin_slsqp(self.errorfunc, p0=init_values, - args=(y, x), - bounds=self.bounds, - eqcons=self.eqcons, - ineqcons=self.ineqcons) - return model_copy - - -Using a Custom Statistic Function -================================= - -This section describes how to write a new fitter with a user-defined statistic -function. The example below shows a specialized class which fits a straight -line with uncertainties in both variables. - -The following import statements are needed:: - - import numpy as np - from astropy.modeling.fitting import (_validate_model, - _fitter_to_model_params, - _model_to_fit_params, Fitter, - _convert_input) - from astropy.modeling.optimizers import Simplex - -First one needs to define a statistic. This can be a function or a callable -class.:: - - def chi_line(measured_vals, updated_model, x_sigma, y_sigma, x): - """ - Chi^2 statistic for fitting a straight line with uncertainties in x and - y. - - Parameters - ---------- - measured_vals : array - updated_model : `~astropy.modeling.ParametricModel` - model with parameters set by the current iteration of the optimizer - x_sigma : array - uncertainties in x - y_sigma : array - uncertainties in y - - """ - model_vals = updated_model(x) - if x_sigma is None and y_sigma is None: - return np.sum((model_vals - measured_vals) ** 2) - elif x_sigma is not None and y_sigma is not None: - weights = 1 / (y_sigma ** 2 + updated_model.parameters[1] ** 2 * - x_sigma ** 2) - return np.sum((weights * (model_vals - measured_vals)) ** 2) - else: - if x_sigma is not None: - weights = 1 / x_sigma ** 2 - else: - weights = 1 / y_sigma ** 2 - return np.sum((weights * (model_vals - measured_vals)) ** 2) - -In general, to define a new fitter, all one needs to do is provide a statistic -function and an optimizer. In this example we will let the optimizer be an -optional argument to the fitter and will set the statistic to ``chi_line`` -above:: - - class LineFitter(Fitter): - """ - Fit a straight line with uncertainties in both variables - - Parameters - ---------- - optimizer : class or callable - one of the classes in optimizers.py (default: Simplex) - """ - - def __init__(self, optimizer=Simplex): - self.statistic = chi_line - super(LineFitter, self).__init__(optimizer, - statistic=self.statistic) - -The last thing to define is the ``__call__`` method:: - - def __call__(self, model, x, y, x_sigma=None, y_sigma=None, **kwargs): - """ - Fit data to this model. - - Parameters - ---------- - model : `~astropy.modeling.core.ParametricModel` - model to fit to x, y - x : array - input coordinates - y : array - input coordinates - x_sigma : array - uncertainties in x - y_sigma : array - uncertainties in y - kwargs : dict - optional keyword arguments to be passed to the optimizer - - Returns - ------ - model_copy : `~astropy.modeling.core.ParametricModel` - a copy of the input model with parameters set by the fitter - - """ - model_copy = _validate_model(model, - self._opt_method.supported_constraints) - - farg = _convert_input(x, y) - farg = (model_copy, x_sigma, y_sigma) + farg - p0, _ = _model_to_fit_params(model_copy) - - fitparams, self.fit_info = self._opt_method( - self.objective_function, p0, farg, **kwargs) - _fitter_to_model_params(model_copy, fitparams) - - return model_copy diff --git a/docs/modeling/parallel-fitting.rst b/docs/modeling/parallel-fitting.rst new file mode 100644 index 000000000000..f3fd277a403a --- /dev/null +++ b/docs/modeling/parallel-fitting.rst @@ -0,0 +1,451 @@ +.. _parallel-fitting: + +Fitting models in parallel with N-dimensional data +************************************************** + +In some cases, you may want to fit a model many times to data. For example, you +may have a spectral cube (with two celestial axes and one spectral axis) and you +want to fit a 1D model (which could be either a simple Gaussian model or a +complex compound model with multiple lines and a continuum) to each individual +spectrum in the cube. Alternatively, you may have a cube with two celestial +axes, one spectral axis, and one time axis, and you want to fit a 2D model to +each 2D celestial plane in the cube. Provided each model fit can be treated as +independent, there are significant performance benefits to carrying out these +model fits in parallel. + +The :func:`~astropy.modeling.fitting.parallel_fit_dask` function is ideally +suited to these use cases. It makes it simple to set up fitting of M-dimensional +models to N-dimensional datasets and leverages the power of the `dask +`_ package to efficiently parallelize the problem, +running it either on multiple processes of a single machine or in a distributed +environment. You do not need to know how to use dask in order to use this function, +but you will need to make sure you have `dask `_ +installed. + +Note that the approach here is different from *model sets* which are described +in :ref:`example-fitting-model-sets`, which are a way of fitting a linear model +with a vector of parameters to a data array, as in that specific case the +fitting can be truly vectorized, and will likely not benefit from the approach +described here. + +Getting started +=============== + +To demonstrate the use of this function, we will work through a simple +example of fitting a 1D model to a small spectral cube (if you are +interested in accessing the file, you can find it at +:download:`l1448_13co.fits `, +but the code below will automatically download it). + +.. The following block is to make sure 'data' and 'wcs' are defined if we are not running with --remote-data + +.. plot:: + :context: close-figs + :nofigs: + + >>> import numpy as np + >>> from astropy.wcs import WCS + >>> wcs = WCS(naxis=3) + >>> wcs.wcs.ctype = ['RA---SFL', 'DEC--SFL', 'VOPT'] + >>> wcs.wcs.crval = [57.66, 0., -9959.44378305] + >>> wcs.wcs.crpix = [-799.0, -4741.913, -187.0] + >>> wcs.wcs.cdelt = [-0.006388889, 0.006388889, 66.42361] + >>> wcs.wcs.cunit = ['deg', 'deg', 'm s-1'] + >>> wcs._naxis = [105, 105, 53] + >>> wcs.wcs.set() + >>> data = np.broadcast_to(np.exp(-(np.arange(53) - 25)**2 / 6 ** 2).reshape((53, 1, 1)), (53, 105, 105)) + +We start by downloading the cube and extracting the data and WCS: + +.. plot:: + :context: close-figs + :include-source: + :nofigs: + + >>> from astropy.wcs import WCS + >>> from astropy.io import fits + >>> from astropy.utils.data import get_pkg_data_filename + + >>> filename = get_pkg_data_filename('l1448/l1448_13co.fits') # doctest: +REMOTE_DATA + >>> with fits.open(filename) as hdulist: + ... data = hdulist[0].data + ... wcs = WCS(hdulist[0].header) # doctest: +REMOTE_DATA + +We extract a sub-cube spatially for the purpose of demonstration: + +.. plot:: + :context: close-figs + :include-source: + :nofigs: + + >>> data = data[:, 25:75, 35:85] + >>> wcs = wcs[:, 25:75, 35:85] + +This is a cube of a star-formation region traced by the 13CO line. We can look +at one of the channels: + +.. plot:: + :context: close-figs + :include-source: + :align: center + + >>> import matplotlib.pyplot as plt + >>> fig, ax = plt.subplots(subplot_kw=dict(projection=wcs, slices=('x', 'y', 20))) + >>> ax.imshow(data[20, :, :]) # doctest: +IGNORE_OUTPUT + +We can also extract a spectrum for one of the celestial positions: + +.. plot:: + :context: close-figs + :include-source: + :align: center + + >>> fig, ax = plt.subplots(subplot_kw=dict(projection=wcs, slices=(5, 5, 'x'))) + >>> ax.plot(data[:, 5, 5]) # doctest: +IGNORE_OUTPUT + +We now set up a model to fit this; we will use a simple Gaussian model, +with some reasonable initial guesses for the parameters: + +.. plot:: + :context: close-figs + :include-source: + :nofigs: + + >>> from astropy import units as u + >>> from astropy.modeling.models import Gaussian1D + >>> model = Gaussian1D(amplitude=1 * u.one, mean=4000 * u.m / u.s, stddev=500 * u.m / u.s) + +The data does not have any units in this case, so we use ``u.one`` as +the unit, which indicates it is dimensionless. + +Before fitting this to all spectra in the cube, it’s a good idea to test +the model with at least one of the spectra manually. To do this, we need to extract the x-axis of the spectra: + +.. plot:: + :context: close-figs + :include-source: + :nofigs: + + >>> import numpy as np + >>> x = wcs.pixel_to_world(0, 0, np.arange(data.shape[0]))[1] + >>> x + ) + [2528.19489695, 2594.61850695, 2661.04211695, 2727.46572695, + 2793.88933695, 2860.31294695, 2926.73655695, 2993.16016695, + ... + 5716.52817695, 5782.95178695, 5849.37539695, 5915.79900695, + 5982.22261695] m / s> + +We can now carry out the fit: + +.. plot:: + :context: close-figs + :include-source: + :nofigs: + + >>> from astropy.modeling.fitting import TRFLSQFitter + >>> fitter = TRFLSQFitter() + >>> model_fit_single = fitter(model, x, data[:, 5, 5]) + +.. plot:: + :context: close-figs + :include-source: + :align: center + + >>> fig, ax = plt.subplots() + >>> ax.plot(x, data[:, 5, 5], '.', label='data') # doctest: +IGNORE_OUTPUT + >>> ax.plot(x, model(x), label='initial model') # doctest: +IGNORE_OUTPUT + >>> ax.plot(x, model_fit_single(x), label='fitted model') # doctest: +IGNORE_OUTPUT + >>> ax.legend() # doctest: +IGNORE_OUTPUT + +The model seems to work! We can now use the +:func:`~astropy.modeling.fitting.parallel_fit_dask` function +to fit all spectra in the cube: + +.. plot:: + :context: close-figs + :include-source: + :nofigs: + + >>> from astropy.modeling.fitting import parallel_fit_dask + >>> model_fit = parallel_fit_dask(model=model, + ... fitter=fitter, + ... data=data, + ... world=wcs, + ... fitting_axes=0, + ... data_unit=u.one, + ... scheduler='synchronous') + +The arguments in this case are as follows: + +* ``model=`` is the initial model. While in our case the initial + parameters were specified as scalars, it is possible to pass in a + model that has array parameters if you want to have different initial + parameters as a function of location in the dataset. +* ``fitter=`` is the fitter instance. +* ``data=`` is the N-dimensional dataset, in our case the 3D spectral + cube. +* ``world=`` provides information about the world coordinates for the + fit, for example the spectral coordinates for a spectrum. This can be + specified in different ways, but above we have chosen to pass in the + WCS object for the dataset, from which the spectral axis coordinates + will be extracted. +* ``fitting_axes=`` specifies which axis or axes include the data to + fit. In our example, we are fitting the spectra, + which in NumPy notation is the first axis in the cube, so we specify + ``fitting_axes=0``. +* ``data_unit=`` specifies the unit to use for the data. In our case, + the data has no unit, but because we are using units for the spectral + axis, we need to specify ``u.one`` here. + +We can now take a look at the parameter maps: + +.. plot:: + :context: close-figs + :include-source: + :align: center + + >>> fig, axs = plt.subplots(figsize=(10, 5), ncols=3) + >>> ax1 = axs[0] + >>> ax1.set_title('Amplitude') # doctest: +IGNORE_OUTPUT + >>> ax1.imshow(model_fit.amplitude.value, vmin=0, vmax=5, origin='lower') # doctest: +IGNORE_OUTPUT + >>> ax2 = axs[1] + >>> ax2.set_title('Mean') # doctest: +IGNORE_OUTPUT + >>> ax2.imshow(model_fit.mean.value, vmin=2500, vmax=6000, origin='lower') # doctest: +IGNORE_OUTPUT + >>> ax3 = axs[2] + >>> ax3.set_title('Standard deviation') # doctest: +IGNORE_OUTPUT + >>> ax3.imshow(model_fit.stddev.value, vmin=0, vmax=2000, origin='lower') # doctest: +IGNORE_OUTPUT + +There are a number of pixels that appear to have issues. Inspecting the +histogram of means, we can see that a lot of values are not at all in +the spectral range we are fitting: + +.. plot:: + :context: close-figs + :include-source: + :align: center + + >>> fig, ax = plt.subplots() + >>> ax.hist(model_fit.mean.value.ravel(), bins=100) # doctest: +IGNORE_OUTPUT + >>> ax.set(yscale='log', xlabel='mean', ylabel='number') # doctest: +IGNORE_OUTPUT + +We can set the bounds on the mean and try the fit again + +.. plot:: + :context: close-figs + :include-source: + :nofigs: + + >>> model.mean.bounds = (3000, 6000) * u.km / u.s + >>> model_fit = parallel_fit_dask(model=model, + ... fitter=fitter, + ... data=data, + ... world=wcs, + ... fitting_axes=0, + ... data_unit=u.one, + ... scheduler='synchronous') + +and we can visualize the results: + +.. plot:: + :context: close-figs + :include-source: + :align: center + + >>> fig, axs = plt.subplots(figsize=(10, 5), ncols=3) + >>> ax1 = axs[0] + >>> ax1.set_title('Amplitude') # doctest: +IGNORE_OUTPUT + >>> ax1.imshow(model_fit.amplitude.value, vmin=0, vmax=5, origin='lower') # doctest: +IGNORE_OUTPUT + >>> ax2 = axs[1] + >>> ax2.set_title('Mean') # doctest: +IGNORE_OUTPUT + >>> ax2.imshow(model_fit.mean.value, vmin=2500, vmax=6000, origin='lower') # doctest: +IGNORE_OUTPUT + >>> ax3 = axs[2] + >>> ax3.set_title('Standard deviation') # doctest: +IGNORE_OUTPUT + >>> ax3.imshow(model_fit.stddev.value, vmin=0, vmax=2000, origin='lower') # doctest: +IGNORE_OUTPUT + +The amplitude map no longer contains any problematic pixels. + +World input +=========== + +The example above demonstrated that it is possible to pass in a +:class:`astropy.wcs.WCS` object to the ``world=`` argument in order to determine +the world coordinates for the fit (e.g. the spectral axis values for a spectral +fit). It is also possible to pass in a tuple of arrays - if you do this, the +tuple should have one item per fitting axis. It is most efficient to pass in a +tuple of 1D arrays, but if the world coordinates vary over the axes being +iterated over, you can also pass in a tuple of N-d arrays, giving the +coordinates of each individual pixel (it is also possible to pass in arrays that +are not 1D but also not fully N-d as long as they can be broadcasted to the data +shape). + +Multiprocessing +=============== + +By default, :func:`~astropy.modeling.fitting.parallel_fit_dask` will make use +of multi-processing to parallelize the fitting. If you write a script to +carry out the fitting, you will likely need to move your code inside a:: + + if __name__ == "__main__": + + ... + +clause as otherwise Python will execute the whole code in the script many times, +and potentially recursively, rather than just parallelizing the fitting. + +Performance +=========== + +The :func:`~astropy.modeling.fitting.parallel_fit_dask` function splits the data +into chunks, each of which is then sent to a different process. The size of +these chunks is critical to obtaining good performance. If we split the data +into one chunk per fit, the process would be inefficient due to significant +overhead from inter-process communication. Conversely, if we split the data into +fewer chunks than there are available processes, we will not utilize all the +available computational power. If we split the data into slightly more chunks +than there are processes, inefficiencies can arise as well. For example, +splitting the data into five chunks with four available processes means the four +processes will first fit four chunks, and then a single process will be held up +fitting the remaining chunk. Therefore, it is important to carefully consider +how the data is split. + +To control the splitting of the data, use the ``chunk_n_max=`` keyword argument. +This determines how many individual fits will be carried out in each chunk. For +example, when fitting a model to individual spectra in a spectral cube, setting +``chunk_n_max=100`` means each chunk will contain 100 spectra. As a general +guide, you will likely want to set this to be roughly the number of fits to be +carried out in the data divided by several times the number of available +processes. For example, if you need to fit 100,000 spectra and have 8 processes +available, setting ``chunk_n_max=1000`` would be reasonable. This configuration +would break the data into 100 chunks, meaning each process will need to handle +approximately a dozen chunks. Additionally, fitting 1,000 spectra per chunk will +take enough time to avoid being dominated by communication overhead. + +The default value for ``chunk_n_max`` is 500. + +Fit information +=============== + +When carrying out regular (non-parallel) fitting with astropy, fitters will typically +have a ``.fit_info`` attribute which contains information about the fit, such as +the number of function evaluations, parameter covariance matrix, and so on. The +information available depends on the specific fitter used. + +These fit information objects can in some cases take up more memory than the +data that was being fit in the first place, so when carrying out many fits +in parallel with :func:`~astropy.modeling.fitting.parallel_fit_dask`, this +information is not preserved by default and the ``.fit_info`` parameter on +the fitter instance is set to `None` + +However, since access to this information can be useful in some cases, it is +possible to opt-in to keeping it. Either all of the fit information can be +preserved, by setting ``fit_info=True``: + + >>> model_fit = parallel_fit_dask(model=model, + ... ... + ... fitter=fitter, + ... fit_info=True) # doctest: +SKIP + +or just specific keys (which can help reduce memory usage): + + >>> model_fit = parallel_fit_dask(model=model, + ... ... + ... fitter=fitter, + ... fit_info=('nfev', 'message', 'status')) # doctest: +SKIP + + +In these cases, the fitter's ``.fit_info`` will be set to a +:class:`~astropy.modeling.fitting.FitInfoArrayContainer` object, which internally +has a numpy object array containing all the different fit information objects. +The shape of ``.fit_info`` should be the same as the parameter arrays: + + >>> fitter.fit_info.shape # doctest: +SKIP + (50, 50) + >>> fitter.fit_info.ndim # doctest: +SKIP + 2 + +Indexing the fit info will return a specific fit information object, e.g. + + >>> fitter.fit_info[10, 20] # doctest: +SKIP + message: The maximum number of function evaluations is exceeded. + success: False + status: 0 + fun: [-2.169e-01 -2.398e-01 ... -5.502e-02 2.498e-01] + x: [ 5.352e+02 2.034e+04 3.932e+03] + cost: 0.575174901185717 + jac: [[ 3.514e-05 -2.166e-05 9.810e-05] + [ 3.793e-05 -2.329e-05 1.051e-04] + ... + [ 1.200e-03 -5.990e-04 2.197e-03] + [ 1.277e-03 -6.343e-04 2.316e-03]] + grad: [-5.634e-06 2.866e-06 -1.092e-05] + optimality: 1.0921480583423703e-05 + active_mask: [0 0 0] + nfev: 100 + njev: 93 + param_cov: [[ 5.965e+08 2.262e+09 2.913e+08] + [ 2.262e+09 8.584e+09 1.106e+09] + [ 2.913e+08 1.106e+09 1.427e+08]] + +Indexing the fit info in a way that returns a range of fits, e.g. +``fitter.fit_info[10:20, 20:30]``, will return a +:class:`~astropy.modeling.fitting.FitInfoArrayContainer` object. + +It is also possible to retrieve one of these keys for all fits as an array, e.g.: + + >>> nfev = fitter.fit_info.get_property_as_array('nfev') # doctest: +SKIP + >>> nfev.shape # doctest: +SKIP + (50, 50) + >>> nfev[0:3, 0:3] # doctest: +SKIP + array([[ 9, 8, 10], + [10, 13, 9], + [10, 13, 10]]) + >>> param_cov = fitter.fit_info.get_property_as_array('param_cov') # doctest: +SKIP + >>> param_cov.shape # doctest: +SKIP + (50, 50, 3, 3) + +Diagnostics +=========== + +One of the challenges of fitting a model many different times is understanding +what went wrong when issues arise. By default, if a fit fails with a warning or +an exception, the parameters for that fit will be set to NaN, and no warning or +exception will be shown to the user. However, it can be helpful to have more +information, such as the specific error or exception that occurred. + +You can control this by setting the ``diagnostics=`` argument. This allows you +to choose whether to output information about: + +* Failed fits with errors (``diagnostics='error'``), +* Fits with errors or warnings (``diagnostics='error+warn'``), or +* All fits (``diagnostics='all'``). + +If the ``diagnostics`` option is specified, you will also need to specify +``diagnostics_path``, which should be the path to a folder that will contain all +the output. Each fit that needs to be output will be assigned a sub-folder named +after the indices along the axes of the data (excluding the fitting axes). The +output will include (if appropriate): + +* ``error.log``, containing details of any exceptions that occurred +* ``warn.log``, containing any warnings + +You may also want to automatically create a plot of the fit, inspect the data +being fit, or examine the model. To do this, you can pass a function to +``diagnostics_callable``. See :func:`~astropy.modeling.fitting.parallel_fit_dask` +for more information about the arguments this function should accept. + +Schedulers +========== + +By default, :func:`~astropy.modeling.fitting.parallel_fit_dask` will make use of +the ``'processes'`` scheduler, which means that multiple processes on your local +machine can be used. You can override the scheduler being used with the +``scheduler=`` keyword argument. You can either set this to the name of a +scheduler (such as ``'synchronous'``), or you can set it to ``'default'`` in order +to make use of whatever is the currently active dask scheduler, which allows +you for example to set up a `dask.distributed +`_ scheduler. diff --git a/docs/modeling/parameters.rst b/docs/modeling/parameters.rst index 684d3a455f13..bb4f1c21bded 100644 --- a/docs/modeling/parameters.rst +++ b/docs/modeling/parameters.rst @@ -6,11 +6,14 @@ Parameters ********** +Basics +====== + Most models in this package are "parametric" in the sense that each subclass of `~astropy.modeling.Model` represents an entire family of models, each member of which is distinguished by a fixed set of parameters that fit that -model to some some dependent and independent variable(s) (also referred to -throughout the the package as the outputs and inputs of the model). +model to some dependent and independent variable(s) (also referred to +throughout the package as the outputs and inputs of the model). Parameters are used in three different contexts within this package: Basic evaluation of models, fitting models to data, and providing information about @@ -24,7 +27,7 @@ other property of the model (the degree in the case of polynomials). Models maintain a list of parameter names, `~astropy.modeling.Model.param_names`. Single parameters are instances of -`~astropy.modeling.Parameter` which provide a proxy for the actual parameter +`~astropy.modeling.Parameter` which provides a proxy for the actual parameter values. Simple mathematical operations can be performed with them, but they also contain additional attributes specific to model parameters, such as any constraints on their values and documentation. @@ -36,9 +39,65 @@ cases, however, array-valued parameters have no meaning specific to the model, and are simply combined with input arrays during model evaluation according to the standard `Numpy broadcasting rules`_. +Parameter constraints +===================== + +`astropy.modeling` supports several types of parameter constraints. They are implemented +as properties of `~astropy.modeling.Parameter`, the class which defines all fittable +parameters, and can be set on individual parameters or on model instances. + +The `astropy.modeling.Parameter.fixed` constraint is boolean and indicates +whether a parameter is kept "fixed" or "frozen" during fitting. For example, fixing the +``stddev`` of a :class:`~astropy.modeling.functional_models.Gaussian1D` model +means it will be excluded from the list of fitted parameters:: + + >>> from astropy.modeling.models import Gaussian1D + >>> g = Gaussian1D(amplitude=10.2, mean=2.3, stddev=1.2) + >>> g.stddev.fixed + False + >>> g.stddev.fixed = True + >>> g.stddev.fixed + True + +`astropy.modeling.Parameter.bounds` is a tuple of numbers +setting minimum and maximum value for a parameter. ``(None, None)`` indicates +the parameter values are not bound. ``bounds`` can be set also using the +`~astropy.modeling.Parameter.min` and +`~astropy.modeling.Parameter.max` properties. Assigning ``None`` to +the corresponding property removes the bound on the parameter. For example, setting +bounds on the ``mean`` value of a :class:`~astropy.modeling.functional_models.Gaussian1D` +model can be done either by setting ``min`` and ``max``:: + + >>> g.mean.bounds + (None, None) + >>> g.mean.min = 2.2 + >>> g.mean.bounds + (2.2, None) + >>> g.mean.max = 2.4 + >>> g.mean.bounds + (2.2, 2.4) + +or using the ``bounds`` property:: + + >>> g.mean.bounds = (2.2, 2.4) + +`astropy.modeling.Parameter.tied` is a user supplied callable +which takes a model instance and returns a value for the parameter. It is most useful +with setting constraints on compounds models, for example a ratio between two parameters (:ref:`example`). + +Constraints can also be set when the model is initialized. For example:: + + >>> g = Gaussian1D(amplitude=10.2, mean=2.3, stddev=1.2, + ... fixed={'stddev': True}, + ... bounds={'mean': (2.2, 2.4)}) + >>> g.stddev.fixed + True + >>> g.mean.bounds + (2.2, 2.4) + Parameter examples ------------------- +================== - Model classes can be introspected directly to find out what parameters they accept:: @@ -75,8 +134,8 @@ Parameter examples >>> p1 = models.Polynomial1D(degree=3, c0=1.0, c1=0.0, c2=2.0, c3=3.0) >>> p1.param_names ('c0', 'c1', 'c2', 'c3') - >>> p1 - + >>> p1 # doctest: +FLOAT_CMP + For the basic `~astropy.modeling.polynomial.Polynomial1D` class the parameters are named ``c0`` through ``cN`` where ``N`` is the degree of the @@ -89,17 +148,17 @@ Parameter examples of the coefficients initially:: >>> p2 = models.Polynomial1D(degree=4) - >>> p2 - + >>> p2 # doctest: +FLOAT_CMP + -- Parameters can the be set/updated by accessing attributes on the model of +- Parameters can then be set/updated by accessing attributes on the model of the same names as the parameters:: >>> p2.c4 = 1 >>> p2.c2 = 3.5 >>> p2.c0 = 2.0 - >>> p2 - + >>> p2 # doctest: +FLOAT_CMP + This example now represents the polynomial :math:`x^4 + 3.5x^2 + 2`. @@ -112,19 +171,19 @@ Parameter examples ... for idx, name in enumerate(ch2.param_names)) >>> ch2 = models.Chebyshev2D(x_degree=2, y_degree=3, n_models=2, ... **coeffs) - >>> ch2.param_sets - array([[ 0., 10.], - [ 1., 11.], - [ 2., 12.], - [ 3., 13.], - [ 4., 14.], - [ 5., 15.], - [ 6., 16.], - [ 7., 17.], - [ 8., 18.], - [ 9., 19.], - [ 10., 20.], - [ 11., 21.]]) + >>> ch2.param_sets # doctest: +FLOAT_CMP + array([[ 0., 10.], + [ 1., 11.], + [ 2., 12.], + [ 3., 13.], + [ 4., 14.], + [ 5., 15.], + [ 6., 16.], + [ 7., 17.], + [ 8., 18.], + [ 9., 19.], + [10., 20.], + [11., 21.]]) - Or directly, using keyword arguments:: @@ -140,10 +199,10 @@ Parameter examples >>> p3 = models.Polynomial1D(degree=2, c0=1.0, c1=[2.0, 3.0], ... c2=[[4.0, 5.0], [6.0, 7.0], [8.0, 9.0]]) - >>> p3(2.0) - array([[ 21., 27.], - [ 29., 35.], - [ 37., 43.]]) + >>> p3(2.0) # doctest: +FLOAT_CMP + array([[21., 27.], + [29., 35.], + [37., 43.]]) This is equivalent to evaluating the Numpy expression:: @@ -152,17 +211,17 @@ Parameter examples ... [6.0, 7.0], ... [8.0, 9.0]]) >>> c1 = np.array([2.0, 3.0]) - >>> c2 * 2.0**2 + c1 * 2.0 + 1.0 - array([[ 21., 27.], - [ 29., 35.], - [ 37., 43.]]) + >>> c2 * 2.0**2 + c1 * 2.0 + 1.0 # doctest: +FLOAT_CMP + array([[21., 27.], + [29., 35.], + [37., 43.]]) Note that in most cases, when using array-valued parameters, the parameters must obey the standard broadcasting rules for Numpy arrays with respect to each other:: >>> models.Polynomial1D(degree=2, c0=1.0, c1=[2.0, 3.0], - ... c2=[4.0, 5.0, 6.0]) + ... c2=[4.0, 5.0, 6.0]) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... InputParameterError: Parameter u'c1' of shape (2,) cannot be broadcast diff --git a/docs/modeling/performance.rst b/docs/modeling/performance.rst new file mode 100644 index 000000000000..be84ff7c4857 --- /dev/null +++ b/docs/modeling/performance.rst @@ -0,0 +1,19 @@ + +.. _astropy-modeling-performance: + +Performance Tips +**************** + +Initializing a compound model with many constituent models can be time consuming. +If your code uses the same compound model repeatedly consider initializing it +once and reusing the model. + +Consider the :ref:`performance tips ` that apply to +quantities when initializing and evaluating models with quantities. + +When fitting models with one of the fitter classes, by default a copy of the +model is returned, with parameters set to those determined by the fitting. If +you do not need to preserve the initial model used in the fitting, you can +optionally pass ``inplace=True`` when calling the fitter, and the parameters +will be updated on the model you supply rather than returning a copy of the +model - this can improve performance in some cases. diff --git a/docs/modeling/physical_models.rst b/docs/modeling/physical_models.rst new file mode 100644 index 000000000000..5d666a95fc4f --- /dev/null +++ b/docs/modeling/physical_models.rst @@ -0,0 +1,329 @@ +.. _predef_physicalmodels: + +*************** +Physical Models +*************** + +These are models that are physical motivated, generally as solutions to +physical problems. This is in contrast to those that are mathematically motivated, +generally as solutions to mathematical problems. + +.. _blackbody-planck-law: + +BlackBody +========= + +The :class:`~astropy.modeling.physical_models.BlackBody` model provides a model +for using `Planck's Law `_. +The blackbody function is + +.. math:: + + B_{\nu}(T) = A \frac{2 h \nu^{3} / c^{2}}{exp(h \nu / k T) - 1} + +where :math:`\nu` is the frequency, :math:`T` is the temperature, +:math:`A` is the scaling factor, +:math:`h` is the Plank constant, :math:`c` is the speed of light, and +:math:`k` is the Boltzmann constant. + +The two parameters of the model the scaling factor ``scale`` (A) and +the absolute temperature ``temperature`` (T). If the ``scale`` factor does not +have units, then the result is in units of spectral radiance, specifically +ergs/(cm^2 Hz s sr). If the ``scale`` factor is passed with spectral radiance units, +then the result is in those units (e.g., ergs/(cm^2 A s sr) or MJy/sr). +Setting the ``scale`` factor with units of ergs/(cm^2 A s sr) will give the +Planck function as :math:`B_\lambda`. +The temperature can be passed as a Quantity with any supported temperature unit. + +An example plot for a blackbody with a temperature of 10000 K and a scale of 1 is +shown below. A scale of 1 shows the Planck function with no scaling in the +default units returned by :class:`~astropy.modeling.physical_models.BlackBody`. + +.. plot:: + :include-source: + + import numpy as np + import matplotlib.pyplot as plt + + from astropy.modeling.models import BlackBody + import astropy.units as u + + wavelengths = np.logspace(np.log10(1000), np.log10(3e4), num=1000) * u.AA + + # blackbody parameters + temperature = 10000 * u.K + + # BlackBody provides the results in ergs/(cm^2 Hz s sr) when scale has no units + bb = BlackBody(temperature=temperature, scale=10000.0) + bb_result = bb(wavelengths) + + fig, ax = plt.subplots(layout='tight') + ax.plot(wavelengths, bb_result, '-') + + ax.set( + xscale="log", + xlabel=fr"$\lambda$ [{wavelengths.unit}]", + ylabel=fr"$F(\lambda)$ [{bb_result.unit}]", + ) + + plt.show() + +The :meth:`~astropy.modeling.physical_models.BlackBody.bolometric_flux` member +function gives the bolometric flux using +:math:`\sigma T^4/\pi` where :math:`\sigma` is the Stefan-Boltzmann constant. + +The :meth:`~astropy.modeling.physical_models.BlackBody.lambda_max` and +:meth:`~astropy.modeling.physical_models.BlackBody.nu_max` member functions +give the wavelength and frequency of the maximum for :math:`B_\lambda` +and :math:`B_\nu`, respectively, calculated using `Wien's Law +`_. + +Drude1D +======= + +The :class:`~astropy.modeling.physical_models.Drude1D` model provides a model +for the behavior of an electron in a material +(see `Drude Model `_). +Like the :class:`~astropy.modeling.functional_models.Lorentz1D` model, the Drude model +has broader wings than the :class:`~astropy.modeling.functional_models.Gaussian1D` +model. The Drude profile has been used to model dust features including the +2175 Angstrom extinction feature and the mid-infrared aromatic/PAH features. +The Drude function at :math:`x` is + +.. math:: + + D(x) = A \frac{(f/x_0)^2}{((x/x_0 - x_0/x)^2 + (f/x_0)^2} + +where :math:`A` is the amplitude, :math:`f` is the full width at half maximum, +and :math:`x_0` is the central wavelength. An example of a Drude1D model +with :math:`x_0 = 2175` Angstrom and :math:`f = 400` Angstrom is shown below. + +.. plot:: + :include-source: + + import numpy as np + import matplotlib.pyplot as plt + + from astropy.modeling.models import Drude1D + import astropy.units as u + + wavelengths = np.linspace(1000, 4000, num=1000) * u.AA + + # Parameters and model + mod = Drude1D(amplitude=1.0, x_0=2175. * u.AA, fwhm=400. * u.AA) + mod_result = mod(wavelengths) + + fig, ax = plt.subplots(layout="tight") + ax.plot(wavelengths, mod_result, '-') + + ax.set(xlabel=fr"$\lambda$ [{wavelengths.unit}]", ylabel=r"$D(\lambda)$") + + plt.show() + +.. _NFW: + +NFW +========= + +The :class:`~astropy.modeling.physical_models.NFW` model computes a +1-dimensional Navarro–Frenk–White profile. The dark matter density in an +NFW profile is given by: + + +.. math:: + + \rho(r)=\frac{\delta_c\rho_{c}}{r/r_s(1+r/r_s)^2} + +where :math:`\rho_{c}` is the critical density of the Universe at the redshift +of the profile, :math:`\delta_c` is the over density, and :math:`r_s` is the +scale radius of the profile. + + +This model relies on three parameters: + + ``mass`` : the mass of the profile (in solar masses if no units are provided) + + ``concentration`` : the profile concentration + + ``redshift`` : the redshift of the profile + +As well as two optional initialization variables: + + ``massfactor`` : tuple or string specifying the overdensity type and factor (default ("critical", 200)) + + ``cosmo`` : the cosmology for density calculation (default default_cosmology) + +.. note:: + Initialization of NFW profile object required before evaluation (in order to set mass + overdensity and cosmology). + + +Sample plots of an NFW profile with the following parameters are displayed below: + ``mass`` = :math:`2.0 x 10^{15} M_{sun}` + + ``concentration`` = 8.5 + + ``redshift`` = 0.63 + +The first plot is of the NFW profile density as a function of radius. +The second plot displays the profile density and radius normalized by the NFW scale +density and scale radius, respectively. The scale density and scale radius are available +as attributes ``rho_s`` and ``r_s``, and the overdensity radius can be accessed via ``r_virial``. + +.. plot:: + :include-source: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling.models import NFW + import astropy.units as u + from astropy import cosmology + + # NFW Parameters + mass = u.Quantity(2.0E15, u.M_sun) + concentration = 8.5 + redshift = 0.63 + cosmo = cosmology.Planck15 + massfactor = ("critical", 200) + + # Create NFW Object + n = NFW(mass=mass, concentration=concentration, redshift=redshift, cosmo=cosmo, + massfactor=massfactor) + + # Radial distribution for plotting + radii = range(1,2001,10) * u.kpc + + # Radial NFW density distribution + n_result = n(radii) + + # Plot creation + fig, axs = plt.subplots(nrows=2) + fig.suptitle('1 Dimensional NFW Profile') + + # Density profile subplot + axs[0].plot(radii, n_result, '-') + axs[0].set( + yscale='log', + xlabel=fr"$r$ [{radii.unit}]", + ylabel=fr"$\rho$ [{n_result.unit}]", + ) + + # Create scaled density / scaled radius subplot + # NFW Object + n = NFW(mass=mass, concentration=concentration, redshift=redshift, cosmo=cosmo, + massfactor=massfactor) + + # Radial distribution for plotting + radii = np.logspace(np.log10(1e-5), np.log10(2), num=1000) * u.Mpc + n_result = n(radii) + + # Scaled density / scaled radius subplot + axs[1].plot(radii / n.radius_s, n_result / n.density_s, '-') + axs[1].set( + xscale='log', + yscale='log', + xlabel=r"$r / r_s$", + ylabel=r"$\rho / \rho_s$", + ) + + # Display plot + fig.tight_layout(rect=[0, 0.03, 1, 0.95]) + plt.show() + + + +The :meth:`~astropy.modeling.physical_models.NFW.circular_velocity` member provides the circular +velocity at each position ``r`` via the equation: + + +.. math:: + + v_{circ}(r)^2=\frac{1}{x}\frac{\ln(1+cx)-(cx)/(1+cx)}{\ln(1+c)-c/(1+c)} + +where x is the ratio ``r``:math:`/r_{vir}`. Circular velocities are provided in km/s. + +A sample plot of circular velocities of an NFW profile with the following parameters is displayed +below: + + ``mass`` = :math:`2.0 x 10^{15} M_{sun}` + + ``concentration`` = 8.5 + + ``redshift`` = 0.63 + +The maximum circular velocity and radius of maximum circular velocity are available as attributes +``v_max`` and ``r_max``. + + +.. plot:: + :include-source: + + import matplotlib.pyplot as plt + from astropy.modeling.models import NFW + import astropy.units as u + from astropy import cosmology + + # NFW Parameters + mass = u.Quantity(2.0E15, u.M_sun) + concentration = 8.5 + redshift = 0.63 + cosmo = cosmology.Planck15 + massfactor = ("critical", 200) + + # Create NFW Object + n = NFW(mass=mass, concentration=concentration, redshift=redshift, cosmo=cosmo, + massfactor=massfactor) + + # Radial distribution for plotting + radii = range(1,200001,10) * u.kpc + + # NFW circular velocity distribution + n_result = n.circular_velocity(radii) + + # Plot creation + fig,ax = plt.subplots() + ax.set_title('NFW Profile Circular Velocity') + ax.plot(radii, n_result, '-') + ax.set_xscale('log') + ax.set_xlabel(fr"$r$ [{radii.unit}]") + ax.set_ylabel(r"$v_{circ}$" + f" [{n_result.unit}]") + + # Display plot + plt.tight_layout(rect=[0, 0.03, 1, 0.95]) + plt.show() + + +.. _Cosmologies: + +Cosmologies +=========== + +The instances of the |Cosmology| class (and subclasses) include +|Cosmology.to_format|, a method to convert a Cosmology to another python +object. Specifically, any redshift method can be converted to a +:class:`~astropy.modeling.FittableModel` instance using the argument +``format="astropy.model"``. +During the conversion, each |Cosmology| :class:`~astropy.cosmology.Parameter` +is converted to a :class:`astropy.modeling.Model` +:class:`~astropy.modeling.Parameter`, while the redshift-method becomes the +model's ``__call__`` / ``evaluate`` method. +This means cosmologies can now be fit with data! + +.. code-block:: + + >>> from astropy.cosmology import Planck18 + >>> model = Planck18.to_format(format="astropy.model", method="lookback_time") + >>> model + + +When finished, e.g. fitting, a model can be turned back into a |Cosmology| +using |Cosmology.from_format|. + +.. code-block:: + + >>> from astropy.cosmology import Cosmology + >>> cosmo = Cosmology.from_format(model, format="astropy.model") + >>> cosmo == Planck18 + True diff --git a/docs/modeling/polynomial_models.rst b/docs/modeling/polynomial_models.rst new file mode 100644 index 000000000000..4cf7c62394bf --- /dev/null +++ b/docs/modeling/polynomial_models.rst @@ -0,0 +1,79 @@ +.. include:: links.inc + +.. _polynomial_models: + +***************** +Polynomial Models +***************** + +.. _domain-window-note: + +Notes regarding usage of domain and window +------------------------------------------ + +Most of the polynomial models have optional domain and window attributes. +It is important to understand how they currently are interpreted, which +can be confusing since the terminology often implies something different. + +Both the domain and window attributes for a polynomial consist of a two +element list (this will change to tuples in a future release) that +indicate a range of values for input values. For 2-Dimensional polynomials +the attributes become x_domain, y_domain, x_window, and y_window. +Generally speaking, the main purpose of these attributes is to define +a linear transform between the supplied input variable and the resultant +input variable that is supplied to the polynomial. For example, if +domain = [-2, 2] and window = [-1, 1], input values will be divided by +two so that the domain maps to the window. Correspondingly the pair +domain = [0, 2], window = [-1, 1] implies that 1 will be subtracted from +the input variable before using it in the polynomial. + +Neither domain or window are meant to imply that values that fall outside +of their corresponding ranges will result in an exception, or that +such values are necessarily invalid (the latter depends on the context +of how the polynomial is being used). + +It is the case that the orthogonal polynomials are defined on a range of +[-1, 1], but nothing in the current machinery prevents them from being +evaluated outside that range. + +Domain is used in fitting polynomials to bound the input variable to map +to the defined window so that they fall within the expected [-1, 1] range +for such polynomials. That is, the fitting routine will set the domain to +map to the window range for the range of input x values supplied (so that +domain may change if the minimum and maximum x values being fit change). + +The meaning of these terms may conflict with expectations (e.g., domain +is often meant to mean the range of input values the function is valid +for). For fit results that is somewhat true, but otherwise, it is not. +The default values for ordinary polynomials is [-1, 1] for both domain +and window, which effectively signals no transformation of the input +variable. + +The terminology was adopted from numpy polynomials, which have the same +confusion in meaning. + + +1D Polynomials +-------------- + +- :class:`~astropy.modeling.polynomial.Polynomial1D` + +- :class:`~astropy.modeling.polynomial.Chebyshev1D` + +- :class:`~astropy.modeling.polynomial.Legendre1D` + +- :class:`~astropy.modeling.polynomial.Hermite1D` + +2D Polynomials +-------------- + +- :class:`~astropy.modeling.polynomial.Polynomial2D` + +- :class:`~astropy.modeling.polynomial.Chebyshev2D` + +- :class:`~astropy.modeling.polynomial.Legendre2D` + +- :class:`~astropy.modeling.polynomial.Hermite2D` + +- :class:`~astropy.modeling.polynomial.SIP` model implements the + Simple Imaging Polynomial (`SIP`_) convention diff --git a/docs/modeling/powerlaw_models.rst b/docs/modeling/powerlaw_models.rst new file mode 100644 index 000000000000..25534f174d78 --- /dev/null +++ b/docs/modeling/powerlaw_models.rst @@ -0,0 +1,17 @@ +.. _powerlaw_models: + +*************** +Powerlaw Models +*************** + +- :class:`~astropy.modeling.powerlaws.PowerLaw1D` + +- :class:`~astropy.modeling.powerlaws.BrokenPowerLaw1D` + +- :class:`~astropy.modeling.powerlaws.SmoothlyBrokenPowerLaw1D` + +- :class:`~astropy.modeling.powerlaws.ExponentialCutoffPowerLaw1D` + +- :class:`~astropy.modeling.powerlaws.LogParabola1D` + +- :class:`~astropy.modeling.powerlaws.Schechter1D` diff --git a/docs/modeling/predef_models1D.rst b/docs/modeling/predef_models1D.rst new file mode 100644 index 000000000000..eaa47a671f7c --- /dev/null +++ b/docs/modeling/predef_models1D.rst @@ -0,0 +1,161 @@ +.. _predef_models1D: + +********* +1D Models +********* + +Operations +========== + +These models perform simple mathematical operations. + +- :class:`~astropy.modeling.functional_models.Const1D` model returns the + constant replicated by the number of input x values. + +- :class:`~astropy.modeling.functional_models.Multiply` model multiples the + input x values by a factor and propagates units if the factor is + a :class:`~astropy.units.Quantity`. + +- :class:`~astropy.modeling.functional_models.RedshiftScaleFactor` model + multiples the input x values by a (1 + z) factor. + +- :class:`~astropy.modeling.functional_models.Scale` model multiples by a + factor without changing the units of the result. + +- :class:`~astropy.modeling.functional_models.Shift` model adds a constant + to the input x values. + +Shapes +====== + +These models provide shapes, often used to model general x, y data. + +- :class:`~astropy.modeling.functional_models.Linear1D` model provides a + line parameterizied by the slope and y-intercept + +- :class:`~astropy.modeling.functional_models.Sine1D` model provides a sine + parameterized by an amplitude, frequency, and phase shift. + +- :class:`~astropy.modeling.functional_models.Cosine1D` model provides a + cosine parameterized by an amplitude, frequency, and phase shift. + +.. plot:: + + import numpy as np + import matplotlib.pyplot as plt + + from astropy.modeling.models import (Linear1D, Sine1D, Cosine1D) + + x = np.linspace(-4.0, 6.0, num=100) + + fig, sax = plt.subplots(ncols=3, figsize=(10, 5), layout="tight") + ax = sax.flatten() + + linemod = Linear1D(slope=2., intercept=1.) + ax[0].plot(x, linemod(x), label="Linear1D") + + sinemod = Sine1D(amplitude=10., frequency=0.5, phase=0.) + ax[1].plot(x, sinemod(x), label="Sine1D") + ax[1].set_ylim(-11.0, 13.0) + + cosinemod = Cosine1D(amplitude=10., frequency=0.5, phase=0) + ax[2].plot(x, cosinemod(x), label="Cosine1D") + ax[2].set_ylim(-11.0, 13.0) + + for k in range(3): + ax[k].set_xlabel("x") + ax[k].set_ylabel("y") + ax[k].legend() + + plt.show() + +Profiles +======== + +These models provide profiles, often used for lines in spectra. + +- :class:`~astropy.modeling.functional_models.Box1D` model computes a box + function with an amplitude centered at x_0 with the specified width. + +- :class:`~astropy.modeling.functional_models.Gaussian1D` model computes + a Gaussian with an amplitude centered at x_0 with the specified width. + +- :class:`~astropy.modeling.functional_models.KingProjectedAnalytic1D` model + computes the analytic form of the a King model with an amplitude and + core and tidal radii. + +- :class:`~astropy.modeling.functional_models.Lorentz1D` model computes + a Lorentzian with an amplitude centered at x_0 with the specified width. + +- :class:`~astropy.modeling.functional_models.RickerWavelet1D` model computes + a RickerWavelet function with an amplitude centered at x_0 with the specified width. + +- :class:`~astropy.modeling.functional_models.Moffat1D` model computes a + Moffat function with an amplitude centered at x_0 with the specified width. + +- :class:`~astropy.modeling.functional_models.Sersic1D` model + computes a Sersic model with an amplitude with an effective radius and + the specified sersic index. + +- :class:`~astropy.modeling.functional_models.Trapezoid1D` model computes a + box with sloping sides with an amplitude centered at x_0 with the specified + width and sides with the specified slope. + +- :class:`~astropy.modeling.functional_models.Voigt1D` model computes a + Voigt function with an amplitude centered at x_0 with the specified + Lorentzian and Gaussian widths. + +.. plot:: + + import numpy as np + import matplotlib.pyplot as plt + + from astropy.modeling.models import ( + Box1D, + Gaussian1D, + RickerWavelet1D, + Moffat1D, + Lorentz1D, + Sersic1D, + Trapezoid1D, + KingProjectedAnalytic1D, + Voigt1D, + ) + + x = np.linspace(-4.0, 6.0, num=100) + r = np.logspace(-1.0, 2.0, num=100) + + fig, sax = plt.subplots(nrows=3, ncols=3, figsize=(10, 10), layout="tight") + ax = sax.flatten() + + mods = [ + Box1D(amplitude=10.0, x_0=1.0, width=1.0), + Gaussian1D(amplitude=10.0, mean=1.0, stddev=1.0), + KingProjectedAnalytic1D(amplitude=10.0, r_core=1.0, r_tide=10.0), + Lorentz1D(amplitude=10.0, x_0=1.0, fwhm=1.0), + RickerWavelet1D(amplitude=10.0, x_0=1.0, sigma=1.0), + Moffat1D(amplitude=10.0, x_0=1.0, gamma=1.0, alpha=1.), + Sersic1D(amplitude=10.0, r_eff=1.0 / 2.0, n=5), + Trapezoid1D(amplitude=10.0, x_0=1.0, width=1.0, slope=5.0), + Voigt1D(amplitude_L=10.0, x_0=1.0, fwhm_L=1.0, fwhm_G=1.0), + ] + + for k, mod in enumerate(mods): + cname = mod.__class__.__name__ + ax[k].set_title(cname) + if cname in ["KingProjectedAnalytic1D", "Sersic1D"]: + ax[k].plot(r, mod(r)) + ax[k].set_xscale("log") + ax[k].set_yscale("log") + else: + ax[k].plot(x, mod(x)) + + for k in range(len(mods)): + ax[k].set_xlabel("x") + ax[k].set_ylabel("y") + + # remove axis for any plots not used + for k in range(len(mods), len(ax)): + ax[k].axis("off") + + plt.show() diff --git a/docs/modeling/predef_models2D.rst b/docs/modeling/predef_models2D.rst new file mode 100644 index 000000000000..49801982d013 --- /dev/null +++ b/docs/modeling/predef_models2D.rst @@ -0,0 +1,137 @@ +.. _predef_models2D: + +********* +2D Models +********* + +These models take as input x and y arrays. + +Operations +========== + +These models perform simple mathematical operations. + +- :class:`~astropy.modeling.functional_models.Const2D` model returns the + constant replicated by the number of input x and y values. + +Shapes +====== + +These models provide shapes, often used to model general x, y, z data. + +- :class:`~astropy.modeling.functional_models.Planar2D` model computes + a tilted plan with specified x,y slopes and z intercept + +Profiles +======== + +These models provide profiles, often used sources in images. +All models have parameters giving the x,y location of the center and +an amplitude. + +- :class:`~astropy.modeling.functional_models.AiryDisk2D` model computes + the Airy function for a radius + +- :class:`~astropy.modeling.functional_models.Box2D` model computes a box + with x,y dimensions + +- :class:`~astropy.modeling.functional_models.Disk2D` model computes a + disk a radius + +- :class:`~astropy.modeling.functional_models.Ellipse2D` model computes + an ellipse with major and minor axis and rotation angle + +- :class:`~astropy.modeling.functional_models.Gaussian2D` model computes + a Gaussian with x,y standard deviations and rotation angle + +- :class:`~astropy.modeling.functional_models.Moffat2D` model computes + a Moffat with x,y dimensions and alpha (power index) and gamma (core width) + +- :class:`~astropy.modeling.functional_models.Lorentz2D` model computes + a Lorentz profile with x,y dimensions and full width at half maximum + +- :class:`~astropy.modeling.functional_models.RickerWavelet2D` model computes + a symmetric RickerWavelet function with the specified sigma + +- :class:`~astropy.modeling.functional_models.Sersic2D` model computes + a Sersic profile with an effective half-light radius, rotation, and + Sersic index + +- :class:`~astropy.modeling.functional_models.GeneralSersic2D` model + computes a generalized Sersic profile with an effective half-light + radius, rotation, Sersic index, and a parameter to control the shape of + the isophotes (e.g., boxy or disky) + +- :class:`~astropy.modeling.functional_models.TrapezoidDisk2D` model + computes a disk with a radius and slope + +- :class:`~astropy.modeling.functional_models.Ring2D` model computes + a ring with inner and outer radii + +.. plot:: + + import numpy as np + import math + import matplotlib.pyplot as plt + from matplotlib.colors import LogNorm + + from astropy.modeling.models import (AiryDisk2D, Box2D, Disk2D, Ellipse2D, + Gaussian2D, Moffat2D, Lorentz2D, + RickerWavelet2D, Sersic2D, + GeneralSersic2D, + TrapezoidDisk2D, Ring2D) + + x = np.linspace(-4.0, 6.0, num=100) + r = np.logspace(-1.0, 2.0, num=100) + + fig, sax = plt.subplots(nrows=5, ncols=3, figsize=(9, 12), layout="tight") + ax = sax.flatten() + + # setup the x,y coordinates + x_npts = 100 + y_npts = x_npts + x0, x1 = -4, 6 + y0, y1 = -3, 7 + x = np.linspace(x0, x1, num=x_npts) + y = np.linspace(y0, y1, num=y_npts) + X, Y = np.meshgrid(x, y) + + # plot the different 2D profiles + mods = [AiryDisk2D(amplitude=10.0, x_0=1.0, y_0=2.0, radius=1.0), + Box2D(amplitude=10.0, x_0=1.0, y_0=2.0, x_width=1.0, y_width=2.0), + Disk2D(amplitude=10.0, x_0=1.0, y_0=2.0, R_0=1.0), + Ellipse2D(amplitude=10.0, x_0=1.0, y_0=2.0, a=1.0, b=2.0, theta=math.pi/4.), + Gaussian2D(amplitude=10.0, x_mean=1.0, y_mean=2.0, x_stddev=1.0, y_stddev=2.0, theta=math.pi/4.), + Moffat2D(amplitude=10.0, x_0=1.0, y_0=2.0, alpha=3, gamma=4), + Lorentz2D(amplitude=10.0, x_0=1.0, y_0=2.0, fwhm=3), + RickerWavelet2D(amplitude=10.0, x_0=1.0, y_0=2.0, sigma=1.0), + Sersic2D(amplitude=10.0, x_0=1.0, y_0=2.0, r_eff=1.0, ellip=0.5, theta=math.pi/4.), + GeneralSersic2D(amplitude=10.0, x_0=1.0, y_0=2.0, r_eff=1.0, ellip=0.5, theta=math.pi/4., c=-1), + GeneralSersic2D(amplitude=10.0, x_0=1.0, y_0=2.0, r_eff=1.0, ellip=0.5, theta=math.pi/4., c=1), + TrapezoidDisk2D(amplitude=10.0, x_0=1.0, y_0=2.0, R_0=1.0, slope=5.0), + Ring2D(amplitude=10.0, x_0=1.0, y_0=2.0, r_in=1.0, r_out=2.0)] + + for k, mod in enumerate(mods): + cname = mod.__class__.__name__ + if cname == "AiryDisk2D": + normfunc = LogNorm(vmin=0.001, vmax=10.) + elif cname in ["Gaussian2D", "Sersic2D", "GeneralSersic2D"]: + normfunc = LogNorm(vmin=0.1, vmax=10.) + else: + normfunc = None + if cname == "GeneralSersic2D": + cname = f'{cname}, c={mod.c.value:.1f}' + ax[k].set_title(cname) + + ax[k].imshow(mod(X, Y), extent=[x0, x1, y0, y1], origin="lower", cmap="gray_r", + norm=normfunc) + + for k in range(len(mods)): + ax[k].set_xlabel("x") + ax[k].set_ylabel("y") + + # remove axis for any plots not used + for k in range(len(mods), len(ax)): + ax[k].axis("off") + + plt.show() diff --git a/docs/modeling/reference_api.rst b/docs/modeling/reference_api.rst new file mode 100644 index 000000000000..9212f389d9e7 --- /dev/null +++ b/docs/modeling/reference_api.rst @@ -0,0 +1,30 @@ +Reference/API +************* + +Capabilities +============ + +.. automodapi:: astropy.modeling +.. automodapi:: astropy.modeling.bounding_box +.. automodapi:: astropy.modeling.mappings +.. automodapi:: astropy.modeling.fitting + :inherited-members: True + :skip: SplineExactKnotsFitter + :skip: SplineInterpolateFitter + :skip: SplineSmoothingFitter + :skip: SplineSplrepFitter +.. automodapi:: astropy.modeling.optimizers +.. automodapi:: astropy.modeling.statistic +.. automodapi:: astropy.modeling.separable + +Pre-Defined Models +================== + +.. automodapi:: astropy.modeling.functional_models +.. automodapi:: astropy.modeling.physical_models +.. automodapi:: astropy.modeling.powerlaws +.. automodapi:: astropy.modeling.polynomial +.. automodapi:: astropy.modeling.projections +.. automodapi:: astropy.modeling.rotations +.. automodapi:: astropy.modeling.spline +.. automodapi:: astropy.modeling.tabular diff --git a/docs/modeling/spline_models.rst b/docs/modeling/spline_models.rst new file mode 100644 index 000000000000..b9d9783aa644 --- /dev/null +++ b/docs/modeling/spline_models.rst @@ -0,0 +1,70 @@ +.. include:: links.inc + +.. _spline_models: + +**************** +1D Spline Models +**************** + +`~astropy.modeling.spline.Spline1D` models are models which can be used +to fit a piecewise polynomial to a set of data. This means that splines +are closely tied to the method used to fit the spline to the data. Currently, +we provide three methods for fitting splines to data: + +- :class:`~astropy.modeling.spline.SplineInterpolateFitter`, which + fits an interpolating spline to the data. This means that the spline + will exactly fit all data points. + +- :class:`~astropy.modeling.spline.SplineSmoothingFitter`, which fits + a smoothing spline to the data. This means that the number of knots + is chosen to satisfy the "smoothing condition": + + .. math:: \sum_{i} \left(w_i * (y_i - spl(x_i))\right)^{2} \leq s + +- :class:`~astropy.modeling.spline.SplineExactKnotsFitter`, which fits + a spline to the data using an exact set of knots. This means that the + spline will use least-squares regression using the user supplied (interior) + knots to find the best fit spline to the data. + +.. plot:: + :include-source: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling.models import Spline1D + from astropy.modeling.fitting import (SplineInterpolateFitter, + SplineSmoothingFitter, + SplineExactKnotsFitter) + + rng = np.random.default_rng() + x = np.linspace(-3, 3, 50) + y = np.exp(-x**2) + 0.1 * rng.standard_normal(50) + xs = np.linspace(-3, 3, 1000) + t = [-1, 0, 1] + spl = Spline1D() + + fitter = SplineInterpolateFitter() + spl1 = fitter(spl, x, y) + + fitter = SplineSmoothingFitter() + spl2 = fitter(spl, x, y, s=0.5) + + fitter = SplineExactKnotsFitter() + spl3 = fitter(spl, x, y, t=t) + + fig, ax = plt.subplots() + ax.plot(x, y, 'ro', label="Data") + ax.plot(xs, spl1(xs), 'b-', label="Interpolating") + ax.plot(xs, spl2(xs), 'g-', label="Smoothing") + ax.plot(xs, spl3(xs), 'k-', label="Exact Knots") + ax.legend() + plt.show() + +Note that by default, splines have `degree ` 3. +In the case of these splines, the ``degree - 1`` is the number of derivatives that +are matched by the spline across knot points. So for degree 3 splines, the value, +first, and second derivatives of the spline will match across each knot point. + +.. warning:: + + Splines only support integer degrees, such that ``1 <= degree <= 5``. diff --git a/docs/modeling/units.rst b/docs/modeling/units.rst new file mode 100644 index 000000000000..1f0f9e6850f1 --- /dev/null +++ b/docs/modeling/units.rst @@ -0,0 +1,227 @@ +.. _modeling-units: + +******************************** +Support for units and quantities +******************************** + + +.. note:: The functionality presented here was recently added. If you run into + any issues, please don't hesitate to open an issue in the `issue + tracker `_. + +The `astropy.modeling` package includes partial support for the use of units and +quantities in model parameters, models, and during fitting. At this time, only +some of the built-in models (such as +:class:`~astropy.modeling.functional_models.Gaussian1D`) support units, but this +will be extended in future to all models where this is appropriate. + +Setting parameters to quantities +================================ + +Models can take :class:`~astropy.units.Quantity` objects as parameters:: + + >>> from astropy import units as u + >>> from astropy.modeling.models import Gaussian1D + >>> g1 = Gaussian1D(mean=3 * u.m, stddev=2 * u.cm, amplitude=3 * u.Jy) + +Accessing the parameter then returns a Parameter object that contains the value +and the unit:: + + >>> g1.mean + Parameter('mean', value=3.0, unit=m) + +It is then possible to access the individual properties of the parameter:: + + >>> g1.mean.name + 'mean' + >>> g1.mean.value + np.float64(3.0) + >>> g1.mean.unit + Unit("m") + +If a parameter has been initialized as a Quantity, it should always be set to a +quantity, but the units don't have to be compatible with the initial ones:: + + >>> g1.mean = 3 * u.s + >>> g1 # doctest: +FLOAT_CMP + + +To change the value of a parameter and not the unit, simply set the value +property:: + + >>> g1.mean.value = 2 + >>> g1 # doctest: +FLOAT_CMP + + +Setting a parameter which was originally set to a quantity to a scalar doesn't +work because it's ambiguous whether the user means to change just the value and +preserve the unit, or get rid of the unit:: + + >>> g1.mean = 2 # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + UnitsError : The 'mean' parameter should be given as a Quantity because it + was originally initialized as a Quantity + +On the other hand, if a parameter previously defined without units is given a +Quantity with a unit, this works because it is unambiguous:: + + >>> g2 = Gaussian1D(mean=3) + >>> g2.mean = 3 * u.m + +In other words, once units are attached to a parameter, they can't be removed +due to ambiguous meaning. + +Evaluating models with quantities +================================= + +Quantities can be passed to model during evaluation:: + + >>> g3 = Gaussian1D(mean=3 * u.m, stddev=5 * u.cm) + >>> g3(2.9 * u.m) # doctest: +FLOAT_CMP + + >>> g3(2.9 * u.s) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + UnitsError : Units of input 'x', s (time), could not be converted to + required input units of m (length) + +In this case, since the mean and standard deviation have units, the value passed +during evaluation also needs units:: + + >>> g3(3) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + UnitsError : Units of input 'x', (dimensionless), could not be converted to + required input units of m (length) + +Equivalencies +------------- + +Equivalencies require special care - a Gaussian defined in frequency space is +not a Gaussian in wavelength space for example. For this reason, we don't allow +equivalencies to be attached to the parameters themselves. Instead, we take the +approach of converting the input data to the parameter space, and any +equivalencies should be applied at evaluation time to the data (not the +parameters). + +Let's consider a model that is Gaussian in wavelength space:: + + >>> g4 = Gaussian1D(mean=3 * u.micron, stddev=1 * u.micron, amplitude=3 * u.Jy) + +By default, passing a frequency will not work: + + >>> g4(1e2 * u.THz) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + UnitsError : Units of input 'x', THz (frequency), could not be converted to + required input units of micron (length) + +But you can pass a dictionary of equivalencies to the equivalencies argument +(this needs to be a dictionary since some models can contain multiple inputs):: + + >>> g4(110 * u.THz, equivalencies={'x': u.spectral()}) # doctest: +FLOAT_CMP + + +The key of the dictionary should be the name of the inputs according to:: + + >>> g4.inputs + ('x',) + +It is also possible to set default equivalencies for the input parameters using +the input_units_equivalencies property:: + + >>> g4.input_units_equivalencies = {'x': u.spectral()} + >>> g4(110 * u.THz) # doctest: +FLOAT_CMP + + +Fitting models with units to data +================================= + +Fitting models with units to data with units should be seamless provided that +the model supports fitting with units. To demonstrate this, we start off by +generating synthetic data: + +.. plot:: + :context: reset + :include-source: + + import numpy as np + from astropy import units as u + import matplotlib.pyplot as plt + + x = np.linspace(1, 5, 30) * u.micron + y = np.exp(-0.5 * (x - 2.5 * u.micron)**2 / (200 * u.nm)**2) * u.mJy + + fig, ax = plt.subplots() + ax.plot(x, y, 'ko') + ax.set(xlabel='Wavelength (microns)', ylabel='Flux density (mJy)') + +and we then define the initial guess for the fitting and we carry out the fit as +we would without any units: + +.. plot:: + :context: + :include-source: + + from astropy.modeling import models, fitting + + g5 = models.Gaussian1D(mean=3 * u.micron, stddev=1 * u.micron, amplitude=1 * u.Jy) + + fitter = fitting.TRFLSQFitter() + + g5_fit = fitter(g5, x, y) + + fig, ax = plt.subplots() + ax.plot(x, y, 'ko') + ax.plot(x, g5_fit(x), 'r-') + ax.set(xlabel='Wavelength (microns)', ylabel='Flux density (mJy)') + +Fitting with equivalencies +-------------------------- + +Let's now consider the case where the data is not equivalent to those of the +parameters, but they are convertible via equivalencies. In this case, the +equivalencies can either be passed via a dictionary as shown higher up for the +evaluation examples: + +.. plot:: + :context: + :include-source: + + g6 = models.Gaussian1D(mean=110 * u.THz, stddev=10 * u.THz, amplitude=1 * u.Jy) + + g6_fit = fitter(g6, x, y, equivalencies={'x': u.spectral()}) + + fig, ax = plt.subplots() + ax.plot(x, g6_fit(x, equivalencies={'x': u.spectral()}), 'b-') + ax.set(xlabel='Wavelength (microns)', ylabel='Flux density (mJy)') + +In this case, the fit (in blue) is slightly worse, because a Gaussian in +frequency space (blue) is not a Gaussian in wavelength space (red). As mentioned +previously, you can also set input_units_equivalencies on the model itself to +avoid having to pass extra arguments to the fitter:: + + g6.input_units_equivalencies = {'x': u.spectral()} + g6_fit = fitter(g6, x, y) + + +.. _units-mapping: + +Support for units in otherwise unitless models +============================================== + +Some models, like polynomials, do not work intrinsically with units. Instead, +the :meth:`~astropy.modeling.core.Model.coerce_units` method provides a way to add input and return units to +unitless models by enclosing the unitless model with two instances of :class:`~astropy.modeling.mappings.UnitsMapping`. +Internally the inputs are stripped of the units before passed +to the model and units are attached to the result if ``return_units`` is specified. +The method returns a new composite model:: + + >>> from astropy.modeling import models + >>> from astropy import units as u + >>> model = models.Polynomial1D(1, c0=1, c1=2) + >>> new_model = model.coerce_units(input_units={'x': u.Hz}, return_units={'y': u.s}, + ... input_units_equivalencies={'x':u.spectral()}) + >>> new_model(10 * u.Hz) + diff --git a/docs/nddata/bitmask.rst b/docs/nddata/bitmask.rst new file mode 100644 index 000000000000..eacf7750e28c --- /dev/null +++ b/docs/nddata/bitmask.rst @@ -0,0 +1,292 @@ +.. _bitmask_details: + +******************************************************** +Utility Functions for Handling Bit Masks and Mask Arrays +******************************************************** + +It is common to use `bit fields `_, +such as integer variables whose individual bits represent some attributes, to +characterize the state of data. For example, Hubble Space Telescope (HST) uses +arrays of bit fields to characterize data quality (DQ) of HST images. See, for +example, DQ field values for `WFPC2 image data (see Table 3.3) `_ and `WFC3 image data (see Table 3.3) `_. +As you can see, the meaning assigned to various *bit flags* for the two +instruments is generally different. + +Bit fields can be thought of as tightly packed collections of bit flags. Using +`masking `_ we can "inspect" +the status of individual bits. + +One common operation performed on bit field arrays is their conversion to +boolean masks, for example, by assigning boolean `True` (in the boolean +mask) to those elements that correspond to non-zero-valued bit fields +(bit fields with at least one bit set to ``1``) or, oftentimes, by assigning +`True` to elements whose corresponding bit fields have only *specific fields* +set (to ``1``). This more sophisticated analysis of bit fields can be +accomplished using *bit masks* and the aforementioned masking operation. + +The `~astropy.nddata.bitmask` module provides two functions that facilitate +conversion of bit field arrays (i.e., DQ arrays) to boolean masks: +`~astropy.nddata.bitmask.bitfield_to_boolean_mask` converts an input bit +field array to a boolean mask using an input bit mask (or list of individual +bit flags) and `~astropy.nddata.bitmask.interpret_bit_flags` creates a bit mask +from an input list of individual bit flags. + +Creating Boolean Masks +********************** + +Overview +======== + +`~astropy.nddata.bitmask.bitfield_to_boolean_mask` by default assumes that +all input bit fields that have at least one bit turned "ON" corresponds to +"bad" data (i.e., pixels) and converts them to boolean `True` in the output +boolean mask (otherwise output boolean mask values are set to `False`). + +Often, for specific algorithms and situations, some bit flags are okay and +can be ignored. `~astropy.nddata.bitmask.bitfield_to_boolean_mask` accepts +lists of bit flags that *by default must be ignored* in the input bit fields +when creating boolean masks. + +Fundamentally, *by default*, `~astropy.nddata.bitmask.bitfield_to_boolean_mask` +performs the following operation: + +.. _main_eq: + +``(1) boolean_mask = (bitfield & ~bit_mask) != 0`` + +(Here ``&`` is bitwise ``and`` while ``~`` is the bitwise ``not`` +operation.) In the previous formula, ``bit_mask`` is a bit mask created from +individual bit flags that need to be ignored in the bit field. + +Example +------- + +.. + EXAMPLE START + Creating Boolean Masks from Bit Field Arrays + +.. _table1: + +.. table:: Table 1: Examples of Boolean Mask Computations \ + (default parameters and 8-bit data type) + + +--------------+--------------+--------------+--------------+------------+ + | Bit Field | Bit Mask | ~(Bit Mask) | Bit Field & |Boolean Mask| + | | | | ~(Bit Mask) | | + +==============+==============+==============+==============+============+ + |11011001 (217)|01010000 (80) |10101111 (175)|10001001 (137)| True | + +--------------+--------------+--------------+--------------+------------+ + |11011001 (217)|10101111 (175)|01010000 (80) |01010000 (80) | True | + +--------------+--------------+--------------+--------------+------------+ + |00001001 (9) |01001001 (73) |10110110 (182)|00000000 (0) | False | + +--------------+--------------+--------------+--------------+------------+ + |00001001 (9) |00000000 (0) |11111111 (255)|00001001 (9) | True | + +--------------+--------------+--------------+--------------+------------+ + |00001001 (9) |11111111 (255)|00000000 (0) |00000000 (0) | False | + +--------------+--------------+--------------+--------------+------------+ + +.. + EXAMPLE END + +Specifying Bit Flags +==================== + +`~astropy.nddata.bitmask.bitfield_to_boolean_mask` accepts either an integer +bit mask or lists of bit flags. Lists of bit flags will be combined into a +bit mask and can be provided either as a Python list of +**integer bit flag values** or as a comma-separated (or ``+``-separated) +list of integer bit flag values. Consider the bit mask from the first example +in `Table 1 `_. In this case ``ignore_flags`` can be set either to: + + - An integer value bit mask 80 + - A Python list indicating individual non-zero + *bit flag values:* ``[16, 64]`` + - A string of comma-separated *bit flag values or mnemonic names*: ``'16,64'``, ``'CR,WARM'`` + - A string of ``+``-separated *bit flag values or mnemonic names*: ``'16+64'``, ``'CR+WARM'`` + +Example +------- + +.. + EXAMPLE START + Specifying Bit Flags in NDData + +To specify bit flags: + + >>> from astropy.nddata import bitmask + >>> import numpy as np + >>> bitmask.bitfield_to_boolean_mask(217, ignore_flags=80) + array(True...) + >>> bitmask.bitfield_to_boolean_mask(217, ignore_flags='16,64') + array(True...) + >>> bitmask.bitfield_to_boolean_mask(217, ignore_flags=[16, 64]) + array(True...) + >>> bitmask.bitfield_to_boolean_mask(9, ignore_flags=[1, 8, 64]) + array(False...) + >>> bitmask.bitfield_to_boolean_mask([9, 10, 73, 217], ignore_flags='1,8,64') + array([False, True, False, True]...) + +It is also possible to specify the type of the output mask: + + >>> bitmask.bitfield_to_boolean_mask([9, 10, 73, 217], ignore_flags='1,8,64', dtype=np.uint8) + array([0, 1, 0, 1], dtype=uint8) + +In order to use lists of mnemonic bit flags names, one must provide a map, +a subclass of `~astropy.nddata.bitmask.BitFlagNameMap`, that can be +used to map mnemonic names to bit flag values. Normally these maps should be +provided by a third-party package supporting a specific instrument. Each bit +flag in the map may also contain a string comment following the flag value. +In the example below we define a simple mask map: + + >>> from astropy.nddata.bitmask import BitFlagNameMap + >>> class ST_DQ(BitFlagNameMap): + ... CR = 1 + ... CLOUDY = 4 + ... RAINY = 8, 'Dome closed' + ... HOT = 32 + ... DEAD = 64 + >>> bitmask.bitfield_to_boolean_mask([9, 10, 73, 217], ignore_flags='CR,RAINY,DEAD', + ... dtype=np.uint8, flag_name_map=ST_DQ) + array([0, 1, 0, 1], dtype=uint8) + +.. + EXAMPLE END + +Using Bit Flags Name Maps +========================= + +.. + EXAMPLE START + +In order to allow the use of mnemonic bit flag names to describe the flags +to be taken into consideration or ignored when creating a *boolean* mask, we +use bit flag name maps. These maps perform case-insensitive translation of +mnemonic bit flag names to the corresponding integer value. + +Bit flag name maps are subclasses of `~astropy.nddata.bitmask.BitFlagNameMap` +and can be constructed in two ways, either by directly subclassing +`~astropy.nddata.bitmask.BitFlagNameMap`, e.g., + + >>> from astropy.nddata.bitmask import BitFlagNameMap + >>> class ST_DQ(BitFlagNameMap): + ... CR = 1 + ... CLOUDY = 4 + ... RAINY = 8 + ... + >>> class ST_CAM1_DQ(ST_DQ): + ... HOT = 16 + ... DEAD = 32 + +or by using the `~astropy.nddata.bitmask.extend_bit_flag_map` class factory: + + >>> from astropy.nddata.bitmask import extend_bit_flag_map + >>> ST_DQ = extend_bit_flag_map('ST_DQ', CR=1, CLOUDY=4, RAINY=8) + >>> ST_CAM1_DQ = extend_bit_flag_map('ST_CAM1_DQ', ST_DQ, HOT=16, DEAD=32) + +.. note:: + + Bit flag values must be integer numbers that are powers of 2. + +Once constructed, bit flag values of a map cannot be modified, deleted, or +added. Adding flags to a map is allowed only through subclassing using one of +the two methods shown above or by adding lists of tuples of +the form ``('NAME', value)`` to the class. This will create a new map class +subclassed from the original map but containing the additional flags + + >>> ST_CAM1_DQ = ST_DQ + [('HOT', 16), ('DEAD', 32)] + +would result in an equivalent map as in the subclassing or class factory +examples shown above. + +Once a bit flag name map was created, the bit flag values can be accessed +either as *case-insensitive* class attributes or keys in a dictionary: + + >>> ST_CAM1_DQ.cloudy + 4 + >>> ST_CAM1_DQ['Rainy'] + 8 + +.. + EXAMPLE END + +Modifying the Formula for Creating Boolean Masks +================================================ + +`~astropy.nddata.bitmask.bitfield_to_boolean_mask` provides several parameters +that can be used to modify the formula used to create boolean masks. + +Inverting Bit Masks +------------------- + +Sometimes it is more convenient to be able to specify those bit +flags that *must be considered* when creating the boolean mask, and all other +flags should be ignored. + +Example +^^^^^^^ + +.. + EXAMPLE START + Inverting Bit Masks in NDData + +In `~astropy.nddata.bitmask.bitfield_to_boolean_mask` specifying bit flags that +must be considered when creating the boolean mask can be accomplished by +setting the parameter ``flip_bits`` to `True`. This effectively modifies +`equation (1) `_ to: + +.. _modif_eq2: + +``(2) boolean_mask = (bitfield & bit_mask) != 0`` + +So, instead of: + + >>> bitmask.bitfield_to_boolean_mask([9, 10, 73, 217], ignore_flags=[1, 8, 64]) + array([False, True, False, True]...) + +You can obtain the same result as: + + >>> bitmask.bitfield_to_boolean_mask( + ... [9, 10, 73, 217], ignore_flags=[2, 4, 16, 32, 128], flip_bits=True + ... ) + array([False, True, False, True]...) + +Note however, when ``ignore_flags`` is a comma-separated list of bit flag +values, ``flip_bits`` cannot be set to either `True` or `False`. Instead, +to flip bits of the bit mask formed from a string list of comma-separated +bit flag values, you can prepend a single ``~`` to the list: + + >>> bitmask.bitfield_to_boolean_mask([9, 10, 73, 217], ignore_flags='~2+4+16+32+128') + array([False, True, False, True]...) + +.. + EXAMPLE END + +Inverting Boolean Masks +----------------------- + +Other times, it may be more convenient to obtain an inverted mask in which +flagged data are converted to `False` instead of `True`: + +.. _modif_eq3: + +``(3) boolean_mask = (bitfield & ~bit_mask) == 0`` + +This can be accomplished by changing the ``good_mask_value`` parameter from +its default value (`False`) to `True`. + +Example +^^^^^^^ + +.. + EXAMPLE START + Inverting Boolean Masks in NDData + +To obtain an inverted mask in which flagged data are converted to `False` +instead of `True`: + + >>> bitmask.bitfield_to_boolean_mask([9, 10, 73, 217], ignore_flags=[1, 8, 64], + ... good_mask_value=True) + array([ True, False, True, False]...) + +.. + EXAMPLE END diff --git a/docs/nddata/ccddata.rst b/docs/nddata/ccddata.rst new file mode 100644 index 000000000000..5aa13a0a09a8 --- /dev/null +++ b/docs/nddata/ccddata.rst @@ -0,0 +1,249 @@ +.. _ccddata: + + +CCDData Class +============= + +Getting Started +--------------- + +Getting Data In ++++++++++++++++ + +Creating a `~astropy.nddata.CCDData` object from any array-like data using +`astropy.nddata` is convenient: + + >>> import numpy as np + >>> from astropy.nddata import CCDData + >>> from astropy.utils.data import get_pkg_data_filename + >>> ccd = CCDData(np.arange(10), unit="adu") + +Note that behind the scenes, this creates references to (not copies of) your +data when possible, so modifying the data in ``ccd`` will modify the +underlying data. + +You are **required** to provide a unit for your data. The most frequently used +units for these objects are likely to be ``adu``, ``photon``, and ``electron``, +which can be set either by providing the string name of the unit (as in the +example above) or from unit objects: + + >>> from astropy import units as u + >>> ccd_photon = CCDData([1, 2, 3], unit=u.photon) + >>> ccd_electron = CCDData([1, 2, 3], unit="electron") + +If you prefer *not* to use the unit functionality, then use the special unit +``u.dimensionless_unscaled`` when you create your `~astropy.nddata.CCDData` +images: + + >>> ccd_unitless = CCDData(np.zeros((10, 10)), + ... unit=u.dimensionless_unscaled) + +A `~astropy.nddata.CCDData` object can also be initialized from a FITS filename +or URL: + + >>> ccd = CCDData.read('my_file.fits', unit="adu") # doctest: +SKIP + >>> ccd = CCDData.read(get_pkg_data_filename('tutorials/FITS-images/HorseHead.fits'), unit="adu", cache=True) # doctest: +REMOTE_DATA +IGNORE_WARNINGS + +If there is a unit in the FITS file (in the ``BUNIT`` keyword), that will be +used, but explicitly providing a unit in ``read`` will override any unit in the +FITS file. + +There is no restriction at all on what the unit can be — any unit in +`astropy.units` or another that you create yourself will work. + +In addition, the user can specify the extension in a FITS file to use: + + >>> ccd = CCDData.read('my_file.fits', hdu=1, unit="adu") # doctest: +SKIP + +If ``hdu`` is not specified, it will assume the data is in the primary +extension. If there is no data in the primary extension, the first extension +with image data will be used. + +Metadata +++++++++ + +When initializing from a FITS file, the ``header`` property is initialized using +the header of the FITS file. Metadata is optional, and can be provided by any +dictionary or dict-like object: + + >>> ccd_simple = CCDData(np.arange(10), unit="adu") + >>> my_meta = {'observer': 'Edwin Hubble', 'exposure': 30.0} + >>> ccd_simple.header = my_meta # or use ccd_simple.meta = my_meta + +Whether the metadata is case-sensitive or not depends on how it is +initialized. A FITS header, for example, is not case-sensitive, but a Python +dictionary is. + +Getting Data Out +++++++++++++++++ + +A `~astropy.nddata.CCDData` object behaves like a ``numpy`` array (masked if the +`~astropy.nddata.CCDData` mask is set) in expressions, and the underlying +data (ignoring any mask) is accessed through the ``data`` attribute: + + >>> ccd_masked = CCDData([1, 2, 3], unit="adu", mask=[0, 0, 1]) + >>> 2 * np.ones(3) * ccd_masked # one return value will be masked + masked_array(data=[2.0, 4.0, --], + mask=[False, False, True], + fill_value=1e+20) + >>> 2 * np.ones(3) * ccd_masked.data # ignores the mask # doctest: +FLOAT_CMP + array([2., 4., 6.]) + +You can force conversion to a ``numpy`` array with: + + >>> np.asarray(ccd_masked) + array([1, 2, 3]) + >>> np.ma.array(ccd_masked.data, mask=ccd_masked.mask) + masked_array(data=[1, 2, --], + mask=[False, False, True], + fill_value=999999) + +A method for converting a `~astropy.nddata.CCDData` object to a FITS HDU list +is also available. It converts the metadata to a FITS header: + + >>> hdulist = ccd_masked.to_hdu() + +You can also write directly to a FITS file: + + >>> ccd_masked.write('my_image.fits') + +Masks and Flags ++++++++++++++++ + +Although it is not required when a `~astropy.nddata.CCDData` image is created, +you can also specify a mask and/or flags. + +A mask is a boolean array the same size as the data in which a value of +``True`` indicates that a particular pixel should be masked (*i.e.*, not be +included in arithmetic operations or aggregation). + +Flags are one or more additional arrays (of any type) whose shape matches the +shape of the data. One particularly useful type of flag is a bit planes; for +more details about bit planes and the functions ``astropy`` provides for +converting them to binary masks, see :ref:`bitmask_details`. + +A simple example on how to set flags can be: + + >>> data = np.zeros((10, 10)) + >>> ccd = CCDData(data, unit="electron") + + >>> flags = np.ones((10, 10)) # Create a simple flags array + >>> ccd = CCDData(data, unit='adu', flags=flags) + +Flags can be also set using `~astropy.nddata.FlagCollection`, which provides a +convenient interface for managing multiple flags. + + >>> ccd = CCDData(data, unit="electron") + + >>> # Create a FlagCollection with different flag types + >>> from astropy.nddata import FlagCollection + >>> flags = FlagCollection(shape=(100, 100)) + + >>> # Add different types of flags + >>> flags['COSMIC_RAY'] = np.zeros((100, 100), dtype=float) + >>> flags['SATURATED'] = np.zeros((100, 100), dtype=int) + >>> flags['BAD_PIXEL'] = np.zeros((100, 100), dtype=bool) + >>> flags['BAD_PIXEL'][50:60, 50:60] = True # Mark a region as bad + + +When writing `~astropy.nddata.CCDData` to FITS, flags are stored in additional image HDU extensions. In order +to do this, the user must explicitly provide a key or name for the flags HDU +when creating the `~astropy.nddata.CCDData` object using the ``hdu_flags``. In +case that multiple flags are set using `~astropy.nddata.FlagCollection`, they will be stored in multiple HDUs using the flag collection names, but user +must still provide a non-empty string to ``hdu_flags`` to indicate that flags should be saved. + + +WCS ++++ + +The ``wcs`` attribute of a `~astropy.nddata.CCDData` object can be set two ways. + ++ If the `~astropy.nddata.CCDData` object is created from a FITS file that has + WCS keywords in the header, the ``wcs`` attribute is set to a + `~astropy.wcs.WCS` object using the information in the FITS header. + ++ The WCS can also be provided when the `~astropy.nddata.CCDData` object is + constructed with the ``wcs`` argument. + +Either way, the ``wcs`` attribute is kept up to date if the +`~astropy.nddata.CCDData` image is trimmed. + +PSF ++++ + +The ``psf`` attributes of a `~astropy.nddata.CCDData` object can be set two ways. + ++ If the FITS file has an image HDU extension matching the appropriate name (defaulted to ``"PSFIMAGE"``), the ``psf`` attribute is loaded from that image HDU. + ++ The PSF can also be provided when the `~astropy.nddata.CCDData` object is + constructed with the ``psf`` argument. + +The ``psf`` attribute should be a normalized image representing the PSF at the center of the `~astropy.nddata.CCDData`, sized appropriately for the data; users are responsible for managing and interpreting it in context. +For more on normalizing a PSF image, see :ref:`astropy:kernel_normalization`. + +The ``psf`` attribute is set to `None` in the output of an arithmetic operation, no matter the inputs. A warning message is emitted if either of the input images contain a non-`None` PSF; users are responsible for determining the appropriate thing to do in that context. + +Uncertainty +----------- + +You can set the uncertainty directly, either by creating a +`~astropy.nddata.StdDevUncertainty` object first: + + >>> rng = np.random.default_rng() + >>> data = rng.normal(size=(10, 10), loc=1.0, scale=0.1) + >>> ccd = CCDData(data, unit="electron") + >>> from astropy.nddata.nduncertainty import StdDevUncertainty + >>> uncertainty = 0.1 * ccd.data # can be any array whose shape matches the data + >>> my_uncertainty = StdDevUncertainty(uncertainty) + >>> ccd.uncertainty = my_uncertainty + +Or by providing a `~numpy.ndarray` with the same shape as the data: + + >>> ccd.uncertainty = 0.1 * ccd.data # doctest: +ELLIPSIS + INFO: array provided for uncertainty; assuming it is a StdDevUncertainty. [...] + +In this case, the uncertainty is assumed to be +`~astropy.nddata.StdDevUncertainty`. + +Two other uncertainty classes are available for which error propagation is +also supported: `~astropy.nddata.VarianceUncertainty` and +`~astropy.nddata.InverseVariance`. Using one of these three uncertainties is +required to enable error propagation in `~astropy.nddata.CCDData`. + +If you want access to the underlying uncertainty, use its ``.array`` attribute: + + >>> ccd.uncertainty.array # doctest: +ELLIPSIS + array(...) + +Arithmetic with Images +---------------------- + +Methods are provided to perform arithmetic operations with a +`~astropy.nddata.CCDData` image and a number, an ``astropy`` +`~astropy.units.Quantity` (a number with units), or another +`~astropy.nddata.CCDData` image. + +Using these methods propagates errors correctly (if the errors are +uncorrelated), takes care of any necessary unit conversions, and applies masks +appropriately. Note that the metadata of the result is *not* set if the +operation is between two `~astropy.nddata.CCDData` objects. + + >>> result = ccd.multiply(0.2 * u.adu) + >>> uncertainty_ratio = result.uncertainty.array[0, 0]/ccd.uncertainty.array[0, 0] + >>> round(uncertainty_ratio, 5) # doctest: +FLOAT_CMP + np.float64(0.2) + >>> result.unit + Unit("adu electron") + +.. note:: + The affiliated package `ccdproc `_ provides + functions for many common data reduction operations. Those functions try to + construct a sensible header for the result and provide a mechanism for + logging the action of the function in the header. + + +The arithmetic operators ``*``, ``/``, ``+``, and ``-`` are *not* overridden. + +.. note:: + If two images have different WCS values, the ``wcs`` on the first + `~astropy.nddata.CCDData` object will be used for the resultant object. diff --git a/docs/nddata/covariance.rst b/docs/nddata/covariance.rst new file mode 100644 index 000000000000..7fc4967a735d --- /dev/null +++ b/docs/nddata/covariance.rst @@ -0,0 +1,684 @@ + +.. _nddata-covariance: + +Covariance +********** + +Overview +======== + +For a data vector, :math:`{\mathbf x} = \{x_0, x_1, ...\}`, the covariance +between any two elements :math:`x_i` and :math:`x_j` define the elements of the +*covariance matrix* + +.. math:: + + \Sigma_{ij} = \rho_{ij} \sigma_i \sigma_j, + +where :math:`\rho_{ij}` are the elements of the *correlation matrix* and +:math:`V_i \equiv \sigma^2_i` is the variance in :math:`x_i`. The covariance +matrix is, by definition, symmetric and positive-semidefinite (all eigenvalues +are non-negative). + +The `~astropy.nddata.covariance.Covariance` object is a general utility for +constructing, visualizing, and storing two-dimensional covariance matrices. To +minimize its memory footprint, the class uses sparse matrices (i.e., the module +requires `scipy.sparse`) and only stores the upper triangle of the covariance +matrix. + +The class provides two convenient *static* methods for swapping between a full +covariance matrix (`~astropy.nddata.covariance.Covariance.revert_correlation`) +and the combination of a variance vector and correlation matrix +(`~astropy.nddata.covariance.Covariance.to_correlation`). + +.. _nddata-covariance-intro: + +Introductory Examples +--------------------- + +As a general introduction to covariance matrices, let :math:`{\mathbf x}` +contain 10 measurements. Let the correlation coefficient between adjacent +measurements be 0.5 (:math:`\rho_{ij} = 0.5\ {\rm for}\ |j-i| = 1`), 0.2 for +next but one measurements (:math:`\rho_{ij} = 0.2\ {\rm for}\ |j-i| = 2`), and 0 +otherwise. If we adopt unity variance for all elements of :math:`{\mathbf x}`, +we can directly construct the (banded) covariance matrix in python as follows: + +>>> import numpy as np +>>> +>>> # Create the covariance matrix as a dense array +>>> npts = 10 +>>> c = (np.diag(np.full(npts-2, 0.2, dtype=float), k=-2) +... + np.diag(np.full(npts-1, 0.5, dtype=float), k=-1) +... + np.diag(np.full(npts, 1.0, dtype=float), k=0) +... + np.diag(np.full(npts-1, 0.5, dtype=float), k=1) +... + np.diag(np.full(npts-2, 0.2, dtype=float), k=2)) +>>> c +array([[1. , 0.5, 0.2, 0. , 0. , 0. , 0. , 0. , 0. , 0. ], + [0.5, 1. , 0.5, 0.2, 0. , 0. , 0. , 0. , 0. , 0. ], + [0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. , 0. , 0. , 0. ], + [0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. , 0. , 0. ], + [0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. , 0. ], + [0. , 0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. ], + [0. , 0. , 0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. ], + [0. , 0. , 0. , 0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2], + [0. , 0. , 0. , 0. , 0. , 0. , 0.2, 0.5, 1. , 0.5], + [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0.2, 0.5, 1. ]]) + +In this case, the correlation matrix and the covariance matrix are *identical* +because the elements of the variance vector are all unity. + +With a correlation matrix, we can construct the covariance matrix with any +arbitrary variance vector. Continuing the example above, the following creates +a new covariance matrix with a variance vector with all elements equal to 4: + +>>> new_var = np.full(npts, 4., dtype=float) +>>> new_c = c * np.sqrt(new_var[:,None] * new_var[None,:]) +>>> new_c +array([[4. , 2. , 0.8, 0. , 0. , 0. , 0. , 0. , 0. , 0. ], + [2. , 4. , 2. , 0.8, 0. , 0. , 0. , 0. , 0. , 0. ], + [0.8, 2. , 4. , 2. , 0.8, 0. , 0. , 0. , 0. , 0. ], + [0. , 0.8, 2. , 4. , 2. , 0.8, 0. , 0. , 0. , 0. ], + [0. , 0. , 0.8, 2. , 4. , 2. , 0.8, 0. , 0. , 0. ], + [0. , 0. , 0. , 0.8, 2. , 4. , 2. , 0.8, 0. , 0. ], + [0. , 0. , 0. , 0. , 0.8, 2. , 4. , 2. , 0.8, 0. ], + [0. , 0. , 0. , 0. , 0. , 0.8, 2. , 4. , 2. , 0.8], + [0. , 0. , 0. , 0. , 0. , 0. , 0.8, 2. , 4. , 2. ], + [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0.8, 2. , 4. ]]) + +Or likewise for heteroscedastic data: + +>>> new_var = (1. + np.absolute(np.arange(npts) - npts//2).astype(float))**2 +>>> new_var +array([36., 25., 16., 9., 4., 1., 4., 9., 16., 25.]) +>>> new_c = c * np.sqrt(new_var[:,None] * new_var[None,:]) +>>> new_c +array([[36. , 15. , 4.8, 0. , 0. , 0. , 0. , 0. , 0. , 0. ], + [15. , 25. , 10. , 3. , 0. , 0. , 0. , 0. , 0. , 0. ], + [ 4.8, 10. , 16. , 6. , 1.6, 0. , 0. , 0. , 0. , 0. ], + [ 0. , 3. , 6. , 9. , 3. , 0.6, 0. , 0. , 0. , 0. ], + [ 0. , 0. , 1.6, 3. , 4. , 1. , 0.8, 0. , 0. , 0. ], + [ 0. , 0. , 0. , 0.6, 1. , 1. , 1. , 0.6, 0. , 0. ], + [ 0. , 0. , 0. , 0. , 0.8, 1. , 4. , 3. , 1.6, 0. ], + [ 0. , 0. , 0. , 0. , 0. , 0.6, 3. , 9. , 6. , 3. ], + [ 0. , 0. , 0. , 0. , 0. , 0. , 1.6, 6. , 16. , 10. ], + [ 0. , 0. , 0. , 0. , 0. , 0. , 0. , 3. , 10. , 25. ]]) + +.. note:: + + The `~astropy.nddata.covariance.Covariance` class provides a convenience + function for creating a new `~astropy.nddata.covariance.Covariance` instance + with the same correlation matrix but a new variance vector; see + :ref:`here`. + +Reordering and Subsets +---------------------- + +When reordering or down-selecting subsets of the elements of :math:`\mathbf{x}`, +these changes must be propagated to the associated covariance matrix, just as +would be needed for the error vector for an uncorrelated dataset. + +The example below first generates a vector with a shuffled set of indices. The +reordered :math:`\mathbf{x}` vector would be constructed by setting +``reordered_x = x[i]`` and the covariance matrix would be reordered using +`numpy.ix_`, as follows: + +>>> rng = np.random.default_rng(99) +>>> i = np.arange(npts) +>>> rng.shuffle(i) +>>> i +array([4, 6, 0, 3, 8, 1, 2, 5, 7, 9]) +>>> reordered_c = c[np.ix_(i,i)] +>>> reordered_c +array([[1. , 0.2, 0. , 0.5, 0. , 0. , 0.2, 0.5, 0. , 0. ], + [0.2, 1. , 0. , 0. , 0.2, 0. , 0. , 0.5, 0.5, 0. ], + [0. , 0. , 1. , 0. , 0. , 0.5, 0.2, 0. , 0. , 0. ], + [0.5, 0. , 0. , 1. , 0. , 0.2, 0.5, 0.2, 0. , 0. ], + [0. , 0.2, 0. , 0. , 1. , 0. , 0. , 0. , 0.5, 0.5], + [0. , 0. , 0.5, 0.2, 0. , 1. , 0.5, 0. , 0. , 0. ], + [0.2, 0. , 0.2, 0.5, 0. , 0.5, 1. , 0. , 0. , 0. ], + [0.5, 0.5, 0. , 0.2, 0. , 0. , 0. , 1. , 0.2, 0. ], + [0. , 0.5, 0. , 0. , 0.5, 0. , 0. , 0.2, 1. , 0.2], + [0. , 0. , 0. , 0. , 0.5, 0. , 0. , 0. , 0.2, 1. ]]) + +Note that the diagonal of ``reordered_c`` is still unity (all elements of +:math:`\mathbf{x}` are perfectly correlated with themselves), but the +off-diagonal terms have been rearranged to maintain the pre-existing +correlations. + +Creating a covariance matrix for a subset of data is a very similar operation. +If we want the covariance matrix for the first 3 elements of the data vector, we +can do the following: + +>>> i = np.arange(3) +>>> sub_c = c[np.ix_(i,i)] +>>> sub_c +array([[1. , 0.5, 0.2], + [0.5, 1. , 0.5], + [0.2, 0.5, 1. ]]) + +.. note:: + + The `~astropy.nddata.covariance.Covariance` class provides a convenience + function for matching the covariance data to a slice of its parent data array; + see :ref:`here`. + +In N-dimensions +--------------- + +Covariance matrices can be constructed for arrays of higher dimensionality by +flattening the data arrays. For a row-major array flattening order, one can +adopt the convention that :math:`\Sigma_{ij}` for an image of size +:math:`(N_x,N_y)` is the covariance between image pixels :math:`I_{x_i,y_i}` and +:math:`I_{x_j,y_j}`, where :math:`i = x_i + N_x\ y_i` and :math:`j = x_j + N_x\ +y_j`. + +As an example, let the covariance matrix ``c``, used throughout this section, be +the covariance matrix for a :math:`5 \times 2` array, instead of a 10-element +vector. The complication is determining the mapping from the data array to the +relevant covariance element; we can do this using `numpy` functions as follows. +To determine the covariance between elements ``data[1,0]`` and ``data[2,0]``, we +convert the indices from the ``data`` to find a covariance of 0.2: + +>>> data_array_shape = (5,2) +>>> i_data = (np.array([1]), np.array([0])) +>>> j_data = (np.array([2]), np.array([0])) +>>> i_cov = np.ravel_multi_index(i_data, data_array_shape) +>>> j_cov = np.ravel_multi_index(j_data, data_array_shape) +>>> i_cov, j_cov +(array([2]), array([4])) +>>> c[i_cov, j_cov] +array([0.2]) + +The inverse operation (determining the indices of the data array given the +indices in the covariance matrix) uses `~numpy.unravel_index` (cf. ``i_data``): + +>>> np.unravel_index(i_cov, data_array_shape) +(array([1]), array([0])) + +.. note:: + + The `~astropy.nddata.covariance.Covariance` class provides convenience + functions for switching between the data array and covariance matrix + indexing when working with higher dimensionality data arrays; + see :ref:`here`. + +.. _nddata-covariance-construction: + +Construction +============ + +Many methods are provided to construct a `~astropy.nddata.covariance.Covariance` +object. In *all* of the following examples, the object ``c`` is the banded +covariance array created at the beginning of the :ref:`nddata-covariance-intro` +section. + +Instantiating from pre-existing arrays +-------------------------------------- + +The simplest instantiation methods are based on using data that are already +available. + +To create a `~astropy.nddata.covariance.Covariance` object from a +variance vector: + +.. doctest-requires:: scipy + + >>> from astropy.nddata.covariance import Covariance + >>> # Create from a variance vector + >>> var = np.ones(3, dtype=float) + >>> # Create from the Covariance object + >>> covar = Covariance.from_variance(var) + >>> # Test its contents + >>> print(np.array_equal(covar.to_dense(), np.identity(3))) + True + +In this case, the variance is unity for all elements of the data array such that +the covariance matrix is diagonal and identical to the identity matrix. + +To create a `~astropy.nddata.covariance.Covariance` object from a "dense" (i.e., +fully populated) covariance matrix: + +.. doctest-requires:: scipy + + >>> # Instantiate from a covariance array + >>> covar = Covariance(array=c) + >>> print(np.array_equal(covar.to_dense(), c)) + True + >>> covar.to_dense() + array([[1. , 0.5, 0.2, 0. , 0. , 0. , 0. , 0. , 0. , 0. ], + [0.5, 1. , 0.5, 0.2, 0. , 0. , 0. , 0. , 0. , 0. ], + [0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. , 0. , 0. , 0. ], + [0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. , 0. , 0. ], + [0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. , 0. ], + [0. , 0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. ], + [0. , 0. , 0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. ], + [0. , 0. , 0. , 0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2], + [0. , 0. , 0. , 0. , 0. , 0. , 0.2, 0.5, 1. , 0.5], + [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0.2, 0.5, 1. ]]) + +.. important:: + + The last statement uses `~astropy.nddata.covariance.Covariance.to_dense` to + access the array; see :ref:`nddata-covariance-data-access`. + +Above, the base instantiation method is used; however, the +`~astropy.nddata.covariance.Covariance.from_array` method is also provided. The +primary difference is that the latter allows limits to be imposed on the +(absolute value of the) correlation or covariance values. + +Finally, note that, by default, all instantiations of a +`~astropy.nddata.covariance.Covariance` object check that the input matrix is +symmetric. If it is not, a warning is issued. To skip the check and the +warning, set ``assume_symmetric=True``. Regardless of whether or not the check +is performed, the object *only stores the upper triangle of the input matrix* +effectively meaning that any asymmetry in the matrix is lost when it is +ingested. + +Instantiating from random samples +--------------------------------- + +You can construct a covariance matrix based on samples from a distribution using +`~astropy.nddata.covariance.Covariance.from_samples`: + +.. doctest-requires:: scipy + + >>> # Set the mean to 0 for all elements + >>> m = np.zeros(npts, dtype=float) + >>> + >>> # Sample the multivariate normal distribution with the provided + >>> # mean and covariance. + >>> s = rng.multivariate_normal(m, c, size=100000) + >>> + >>> # Construct the covariance matrix from the random samples + >>> covar = Covariance.from_samples(s.T, cov_tol=0.1) + >>> + >>> # Test that the known input covariance matrix is close to the + >>> # measured covariance from the random samples + >>> print(np.all(np.absolute(c - covar.to_dense()) < 0.02)) + True + +Here, we have drawn samples from a known multivariate normal distribution with a +mean of zero (``m``) and a known covariance matrix (``c``), defined for the 10 +(``npts``) elements in the dataset (e.g., 10 pixels in a spectrum). The code +checks the reconstruction of the known covariance matrix against the result +built from these random samples. + +Instantiating from a matrix multiplication +------------------------------------------ + +Linear operations on a dataset (e.g., binning or smoothing) can be written as +matrix multiplications of the form + +.. math:: + + {\mathbf y} = {\mathbf T}\ {\mathbf x}, + +where :math:`{\mathbf T}` is a transfer matrix of size :math:`N_y\times N_x`, +:math:`{\mathbf x}` is a vector of size :math:`N_x`, and :math:`{\mathbf y}` is +a vector of length :math:`{N_y}` that results from the multiplication. If +:math:`{\mathbf \Sigma}_x` is the covariance matrix for :math:`{\mathbf x}`, then +the covariance matrix for :math:`{\mathbf y}` is + +.. math:: + + {\mathbf \Sigma}_y = {\mathbf T}\ {\mathbf \Sigma}_x\ {\mathbf T}^\top. + +The example below shows how to build a covariance matrix from a matrix +multiplication using +`~astropy.nddata.covariance.Covariance.from_matrix_multiplication`: + +.. doctest-requires:: scipy + + >>> # Construct a dataset + >>> x = np.arange(npts, dtype=float) + >>> + >>> # Construct a transfer matrix that simply selects the elements at + >>> # indices 0, 2, and 4 + >>> t = np.zeros((3,npts), dtype=float) + >>> t[0,0] = 1.0 + >>> t[1,2] = 1.0 + >>> t[2,4] = 1.0 + >>> + >>> # Get y + >>> y = np.dot(t, x) + >>> y + array([0., 2., 4.]) + >>> + >>> # Construct the covariance matrix + >>> covar = Covariance.from_matrix_multiplication(t, c) + >>> + >>> # Test the result + >>> _c = (np.diag(np.full(3-1, 0.2, dtype=float), k=-1) + ... + np.diag(np.full(3, 1.0, dtype=float), k=0) + ... + np.diag(np.full(3-1, 0.2, dtype=float), k=1)) + >>> _c + array([[1. , 0.2, 0. ], + [0.2, 1. , 0.2], + [0. , 0.2, 1. ]]) + >>> print(np.array_equal(covar.to_dense(), _c)) + True + +In N-dimensions +--------------- + +All of the instantiation methods above allow you to define the "data shape" of +the data array for the associated covariance matrix. Following the previous +N-dimensional example, let ``c`` be the covariance matrix for a :math:`5 \times +2` array, instead of a 10-element vector. + +.. doctest-requires:: scipy + + >>> data_array_shape + (5, 2) + >>> covar = Covariance(array=c, data_shape=data_array_shape) + >>> covar.to_dense() + array([[1. , 0.5, 0.2, 0. , 0. , 0. , 0. , 0. , 0. , 0. ], + [0.5, 1. , 0.5, 0.2, 0. , 0. , 0. , 0. , 0. , 0. ], + [0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. , 0. , 0. , 0. ], + [0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. , 0. , 0. ], + [0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. , 0. ], + [0. , 0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. ], + [0. , 0. , 0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. ], + [0. , 0. , 0. , 0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2], + [0. , 0. , 0. , 0. , 0. , 0. , 0.2, 0.5, 1. , 0.5], + [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0.2, 0.5, 1. ]]) + +The covariance matrix looks identical, but the higher dimensionality will affect +its :ref:`nddata-covariance-coord-access`. + +.. _nddata-covariance-data-access: + +Accessing the data +================== + +The `~astropy.nddata.covariance.Covariance` object is primarily a storage +utility. Internally, the object only stores the upper triangle of the covariance +matrix. **This means that you should not directly access a covariance value +within the object itself**; you must use the functions described below. + +.. _nddata-covariance-covariance-access: + +Covariance Matrix +----------------- + +There are two ways to access the full covariance matrix: + +- Use `~astropy.nddata.covariance.Covariance.to_sparse` to produce a sparse matrix or + +- Use `~astropy.nddata.covariance.Covariance.to_dense` for a dense matrix. + +The output of these two methods can be used as you would use any +`scipy.sparse.csr_matrix` or `numpy.ndarray` object, respectively. + +.. _nddata-covariance-correl-access: + +Variance Vector and Correlation Matrix +-------------------------------------- + +The variance vector is stored as an accessible property +(`~astropy.nddata.covariance.Covariance.variance`), but note that the property +is immutable. + +Access to the full correlation matrix is provided using +`~astropy.nddata.covariance.Covariance.to_sparse` to produce a sparse matrix or +`~astropy.nddata.covariance.Covariance.to_dense` for a dense matrix by setting +the keyword argument ``correlation = True``. + +.. _nddata-covariance-coord-access: + +Coordinate Data +--------------- + +Although more useful as preparation for storage, the covariance data can also be +accessed in coordinate format: + +.. doctest-requires:: scipy + + >>> covar = Covariance(array=c) + >>> i, j, cij = covar.coordinate_data() + >>> print(np.array_equal(covar.to_dense()[i,j], cij)) + True + +The arrays returned by `~astropy.nddata.covariance.Covariance.coordinate_data` +provide the matrix coordinates (``i`` and ``j``) for the non-zero covariance +values (``cij``). + +.. _nddata-covariance-table: + +File IO +======= + +The primary way to write/read `~astropy.nddata.covariance.Covariance` objects is +by first parsing the data into a `~astropy.table.Table` using the +`~astropy.nddata.covariance.Covariance.to_table` method: + +.. doctest-requires:: scipy + + >>> covar = Covariance(array=c) + >>> tbl = covar.to_table() + >>> tbl.meta + {'COVSHAPE': '(10, 10)'} + >>> tbl[:3] + + INDXI INDXJ COVARIJ + int64 int64 float64 + ----- ----- ------- + 0 0 1.0 + 0 1 0.5 + 0 2 0.2 + +The output above just shows the first 3 rows of the table to demonstrate that +the non-zero elements of the covariance matrix are stored in "coordinate +format." Specifically, the data is provided in three columns: + +- ``'INDXI'``: The row index in the covariance matrix (:math:`i`). + +- ``'INDXJ'``: The column index in the covariance matrix (:math:`j`). + +- ``'COVARIJ'``: The covariance value (:math:`\Sigma_{ij}`). + +The table also contains the following metadata: + +- ``'COVSHAPE'``: The shape of the covariance matrix. + +- ``'BUNIT'``: (If defined) The string representation of the covariance units. + +- ``'COVDSHP'``: (If the dimensionality is greater than 1) The shape of the + associated data array. + +For higher dimensional arrays, the coordinate data are automatically reshaped so +that the indices correspond to the data array. For example, + +.. doctest-requires:: scipy + + >>> data_array_shape + (5, 2) + >>> covar = Covariance(array=c, data_shape=data_array_shape) + >>> tbl = covar.to_table() + >>> tbl.meta + {'COVSHAPE': '(10, 10)', 'COVDSHP': '(5, 2)'} + >>> tbl[:3] +
+ INDXI INDXJ COVARIJ + int64[2] int64[2] float64 + -------- -------- ------- + 0 .. 0 0 .. 0 1.0 + 0 .. 0 0 .. 1 0.5 + 0 .. 0 1 .. 0 0.2 + >>> tbl['INDXI'][0] + array([0, 0]) + +.. warning:: + + Recall that the storage of covariance matrices for higher + dimensional data always assumes a row-major storage order. + +The inverse operation is also provided to instantiate a +`~astropy.nddata.covariance.Covariance` object from a table. Continuing the +N-dimensional example above: + +.. doctest-requires:: scipy + + >>> _covar = Covariance.from_table(tbl) + >>> _covar.data_shape + (5, 2) + >>> _covar.to_dense() + array([[1. , 0.5, 0.2, 0. , 0. , 0. , 0. , 0. , 0. , 0. ], + [0.5, 1. , 0.5, 0.2, 0. , 0. , 0. , 0. , 0. , 0. ], + [0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. , 0. , 0. , 0. ], + [0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. , 0. , 0. ], + [0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. , 0. ], + [0. , 0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. ], + [0. , 0. , 0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. ], + [0. , 0. , 0. , 0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2], + [0. , 0. , 0. , 0. , 0. , 0. , 0.2, 0.5, 1. , 0.5], + [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0.2, 0.5, 1. ]]) + +Use of the `~astropy.nddata.covariance.Covariance.to_table` and +`~astropy.nddata.covariance.Covariance.from_table` methods can be used with +Astropy's unified file I/O system to read and write the covariance matrices. + +For example, to write the covariance matrix to table and reload it: + +.. doctest-requires:: scipy + + >>> ofile = 'test_covar_io.fits' + >>> covar = Covariance(array=c) + >>> tbl = covar.to_table() + >>> tbl.write(ofile, format='fits') + >>> from astropy.io import fits + >>> with fits.open(ofile) as hdu: + ... hdu.info() + ... + Filename: test_covar_io.fits + No. Name Ver Type Cards Dimensions Format + 0 PRIMARY 1 PrimaryHDU 4 () + 1 1 BinTableHDU 15 27R x 3C [K, K, D] + >>> from astropy.table import Table + >>> _tbl = Table.read(ofile, format='fits') + >>> _covar = Covariance.from_table(_tbl) + >>> print(np.array_equal(covar.to_dense(), _covar.to_dense())) + True + +Utility Functions +================= + +.. _covariance-apply-new-variance: + +Renormalizing the variance +-------------------------- + +To create a new covariance matrix that maintains the same correlations as an +existing matrix but a different variance, you can apply a new variance +normalization (following the examples in the :ref:`introductory section +`). The `~astropy.nddata.covariance.Covariance` object +provides a convenience function for this. + +.. doctest-requires:: scipy + + >>> covar_var1 = Covariance(array=c) + >>> covar_var1.to_dense() + array([[1. , 0.5, 0.2, 0. , 0. , 0. , 0. , 0. , 0. , 0. ], + [0.5, 1. , 0.5, 0.2, 0. , 0. , 0. , 0. , 0. , 0. ], + [0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. , 0. , 0. , 0. ], + [0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. , 0. , 0. ], + [0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. , 0. ], + [0. , 0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. , 0. ], + [0. , 0. , 0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2, 0. ], + [0. , 0. , 0. , 0. , 0. , 0.2, 0.5, 1. , 0.5, 0.2], + [0. , 0. , 0. , 0. , 0. , 0. , 0.2, 0.5, 1. , 0.5], + [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0.2, 0.5, 1. ]]) + >>> var4 = np.full(c.shape[0], 4.0, dtype=float) + >>> covar_var4 = covar_var1.apply_new_variance(var4) + >>> covar_var4.to_dense() + array([[4. , 2. , 0.8, 0. , 0. , 0. , 0. , 0. , 0. , 0. ], + [2. , 4. , 2. , 0.8, 0. , 0. , 0. , 0. , 0. , 0. ], + [0.8, 2. , 4. , 2. , 0.8, 0. , 0. , 0. , 0. , 0. ], + [0. , 0.8, 2. , 4. , 2. , 0.8, 0. , 0. , 0. , 0. ], + [0. , 0. , 0.8, 2. , 4. , 2. , 0.8, 0. , 0. , 0. ], + [0. , 0. , 0. , 0.8, 2. , 4. , 2. , 0.8, 0. , 0. ], + [0. , 0. , 0. , 0. , 0.8, 2. , 4. , 2. , 0.8, 0. ], + [0. , 0. , 0. , 0. , 0. , 0.8, 2. , 4. , 2. , 0.8], + [0. , 0. , 0. , 0. , 0. , 0. , 0.8, 2. , 4. , 2. ], + [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0.8, 2. , 4. ]]) + +.. _covariance-match-to-data-slice: + +Matching the covariance data to a slice of its parent data array +---------------------------------------------------------------- + +To adjust a `~astropy.nddata.covariance.Covariance` object so that it is +appropriate for a slice of its parent data array, use +`~astropy.nddata.covariance.Covariance.match_to_data_slice`. For example, to +create a matrix with every other entry: + +.. doctest-requires:: scipy + + >>> covar = Covariance(array=c) + >>> sub_covar = covar.match_to_data_slice(np.s_[::2]) + >>> sub_covar + + >>> sub_covar.to_dense() + array([[1. , 0.2, 0. , 0. , 0. ], + [0.2, 1. , 0.2, 0. , 0. ], + [0. , 0.2, 1. , 0.2, 0. ], + [0. , 0. , 0.2, 1. , 0.2], + [0. , 0. , 0. , 0.2, 1. ]]) + +or to adjust for a reordering of the parent data array: + +.. doctest-requires:: scipy + + >>> covar = Covariance(array=c) + >>> rng = np.random.default_rng(99) + >>> reorder = np.arange(covar.shape[0]) + >>> rng.shuffle(reorder) + >>> reorder + array([4, 6, 0, 3, 8, 1, 2, 5, 7, 9]) + >>> reorder_covar = covar.match_to_data_slice(reorder) + >>> reorder_covar.to_dense() + array([[1. , 0.2, 0. , 0.5, 0. , 0. , 0.2, 0.5, 0. , 0. ], + [0.2, 1. , 0. , 0. , 0.2, 0. , 0. , 0.5, 0.5, 0. ], + [0. , 0. , 1. , 0. , 0. , 0.5, 0.2, 0. , 0. , 0. ], + [0.5, 0. , 0. , 1. , 0. , 0.2, 0.5, 0.2, 0. , 0. ], + [0. , 0.2, 0. , 0. , 1. , 0. , 0. , 0. , 0.5, 0.5], + [0. , 0. , 0.5, 0.2, 0. , 1. , 0.5, 0. , 0. , 0. ], + [0.2, 0. , 0.2, 0.5, 0. , 0.5, 1. , 0. , 0. , 0. ], + [0.5, 0.5, 0. , 0.2, 0. , 0. , 0. , 1. , 0.2, 0. ], + [0. , 0.5, 0. , 0. , 0.5, 0. , 0. , 0.2, 1. , 0.2], + [0. , 0. , 0. , 0. , 0.5, 0. , 0. , 0. , 0.2, 1. ]]) + +.. _covariance-nd-indexing: + +Data-to-covariance Indexing Transformations +------------------------------------------- + +For higher dimensional arrays, two methods are provided to ease conversion +between data array and covariance matrix indexing. Following examples above, +define the ten elements in the covariance matrix as coming from a :math:`5 +\times 2` array, then find the indices in the data array for the covariance +values at indices covariance values at matrix locations ``(0,3)``, ``(1,4)``, +and ``(2,3)``: + +.. doctest-requires:: scipy + + >>> covar = Covariance(array=c, data_shape=data_array_shape) + >>> i_data, j_data = covar.covariance_to_data_indices([0,1,2], [3,4,3]) + >>> i_data + (array([0, 0, 1]), array([0, 1, 0])) + >>> j_data + (array([1, 2, 1]), array([1, 0, 1])) + +This shows that the covariance elements provide the covariance between +``data[0,0]`` and ``data[1,1]``, elements ``data[0,1]`` and ``data[2,0]``, and +elements ``data[1,0]`` and ``data[1,1]``. + +The inverse operation gives the covariance indices for a specified set of +data-array indices. Keeping the indices we defined above: + +.. doctest-requires:: scipy + + >>> i_cov, j_cov = covar.data_to_covariance_indices(i_data, j_data) + >>> i_cov, j_cov + (array([0, 1, 2]), array([3, 4, 3])) + diff --git a/docs/nddata/decorator.rst b/docs/nddata/decorator.rst index c79c1a4dc9ac..2077134fa65a 100644 --- a/docs/nddata/decorator.rst +++ b/docs/nddata/decorator.rst @@ -1,28 +1,19 @@ ********************************************* -Decorating functions to accept NDData objects +Decorating Functions to Accept NDData Objects ********************************************* -.. important:: The functionality described here is still experimental and will - likely evolve over time as more packages make use of it. - -Introduction -============ - The `astropy.nddata` module includes a decorator -:func:`~astropy.nddata.support_nddata` that makes it easy for developers and -users to write functions that can accept either :class:`~astropy.nddata.NDData` +:func:`~astropy.nddata.support_nddata` that makes it convenient for developers +and users to write functions that can accept :class:`~astropy.nddata.NDData` objects and also separate arguments. -Getting started -=============== - -Let's consider the following function:: +Consider the following function:: def test(data, wcs=None, unit=None, n_iterations=3): ... -Now let's say that we want to be able to call the function as ``test(nd)`` -where ``nd`` is a :class:`~astropy.nddata.NDData` instance. We can decorate +Now say that we want to be able to call the function as ``test(nd)`` +where ``nd`` is an :class:`~astropy.nddata.NDData` instance. We can decorate this function using :func:`~astropy.nddata.support_nddata`:: from astropy.nddata import support_nddata @@ -31,21 +22,20 @@ this function using :func:`~astropy.nddata.support_nddata`:: def test(data, wcs=None, unit=None, n_iterations=3): ... -which makes it so that when the user calls ``test(nd)``, the function would +Which makes it so that when the user calls ``test(nd)``, the function would automatically be called with:: test(nd.data, wcs=nd.wcs, unit=nd.unit) -That is, the decorator looks at the signature of the function and checks if any +The decorator looks at the signature of the function and checks if any of the arguments are also properties of the ``NDData`` object, and passes them as individual arguments. The function can also be called with separate -arguments as if it wasn't decorated. +arguments as if it was not decorated. -An exception is raised if an ``NDData`` property is set but the function does -not accept it - for example, if ``wcs`` is set, but the function cannot support -WCS objects, an error would be raised. On the other hand, if an argument in the -function does not exist in the ``NDData`` object or is not set, it is simply -left to its default value. +A warning is emitted if an ``NDData`` property is set but the function does +not accept it — for example, if ``wcs`` is set, but the function cannot support +WCS objects. On the other hand, if an argument in the function does not exist +in the ``NDData`` object or is not set, it is left to its default value. If the function call succeeds, then the decorator returns the values from the function unmodified by default. However, in some cases we may want to return @@ -63,9 +53,8 @@ separate arguments, and an object with the same class type as the input if the input is an :class:`~astropy.nddata.NDData` or subclass instance. Finally, the decorator can be made to restrict input to specific ``NDData`` -sub-classes (and sub-classes of those) using the ``accepts`` option:: +subclasses (and the subclasses of those) using the ``accepts`` option:: @support_nddata(accepts=CCDImage) def test(data, wcs=None, unit=None, n_iterations=3): ... - diff --git a/docs/nddata/examples/cutout2d_tofits.py b/docs/nddata/examples/cutout2d_tofits.py new file mode 100644 index 000000000000..dac61aec05f9 --- /dev/null +++ b/docs/nddata/examples/cutout2d_tofits.py @@ -0,0 +1,36 @@ +# Download an example FITS file, create a 2D cutout, and save it to a +# new FITS file, including the updated cutout WCS. +from astropy.io import fits +from astropy.nddata import Cutout2D +from astropy.utils.data import download_file +from astropy.wcs import WCS + + +def download_image_save_cutout(url, position, size): + # Download the image + filename = download_file(url) + + # Load the image and the WCS + hdu = fits.open(filename)[0] + wcs = WCS(hdu.header) + + # Make the cutout, including the WCS + cutout = Cutout2D(hdu.data, position=position, size=size, wcs=wcs) + + # Put the cutout image in the FITS HDU + hdu.data = cutout.data + + # Update the FITS header with the cutout WCS + hdu.header.update(cutout.wcs.to_header()) + + # Write the cutout to a new FITS file + cutout_filename = "example_cutout.fits" + hdu.writeto(cutout_filename, overwrite=True) + + +if __name__ == "__main__": + url = "https://astropy.stsci.edu/data/photometry/spitzer_example_image.fits" + + position = (500, 300) + size = (400, 400) + download_image_save_cutout(url, position, size) diff --git a/docs/nddata/index.rst b/docs/nddata/index.rst index b8f91d52a187..894b777fa838 100644 --- a/docs/nddata/index.rst +++ b/docs/nddata/index.rst @@ -1,38 +1,22 @@ .. _astropy_nddata: ***************************************** -N-dimensional datasets (`astropy.nddata`) +N-Dimensional Datasets (`astropy.nddata`) ***************************************** Introduction ============ -The `~astropy.nddata` package provides a uniform interface to N-dimensional -datasets in astropy through: +The `~astropy.nddata` package provides classes to represent images and other +gridded data, some essential functions for manipulating images, and the +infrastructure for package developers who wish to include support for the +image classes. This subpackage was developed based on `APE 7`_. -+ The `~astropy.nddata.NDDataBase` metaclass to define an astropy-wide - interface to N-dimensional data sets while allowing flexibility in - how those datasets are represented internally. -+ The `~astropy.nddata.NDData` class, which provides a basic container for - gridded N-dimensional datasets. -+ Several mixin classes for adding functionality to `~astropy.nddata.NDData` - containers. -+ A decorator, `~astropy.nddata.support_nddata`, for facilitating use of - `~astropy.nddata` objects in functions in astropy and affiliated packages. +.. _astropy_nddata_getting_started: -.. warning:: - - `~astropy.nddata` has changed significantly in astropy 1.0. See the section - :ref:`nddata_transition` for more information. - -Getting started +Getting Started =============== -Of the classes provided by `~astropy.nddata`, the place to start for most -users will be `~astropy.nddata.NDData`, which by default uses a numpy array to -store the data. Designers of new classes should also look at -`~astropy.nddata.NDDataBase` before deciding what to subclass from. - NDData ------ @@ -40,121 +24,465 @@ The primary purpose of `~astropy.nddata.NDData` is to act as a *container* for data, metadata, and other related information like a mask. An `~astropy.nddata.NDData` object can be instantiated by passing it an -n-dimensional Numpy array:: +n-dimensional `numpy` array:: >>> import numpy as np >>> from astropy.nddata import NDData >>> array = np.zeros((12, 12, 12)) # a 3-dimensional array with all zeros - >>> ndd = NDData(array) + >>> ndd1 = NDData(array) -or something that can be converted to an array:: +Or something that can be converted to a `numpy.ndarray`:: >>> ndd2 = NDData([1, 2, 3, 4]) + >>> ndd2 + NDData([1, 2, 3, 4]) + +And can be accessed again via the ``data`` attribute:: -It is also possible to initialize `~astropy.nddata.NDData` with more exotic -objects; see :ref:`nddata_details` for more information. + >>> ndd2.data + array([1, 2, 3, 4]) -The underlying Numpy array can be accessed via the ``data`` attribute:: +It also supports additional properties like a ``unit`` or ``mask`` for the +data, a ``wcs`` (World Coordinate System) and ``uncertainty`` of the data and +additional ``meta`` attributes: - >>> ndd.data - array([[[ 0., 0., 0., ... - ... + >>> data = np.array([1,2,3,4]) + >>> mask = data > 2 + >>> unit = 'erg / s' + >>> from astropy.nddata import StdDevUncertainty + >>> uncertainty = StdDevUncertainty(np.sqrt(data)) # representing standard deviation + >>> meta = {'object': 'fictional data.'} + >>> ndd = NDData(data, mask=mask, unit=unit, uncertainty=uncertainty, + ... meta=meta) + >>> ndd + NDData([1, 2, —, —], unit='erg / s') -Values can be masked using the ``mask`` attribute:: +The representation only displays the ``data``; the other attributes need to be +accessed directly, for example, ``ndd.mask`` to access the mask. - >>> ndd_masked = NDData(ndd, mask = ndd.data > 0.9) - INFO: Overwriting NDData's current mask with specified mask [astropy.nddata.nddata] -A mask value of `True` indicates a value that should be ignored, while a mask -value of `False` indicates a valid value. +NDDataRef +--------- +Building upon this pure container, `~astropy.nddata.NDDataRef` implements: -Similar attributes are available to store: ++ A ``read`` and ``write`` method to access ``astropy``'s unified file I/O + interface. ++ Simple arithmetic like addition, subtraction, division, and multiplication. ++ Slicing. -+ generic meta-data, in ``meta``, -+ a unit for the data values, in ``unit`` and -+ an uncertainty for the data values, in ``uncertainty``. Note that the - ``uncertainty`` must have a string attribute called ``uncertainty_type``. +Instances are created in the same way:: -Note that a `~astropy.nddata.NDData` object is not sliceable:: + >>> from astropy.nddata import NDDataRef + >>> ndd = NDDataRef(ndd) + >>> ndd + NDDataRef([1, 2, —, —], unit='erg / s') - >>> ndd2[1:3] # doctest: +SKIP - Traceback (most recent call last): - ... - TypeError: 'NDData' object has no attribute '__getitem__' +But also support arithmetic (:ref:`nddata_arithmetic`) like addition:: + >>> import astropy.units as u + >>> ndd2 = ndd.add([4, -3.5, 3, 2.5] * u.erg / u.s) + >>> ndd2 + NDDataRef([ 5. , -1.5, ———, ———], unit='erg / s') +Because these operations have a wide range of options, these are not available +using arithmetic operators like ``+``. -Mixins for additional functionality ------------------------------------ +Slicing or indexing (:ref:`nddata_slicing`) is possible (with warnings issued if +some attribute cannot be sliced):: -Several classes are provided to add functionality to the basic ``NDData`` -container. They include: + >>> ndd2[2:] # discard the first two elements # doctest: +FLOAT_CMP + NDDataRef([———, ———], unit='erg / s') + >>> ndd2[1] # get the second element # doctest: +FLOAT_CMP + NDDataRef(-1.5, unit='erg / s') -+ `~astropy.nddata.NDSlicingMixin` to handle slicing of N-dimensional data. -+ `~astropy.nddata.NDArithmeticMixin` to allow arithmetic operations on - `~astropy.nddata.NDData` objects that include support propagation of - uncertainties (in limited cases). -+ `~astropy.nddata.NDIOMixin` to use existing astropy functionality for input - (with the method ``read``) and output (with the method ``write``). -To use these mixins, create a new class that includes the appropriate mixins -as subclasses. For example, to make a class that includes slicing, but not -arithmetic or I/O:: +Working with Two-Dimensional Data Like Images +--------------------------------------------- - >>> from astropy.nddata import NDData, NDSlicingMixin - >>> class NDDataSliceable(NDSlicingMixin, NDData): pass +Though the `~astropy.nddata` package supports any kind of gridded data, this +introduction will focus on the use of `~astropy.nddata` for two-dimensional +images. To get started, we will construct a two-dimensional image with a few +sources, some Gaussian noise, and a "cosmic ray" which we will later mask out. -Note that the body of the class need not contain any code; all of the -functionality is provided by the ``NDData`` container and the mixins. The -order of the classes is important because python works from right to left in -determining the order in which methods are resolved. +Examples +^^^^^^^^ + +.. + EXAMPLE START + Working with Two-Dimensional Data Using NDData + +First, construct a two-dimensional image with a few sources, some Gaussian +noise, and a "cosmic ray":: + + >>> import numpy as np + >>> from astropy.modeling.models import Gaussian2D + >>> rng = np.random.default_rng() + >>> y, x = np.mgrid[0:500, 0:600] + >>> data = (Gaussian2D(1, 150, 100, 20, 10, theta=0.5)(x, y) + + ... Gaussian2D(0.5, 400, 300, 8, 12, theta=1.2)(x,y) + + ... Gaussian2D(0.75, 250, 400, 5, 7, theta=0.23)(x,y) + + ... Gaussian2D(0.9, 525, 150, 3, 3)(x,y) + + ... Gaussian2D(0.6, 200, 225, 3, 3)(x,y)) + >>> data += 0.01 * rng.standard_normal((500, 600)) + >>> cosmic_ray_value = 0.997 + >>> data[100, 300:310] = cosmic_ray_value + +This image has a large "galaxy" in the lower left and the "cosmic ray" is the +horizontal line in the lower middle of the image: + +.. doctest-skip:: + + >>> import matplotlib.pyplot as plt + >>> fig, ax = plt.subplots() + >>> ax.imshow(data, origin='lower') + +.. plot:: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling.models import Gaussian2D + y, x = np.mgrid[0:500, 0:600] + data = (Gaussian2D(1, 150, 100, 20, 10, theta=0.5)(x, y) + + Gaussian2D(0.5, 400, 300, 8, 12, theta=1.2)(x,y) + + Gaussian2D(0.75, 250, 400, 5, 7, theta=0.23)(x,y) + + Gaussian2D(0.9, 525, 150, 3, 3)(x,y) + + Gaussian2D(0.6, 200, 225, 3, 3)(x,y)) + rng = np.random.default_rng(123456) + data += 0.01 * rng.standard_normal((500, 600)) + cosmic_ray_value = 0.997 + data[100, 300:310] = cosmic_ray_value + fig, ax = plt.subplots() + ax.imshow(data, origin='lower') -``NDDataSliceable`` is initialized the same way that `~astropy.nddata.NDData` is:: - >>> ndd_sliceable = NDDataSliceable([1, 2, 3, 4]) +The "cosmic ray" can be masked out in this test image, like this:: -but can be sliced:: + >>> mask = (data == cosmic_ray_value) - >>> ndd_sliceable[1:3] - NDDataSliceable([2, 3]) +.. + EXAMPLE END -The class `~astropy.nddata.NDDataArray` is an example of a class which -utilizes mixins *and* adds functionality. +`~astropy.nddata.CCDData` Class for Images +------------------------------------------ -NDDataBase for making new subclasses ------------------------------------- +The `~astropy.nddata.CCDData` object, like the other objects in this package, +can store the data, a mask, and metadata. The `~astropy.nddata.CCDData` object +requires that a unit be specified:: -`~astropy.nddata.NDDataBase` is a metaclass provided to support the creation -of objects that support the NDData interface but need the freedom to define -their own ways of storing data, unit, metadata and/or other properties. It -should be used instead of `~astropy.nddata.NDData` as the starting point for -any class for which the `~astropy.nddata.NDData` class is too restrictive. + >>> from astropy.nddata import CCDData + >>> ccd = CCDData(data, mask=mask, + ... meta={'object': 'fake galaxy', 'filter': 'R'}, + ... unit='adu') -.. _nddata_transition: +Slicing +------- -Transition to astropy 1.0 -========================= +Slicing works the way you would expect with the mask and, if present, +WCS, sliced appropriately:: -The nddata package underwent substantial revision as a result of `APE 7`_; -please see that APE for an extensive discussion of the motivation and the -changes. + >>> ccd2 = ccd[:200, :] + >>> ccd2.data.shape + (200, 600) + >>> ccd2.mask.shape + (200, 600) + >>> # Show the mask in a region around the cosmic ray: + >>> ccd2.mask[99:102, 299:311] + array([[False, False, False, False, False, False, False, False, False, + False, False, False], + [False, True, True, True, True, True, True, True, True, + True, True, False], + [False, False, False, False, False, False, False, False, False, + False, False, False]]...) -The most important changes are that: +For many applications it may be more convenient to use +`~astropy.nddata.Cutout2D`, described in `image_utilities`_. -+ ``NDData`` does not provide a numpy-like interface; to use its data use the - ``data`` attribute instead. -+ Slicing is no provided in the base `~astropy.nddata.NDData`. -+ Arithmetic is no longer included in the base `~astropy.nddata.NDData` class. +Image Arithmetic, Including Uncertainty +--------------------------------------- -Code that only uses the metadata features of `~astropy.nddata.NDData` should -not need to be modified. +Methods are provided for basic arithmetic operations between images, including +propagation of uncertainties. Three uncertainty types are supported: variance +(`~astropy.nddata.VarianceUncertainty`), standard deviation +(`~astropy.nddata.StdDevUncertainty`), and inverse variance +(`~astropy.nddata.InverseVariance`). -Code that uses the arithemtic methods that used to be included in -`~astropy.nddata.NDData` and relied on it to behave like a numpy array should -instead subclass `~astropy.nddata.NDDataArray`; that class is equivalent to -the original `~astropy.nddata.NDData` class. +Examples +^^^^^^^^ + +.. + EXAMPLE START + Image Arithmetic Including Uncertainty in NDData + +This example creates an uncertainty that is Poisson error, stored as a +variance:: + + >>> from astropy.nddata import VarianceUncertainty + >>> poisson_noise = np.ma.sqrt(np.ma.abs(ccd.data)) + >>> ccd.uncertainty = VarianceUncertainty(poisson_noise ** 2) + +As a convenience, the uncertainty can also be set with a ``numpy`` array. In +that case, the uncertainty is assumed to be the standard deviation:: + >>> ccd.uncertainty = poisson_noise + INFO: array provided for uncertainty; assuming it is a StdDevUncertainty. [astropy.nddata.ccddata] + +If we make a copy of the image and add that to the original, the uncertainty +changes as expected:: + + >>> ccd2 = ccd.copy() + >>> added_ccds = ccd.add(ccd2, handle_meta='first_found') + >>> added_ccds.uncertainty.array[0, 0] / ccd.uncertainty.array[0, 0] / np.sqrt(2) # doctest: +FLOAT_CMP + np.float64(0.99999999999999989) + +.. + EXAMPLE END + +.. _nddata_reading_writing: + +Reading and Writing +------------------- + +A `~astropy.nddata.CCDData` can be saved to a FITS file:: + + >>> ccd.write('test_file.fits') + +And can also be read in from a FITS file:: + + >>> ccd2 = CCDData.read('test_file.fits') + +Note the unit is stored in the ``BUNIT`` keyword in the header on saving, and is +read from the header if it is present. + +Detailed help on the available keyword arguments for reading and writing +can be obtained via the ``help()`` method as follows: + +.. doctest-skip:: + + >>> CCDData.read.help('fits') # Get help on the CCDData FITS reader + >>> CCDData.writer.help('fits') # Get help on the CCDData FITS writer + +.. _image_utilities: + +Image Utilities +--------------- + +Cutouts +^^^^^^^ + +Though slicing directly is one way to extract a subframe, +`~astropy.nddata.Cutout2D` provides more convenient access to cutouts from the +data. + +Examples +~~~~~~~~ + +.. + EXAMPLE START + Accessing Cutouts in NDData + +This example pulls out the large "galaxy" in the lower left of the image, with +the center of the cutout at ``position``:: + + >>> from astropy.nddata import Cutout2D + >>> position = (149.7, 100.1) + >>> size = (81, 101) # pixels + >>> cutout = Cutout2D(ccd, position, size) + >>> fig, ax = plt.subplots() # doctest: +SKIP + >>> ax.imshow(cutout.data, origin='lower') # doctest: +SKIP + +.. plot:: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling.models import Gaussian2D + from astropy.nddata import CCDData + from astropy.nddata import Cutout2D + y, x = np.mgrid[0:500, 0:600] + data = (Gaussian2D(1, 150, 100, 20, 10, theta=0.5)(x, y) + + Gaussian2D(0.5, 400, 300, 8, 12, theta=1.2)(x,y) + + Gaussian2D(0.75, 250, 400, 5, 7, theta=0.23)(x,y) + + Gaussian2D(0.9, 525, 150, 3, 3)(x,y) + + Gaussian2D(0.6, 200, 225, 3, 3)(x,y)) + rng = np.random.default_rng(123456) + data += 0.01 * rng.standard_normal((500, 600)) + cosmic_ray_value = 0.997 + data[100, 300:310] = cosmic_ray_value + mask = (data == cosmic_ray_value) + ccd = CCDData(data, mask=mask, + meta={'object': 'fake galaxy', 'filter': 'R'}, + unit='adu') + position = (149.7, 100.1) + size = (81, 101) # pixels + cutout = Cutout2D(ccd, position, size) + fig, ax = plt.subplots() + ax.imshow(cutout.data, origin='lower') + +This cutout can also plot itself on the original image:: + + >>> plt.imshow(ccd, origin='lower') # doctest: +SKIP + >>> cutout.plot_on_original(color='white') # doctest: +SKIP + +.. plot:: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling.models import Gaussian2D + from astropy.nddata import CCDData, Cutout2D + y, x = np.mgrid[0:500, 0:600] + data = (Gaussian2D(1, 150, 100, 20, 10, theta=0.5)(x, y) + + Gaussian2D(0.5, 400, 300, 8, 12, theta=1.2)(x,y) + + Gaussian2D(0.75, 250, 400, 5, 7, theta=0.23)(x,y) + + Gaussian2D(0.9, 525, 150, 3, 3)(x,y) + + Gaussian2D(0.6, 200, 225, 3, 3)(x,y)) + rng = np.random.default_rng(123456) + data += 0.01 * rng.standard_normal((500, 600)) + cosmic_ray_value = 0.997 + data[100, 300:310] = cosmic_ray_value + mask = (data == cosmic_ray_value) + ccd = CCDData(data, mask=mask, + meta={'object': 'fake galaxy', 'filter': 'R'}, + unit='adu') + position = (149.7, 100.1) + size = (81, 101) # pixels + cutout = Cutout2D(ccd, position, size) + fig, ax = plt.subplots() + ax.imshow(ccd, origin='lower') + cutout.plot_on_original(color='white') + +The cutout also provides methods for finding pixel coordinates in the original +or in the cutout; recall that ``position`` is the center of the cutout in the +original image:: + + >>> position + (149.7, 100.1) + >>> cutout.to_cutout_position(position) # doctest: +FLOAT_CMP + (49.7, 40.099999999999994) + >>> cutout.to_original_position((49.7, 40.099999999999994)) # doctest: +FLOAT_CMP + (149.7, 100.1) + +For more details, including constructing a cutout from World Coordinates and +the options for handling cutouts that go beyond the bounds of the original +image, see :ref:`cutout_images`. + +.. + EXAMPLE END + +Image Resizing +^^^^^^^^^^^^^^ + +The functions `~astropy.nddata.block_reduce` and +`~astropy.nddata.block_replicate` resize images. + +Example +~~~~~~~ + +.. + EXAMPLE START + Image Resizing in NDData + +This example reduces the size of the image by a factor of 4. Note that the +result is a `numpy.ndarray`; the mask, metadata, etc. are discarded: + +.. doctest-requires:: skimage + + >>> from astropy.nddata import block_reduce, block_replicate + >>> smaller = block_reduce(ccd, 4) # doctest: +IGNORE_WARNINGS + >>> smaller + array(...) + >>> fig, ax = plt.subplots() + >>> ax.imshow(smaller, origin='lower') # doctest: +SKIP + +.. plot:: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling.models import Gaussian2D + from astropy.nddata import block_reduce, block_replicate + from astropy.nddata import CCDData, Cutout2D + y, x = np.mgrid[0:500, 0:600] + data = (Gaussian2D(1, 150, 100, 20, 10, theta=0.5)(x, y) + + Gaussian2D(0.5, 400, 300, 8, 12, theta=1.2)(x,y) + + Gaussian2D(0.75, 250, 400, 5, 7, theta=0.23)(x,y) + + Gaussian2D(0.9, 525, 150, 3, 3)(x,y) + + Gaussian2D(0.6, 200, 225, 3, 3)(x,y)) + rng = np.random.default_rng(123456) + data += 0.01 * rng.standard_normal((500, 600)) + cosmic_ray_value = 0.997 + data[100, 300:310] = cosmic_ray_value + mask = (data == cosmic_ray_value) + ccd = CCDData(data, mask=mask, + meta={'object': 'fake galaxy', 'filter': 'R'}, + unit='adu') + smaller = block_reduce(ccd.data, 4) + fig, ax = plt.subplots() + ax.imshow(smaller, origin='lower') + +By default, both `~astropy.nddata.block_reduce` and +`~astropy.nddata.block_replicate` conserve flux. + +.. + EXAMPLE END + +Other Image Classes +------------------- + +There are two less restrictive classes, `~astropy.nddata.NDDataArray` and +`~astropy.nddata.NDDataRef`, that can be used to hold image data. They are +primarily of interest to those who may want to create their own image class by +subclassing from one of the classes in the `~astropy.nddata` package. The main +differences between them are: + ++ `~astropy.nddata.NDDataRef` can be sliced and has methods for basic + arithmetic operations, but the user needs to use one of the uncertainty + classes to define an uncertainty. See :ref:`NDDataRef` for more detail. + Most of its properties must be set when the object is created because they + are not mutable. ++ `~astropy.nddata.NDDataArray` extends `~astropy.nddata.NDDataRef` by adding + the methods necessary for it to behave like a ``numpy`` array in expressions + and adds setters for several properties. It lacks the ability to + automatically recognize and read data from FITS files and does not attempt + to automatically set the WCS property. ++ `~astropy.nddata.CCDData` extends `~astropy.nddata.NDDataArray` by setting + up a default uncertainty class, setting up straightforward read/write to FITS + files, and automatically setting up a WCS property. + +More General Gridded Data Classes +--------------------------------- + +There are two generic classes in the ``nddata`` package that are of +interest primarily to users who either need a custom image class that goes +beyond the classes discussed so far, or who are working with gridded data that +is not an image. + ++ `~astropy.nddata.NDData` is a container class for holding general gridded + data. It includes a handful of basic attributes, but no slicing or arithmetic. + More information about this class is in :ref:`nddata_details`. ++ `~astropy.nddata.NDDataBase` is an abstract base class that developers of new + gridded data classes can subclass to declare that the new class follows the + `~astropy.nddata.NDData` interface. More details are in + :ref:`nddata_subclassing`. + +Additional Examples +=================== + +The list of packages below that use the ``nddata`` framework is intended to be +useful to either users writing their own image classes or those looking +for an image class that goes beyond what `~astropy.nddata.CCDData` does. + ++ The `SunPy project `_ uses `~astropy.nddata.NDData` as the + foundation for its + `Map classes `_. ++ The class `~astropy.nddata.NDDataRef` is used in + `specutils `_ as the basis for + `Spectrum1D `_, which adds several methods useful for + spectra. ++ The package `ndmapper `_, which + makes it easy to build reduction pipelines for optical data, uses + `~astropy.nddata.NDDataArray` as its image object. ++ The package `ccdproc `_ uses the + `~astropy.nddata.CCDData` class throughout for implementing optical/IR image + reduction. Using ``nddata`` ================ @@ -162,18 +490,26 @@ Using ``nddata`` .. toctree:: :maxdepth: 2 - nddata.rst + ccddata.rst + utils.rst + bitmask.rst decorator.rst + nddata.rst + covariance.rst mixins/index.rst subclassing.rst +.. note that if this section gets too long, it should be moved to a separate + doc page - see the top of performance.inc.rst for the instructions on how to do + that +.. include:: performance.inc.rst + Reference/API ============= -.. automodapi:: astropy.nddata - :no-inheritance-diagram: +.. toctree:: + :maxdepth: 2 -.. automodapi:: astropy.nddata.utils - :no-inheritance-diagram: + ref_api -.. _APE 7: https://github.com/astropy/astropy-APEs/blob/master/APE7.rst +.. _APE 7: https://github.com/astropy/astropy-APEs/blob/main/APE7.rst diff --git a/docs/nddata/mixins/index.rst b/docs/nddata/mixins/index.rst index 27e4ef3bf312..1be69e869c32 100644 --- a/docs/nddata/mixins/index.rst +++ b/docs/nddata/mixins/index.rst @@ -1,5 +1,5 @@ -Mixins for added functionality -============================== +Mixins for Added Functionality +****************************** .. toctree:: :maxdepth: 2 diff --git a/docs/nddata/mixins/ndarithmetic.rst b/docs/nddata/mixins/ndarithmetic.rst index 46042e5946b0..6c702b5f9bae 100644 --- a/docs/nddata/mixins/ndarithmetic.rst +++ b/docs/nddata/mixins/ndarithmetic.rst @@ -1,49 +1,412 @@ -Arithmetic mixin +.. _nddata_arithmetic: + +NDData Arithmetic +***************** + +Introduction +============ + +`~astropy.nddata.NDDataRef` implements the following arithmetic operations: + +- Addition: :meth:`~astropy.nddata.NDArithmeticMixin.add` +- Subtraction: :meth:`~astropy.nddata.NDArithmeticMixin.subtract` +- Multiplication: :meth:`~astropy.nddata.NDArithmeticMixin.multiply` +- Division: :meth:`~astropy.nddata.NDArithmeticMixin.divide` + +Using Basic Arithmetic Methods +============================== + +Using the standard arithmetic methods requires that the first operand +is an `~astropy.nddata.NDDataRef` instance: + + >>> from astropy.nddata import NDDataRef + >>> from astropy.wcs import WCS + >>> import numpy as np + >>> ndd1 = NDDataRef([1, 2, 3, 4]) + +While the requirement for the second operand is that it must be convertible +to the first operand. It can be a number:: + + >>> ndd1.add(3) + NDDataRef([4, 5, 6, 7]) + +Or a `list`:: + + >>> ndd1.subtract([1,1,1,1]) + NDDataRef([0, 1, 2, 3]) + +Or a `numpy.ndarray`:: + + >>> ndd1.multiply(np.arange(4, 8)) + NDDataRef([ 4, 10, 18, 28]) + >>> ndd1.divide(np.arange(1,13).reshape(3,4)) # a 3 x 4 numpy array # doctest: +FLOAT_CMP + NDDataRef([[1. , 1. , 1. , 1. ], + [0.2 , 0.33333333, 0.42857143, 0.5 ], + [0.11111111, 0.2 , 0.27272727, 0.33333333]]) + +Here, broadcasting takes care of the different dimensions. Several other +types of operands are also accepted. + +Using Arithmetic Classmethods +============================= + +Here both operands do not need to be `~astropy.nddata.NDDataRef`-like:: + + >>> NDDataRef.add(1, 3) + NDDataRef(4) + +To wrap the result of an arithmetic operation between two Quantities:: + + >>> import astropy.units as u + >>> ndd = NDDataRef.multiply([1,2] * u.m, [10, 20] * u.cm) + >>> ndd # doctest: +FLOAT_CMP + NDDataRef([10., 40.], unit='cm m') + >>> ndd.unit + Unit("cm m") + +Or take the inverse of an `~astropy.nddata.NDDataRef` object:: + + >>> NDDataRef.divide(1, ndd1) # doctest: +FLOAT_CMP + NDDataRef([1. , 0.5 , 0.33333333, 0.25 ]) + + +Possible Operands +----------------- + +The possible types of input for operands are: + ++ Scalars of any type ++ Lists containing numbers (or nested lists) ++ ``numpy`` arrays ++ ``numpy`` masked arrays ++ ``astropy`` quantities ++ Other ``nddata`` classes or subclasses + +Advanced Options ================ -Overview --------- +The normal Python operators ``+``, ``-``, etc. are not implemented because +the methods provide several options on how to proceed with the additional +attributes. + +Data and Unit +------------- + +For ``data`` and ``unit`` there are no parameters. Every arithmetic +operation lets the `astropy.units.Quantity`-framework evaluate the result +or fail and abort the operation. + +Adding two `~astropy.nddata.NDData` objects with the same unit works:: + + >>> ndd1 = NDDataRef([1,2,3,4,5], unit='m') + >>> ndd2 = NDDataRef([100,150,200,50,500], unit='m') + + >>> ndd = ndd1.add(ndd2) + >>> ndd.data # doctest: +FLOAT_CMP + array([101, 152, 203, 54, 505]) + >>> ndd.unit + Unit("m") + +Adding two `~astropy.nddata.NDData` objects with compatible units also works:: + + >>> ndd1 = NDDataRef(ndd1, unit='pc') + INFO: overwriting NDData's current unit with specified unit. [astropy.nddata.nddata] + >>> ndd2 = NDDataRef(ndd2, unit='lyr') + INFO: overwriting NDData's current unit with specified unit. [astropy.nddata.nddata] + + >>> ndd = ndd1.subtract(ndd2) + >>> ndd.data # doctest: +FLOAT_CMP + array([ -29.66013938, -43.99020907, -58.32027876, -11.33006969, + -148.30069689]) + >>> ndd.unit + Unit("pc") + +This will keep by default the unit of the first operand. However, units will +not be decomposed during division:: + + >>> ndd = ndd2.divide(ndd1) + >>> ndd.data # doctest: +FLOAT_CMP + array([100. , 75. , 66.66666667, 12.5 , 100. ]) + >>> ndd.unit + Unit("lyr / pc") + +Mask +---- + +The ``handle_mask`` parameter for the arithmetic operations implements what the +resulting mask will be. There are several options. + +- ``None``, the result will have no ``mask``:: + + >>> ndd1 = NDDataRef(1, mask=True) + >>> ndd2 = NDDataRef(1, mask=False) + >>> ndd1.add(ndd2, handle_mask=None).mask is None + True + +- ``"first_found"`` or ``"ff"``, the result will have the ``mask`` of the first + operand or if that is ``None``, the ``mask`` of the second operand:: + + >>> ndd1 = NDDataRef(1, mask=True) + >>> ndd2 = NDDataRef(1, mask=False) + >>> ndd1.add(ndd2, handle_mask="first_found").mask + True + >>> ndd3 = NDDataRef(1) + >>> ndd3.add(ndd2, handle_mask="first_found").mask + False + +- A function (or an arbitrary callable) that takes at least two arguments. + For example, `numpy.logical_or` is the default:: + + >>> ndd1 = NDDataRef(1, mask=np.array([True, False, True, False])) + >>> ndd2 = NDDataRef(1, mask=np.array([True, False, False, True])) + >>> ndd1.add(ndd2).mask + array([ True, False, True, True]...) + + This defaults to ``"first_found"`` in case only one ``mask`` is not None:: + + >>> ndd1 = NDDataRef(1) + >>> ndd2 = NDDataRef(1, mask=np.array([True, False, False, True])) + >>> ndd1.add(ndd2).mask + array([ True, False, False, True]...) + + Custom functions are also possible:: + + >>> def take_alternating_values(mask1, mask2, start=0): + ... result = np.zeros(mask1.shape, dtype=np.bool_) + ... result[start::2] = mask1[start::2] + ... result[start+1::2] = mask2[start+1::2] + ... return result + + This function is nonsense, but we can still see how it performs:: + + >>> ndd1 = NDDataRef(1, mask=np.array([True, False, True, False])) + >>> ndd2 = NDDataRef(1, mask=np.array([True, False, False, True])) + >>> ndd1.add(ndd2, handle_mask=take_alternating_values).mask + array([ True, False, True, True]...) + + Additional parameters can be given by prefixing them with ``mask_`` + (which will be stripped before passing it to the function):: + + >>> ndd1.add(ndd2, handle_mask=take_alternating_values, mask_start=1).mask + array([False, False, False, False]...) + >>> ndd1.add(ndd2, handle_mask=take_alternating_values, mask_start=2).mask + array([False, False, True, True]...) + +Meta +---- + +The ``handle_meta`` parameter for the arithmetic operations implements what the +resulting ``meta`` will be. The options are the same as for the ``mask``: + +- If ``None`` the resulting ``meta`` will be an empty `collections.OrderedDict`. + + >>> ndd1 = NDDataRef(1, meta={'object': 'sun'}) + >>> ndd2 = NDDataRef(1, meta={'object': 'moon'}) + >>> ndd1.add(ndd2, handle_meta=None).meta + OrderedDict() + + For ``meta`` this is the default so you do not need to pass it in this case:: + + >>> ndd1.add(ndd2).meta + OrderedDict() + +- If ``"first_found"`` or ``"ff"``, the resulting ``meta`` will be the ``meta`` + of the first operand or if that contains no keys, the ``meta`` of the second + operand is taken. + + >>> ndd1 = NDDataRef(1, meta={'object': 'sun'}) + >>> ndd2 = NDDataRef(1, meta={'object': 'moon'}) + >>> ndd1.add(ndd2, handle_meta='ff').meta + {'object': 'sun'} + +- If it is a ``callable`` it must take at least two arguments. Both ``meta`` + attributes will be passed to this function (even if one or both of them are + empty) and the callable evaluates the result's ``meta``. For example, a + function that merges these two:: + + >>> # It's expected with arithmetic that the result is not a reference, + >>> # so we need to copy + >>> from copy import deepcopy + + >>> def combine_meta(meta1, meta2): + ... if not meta1: + ... return deepcopy(meta2) + ... elif not meta2: + ... return deepcopy(meta1) + ... else: + ... meta_final = deepcopy(meta1) + ... meta_final.update(meta2) + ... return meta_final + + >>> ndd1 = NDDataRef(1, meta={'time': 'today'}) + >>> ndd2 = NDDataRef(1, meta={'object': 'moon'}) + >>> ndd1.subtract(ndd2, handle_meta=combine_meta).meta # doctest: +SKIP + {'object': 'moon', 'time': 'today'} + + Here again additional arguments for the function can be passed in using + the prefix ``meta_`` (which will be stripped away before passing it to this + function). See the description for the mask-attribute for further details. + +World Coordinate System (WCS) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``compare_wcs`` argument will determine what the result's ``wcs`` will be +or if the operation should be forbidden. The possible values are identical to +``mask`` and ``meta``: + +- If ``None`` the resulting ``wcs`` will be an empty ``None``. + + >>> ndd1 = NDDataRef(1, wcs=None) + >>> ndd2 = NDDataRef(1, wcs=WCS()) + >>> ndd1.add(ndd2, compare_wcs=None).wcs is None + True + +- If ``"first_found"`` or ``"ff"`` the resulting ``wcs`` will be the ``wcs`` of + the first operand or if that is ``None``, the ``meta`` of the second operand + is taken. + + >>> wcs = WCS() + >>> ndd1 = NDDataRef(1, wcs=wcs) + >>> ndd2 = NDDataRef(1, wcs=None) + >>> str(ndd1.add(ndd2, compare_wcs='ff').wcs) == str(wcs) + True + +- If it is a ``callable`` it must take at least two arguments. Both ``wcs`` + attributes will be passed to this function (even if one or both of them are + ``None``) and the callable should return ``True`` if these ``wcs`` are + identical (enough) to allow the arithmetic operation or ``False`` if the + arithmetic operation should be aborted with a ``ValueError``. If ``True`` the + ``wcs`` are identical and the first one is used for the result:: + + >>> def compare_wcs_scalar(wcs1, wcs2, allowed_deviation=0.1): + ... if wcs1 is None and wcs2 is None: + ... return True # both have no WCS so they are identical + ... if wcs1 is None or wcs2 is None: + ... return False # one has WCS, the other doesn't not possible + ... else: + ... # Consider wcs close if centers are close enough + ... return all(abs(wcs1.wcs.crpix - wcs2.wcs.crpix) < allowed_deviation) + + >>> ndd1 = NDDataRef(1, wcs=None) + >>> ndd2 = NDDataRef(1, wcs=None) + >>> ndd1.subtract(ndd2, compare_wcs=compare_wcs_scalar).wcs + + + Additional arguments can be passed in prefixing them with ``wcs_`` (this + prefix will be stripped away before passing it to the function):: + + >>> ndd1 = NDDataRef(1, wcs=WCS()) + >>> ndd1.wcs.wcs.crpix = [1, 1] + >>> ndd2 = NDDataRef(1, wcs=WCS()) + >>> ndd1.subtract(ndd2, compare_wcs=compare_wcs_scalar, wcs_allowed_deviation=2).wcs.wcs.crpix + array([1., 1.]) + + If you are using `~astropy.wcs.WCS` objects, a very handy function to use + might be:: + + >>> def wcs_compare(wcs1, wcs2, *args, **kwargs): + ... return wcs1.wcs.compare(wcs2.wcs, *args, **kwargs) + + See :meth:`astropy.wcs.Wcsprm.compare` for the arguments this comparison + allows. + +Uncertainty +----------- + +The ``propagate_uncertainties`` argument can be used to turn the propagation +of uncertainties on or off. + +- If ``None`` the result will have no uncertainty:: + + >>> from astropy.nddata import StdDevUncertainty + >>> ndd1 = NDDataRef(1, uncertainty=StdDevUncertainty(0)) + >>> ndd2 = NDDataRef(1, uncertainty=StdDevUncertainty(1)) + >>> ndd1.add(ndd2, propagate_uncertainties=None).uncertainty is None + True + +- If ``False`` the result will have the first found uncertainty. + + .. note:: + Setting ``propagate_uncertainties=False`` is generally not + recommended. + +- If ``True`` both uncertainties must be ``NDUncertainty`` subclasses that + implement propagation. This is possible for + `~astropy.nddata.StdDevUncertainty`:: + + >>> ndd1 = NDDataRef(1, uncertainty=StdDevUncertainty([10])) + >>> ndd2 = NDDataRef(1, uncertainty=StdDevUncertainty([10])) + >>> ndd1.add(ndd2, propagate_uncertainties=True).uncertainty # doctest: +FLOAT_CMP + StdDevUncertainty([14.14213562]) + +Uncertainty with Correlation +---------------------------- + +If ``propagate_uncertainties`` is ``True`` you can also give an argument +for ``uncertainty_correlation``. `~astropy.nddata.StdDevUncertainty` cannot +keep track of its correlations by itself, but it can evaluate the correct +resulting uncertainty if the correct ``correlation`` is given. + +The default (``0``) represents uncorrelated while ``1`` means correlated and +``-1`` anti-correlated. If given a `numpy.ndarray` it should represent the +element-wise correlation coefficient. + +Examples +^^^^^^^^ + +.. + EXAMPLE START + Uncertainty with Correlation in NDData + +Without correlation, subtracting an `~astropy.nddata.NDDataRef` instance from +itself results in a non-zero uncertainty:: + + >>> ndd1 = NDDataRef(1, uncertainty=StdDevUncertainty([10])) + >>> ndd1.subtract(ndd1, propagate_uncertainties=True).uncertainty # doctest: +FLOAT_CMP + StdDevUncertainty([14.14213562]) + +Given a correlation of ``1`` (because they clearly correlate) gives the +correct uncertainty of ``0``:: -The `~astropy.nddata.NDArithmeticMixin` adds methods for performing basic -operations on `~astropy.nddata.NDData` objects: -:meth:`~astropy.nddata.NDArithmeticMixin.add`, -:meth:`~astropy.nddata.NDArithmeticMixin.subtract`, -:meth:`~astropy.nddata.NDArithmeticMixin.multiply` and -:meth:`~astropy.nddata.NDArithmeticMixin.divide`. + >>> ndd1 = NDDataRef(1, uncertainty=StdDevUncertainty([10])) + >>> ndd1.subtract(ndd1, propagate_uncertainties=True, + ... uncertainty_correlation=1).uncertainty # doctest: +FLOAT_CMP + StdDevUncertainty([0.]) -The operations are permitted only if the two operands have the same WCS and -shape and the units, if any, consistent with the operation performed. The -result is masked at a particular grid point if either of the operands is -masked at that point. +Which would be consistent with the equivalent operation ``ndd1 * 0``:: -The operations include a framework to propagate uncertainties that are based -on the classes `~astropy.nddata.NDUncertainty`. + >>> ndd1.multiply(0, propagate_uncertainties=True).uncertainty # doctest: +FLOAT_CMP + StdDevUncertainty([0.]) -.. warning:: Uncertainty propagation is still experimental, and does not take - into account correlated uncertainties. +.. warning:: + The user needs to calculate or know the appropriate value or array manually + and pass it to ``uncertainty_correlation``. The implementation follows + general first order error propagation formulas. See, for example: + `Wikipedia `_. -Usage ------ +You can also give element-wise correlations:: -As with other mixins, using the arithmetic mixin starts with defining your own -class:: + >>> ndd1 = NDDataRef([1,1,1,1], uncertainty=StdDevUncertainty([1,1,1,1])) + >>> ndd2 = NDDataRef([2,2,2,2], uncertainty=StdDevUncertainty([2,2,2,2])) + >>> ndd1.add(ndd2,uncertainty_correlation=np.array([1,0.5,0,-1])).uncertainty # doctest: +FLOAT_CMP + StdDevUncertainty([3. , 2.64575131, 2.23606798, 1. ]) - >>> from astropy.nddata import NDData, NDArithmeticMixin - >>> class MyNDDataArithmetic(NDArithmeticMixin, NDData): pass +The correlation ``np.array([1, 0.5, 0, -1])`` would indicate that the first +element is fully correlated and the second element partially correlates, while +the third element is uncorrelated, and the fourth is anti-correlated. -Then, create instances of this new object with your data the way you would -with a plain `~astropy.nddata.NDData` object:: +.. + EXAMPLE END - >>> ndd1 = MyNDDataArithmetic([1, 2]) - >>> ndd2 = MyNDDataArithmetic([3, 4]) +Uncertainty with Unit +--------------------- -The mixin adds methods on these instances for combining them:: +`~astropy.nddata.StdDevUncertainty` implements correct error propagation even +if the unit of the data differs from the unit of the uncertainty:: - >>> ndd1.add(ndd2) - MyNDDataArithmetic([ 4., 6.]) - >>> ndd2.multiply(ndd1) - MyNDDataArithmetic([ 3., 8.]) + >>> ndd1 = NDDataRef([10], unit='m', uncertainty=StdDevUncertainty([10], unit='cm')) + >>> ndd2 = NDDataRef([20], unit='m', uncertainty=StdDevUncertainty([10])) + >>> ndd1.subtract(ndd2, propagate_uncertainties=True).uncertainty # doctest: +FLOAT_CMP + StdDevUncertainty([10.00049999]) -One important note: the order you list the mixins and `~astropy.nddata.NDData` -matters; the base class, `~astropy.nddata.NDData` should be on the far -right. +But it needs to be convertible to the unit for the data. diff --git a/docs/nddata/mixins/ndio.rst b/docs/nddata/mixins/ndio.rst index 4863d59cd7b2..d4ae61204d34 100644 --- a/docs/nddata/mixins/ndio.rst +++ b/docs/nddata/mixins/ndio.rst @@ -1,10 +1,12 @@ -I/O mixin -========= +.. _nddata_io: + +I/O Mixin +********* The I/O mixin, `~astropy.nddata.NDIOMixin`, adds ``read`` and ``write`` -methods that us the astropy I/O registry. +methods that use the ``astropy`` I/O registry. -The mixin itself simply creates the read/write methods; it does not register +The mixin itself creates the read/write methods; it does not register any readers or writers with the I/O registry. Subclasses of `~astropy.nddata.NDDataBase` or `~astropy.nddata.NDData` need to include this mixin, implement a reader and writer, *and* register it with the I/O diff --git a/docs/nddata/mixins/ndslicing.rst b/docs/nddata/mixins/ndslicing.rst index d1ed830ba9b5..c7b4ee98becf 100644 --- a/docs/nddata/mixins/ndslicing.rst +++ b/docs/nddata/mixins/ndslicing.rst @@ -1,22 +1,162 @@ -Slicing mixin -============= +.. _nddata_slicing: -The slicing mixin adds the ability to index an `~astropy.nddata.NDData` object -in a manner similar to a numpy array. Slicing returns a new -`~astropy.nddata.NDData` object with the shape indicated by the slice. +Slicing and Indexing NDData +*************************** -The first step in creating an `~astropy.nddata.NDData`-based object with -slicing is to create a new class:: +Introduction +============ - >>> from astropy.nddata import NDData, NDSlicingMixin - >>> class MyNDDataWithSlicing(NDSlicingMixin, NDData): pass +This page only deals with peculiarities that apply to +`~astropy.nddata.NDData`-like classes. For a tutorial about slicing/indexing see the +`python documentation `_ +and `numpy documentation `_. -Then, initialize the new object the same way you would initialize a plain -`~astropy.nddata.NDData` object:: +.. warning:: + `~astropy.nddata.NDData` and `~astropy.nddata.NDDataRef` enforce almost no + restrictions on the properties, so it might happen that some **valid but + unusual** combinations of properties always result in an IndexError or + incorrect results. In this case, see :ref:`nddata_subclassing` on how to + customize slicing for a particular property. - >>> sliceable_ndd = MyNDDataWithSlicing([1, 2, 3, 4]) - >>> sliceable_ndd[1:3] - MyNDDataWithSlicing([2, 3]) -One important note: the order you list the mixins and `~astropy.nddata.NDData` -matters; the base class, `~astropy.nddata.NDData` should be on the far right. +Slicing NDDataRef +================= + +Unlike `~astropy.nddata.NDData` the class `~astropy.nddata.NDDataRef` +implements slicing or indexing. The result will be wrapped inside the same +class as the sliced object. + +Getting one element:: + + >>> import numpy as np + >>> from astropy.nddata import NDDataRef + + >>> data = np.array([1, 2, 3, 4]) + >>> ndd = NDDataRef(data) + >>> ndd[1] + NDDataRef(2) + +Getting a sliced portion of the original:: + + >>> ndd[1:3] # Get element 1 (inclusive) to 3 (exclusive) + NDDataRef([2, 3]) + +This will return a reference (and as such **not a copy**) of the original +properties, so changing a slice will affect the original:: + + >>> ndd_sliced = ndd[1:3] + >>> ndd_sliced.data[0] = 5 + >>> ndd_sliced + NDDataRef([5, 3]) + >>> ndd + NDDataRef([1, 5, 3, 4]) + +But only the one element that was indexed is affected (for example, +``ndd_sliced = ndd[1]``). The element is a scalar and changes will not +propagate to the original. + +Slicing NDDataRef Including Attributes +====================================== + +In the case that a ``mask``, or ``uncertainty`` is present, this +attribute will be sliced too:: + + >>> from astropy.nddata import StdDevUncertainty + >>> data = np.array([1, 2, 3, 4]) + >>> mask = data > 2 + >>> uncertainty = StdDevUncertainty(np.sqrt(data)) + >>> ndd = NDDataRef(data, mask=mask, uncertainty=uncertainty) + >>> ndd_sliced = ndd[1:3] + + >>> ndd_sliced.data + array([2, 3]) + + >>> ndd_sliced.mask + array([False, True]...) + + >>> ndd_sliced.uncertainty # doctest: +FLOAT_CMP + StdDevUncertainty([1.41421356, 1.73205081]) + +``unit`` and ``meta``, however, will be unaffected. + +If any of the attributes are set but do not implement slicing, an info will be +printed and the property will be kept as is:: + + >>> data = np.array([1, 2, 3, 4]) + >>> mask = False + >>> uncertainty = StdDevUncertainty(0) + >>> ndd = NDDataRef(data, mask=mask, uncertainty=uncertainty) + >>> ndd_sliced = ndd[1:3] + INFO: uncertainty cannot be sliced. [astropy.nddata.mixins.ndslicing] + INFO: mask cannot be sliced. [astropy.nddata.mixins.ndslicing] + + >>> ndd_sliced.mask + False + + +Slicing NDData with World Coordinates +------------------------------------- + +If ``wcs`` is set, it must be either implement +`~astropy.wcs.wcsapi.BaseLowLevelWCS` or `~astropy.wcs.wcsapi.BaseHighLevelWCS`. +This means that only integer or range slices without a step are supported. So +slices like ``[::10]`` or array or boolean based slices will not work. + +If you want to slice an ``NDData`` object called ``ndd`` without the WCS you can remove the +WCS from the ``NDData`` object by running: + + >>> ndd.wcs = None + + +Removing Masked Data +-------------------- + +.. warning:: + If ``wcs`` is set this will **NOT** be possible. But you can work around + this by setting the wcs attribute to `None` with ``ndd.wcs = None`` before slicing. + +By convention, the ``mask`` attribute indicates if a point is valid or invalid. +So we are able to get all valid data points by slicing with the mask. + +Examples +^^^^^^^^ + +.. + EXAMPLE START + Removing Masked Data in NDDataRef + +To get all of the valid data points by slicing with the mask:: + + >>> data = np.array([[1,2,3],[4,5,6],[7,8,9]]) + >>> mask = np.array([[0,1,0],[1,1,1],[0,0,1]], dtype=bool) + >>> uncertainty = StdDevUncertainty(np.sqrt(data)) + >>> ndd = NDDataRef(data, mask=mask, uncertainty=uncertainty) + >>> # don't forget that ~ or you'll get the invalid points + >>> ndd_sliced = ndd[~ndd.mask] + >>> ndd_sliced + NDDataRef([1, 3, 7, 8]) + + >>> ndd_sliced.mask + array([False, False, False, False]...) + + >>> ndd_sliced.uncertainty # doctest: +FLOAT_CMP + StdDevUncertainty([1. , 1.73205081, 2.64575131, 2.82842712]) + +Or all invalid points:: + + >>> ndd_sliced = ndd[ndd.mask] # without the ~ now! + >>> ndd_sliced + NDDataRef([—, —, —, —, —]) + + >>> ndd_sliced.mask + array([ True, True, True, True, True]...) + + >>> ndd_sliced.uncertainty # doctest: +FLOAT_CMP + StdDevUncertainty([1.41421356, 2. , 2.23606798, 2.44948974, 3. ]) + +.. note:: + The result of this kind of indexing (boolean indexing) will always be + one-dimensional! + +.. + EXAMPLE END diff --git a/docs/nddata/nddata.rst b/docs/nddata/nddata.rst index b31b9779b4fd..2bde1261d9b9 100644 --- a/docs/nddata/nddata.rst +++ b/docs/nddata/nddata.rst @@ -1,145 +1,309 @@ .. _nddata_details: NDData -====== +****** Overview --------- +======== + +:class:`~astropy.nddata.NDData` is based on `numpy.ndarray`-like ``data`` with +additional meta attributes: + ++ ``meta`` for general metadata ++ ``unit`` represents the physical unit of the data ++ ``uncertainty`` for the uncertainty of the data ++ ``mask`` indicates invalid points in the data ++ ``wcs`` represents the relationship between the data grid and world + coordinates ++ ``psf`` holds an image representation of the point spread function (PSF) -The `~astropy.nddata.NDData` class is a container for gridded N-dimensional -data. It has a ``data`` attribute, which can be any object that presents an -array-like interface, and optional attributes: +Each of these attributes can be set during initialization or directly on the +instance. Only the ``data`` cannot be directly set after creating the instance. -+ ``meta``, for metadata -+ ``unit`` for the ``data`` unit -+ ``uncertainty`` for the uncertainty of the data (which could be standard - deviation,variance, or something else), -+ ``mask`` for the ``data`` -+ ``wcs``, representing the relationship between ``data`` and world - coordinates. +Data +==== -Of these, only ``mask`` and ``uncertainty`` may be changed after the NDData -object is created. +The data is the base of `~astropy.nddata.NDData` and is required to be +`numpy.ndarray`-like. It is the only property that is required to create an +instance and it cannot be directly set on the instance. -Initializing ------------- +Example +------- -An `~astropy.nddata.NDData` object can be instantiated by passing it an -n-dimensional Numpy array:: +.. + EXAMPLE START + Creating Instances with NumPy NDarray-like Data + +To create an instance:: >>> import numpy as np >>> from astropy.nddata import NDData - >>> array = np.zeros((12, 12, 12)) # a 3-dimensional array with all zeros + >>> array = np.array([[0, 1, 0], [1, 0, 1], [0, 1, 0]]) >>> ndd = NDData(array) + >>> ndd + NDData([[0, 1, 0], + [1, 0, 1], + [0, 1, 0]]) + +And access by the ``data`` attribute:: + + >>> ndd.data + array([[0, 1, 0], + [1, 0, 1], + [0, 1, 0]]) + +As already mentioned, it is not possible to set the data directly. So +``ndd.data = np.arange(9)`` will raise an exception. But the data can be +modified in place:: + + >>> ndd.data[1,1] = 100 + >>> ndd.data + array([[ 0, 1, 0], + [ 1, 100, 1], + [ 0, 1, 0]]) + +.. + EXAMPLE END + +Data During Initialization +-------------------------- -Note that the data in ``ndd`` is a reference to the original ``array``, so -changing the data in ``ndd`` will change the corresponding data in ``array`` -in most circumstances. +During initialization it is possible to provide data that is not a +`numpy.ndarray` but convertible to one. -An `~astropy.nddata.NDData` object can also be instantiated by passing it an -`~astropy.nddata.NDData` object: +Examples +^^^^^^^^ - >>> ndd1 = NDData(array) - >>> ndd2 = NDData(ndd1) +.. + EXAMPLE START + Data Convertible to a NumPy NDarray During Initialization -As above, the data in``ndd2`` is a reference to the data in ``ndd1``, so -changes to one will affect the other. +To provide data that is convertible to a `numpy.ndarray`, you can pass a `list` +containing numerical values:: -It can also be instantiated by passing in an object that can be converted to a -numpy numerical array:: + >>> alist = [1, 2, 3, 4] + >>> ndd = NDData(alist) + >>> ndd.data # data will be a numpy-array: + array([1, 2, 3, 4]) - >>> ndd3 = NDData([1, 2, 3, 4]) +A nested `list` or `tuple` is possible, but if these contain non-numerical +values the conversion might fail. -The final way to instantiate an `~astropy.nddata.NDData` object is with a data -object that presents a numpy array-like interface. If the object passed to the -intializer has all three of the attributes ``shape``, ``__getitem__`` (so it -is indexable) and ``__array__`` (so that it can act like a numpy array in -expression) then the ``data`` attribute will be set to that object. +Besides input that is convertible to such an array, you can also use the +``data`` parameter to pass implicit additional information. For example, if the +data is another `~astropy.nddata.NDData` object it implicitly uses its +properties:: + + >>> ndd = NDData(ndd, unit = 'm') + >>> ndd2 = NDData(ndd) + >>> ndd2.data # It has the same data as ndd + array([1, 2, 3, 4]) + >>> ndd2.unit # but it also has the same unit as ndd + Unit("m") + +Another possibility is to use a `~astropy.units.Quantity` as a ``data`` +parameter:: + + >>> import astropy.units as u + >>> quantity = np.ones(3) * u.cm # this will create a Quantity + >>> ndd3 = NDData(quantity) + >>> ndd3.data # doctest: +FLOAT_CMP + array([1., 1., 1.]) + >>> ndd3.unit + Unit("cm") + +Or a `numpy.ma.MaskedArray`:: + + >>> masked_array = np.ma.array([5,10,15], mask=[False, True, False]) + >>> ndd4 = NDData(masked_array) + >>> ndd4.data + array([ 5, 10, 15]) + >>> ndd4.mask + array([False, True, False]...) + +If such an implicitly passed property conflicts with an explicit parameter, the +explicit parameter will be used and an info message will be issued:: + + >>> quantity = np.ones(3) * u.cm + >>> ndd6 = NDData(quantity, unit='m') + INFO: overwriting Quantity's current unit with specified unit. [astropy.nddata.nddata] + >>> ndd6.data # doctest: +FLOAT_CMP + array([0.01, 0.01, 0.01]) + >>> ndd6.unit + Unit("m") + +The unit of the `~astropy.units.Quantity` is being ignored and the unit is set +to the explicitly passed one. + +It might be possible to pass other classes as a ``data`` parameter as long as +they have the properties ``shape``, ``dtype``, ``__getitem__``, and +``__array__``. The purpose of this mechanism is to allow considerable flexibility in the -objects used to store the data while providing a useful to default (numpy +objects used to store the data while providing a useful default (``numpy`` array). +.. + EXAMPLE END + Mask ----- +==== -Values can be masked using the ``mask`` attribute. One straightforward way to -provide a mask is to use a boolean numpy array:: +The ``mask`` is being used to indicate if data points are valid or invalid. +`~astropy.nddata.NDData` does not restrict this mask in any way but it is +expected to follow the `numpy.ma.MaskedArray` convention in that the mask: - >>> ndd_masked = NDData(ndd, mask = ndd.data > 0.9) - INFO: Overwriting NDData's current mask with specified mask [astropy.nddata.nddata] ++ Returns ``True`` for data points that are considered **invalid**. ++ Returns ``False`` for those points that are **valid**. -Another is to simply initialize an `~astropy.nddata.NDData` object with a -masked numpy array:: +Examples +-------- - >>> masked_array = np.ma.array([1, 2, 3, 4], mask=[1, 0, 0, 1]) - >>> ndd_masked = NDData(masked_array) - >>> ndd_masked.mask - array([ True, False, False, True], dtype=bool) +.. + EXAMPLE START + Masks Used to Indicate Valid or Invalid Data Points in NDData -A mask value of `True` indicates a value that should be ignored, while a mask -value of `False` indicates a valid value. +One possibility is to create a mask by using ``numpy``'s comparison operators:: -There is no requirement that the mask actually be a numpy array; for example, -a function which evaluates a mask value as needed is acceptable as long as it -follows the convention that `True` indicates a value that should be ignored. + >>> array = np.array([0, 1, 4, 0, 2]) + + >>> mask = array == 0 # Mask points containing 0 + >>> mask + array([ True, False, False, True, False]...) + + >>> other_mask = array > 1 # Mask points with a value greater than 1 + >>> other_mask + array([False, False, True, False, True]...) + +And initialize the `~astropy.nddata.NDData` instance using the ``mask`` +parameter:: + + >>> ndd = NDData(array, mask=mask) + >>> ndd.mask + array([ True, False, False, True, False]...) + +Or by replacing the mask:: + + >>> ndd.mask = other_mask + >>> ndd.mask + array([False, False, True, False, True]...) + +There is no requirement that the mask actually be a ``numpy`` array; for +example, a function which evaluates a mask value as needed is acceptable as +long as it follows the convention that ``True`` indicates a value that should +be ignored. + +.. + EXAMPLE END Unit ----- +==== -The unit of the data can be set by either explicitly providing an astropy unit -when creating the ``NDData`` object:: +The ``unit`` represents the unit of the data values. It is required to be +`~astropy.units.Unit`-like or a string that can be converted to such a +`~astropy.units.Unit`:: >>> import astropy.units as u - >>> ndd_unit = NDData([1, 2, 3, 4], unit="meter") - >>> ndd_unit.unit + >>> ndd = NDData([1, 2, 3, 4], unit="meter") # using a string + >>> ndd.unit Unit("m") -or by initializing with data that is an astropy `~astropy.units.Quantity`:: - - >>> q = [1, 2, 3, 4] * u.meter - >>> ndd_unit2 = NDData(q) - >>> ndd_unit2.unit - Unit("m") +..note:: + Setting the ``unit`` on an instance is not possible. Uncertainties -------------- +============= -`~astropy.nddata.NDData` objects have an ``uncertainty`` attribute that can be -used to set the uncertainty on the data values. The ``uncertainty`` must have -an attribute ``uncertainty_type`` which is a string. +The ``uncertainty`` represents an arbitrary representation of the error of the +data values. To indicate which kind of uncertainty representation is used, the +``uncertainty`` should have an ``uncertainty_type`` property. If no such +property is found it will be wrapped inside a +`~astropy.nddata.UnknownUncertainty`. -While not a requirement, the following ``uncertainty_type`` strings -are strongly recommended for common ways of specifying normal -distributions: +The ``uncertainty_type`` should follow the `~astropy.nddata.StdDevUncertainty` +convention in that it returns a short string like ``"std"`` for an uncertainty +given in standard deviation. Other examples are +`~astropy.nddata.VarianceUncertainty` and `~astropy.nddata.InverseVariance`. -+ ``"std"``: if ``uncertainty`` stores the standard deviation/sigma - (either a single value or on a per-pixel basis). -+ ``"var"``: if ``uncertainty`` stores the variance (either a single - value or on a per-pixel basis). -+ ``"ivar"``: if ``uncertainty`` stores the inverse variance (either a - single value or on a per-pixel basis). +Examples +-------- + +.. + EXAMPLE START + Setting Uncertainties During Initialization in NDData + +Like the other properties the ``uncertainty`` can be set during +initialization:: + + >>> from astropy.nddata import StdDevUncertainty, InverseVariance + >>> array = np.array([10, 7, 12, 22]) + >>> uncert = StdDevUncertainty(np.sqrt(array)) + >>> ndd = NDData(array, uncertainty=uncert) + >>> ndd.uncertainty # doctest: +FLOAT_CMP + StdDevUncertainty([3.16227766, 2.64575131, 3.46410162, 4.69041576]) + +Or on the instance directly:: + >>> other_uncert = StdDevUncertainty([2,2,2,2]) + >>> ndd.uncertainty = other_uncert + >>> ndd.uncertainty + StdDevUncertainty([2, 2, 2, 2]) -.. note:: For information on creating your own uncertainty classes, - see :doc:`subclassing`. +But it will print an info message if there is no ``uncertainty_type``:: -Meta-data ---------- + >>> ndd.uncertainty = np.array([5, 1, 2, 10]) + INFO: uncertainty should have attribute uncertainty_type. [astropy.nddata.nddata] + >>> ndd.uncertainty + UnknownUncertainty([ 5, 1, 2, 10]) -The :class:`~astropy.nddata.NDData` class includes a ``meta`` attribute that -defaults to an empty ordered dictionary, and can be used to set overall meta- -data for the dataset:: +It is also possible to convert between uncertainty types:: - ndd.meta['exposure_time'] = 340. - ndd.meta['filter'] = 'J' + >>> uncert.represent_as(InverseVariance) + InverseVariance([0.1 , 0.14285714, 0.08333333, 0.04545455]) + +.. + EXAMPLE END + +Covariance +---------- + +A `~astropy.nddata.Covariance` uncertainty type is also implemented; however, +its functionality is generally limited to construction and storage of sparse +covariance matrices. Additional functionality will be implemented as requested. +See :ref:`nddata-covariance` for more description and example usage. + +WCS +=== -Elements of the meta-data dictionary can be set to any valid Python object:: +The ``wcs`` should contain a mapping from the gridded data to world +coordinates. There are no restrictions placed on the property currently but it +may be restricted to an `~astropy.wcs.WCS` object or a more generalized WCS +object in the future. - ndd.meta['history'] = ['calibrated', 'aligned', 'flat-fielded'] +.. note:: + Like the unit the ``wcs`` cannot be set on an instance. -The metadata can be any python object that presents a dict-like interface. For -example, a FITS header can be used as the metadata:: +Metadata +========= + +The ``meta`` property contains all further meta information that does not fit +any other property. + +Examples +-------- + +.. + EXAMPLE START + Metadata in NDData + +If the ``meta`` property is given it must be `dict`-like:: + + >>> ndd = NDData([1,2,3], meta={'observer': 'myself'}) + >>> ndd.meta + {'observer': 'myself'} + +`dict`-like means it must be a mapping from some keys to some values. This +also includes `~astropy.io.fits.Header` objects:: >>> from astropy.io import fits >>> header = fits.Header() @@ -148,26 +312,217 @@ example, a FITS header can be used as the metadata:: >>> ndd.meta['observer'] 'Edwin Hubble' -WCS ---- +If the ``meta`` property is not provided or explicitly set to ``None``, it will +default to an empty `collections.OrderedDict`:: -At the moment the ``wcs`` attribute can be set to any object, though in the -future it may be restricted to an `~astropy.wcs.WCS` object once a generalized -WCS object is developed. + >>> ndd.meta = None + >>> ndd.meta + OrderedDict() -Converting to Numpy arrays --------------------------- + >>> ndd = NDData([1,2,3]) + >>> ndd.meta + OrderedDict() + +The ``meta`` object therefore supports adding or updating these values:: + + >>> ndd.meta['exposure_time'] = 340. + >>> ndd.meta['filter'] = 'J' + +Elements of the metadata dictionary can be set to any valid Python object:: + + >>> ndd.meta['history'] = ['calibrated', 'aligned', 'flat-fielded'] + +.. + EXAMPLE END + +Initialization with Copy +======================== + +The default way to create an `~astropy.nddata.NDData` instance is to try saving +the parameters as references to the original rather than as copy. Sometimes +this is not possible because the internal mechanics do not allow for this. + +Examples +-------- + +.. + EXAMPLE START + Creating an NDData Instance with Copy + +If the ``data`` is a `list` then during initialization this is copied +while converting to a `~numpy.ndarray`. But it is also possible to enforce +copies during initialization by setting the ``copy`` parameter to ``True``:: -Data should be accessed through the ``data`` attribute:: + >>> array = np.array([1, 2, 3, 4]) + >>> ndd = NDData(array) + >>> ndd.data[2] = 10 + >>> array[2] # Original array has changed + np.int64(10) + + >>> ndd2 = NDData(array, copy=True) + >>> ndd2.data[2] = 3 + >>> array[2] # Original array hasn't changed. + np.int64(10) + +.. note:: + In some cases setting ``copy=True`` will copy the ``data`` twice. Known + cases are if the ``data`` is a `list` or `tuple`. + +.. + EXAMPLE END + + +Collapsing an NDData object along one or more axes +================================================== + +.. + EXAMPLE START + Collapsing an NDData object along one or more axes + +A common operation on an `~numpy.ndarray` is to take the sum, mean, +maximum, or minimum along one or more axes, reducing the dimensions +of the output. These four operations are implemented on +`~astropy.nddata.NDData` with appropriate propagation of uncertainties, +masks, and units. + +For example, let's work on the following ``data`` with a mask, unit, and +(uniform) uncertainty:: + + >>> import numpy as np + >>> import astropy.units as u + >>> from astropy.nddata import NDDataArray, StdDevUncertainty + >>> + >>> data = [ + ... [1, 2, 3], + ... [2, 3, 4] + ... ] + >>> mask = [ + ... [True, False, False], + ... [False, False, False] + ... ] + >>> uncertainty = StdDevUncertainty(np.ones_like(data)) + >>> nddata = NDDataArray(data=data, uncertainty=uncertainty, mask=mask, unit='m') + +The sum along axis ``1`` gives one result per row:: + + >>> sum_axis_1 = nddata.sum(axis=1) # this is a new NDDataArray + >>> print(np.asanyarray(sum_axis_1)) # this converts data to a numpy masked array. doctest: +FLOAT_CMP + [-- 9.0] + >>> print(sum_axis_1.uncertainty) # doctest: +FLOAT_CMP + StdDevUncertainty([1.41421356, 1.73205081]) + +The result has one masked value derived from the logical OR of the original mask +along ``axis=1``. The uncertainties are the square-root of the sum of the squares +of the input uncertainties. Since the original uncertainties were all unity, the +result is the square root of the number of unmasked data entries, +:math:`[\sqrt{2},\,\sqrt{3}]`. + +We can similarly take the mean along ``axis=1``:: + + >>> mean_axis_1 = nddata.mean(axis=1) + >>> print(np.asanyarray(mean_axis_1)) # doctest: +FLOAT_CMP + [2.5 3.0] + >>> print(mean_axis_1.uncertainty) # doctest: +FLOAT_CMP + StdDevUncertainty([0.70710678, 0.57735027]) + +The result is the mean of the values where ``mask==False``, and in this example, +the result would only have ``mask==True`` if an entire row was masked. Since the +uncertainties were given as `~astropy.nddata.StdDevUncertainty`, the propagated +uncertainties decrease proportional to the number of unmasked measurements in each +row, following :math:`[2^{-1/2},\,3^{-1/2}]`. + +There's no single, correct way of defining the uncertainties associated +with the ``min`` or ``max`` of a set of measurements, so +`~astropy.nddata.NDData` resists the temptation to guess, and returns +the minimum data value along the axis/axes, and the propagated mask, but +no uncertainties:: + + >>> min_axis_1 = nddata.min(axis=1) + >>> print(np.asanyarray(min_axis_1)) # doctest: +FLOAT_CMP + [2.0 2.0] + >>> print(min_axis_1.uncertainty) + None + +For some use cases, it may be helpful to return the uncertainty +at the same index as the minimum/maximum ``data`` value, so that +the original ``data`` retains its uncertainty. You can get this +behavior with:: + + >>> min_axis_1 = nddata.min(axis=1, propagate_uncertainties=True) + + >>> print(np.asanyarray(min_axis_1)) # doctest: +FLOAT_CMP + [2.0 2.0] + >>> print(min_axis_1.uncertainty) # doctest: +FLOAT_CMP + StdDevUncertainty([1, 1]) + +Finally, in some cases it may be useful to do perform a collapse +operation only on the unmasked values, and only return a masked +result when all of the input values are masked. If we refer back to +the first example in this section, we see that the underlying +``data`` attribute has been summed over all values, including +masked ones:: + + >>> sum_axis_1 # doctest: +FLOAT_CMP + NDDataArray([——, 9.], unit='m') + +where the first data element is masked. We can instead get the sum +for only unmasked values with the ``operation_ignores_mask`` option:: + + >>> nddata.sum(axis=1, operation_ignores_mask=True) + NDDataArray([5, 9], unit='m') + +.. + EXAMPLE END + +Converting NDData to Other Classes +================================== + +There is limited support to convert a `~astropy.nddata.NDData` instance to +other classes. In the process some properties might be lost. + + >>> data = np.array([1, 2, 3, 4]) + >>> mask = np.array([True, False, False, True]) + >>> unit = 'm' + >>> ndd = NDData(data, mask=mask, unit=unit) + +`numpy.ndarray` +--------------- + +Converting the ``data`` to an array:: >>> array = np.asarray(ndd.data) + >>> array + array([1, 2, 3, 4]) + +Though using ``np.asarray`` is not required, in most cases it will ensure that +the result is always a `numpy.ndarray` + +`numpy.ma.MaskedArray` +---------------------- + +Converting the ``data`` and ``mask`` to a MaskedArray:: + + + >>> masked_array = np.ma.array(ndd.data, mask=ndd.mask) + >>> masked_array + masked_array(data=[--, 2, 3, --], + mask=[ True, False, False, True], + fill_value=999999) + +`~astropy.units.Quantity` +------------------------- + +Converting the ``data`` and ``unit`` to a Quantity:: + + >>> quantity = u.Quantity(ndd.data, unit=ndd.unit) + >>> quantity # doctest: +FLOAT_CMP + -Though using ``np.asarray`` is not required it will ensure that an additional -copy of the data is not made if the data is a numpy array. +MaskedQuantity +-------------- -Note that if the data is masked you must explicitly construct a numpy masked -array like this:: +Converting the ``data``, ``unit``, and ``mask`` to a ``MaskedQuantity``:: - >>> input_array = np.ma.array([1, 2, 3, 4], mask=[1, 0, 0, 1]) - >>> ndd_masked = NDData(input_array) - >>> masked_array = np.ma.array(ndd_masked.data, mask=ndd_masked.mask) + >>> from astropy.utils.masked import Masked + >>> Masked(u.Quantity(ndd.data, ndd.unit), ndd.mask) # doctest: +FLOAT_CMP + diff --git a/docs/nddata/performance.inc.rst b/docs/nddata/performance.inc.rst new file mode 100644 index 000000000000..aef9ca9e37fe --- /dev/null +++ b/docs/nddata/performance.inc.rst @@ -0,0 +1,20 @@ +.. note that if this is changed from the default approach of using an *include* + (in index.rst) to a separate performance page, the header needs to be changed + from === to ***, the filename extension needs to be changed from .inc.rst to + .rst, and a link needs to be added in the subpackage toctree + +.. _astropy-nddata-performance: + +Performance Tips +================ + ++ Using the uncertainty class `~astropy.nddata.VarianceUncertainty` will + be somewhat more efficient than the other two uncertainty classes, + `~astropy.nddata.InverseVariance` and `~astropy.nddata.StdDevUncertainty`. + The latter two are converted to variance for the purposes of error + propagation and then converted from variance back to the original + uncertainty type. The performance difference should be small. ++ When possible, mask values by setting them to ``np.nan`` and use the + ``numpy`` functions and methods that automatically exclude ``np.nan``, + like ``np.nanmedian`` and ``np.nanstd``. This will typically be much + faster than using `numpy.ma.MaskedArray`. diff --git a/docs/nddata/ref_api.rst b/docs/nddata/ref_api.rst new file mode 100644 index 000000000000..2a87976a34c1 --- /dev/null +++ b/docs/nddata/ref_api.rst @@ -0,0 +1,11 @@ +Reference/API +************* + +.. automodapi:: astropy.nddata + :no-inheritance-diagram: + +.. automodapi:: astropy.nddata.bitmask + :no-inheritance-diagram: + +.. automodapi:: astropy.nddata.utils + :no-inheritance-diagram: diff --git a/docs/nddata/subclassing.rst b/docs/nddata/subclassing.rst index 060e448a9ff5..20613b4cf0ec 100644 --- a/docs/nddata/subclassing.rst +++ b/docs/nddata/subclassing.rst @@ -1,101 +1,531 @@ +.. _nddata_subclassing: + Subclassing -=========== +*********** + +`~astropy.nddata.NDData` +======================== -There are a couple of choices to be made in subclassing from the nddata -package. For the greatest flexibility, subclass from -`~astropy.nddata.NDDataBase`, which places (almost) no restrictions on any of -its attributes. In many cases, subclassing `~astropy.nddata.NDData` will work -instead; it is more straightforward but places some minimal restrictions on -how the data can be represented. +This class serves as the base for subclasses that use a `numpy.ndarray` (or +something that presents a ``numpy``-like interface) as the ``data`` attribute. -`~astropy.nddata.NDDataBase` +.. note:: + Each attribute is saved as an attribute with one leading underscore. For + example, the ``data`` is saved as ``_data`` and the ``mask`` as ``_mask``, + and so on. + +Adding Another Property +----------------------- + + >>> from astropy.nddata import NDData + + >>> class NDDataWithFlags(NDData): + ... def __init__(self, *args, **kwargs): + ... # Remove flags attribute if given and pass it to the setter. + ... self.flags = kwargs.pop('flags') if 'flags' in kwargs else None + ... super().__init__(*args, **kwargs) + ... + ... @property + ... def flags(self): + ... return self._flags + ... + ... @flags.setter + ... def flags(self, value): + ... self._flags = value + + >>> ndd = NDDataWithFlags([1,2,3]) + >>> ndd.flags is None + True + + >>> ndd = NDDataWithFlags([1,2,3], flags=[0, 0.2, 0.3]) + >>> ndd.flags + [0, 0.2, 0.3] + +.. note:: + To simplify subclassing, each setter (except for ``data``) is called during + ``__init__`` so putting restrictions on any attribute can be done inside + the setter and will also apply during instance creation. + +Customize the Setter for a Property +----------------------------------- + + >>> import numpy as np + + >>> class NDDataMaskBoolNumpy(NDData): + ... + ... @NDData.mask.setter + ... def mask(self, value): + ... # Convert mask to boolean numpy array. + ... self._mask = np.array(value, dtype=np.bool_) + + >>> ndd = NDDataMaskBoolNumpy([1,2,3]) + >>> ndd.mask = [True, False, True] + >>> ndd.mask + array([ True, False, True]...) + +Extend the Setter for a Property +-------------------------------- + +``unit``, ``meta``, and ``uncertainty`` implement some additional logic in their +setter so subclasses might define a call to the superclass and let the +super property set the attribute afterwards:: + + >>> import numpy as np + + >>> class NDDataUncertaintyShapeChecker(NDData): + ... + ... @NDData.uncertainty.setter + ... def uncertainty(self, value): + ... value = np.asarray(value) + ... if value.shape != self.data.shape: + ... raise ValueError('uncertainty must have the same shape as the data.') + ... # Call the setter of the super class in case it might contain some + ... # important logic (only True for meta, unit and uncertainty) + ... super(NDDataUncertaintyShapeChecker, self.__class__).uncertainty.fset(self, value) + ... # Unlike "super(cls_name, cls_name).uncertainty.fset" or + ... # or "NDData.uncertainty.fset" this will respect Pythons method + ... # resolution order. + + >>> ndd = NDDataUncertaintyShapeChecker([1,2,3], uncertainty=[2,3,4]) + INFO: uncertainty should have attribute uncertainty_type. [astropy.nddata.nddata] + >>> ndd.uncertainty + UnknownUncertainty([2, 3, 4]) + +Having a Setter for the Data ---------------------------- -The class `~astropy.nddata.NDDataBase` is a metaclass -- when subclassing it, -all properties of `~astropy.nddata.NDDataBase` except ``uncertainty`` *must* -be overriden in the subclass. For an example of how to do this, see the source -code for `astropy.nddata.NDData`. + >>> class NDDataWithDataSetter(NDData): + ... + ... @NDData.data.setter + ... def data(self, value): + ... self._data = np.asarray(value) -Subclassing from `~astropy.nddata.NDDataBase` gives you complete flexibility -in how you implement data storage and the other properties. If your data is -stored in a numpy array (or something that behaves like a numpy array), it may -be more straightforward to subclass `~astropy.nddata.NDData` instead of -`~astropy.nddata.NDDataBase`. + >>> ndd = NDDataWithDataSetter([1,2,3]) + >>> ndd.data = [3,2,1] + >>> ndd.data + array([3, 2, 1]) -`~astropy.nddata.NDData` ------------------------- +.. _NDDataRef: -This class serves as the base for subclasses that use a numpy array (or -something that presents a numpy-like interface) as the ``data`` attribute. +`~astropy.nddata.NDDataRef` +=========================== -For an example of a class that includes mixins and subclasses -`~astropy.nddata.NDData` to add additional functionality, see -`~astropy.nddata.NDDataArray`. +`~astropy.nddata.NDDataRef` itself inherits from `~astropy.nddata.NDData` so +any of the possibilities there also apply to NDDataRef. But NDDataRef also +inherits from the Mixins: -Subclassing `~astropy.nddata.NDUncertainty` -------------------------------------------- +- `~astropy.nddata.NDSlicingMixin` +- `~astropy.nddata.NDArithmeticMixin` +- `~astropy.nddata.NDIOMixin` + +Which allow additional operations. + +Add Another Arithmetic Operation +-------------------------------- + +Adding another operation is possible provided the ``data`` and ``unit`` allow +it within the framework of `~astropy.units.Quantity`. + +Examples +^^^^^^^^ + +.. + EXAMPLE START + Adding Operations When Working with NDDataRef -This is done by using classes to represent the uncertainties of a given type. -For example, to set standard deviation uncertainties on the pixel values, you -can do:: +To add a power function:: + >>> from astropy.nddata import NDDataRef >>> import numpy as np - >>> from astropy.nddata import NDData, StdDevUncertainty - >>> array = np.zeros((12, 12, 12)) # a 3-dimensional array with all zeros - >>> ndd = NDData(array) - >>> uncertainty = StdDevUncertainty(np.ones((12, 12, 12)) * 0.1) - >>> ndd_uncertainty = NDData(ndd, uncertainty=uncertainty) - INFO: Overwriting NDData's current uncertainty being overwritten with specified uncertainty [astropy.nddata.nddata] + >>> from astropy.utils import sharedmethod + + >>> class NDDataPower(NDDataRef): + ... @sharedmethod # sharedmethod to allow it also as classmethod + ... def pow(self, operand, operand2=None, **kwargs): + ... # the uncertainty doesn't allow propagation so set it to None + ... kwargs['propagate_uncertainties'] = None + ... # Call the _prepare_then_do_arithmetic function with the + ... # numpy.power ufunc. + ... return self._prepare_then_do_arithmetic(np.power, operand, + ... operand2, **kwargs) + +This can be used like the other arithmetic methods similar to +:meth:`~astropy.nddata.NDArithmeticMixin.add`. So it works when calling it +on the class or the instance:: + + >>> ndd = NDDataPower([1,2,3]) + + >>> # using it on the instance with one operand + >>> ndd.pow(3) # doctest: +ELLIPSIS + NDDataPower([ 1, 8, 27]...) + + >>> # using it on the instance with two operands + >>> ndd.pow([1,2,3], [3,4,5]) # doctest: +ELLIPSIS + NDDataPower([ 1, 16, 243]...) + + >>> # or using it as classmethod + >>> NDDataPower.pow(6, [1,2,3]) # doctest: +ELLIPSIS + NDDataPower([ 6, 36, 216]...) + +To allow propagation also with ``uncertainty`` see subclassing +`~astropy.nddata.NDUncertainty`. + +.. + EXAMPLE END + +The ``_prepare_then_do_arithmetic`` implements the relevant checks if it was +called on the class or the instance, and if one or two operands were given, +converts the operands, if necessary, to the appropriate classes. Overriding +``_prepare_then_do_arithmetic`` in subclasses should be avoided if +possible. + +Arithmetic on an Existing Property +---------------------------------- + +Customizing how an existing property is handled during arithmetic is possible +with some arguments to the function calls such as +:meth:`~astropy.nddata.NDArithmeticMixin.add`, but it is possible to hardcode +behavior too. The actual operation on the attribute (except for ``unit``) is +done in a method ``_arithmetic_*`` where ``*`` is the name of the property. + +Examples +^^^^^^^^ + +.. + EXAMPLE START + Customizing Existing Properties During Arithmetic in NDData + +To customize how the ``meta`` will be affected during arithmetic:: + + >>> from astropy.nddata import NDDataRef + + >>> from copy import deepcopy + >>> class NDDataWithMetaArithmetics(NDDataRef): + ... + ... def _arithmetic_meta(self, operation, operand, handle_mask, **kwds): + ... # the function must take the arguments: + ... # operation (numpy-ufunc like np.add, np.subtract, ...) + ... # operand (the other NDData-like object, already wrapped as NDData) + ... # handle_mask (see description for "add") + ... + ... # The meta is dict like but we want the keywords exposure to change + ... # Anticipate that one or both might have no meta and take the first one that has + ... result_meta = deepcopy(self.meta) if self.meta else deepcopy(operand.meta) + ... # Do the operation on the keyword if the keyword exists + ... if result_meta and 'exposure' in result_meta: + ... result_meta['exposure'] = operation(result_meta['exposure'], operand.data) + ... return result_meta # return it + +To trigger this method, the ``handle_meta`` argument to arithmetic methods can +be anything except ``None`` or ``"first_found"``:: + + >>> ndd = NDDataWithMetaArithmetics([1,2,3], meta={'exposure': 10}) + >>> ndd2 = ndd.add(10, handle_meta='') + >>> ndd2.meta + {'exposure': np.int64(20)} -New error classes should sub-class from `~astropy.nddata.NDUncertainty`, and -should provide methods with the following API:: + >>> ndd3 = ndd.multiply(0.5, handle_meta='') + >>> ndd3.meta + {'exposure': np.float64(5.0)} - class MyUncertainty(NDUncertainty): +.. warning:: + To use these internal `_arithmetic_*` methods there are some restrictions on + the attributes when calling the operation: - def propagate_add(self, other_nddata, result_data): - ... - result_uncertainty = MyUncertainty(...) - return result_uncertainty + - ``mask``: ``handle_mask`` must not be ``None``, ``"ff"``, or + ``"first_found"``. + - ``wcs``: ``compare_wcs`` argument with the same restrictions as mask. + - ``meta``: ``handle_meta`` argument with the same restrictions as mask. + - ``uncertainty``: ``propagate_uncertainties`` must be ``None`` or evaluate + to ``False``. ``arithmetic_uncertainty`` must also accept different + arguments: ``operation``, ``operand``, ``result``, ``correlation``, + ``**kwargs``. - def propagate_subtract(self, other_nddata, result_data): - ... - result_uncertainty = MyUncertainty(...) - return result_uncertainty +.. + EXAMPLE END - def propagate_multiply(self, other_nddata, result_data): - ... - result_uncertainty = MyUncertainty(...) - return result_uncertainty +Changing the Default Argument for Arithmetic Operations +------------------------------------------------------- - def propagate_divide(self, other_nddata, result_data): - ... - result_uncertainty = MyUncertainty(...) - return result_uncertainty +If the goal is to change the default value of an existing parameter for +arithmetic methods, such as when explicitly specifying the parameter each +time you call an arithmetic operation is too much effort, you can change the +default value of existing parameters by changing it in the method signature of +``_arithmetic``. -All error sub-classes inherit an attribute ``self.parent_nddata`` that is -automatically set to the parent `~astropy.nddata.NDData` object that they -are attached to. The arguments passed to the error propagation methods are -``other_nddata``, which is the `~astropy.nddata.NDData` object that is being -combined with ``self.parent_nddata``, and ``result_data``, which is a Numpy -array that contains the data array after the arithmetic operation. All these -methods should return an error instance ``result_uncertainty``, and should not -modify ``parent_nddata`` directly. For subtraction and division, the order of -the operations is ``parent_nddata - other_nddata`` and ``parent_nddata / -other_nddata``. +Example +^^^^^^^ -To make it easier and clearer to code up the error propagation, you can use -variables with more explicit names, e.g:: +.. + EXAMPLE START + Changing the Default Argument for Arithmetic Operations in NDData - class MyUncertainty(NDUncertainty): +To change the default value of an existing parameter for arithmetic methods:: - def propogate_add(self, other_nddata, result_data): + >>> from astropy.nddata import NDDataRef + >>> import numpy as np + + >>> class NDDDiffAritDefaults(NDDataRef): + ... def _arithmetic(self, *args, **kwargs): + ... # Changing the default of handle_mask to None + ... if 'handle_mask' not in kwargs: + ... kwargs['handle_mask'] = None + ... # Call the original with the updated kwargs + ... return super()._arithmetic(*args, **kwargs) + + >>> ndd1 = NDDDiffAritDefaults(1, mask=False) + >>> ndd2 = NDDDiffAritDefaults(1, mask=True) + >>> # No mask handling logic will return no mask: + >>> ndd1.add(ndd2).mask + + >>> # But giving other values is still possible: + >>> ndd1.add(ndd2, handle_mask=np.logical_or).mask + np.True_ + + >>> ndd1.add(ndd2, handle_mask="ff").mask + False + +The parameter controlling how properties are handled are all keyword-only +so using the ``*args``, ``**kwargs`` approach allows you to only alter one +default without needing to care about the positional order of arguments. + +.. + EXAMPLE END + +Arithmetic with an Additional Property +-------------------------------------- + +This also requires overriding the ``_arithmetic`` method. Suppose we have a +``flags`` attribute again:: + + >>> from copy import deepcopy + >>> import numpy as np + + >>> class NDDataWithFlags(NDDataRef): + ... def __init__(self, *args, **kwargs): + ... # Remove flags attribute if given and pass it to the setter. + ... self.flags = kwargs.pop('flags') if 'flags' in kwargs else None + ... super().__init__(*args, **kwargs) + ... + ... @property + ... def flags(self): + ... return self._flags + ... + ... @flags.setter + ... def flags(self, value): + ... self._flags = value + ... + ... def _arithmetic(self, operation, operand, *args, **kwargs): + ... # take all args and kwargs to allow arithmetic on the other properties + ... # to work like before. + ... + ... # do the arithmetic on the flags (pop the relevant kwargs, if any!!!) + ... if self.flags is not None and operand.flags is not None: + ... result_flags = np.logical_or(self.flags, operand.flags) + ... # np.logical_or is just a suggestion you can do what you want + ... else: + ... if self.flags is not None: + ... result_flags = deepcopy(self.flags) + ... else: + ... result_flags = deepcopy(operand.flags) + ... + ... # Let the superclass do all the other attributes note that + ... # this returns the result and a dictionary containing other attributes + ... result, kwargs = super()._arithmetic(operation, operand, *args, **kwargs) + ... # The arguments for creating a new instance are saved in kwargs + ... # so we need to add another keyword "flags" and add the processed flags + ... kwargs['flags'] = result_flags + ... return result, kwargs # these must be returned + + >>> ndd1 = NDDataWithFlags([1,2,3], flags=np.array([1,0,1], dtype=bool)) + >>> ndd2 = NDDataWithFlags([1,2,3], flags=np.array([0,0,1], dtype=bool)) + >>> ndd3 = ndd1.add(ndd2) + >>> ndd3.flags + array([ True, False, True]...) + +Slicing an Existing Property +---------------------------- + +Suppose you have a class expecting a 2D ``data`` but the mask is +only 1D. This would lead to problems if you were to slice in two dimensions. + + >>> from astropy.nddata import NDDataRef + >>> import numpy as np + + >>> class NDDataMask1D(NDDataRef): + ... def _slice_mask(self, item): + ... # Multidimensional slices are represented by tuples: + ... if isinstance(item, tuple): + ... # only use the first dimension of the slice + ... return self.mask[item[0]] + ... # Let the superclass deal with the other cases + ... return super()._slice_mask(item) + + >>> ndd = NDDataMask1D(np.ones((3,3)), mask=np.ones(3, dtype=bool)) + >>> nddsliced = ndd[1:3,1:3] + >>> nddsliced.mask + array([ True, True]...) + +.. note:: + The methods slicing the attributes are prefixed by a ``_slice_*`` where ``*`` + can be ``mask``, ``uncertainty``, or ``wcs``. So overriding them is the + most convenient way to customize how the attributes are sliced. + +.. note:: + If slicing should affect the ``unit`` or ``meta`` see the next example. + + +Slicing an Additional Property +------------------------------ + +Building on the added property ``flags``, we want them to be sliceable: + + >>> class NDDataWithFlags(NDDataRef): + ... def __init__(self, *args, **kwargs): + ... # Remove flags attribute if given and pass it to the setter. + ... self.flags = kwargs.pop('flags') if 'flags' in kwargs else None + ... super().__init__(*args, **kwargs) + ... + ... @property + ... def flags(self): + ... return self._flags + ... + ... @flags.setter + ... def flags(self, value): + ... self._flags = value + ... + ... def _slice(self, item): + ... # slice all normal attributes + ... kwargs = super()._slice(item) + ... # The arguments for creating a new instance are saved in kwargs + ... # so we need to add another keyword "flags" and add the sliced flags + ... kwargs['flags'] = self.flags[item] + ... return kwargs # these must be returned + + >>> ndd = NDDataWithFlags([1,2,3], flags=[0, 0.2, 0.3]) + >>> ndd2 = ndd[1:3] + >>> ndd2.flags + [0.2, 0.3] + +If you wanted to keep just the original ``flags`` instead of the sliced ones, +you could use ``kwargs['flags'] = self.flags`` and omit the ``[item]``. + +`~astropy.nddata.NDDataBase` +============================ + +The class `~astropy.nddata.NDDataBase` is a metaclass — when subclassing it, +all properties of `~astropy.nddata.NDDataBase` *must* be overridden in the +subclass. + +Subclassing from `~astropy.nddata.NDDataBase` gives you complete flexibility +in how you implement data storage and the other properties. If your data is +stored in a ``numpy`` array (or something that behaves like a ``numpy`` array), +it may be more convenient to subclass `~astropy.nddata.NDData` instead of +`~astropy.nddata.NDDataBase`. + +Example +------- + +.. + EXAMPLE START + Implementing the NDDataBase Interface + +To implement the NDDataBase interface by creating a read-only container:: + + >>> from astropy.nddata import NDDataBase + + >>> class NDDataReadOnlyNoRestrictions(NDDataBase): + ... def __init__(self, data, unit, mask, uncertainty, meta, wcs, psf): + ... self._data = data + ... self._unit = unit + ... self._mask = mask + ... self._uncertainty = uncertainty + ... self._meta = meta + ... self._wcs = wcs + ... self._psf = psf + ... + ... @property + ... def data(self): + ... return self._data + ... + ... @property + ... def unit(self): + ... return self._unit + ... + ... @property + ... def mask(self): + ... return self._mask + ... + ... @property + ... def uncertainty(self): + ... return self._uncertainty + ... + ... @property + ... def meta(self): + ... return self._meta + ... + ... @property + ... def wcs(self): + ... return self._wcs + ... + ... @property + ... def psf(self): + ... return self._psf + + >>> # A meaningless test to show that creating this class is possible: + >>> NDDataReadOnlyNoRestrictions(1,2,3,4,5,6,7) is not None + True + +.. note:: + Actually defining an ``__init__`` is not necessary and the properties could + return arbitrary values but the properties **must** be defined. + +.. + EXAMPLE END + +Subclassing `~astropy.nddata.NDUncertainty` +=========================================== + +.. warning:: + The internal interface of NDUncertainty and subclasses is experimental and + might change in future versions. + +Subclasses deriving from `~astropy.nddata.NDUncertainty` need in order to +implement: + +- Property ``uncertainty_type`` should return a string describing the + uncertainty, for example, ``"ivar"`` for inverse variance. +- Methods for propagation: `_propagate_*` where ``*`` is the name of the + universal function (ufunc) that is used on the ``NDData`` parent. + +Creating an Uncertainty without Propagation +------------------------------------------- - left_uncertainty = self.parent.uncertainty.array - right_uncertainty = other_nddata.uncertainty.array +`~astropy.nddata.UnknownUncertainty` is a minimal working implementation +without error propagation. We can create an uncertainty by storing +systematic uncertainties:: - ... + >>> from astropy.nddata import NDUncertainty -Note that the above example assumes that the errors are stored in an ``array`` -attribute, but this does not have to be the case. + >>> class SystematicUncertainty(NDUncertainty): + ... @property + ... def uncertainty_type(self): + ... return 'systematic' + ... + ... def _data_unit_to_uncertainty_unit(self, value): + ... return None + ... + ... def _propagate_add(self, other_uncert, *args, **kwargs): + ... return None + ... + ... def _propagate_subtract(self, other_uncert, *args, **kwargs): + ... return None + ... + ... def _propagate_multiply(self, other_uncert, *args, **kwargs): + ... return None + ... + ... def _propagate_divide(self, other_uncert, *args, **kwargs): + ... return None -For an example of a complete implementation, see `~astropy.nddata.StdDevUncertainty`. + >>> SystematicUncertainty([10]) + SystematicUncertainty([10]) diff --git a/docs/nddata/utils.rst b/docs/nddata/utils.rst new file mode 100644 index 000000000000..d72af059629e --- /dev/null +++ b/docs/nddata/utils.rst @@ -0,0 +1,367 @@ +.. _nddata_utils: + +Image Utilities +*************** + +Overview +======== + +The `astropy.nddata.utils` module includes general utility functions +for array operations. + +.. _cutout_images: + +2D Cutout Images +================ + +Getting Started +--------------- + +The `~astropy.nddata.utils.Cutout2D` class can be used to create a +postage stamp cutout image from a 2D array. If an optional +`~astropy.wcs.WCS` object is input to +`~astropy.nddata.utils.Cutout2D`, then the +`~astropy.nddata.utils.Cutout2D` object will contain an updated +`~astropy.wcs.WCS` corresponding to the cutout array. + +First, we simulate a single source on a 2D data array. If you would like to +simulate many sources, see :ref:`bounding-boxes`. + +Note: The pair convention is different for **size** and **position**! The +position is specified as (x,y), but the size is specified as (y,x). + + >>> import numpy as np + >>> from astropy.modeling.models import Gaussian2D + >>> y, x = np.mgrid[0:500, 0:500] + >>> data = Gaussian2D(1, 50, 100, 10, 5, theta=0.5)(x, y) + +Now, we can display the image: + +.. doctest-skip:: + + >>> import matplotlib.pyplot as plt + >>> fig, ax = plt.subplots() + >>> ax.imshow(data, origin='lower') + +.. plot:: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling.models import Gaussian2D + y, x = np.mgrid[0:500, 0:500] + data = Gaussian2D(1, 50, 100, 10, 5, theta=0.5)(x, y) + fig, ax = plt.subplots() + ax.imshow(data, origin='lower') + +Next we can create a cutout for the single object in this image. We +create a cutout centered at position ``(x, y) = (49.7, 100.1)`` with a +size of ``(ny, nx) = (41, 51)`` pixels:: + + >>> from astropy.nddata import Cutout2D + >>> from astropy import units as u + >>> position = (49.7, 100.1) + >>> size = (41, 51) # pixels + >>> cutout = Cutout2D(data, position, size) + +The ``size`` keyword can also be a `~astropy.units.Quantity` object:: + + >>> size = u.Quantity((41, 51), u.pixel) + >>> cutout = Cutout2D(data, position, size) + +Or contain `~astropy.units.Quantity` objects:: + + >>> size = (41*u.pixel, 51*u.pixel) + >>> cutout = Cutout2D(data, position, size) + +A square cutout image can be generated by passing an integer or +a scalar `~astropy.units.Quantity`:: + + >>> size = 41 + >>> cutout2 = Cutout2D(data, position, size) + + >>> size = 41 * u.pixel + >>> cutout2 = Cutout2D(data, position, size) + +The cutout array is stored in the ``data`` attribute of the +`~astropy.nddata.utils.Cutout2D` instance. If the ``copy`` keyword is +`False` (default), then ``cutout.data`` will be a view into the +original ``data`` array. If ``copy=True``, then ``cutout.data`` will +hold a copy of the original ``data``. Now we display the cutout +image: + +.. doctest-skip:: + + >>> cutout = Cutout2D(data, position, (41, 51)) + >>> fig, ax = plt.subplots() + >>> ax.imshow(cutout.data, origin='lower') + +.. plot:: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling.models import Gaussian2D + from astropy.nddata import Cutout2D + y, x = np.mgrid[0:500, 0:500] + data = Gaussian2D(1, 50, 100, 10, 5, theta=0.5)(x, y) + position = (49.7, 100.1) + cutout = Cutout2D(data, position, (41, 51)) + fig, ax = plt.subplots() + ax.imshow(cutout.data, origin='lower') + +The cutout object can plot its bounding box on the original data using +the :meth:`~astropy.nddata.utils.Cutout2D.plot_on_original` method: + +.. doctest-skip:: + + >>> plt.imshow(data, origin='lower') + >>> cutout.plot_on_original(color='white') + +.. plot:: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling.models import Gaussian2D + from astropy.nddata import Cutout2D + y, x = np.mgrid[0:500, 0:500] + data = Gaussian2D(1, 50, 100, 10, 5, theta=0.5)(x, y) + position = (49.7, 100.1) + size = (41, 51) + cutout = Cutout2D(data, position, size) + fig, ax = plt.subplots() + ax.imshow(data, origin='lower') + cutout.plot_on_original(color='white') + +Many properties of the cutout array are also stored as attributes, +including:: + + >>> # shape of the cutout array + >>> print(cutout.shape) + (41, 51) + + >>> # rounded pixel index of the input position + >>> print(cutout.position_original) + (50, 100) + + >>> # corresponding position in the cutout array + >>> print(cutout.position_cutout) + (25, 20) + + >>> # (non-rounded) input position in both the original and cutout arrays + >>> print((cutout.input_position_original, cutout.input_position_cutout)) # doctest: +FLOAT_CMP + ((49.7, 100.1), (24.700000000000003, 20.099999999999994)) + + >>> # the origin pixel in both arrays + >>> print((cutout.origin_original, cutout.origin_cutout)) + ((25, 80), (0, 0)) + + >>> # tuple of slice objects for the original array + >>> print(cutout.slices_original) + (slice(80, 121, None), slice(25, 76, None)) + + >>> # tuple of slice objects for the cutout array + >>> print(cutout.slices_cutout) + (slice(0, 41, None), slice(0, 51, None)) + +There are also two `~astropy.nddata.utils.Cutout2D` methods to convert +pixel positions between the original and cutout arrays:: + + >>> print(cutout.to_original_position((2, 1))) + (27, 81) + + >>> print(cutout.to_cutout_position((27, 81))) + (2, 1) + + +2D Cutout Modes +--------------- + +There are three modes for creating cutout arrays: ``'trim'``, +``'partial'``, and ``'strict'``. For the ``'partial'`` and ``'trim'`` +modes, a partial overlap of the cutout array and the input ``data`` +array is sufficient. For the ``'strict'`` mode, the cutout array has +to be fully contained within the ``data`` array, otherwise an +`~astropy.nddata.utils.PartialOverlapError` is raised. In all modes, +non-overlapping arrays will raise a +`~astropy.nddata.utils.NoOverlapError`. In ``'partial'`` mode, +positions in the cutout array that do not overlap with the ``data`` +array will be filled with ``fill_value``. In ``'trim'`` mode only the +overlapping elements are returned, thus the resulting cutout array may +be smaller than the requested ``size``. + +The default uses ``mode='trim'``, which can result in cutout arrays +that are smaller than the requested ``size``:: + + >>> data2 = np.arange(20.).reshape(5, 4) + >>> cutout1 = Cutout2D(data2, (0, 0), (3, 3), mode='trim') + >>> print(cutout1.data) # doctest: +FLOAT_CMP + [[0. 1.] + [4. 5.]] + >>> print(cutout1.shape) + (2, 2) + >>> print((cutout1.position_original, cutout1.position_cutout)) + ((0, 0), (0, 0)) + +With ``mode='partial'``, the cutout will never be trimmed. Instead it +will be filled with ``fill_value`` (the default is ``numpy.nan``) if +the cutout is not fully contained in the data array:: + + >>> cutout2 = Cutout2D(data2, (0, 0), (3, 3), mode='partial') + >>> print(cutout2.data) # doctest: +FLOAT_CMP + [[nan nan nan] + [nan 0. 1.] + [nan 4. 5.]] + +Note that for the ``'partial'`` mode, the positions (and several other +attributes) are calculated for on the *valid* (non-filled) cutout +values:: + + >>> print((cutout2.position_original, cutout2.position_cutout)) + ((0, 0), (1, 1)) + >>> print((cutout2.origin_original, cutout2.origin_cutout)) + ((0, 0), (1, 1)) + >>> print(cutout2.slices_original) + (slice(0, 2, None), slice(0, 2, None)) + >>> print(cutout2.slices_cutout) + (slice(1, 3, None), slice(1, 3, None)) + +Using ``mode='strict'`` will raise an exception if the cutout is not +fully contained in the data array: + +.. doctest-skip:: + + >>> cutout3 = Cutout2D(data2, (0, 0), (3, 3), mode='strict') + PartialOverlapError: Arrays overlap only partially. + + +2D Cutout from a `~astropy.coordinates.SkyCoord` Position +--------------------------------------------------------- + +The input ``position`` can also be specified as a +`~astropy.coordinates.SkyCoord`, in which case a `~astropy.wcs.WCS` +object must be input via the ``wcs`` keyword. + +First, we define a `~astropy.coordinates.SkyCoord` position and a +`~astropy.wcs.WCS` object for our data (usually this would come from +your FITS header):: + + >>> from astropy.coordinates import SkyCoord + >>> from astropy.wcs import WCS + >>> position = SkyCoord('13h11m29.96s -01d19m18.7s', frame='icrs') + >>> wcs = WCS(naxis=2) + >>> rho = np.pi / 3. + >>> scale = 0.05 / 3600. + >>> wcs.wcs.cd = [[scale*np.cos(rho), -scale*np.sin(rho)], + ... [scale*np.sin(rho), scale*np.cos(rho)]] + >>> wcs.wcs.ctype = ['RA---TAN', 'DEC--TAN'] + >>> wcs.wcs.crval = [position.ra.to_value(u.deg), + ... position.dec.to_value(u.deg)] + >>> wcs.wcs.crpix = [50, 100] + +Now we can create the cutout array using the +`~astropy.coordinates.SkyCoord` position and ``wcs`` object:: + + >>> cutout = Cutout2D(data, position, (30, 40), wcs=wcs) + >>> fig, ax = plt.subplots() # doctest: +SKIP + >>> ax.imshow(cutout.data, origin='lower') # doctest: +SKIP + +.. plot:: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling.models import Gaussian2D + from astropy.nddata import Cutout2D + from astropy.coordinates import SkyCoord + from astropy.wcs import WCS + y, x = np.mgrid[0:500, 0:500] + data = Gaussian2D(1, 50, 100, 10, 5, theta=0.5)(x, y) + position = SkyCoord('13h11m29.96s -01d19m18.7s', frame='icrs') + wcs = WCS(naxis=2) + rho = np.pi / 3. + scale = 0.05 / 3600. + wcs.wcs.cd = [[scale*np.cos(rho), -scale*np.sin(rho)], + [scale*np.sin(rho), scale*np.cos(rho)]] + wcs.wcs.ctype = ['RA---TAN', 'DEC--TAN'] + wcs.wcs.crval = [position.ra.value, position.dec.value] + wcs.wcs.crpix = [50, 100] + cutout = Cutout2D(data, position, (30, 40), wcs=wcs) + fig, ax = plt.subplots() + ax.imshow(cutout.data, origin='lower') + +The ``wcs`` attribute of the `~astropy.nddata.utils.Cutout2D` object now +contains the propagated `~astropy.wcs.WCS` for the cutout array. +Now we can find the sky coordinates for a given pixel in the cutout array. +Note that we need to use the ``cutout.wcs`` object for the cutout +positions:: + + >>> from astropy.wcs.utils import pixel_to_skycoord + >>> x_cutout, y_cutout = (5, 10) + >>> pixel_to_skycoord(x_cutout, y_cutout, cutout.wcs) # doctest: +FLOAT_CMP + + +We now find the corresponding pixel in the original ``data`` array and +its sky coordinates:: + + >>> x_data, y_data = cutout.to_original_position((x_cutout, y_cutout)) + >>> pixel_to_skycoord(x_data, y_data, wcs) # doctest: +FLOAT_CMP + + +As expected, the sky coordinates in the original ``data`` and the +cutout array agree. + + +2D Cutout Using an Angular ``size`` +----------------------------------- + +The input ``size`` can also be specified as a +`~astropy.units.Quantity` in angular units (e.g., degrees, arcminutes, +arcseconds, etc.). For this case, a `~astropy.wcs.WCS` object must be +input via the ``wcs`` keyword. + +For this example, we will use the data, `~astropy.coordinates.SkyCoord` +position, and ``wcs`` object from above to create a cutout with size +1.5 x 2.5 arcseconds:: + + >>> size = u.Quantity((1.5, 2.5), u.arcsec) + >>> cutout = Cutout2D(data, position, size, wcs=wcs) + >>> fig, ax = plt.subplots() # doctest: +SKIP + >>> ax.imshow(cutout.data, origin='lower') # doctest: +SKIP + +.. plot:: + + import numpy as np + import matplotlib.pyplot as plt + from astropy.modeling.models import Gaussian2D + from astropy.nddata import Cutout2D + from astropy.coordinates import SkyCoord + from astropy.wcs import WCS + from astropy import units as u + y, x = np.mgrid[0:500, 0:500] + data = Gaussian2D(1, 50, 100, 10, 5, theta=0.5)(x, y) + position = SkyCoord('13h11m29.96s -01d19m18.7s', frame='icrs') + wcs = WCS(naxis=2) + rho = np.pi / 3. + scale = 0.05 / 3600. + wcs.wcs.cd = [[scale*np.cos(rho), -scale*np.sin(rho)], + [scale*np.sin(rho), scale*np.cos(rho)]] + wcs.wcs.ctype = ['RA---TAN', 'DEC--TAN'] + wcs.wcs.crval = [position.ra.value, position.dec.value] + wcs.wcs.crpix = [50, 100] + size = u.Quantity((1.5, 2.5), u.arcsec) + cutout = Cutout2D(data, position, size, wcs=wcs) + fig, ax = plt.subplots() + ax.imshow(cutout.data, origin='lower') + + +Saving a 2D Cutout to a FITS File with an Updated WCS +===================================================== + +A `~astropy.nddata.utils.Cutout2D` object can be saved to a FITS file, +including the updated WCS object for the cutout region. In this example, we +download an example FITS image and create a cutout image. The resulting +`~astropy.nddata.utils.Cutout2D` object is then saved to a new FITS file with +the updated WCS for the cutout region. + +.. literalinclude:: examples/cutout2d_tofits.py + :language: python diff --git a/docs/nitpick-exceptions b/docs/nitpick-exceptions index 3b539c3d8285..2658d709051d 100644 --- a/docs/nitpick-exceptions +++ b/docs/nitpick-exceptions @@ -1,58 +1,70 @@ # astropy.cosmology -py:class astropy.cosmology.Cosmology -py:class astropy.cosmology.core.Cosmology +py:obj astropy.cosmology.realizations.default_cosmology # astropy.io.votable -py:class astropy.io.votable.tree.Element py:class astropy.io.votable.tree.SimpleElement py:class astropy.io.votable.tree.SimpleElementWithContent # astropy.modeling -py:class astropy.modeling.projections.Zenithal -py:class astropy.modeling.projections.Cylindrical py:class astropy.modeling.polynomial.PolynomialBase -py:class astropy.modeling.rotations.EulerAngleRotation -py:class astropy.modeling.projections.Projection # astropy.io.fits py:class astropy.io.fits.hdu.base.ExtensionHDU py:class astropy.io.fits.util.NotifierMixin +py:class astropy.io.fits.hdu.compressed._codecs.Codec + +# astropy.io.misc.yaml +py:class yaml.dumper.SafeDumper +py:class yaml.loader.SafeLoader +py:class yaml.representer.SafeRepresenter +py:class yaml.scanner.Scanner +py:class yaml.constructor.SafeConstructor +py:class yaml.constructor.BaseConstructor +py:class yaml.parser.Parser +py:class yaml.representer.BaseRepresenter +py:class yaml.reader.Reader +py:class yaml.resolver.BaseResolver +py:class yaml.serializer.Serializer +py:class yaml.composer.Composer +py:class yaml.resolver.Resolver +py:class yaml.emitter.Emitter + +# astropy.units +# This is required on macOS (#9040 and #10026) and Windows (#16288). +py:obj astropy.units.function.logarithmic.m_bol # astropy.utils -py:class astropy.extern.six.Iterator -py:class type py:class json.encoder.JSONEncoder # astropy.table py:class astropy.table.column.BaseColumn py:class astropy.table.groups.BaseGroups -# astropy.time -py:class astropy.time.core.TimeUnique +# astropy.visualization +py:obj Bbox +py:obj Transform +py:obj Figure +py:obj AbstractPathEffect +py:obj N +py:obj masked + +# astropy.wcs +py:class astropy.wcs.wcsapi.fitswcs.FITSWCSAPIMixin +py:class astropy.wcs.wcsapi.fitswcs.custom_ctype_to_ucd_mapping # numpy inherited docstrings py:obj dtype py:obj a -py:obj a.size == 1 py:obj n +py:obj v py:obj ndarray py:obj args +py:obj numpy._typing.ArrayLike # other classes and functions that cannot be linked to -py:class numpy.ma.core.MaskedArray -py:class numpy.core.records.recarray -py:class xmlrpclib.Fault -py:class xmlrpclib.Error -py:class xmlrpc.client.Fault py:class xmlrpc.client.Error -py:obj distutils.version.LooseVersion -py:obj pkg_resources.parse_version -py:class pandas.DataFrame - # Pending on python docs links issue #11975 -py:class list -py:obj list.append py:obj list.append py:obj list.count py:obj list.extend @@ -60,4 +72,15 @@ py:obj list.index py:obj list.insert py:meth list.pop py:obj list.remove -py:class classmethod +py:obj RendererBase +py:obj Artist +py:obj BboxBase + +# This list is from https://github.com/numpy/numpydoc/issues/275 +py:class None. Remove all items from D. +py:class a set-like object providing a view on D's items +py:class a set-like object providing a view on D's keys +py:class None. Update D from dict/iterable E and F. +py:class None. Update D from mapping/iterable E and F. +py:class an object providing a view on D's values +py:class a shallow copy of D diff --git a/docs/overview.rst b/docs/overview.rst deleted file mode 100644 index 33860e19ca2b..000000000000 --- a/docs/overview.rst +++ /dev/null @@ -1,80 +0,0 @@ -******** -Overview -******** - -Here we describe a broad overview of the Astropy project and its parts. - -Astropy Project Concept -======================= - -The "Astropy Project" is distinct from the ``astropy`` package. The -Astropy Project is a process intended to facilitate communication and -interoperability of python packages/codes in astronomy and astrophysics. -The project thus encompasses the ``astropy`` core package (which provides -a common framework), all "affiliated packages" (described below in -`Affiliated Packages`_), and a general community aimed at bringing -resources together and not duplicating efforts. - - -``astropy`` Core Package -======================== - -The ``astropy`` package (alternatively known as the "core" package) -contains various classes, utilities, and a packaging framework intended -to provide commonly-used astronomy tools. It is divided into a variety -of sub-packages, which are documented in the remainder of this -documentation (see :ref:`user-docs` for documentation of these -components). - -The core also provides this documentation, and a variety of utilities -that simplify starting other python astronomy/astrophysics packages. As -described in the following section, these simplify the process of -creating affiliated packages. - - -Affiliated Packages -=================== - -The Astropy project includes the concept of "affiliated packages." An -affiliated package is an astronomy-related python package that is not -part of the ``astropy`` core source code, but has requested to be included -in the general community effort of the Astropy project. Such a package -may be a candidate for eventual inclusion in the main ``astropy`` package -(although this is not required). Until then, however, it is a separate -package, and may not be in the ``astropy`` namespace. - -The authoritative list of current affiliated packages is available at -http://affiliated.astropy.org, including a machine-readable `JSON file -`_. - -If you are interested in starting an affiliated package, or have a -package you are interested in making more compatible with astropy, the -``astropy`` core package includes features that simplify and homogenize -package management. Astropy provides a `package template -`_ that provides a common -way to organize a package, to make your life simpler. You can use this -template either with a new package you are starting or an existing -package to give it most of the organizational tools Astropy provides, -including the documentation, testing, and Cython-building tools. See -the `usage instructions in the template `_ for further details. - -To then get your package listed on the registry, take a look at the -`guidelines for becoming an affiliated package -`_, and then post -your intent on the `astropy-dev mailing list`_. The Astropy -coordination committee, in consultation with the community, will provide -you feedback on the package, and will add it to the registry when it is -approved. - - -Community -========= - -Aside from the actual code, Astropy is also a community of astronomy- -associated users and developers that agree that sharing utilities is -healthy for the community and the science it produces. This community is -of course central to accomplishing anything with the code itself. We -follow the `Python Software Foundation Code of Conduct -`_ and welcome anyone who -wishes to contribute to the project. diff --git a/docs/robots.txt b/docs/robots.txt new file mode 100644 index 000000000000..df810cf3bbb1 --- /dev/null +++ b/docs/robots.txt @@ -0,0 +1,6 @@ +User-agent: * +Allow: /*/latest/ +Allow: /en/latest/ # Fallback for bots that don't understand wildcards +Allow: /*/stable/ +Allow: /en/stable/ # Fallback for bots that don't understand wildcards +Disallow: / diff --git a/docs/rtd-pip-requirements b/docs/rtd-pip-requirements deleted file mode 100644 index 1d066a713e7e..000000000000 --- a/docs/rtd-pip-requirements +++ /dev/null @@ -1,4 +0,0 @@ --e git+http://github.com/astropy/astropy-helpers.git#egg=astropy_helpers -numpy>=1.6.0 -matplotlib -Cython diff --git a/docs/rtd_environment.yaml b/docs/rtd_environment.yaml new file mode 100644 index 000000000000..3d8fd7179dc4 --- /dev/null +++ b/docs/rtd_environment.yaml @@ -0,0 +1,7 @@ +name: rtd313 +channels: + - conda-forge +dependencies: + - python=3.13 + - pip + - graphviz diff --git a/docs/samp/advanced_embed_samp_hub.rst b/docs/samp/advanced_embed_samp_hub.rst new file mode 100644 index 000000000000..0f0df50f5f25 --- /dev/null +++ b/docs/samp/advanced_embed_samp_hub.rst @@ -0,0 +1,138 @@ +.. doctest-skip-all + +Embedding a SAMP Hub in a GUI +***************************** + +Overview +======== + +If you wish to embed a SAMP hub in your Python Graphical User Interface (GUI) +tool, you will need to start the hub programmatically using:: + + from astropy.samp import SAMPHubServer + hub = SAMPHubServer() + hub.start() + +This launches the hub in a thread and is non-blocking. If you are not +interested in connections from web SAMP clients, then you can use:: + + from astropy.samp import SAMPHubServer + hub = SAMPHubServer(web_profile=False) + hub.start() + +This should be all you need to do. However, if you want to keep the Web +Profile active, there is an additional consideration: when a web +SAMP client connects, you will need to ask the user whether they accept the +connection (for security reasons). By default, the confirmation message is a +text-based message in the terminal, but if you have a GUI tool, you will +likely want to open a GUI dialog instead. + +To do this, you will need to define a class that handles the dialog, and then +pass an **instance** of the class to |SAMPHubServer| (not the class itself). +This class should inherit from `astropy.samp.WebProfileDialog` and add the +following: + + 1) A GUI timer callback that periodically calls + ``WebProfileDialog.handle_queue`` (available as + ``self.handle_queue``). + + 2) A ``show_dialog`` method to display a consent dialog. + It should take the following arguments: + + - ``samp_name``: The name of the application making the request. + + - ``details``: A dictionary of details about the client + making the request. The only key in this dictionary required by + the SAMP standard is ``samp.name`` which gives the name of the + client making the request. + + - ``client``: A hostname, port pair containing the client + address. + + - ``origin``: A string containing the origin of the + request. + + 3) Based on the user response, the ``show_dialog`` should call + ``WebProfileDialog.consent`` or ``WebProfileDialog.reject``. + This may, in some cases, be the result of another GUI callback. + +Example of embedding a SAMP hub in a Tk application +--------------------------------------------------- + +.. + EXAMPLE START + Embedding a SAMP Hub in a Tk Application + +The following code is a full example of a Tk application that watches for web +SAMP connections and opens the appropriate dialog:: + + import tkinter as tk + import tkinter.messagebox as tkMessageBox + + from astropy.samp import SAMPHubServer + from astropy.samp.hub import WebProfileDialog + + MESSAGE = """ + A Web application which declares to be + + Name: {name} + Origin: {origin} + + is requesting to be registered with the SAMP Hub. Pay attention + that if you permit its registration, such application will acquire + all current user privileges, like file read/write. + + Do you give your consent? + """ + + class TkWebProfileDialog(WebProfileDialog): + def __init__(self, root): + self.root = root + self.wait_for_dialog() + + def wait_for_dialog(self): + self.handle_queue() + self.root.after(100, self.wait_for_dialog) + + def show_dialog(self, samp_name, details, client, origin): + text = MESSAGE.format(name=samp_name, origin=origin) + + response = tkMessageBox.askyesno( + 'SAMP Hub', text, + default=tkMessageBox.NO) + + if response: + self.consent() + else: + self.reject() + + # Start up Tk application + root = tk.Tk() + tk.Label(root, text="Example SAMP Tk application", + font=("Helvetica", 36), justify=tk.CENTER).pack(pady=200) + root.geometry("500x500") + root.update() + + # Start up SAMP hub + h = SAMPHubServer(web_profile_dialog=TkWebProfileDialog(root)) + h.start() + + try: + # Main GUI loop + root.mainloop() + except KeyboardInterrupt: + pass + + h.stop() + +If you run the above script, a window will open that says "Example SAMP Tk +application." If you then go to the following page, for example: + +http://astrojs.github.io/sampjs/examples/pinger.html + +and click on the Ping button, you will see the dialog open in the Tk +application. Once you click on "CONFIRM," future "Ping" calls will no longer +bring up the dialog. + +.. + EXAMPLE END diff --git a/docs/vo/samp/example_clients.rst b/docs/samp/example_clients.rst similarity index 91% rename from docs/vo/samp/example_clients.rst rename to docs/samp/example_clients.rst index dcd95ae2c696..b6033d3baace 100644 --- a/docs/vo/samp/example_clients.rst +++ b/docs/samp/example_clients.rst @@ -1,18 +1,14 @@ -.. include:: references.txt - .. doctest-skip-all .. _vo-samp-example_clients: -Communication between integrated clients objects ------------------------------------------------- +Communication between Integrated Clients Objects +************************************************ As shown in :doc:`example_table_image`, the |SAMPIntegratedClient| class can be -used to communicate with other SAMP-enabled tools such as `TOPCAT -`_, `SAO Ds9 -`_, or `Aladin Desktop -`_. +used to communicate with other SAMP-enabled tools such as |TOPCAT|, +`SAO DS9 `_, or `Aladin Desktop `_. In this section, we look at how we can set up two |SAMPIntegratedClient| instances and communicate between them. @@ -21,7 +17,7 @@ First, start up a SAMP hub as described in :doc:`example_hub`. Next, we create two clients and connect them to the hub:: - >>> from astropy.vo import samp + >>> from astropy import samp >>> client1 = samp.SAMPIntegratedClient(name="Client 1", description="Test Client 1", ... metadata = {"client1.version":"0.01"}) >>> client2 = samp.SAMPIntegratedClient(name="Client 2", description="Test Client 2", @@ -29,7 +25,8 @@ Next, we create two clients and connect them to the hub:: >>> client1.connect() >>> client2.connect() -We now define functions to call when receiving a notification, call or response:: +We now define functions to call when receiving a notification, call or +response:: >>> def test_receive_notification(private_key, sender_id, mtype, params, extra): ... print("Notification:", private_key, sender_id, mtype, params, extra) @@ -60,8 +57,8 @@ notifies all clients using the "samp.app.echo" message type via the hub:: Notification: 0d7f4500225981c104a197c7666a8e4e cli#2 samp.app.echo {'txt': 'Hello world!'} {'host': 'antigone.lambrate.inaf.it', 'user': 'unknown'} -We can also find a dictionary giving the clients that would currently receive -``samp.app.echo`` messages:: +We can also find a dictionary that specifies which clients would currently +receive ``samp.app.echo`` messages:: >>> print(client2.get_subscribed_clients("samp.app.echo")) {'cli#2': {}} diff --git a/docs/samp/example_hub.rst b/docs/samp/example_hub.rst new file mode 100644 index 000000000000..57a5cd5bdbf2 --- /dev/null +++ b/docs/samp/example_hub.rst @@ -0,0 +1,48 @@ +.. doctest-skip-all + +.. _vo-samp-example_hub: + +Starting and Stopping a SAMP Hub Server +*************************************** + +There are several ways you can start up a SAMP hub: + +Using an Existing Hub +===================== + +You can start up another application that includes a hub, such as +|TOPCAT|, `SAO DS9 `_, or +`Aladin Desktop `_. + +Using the Command-Line Hub Utility +================================== + +You can make use of the ``samp_hub`` command-line utility, which is included in +``astropy``:: + + $ samp_hub + +To get more help on available options for ``samp_hub``:: + + $ samp_hub -h + +To stop the server, press control-C. + +Starting a Hub Programmatically (Advanced) +========================================== + +You can start up a hub by creating a |SAMPHubServer| instance and starting it, +either from the interactive Python prompt, or from a Python script:: + + >>> from astropy.samp import SAMPHubServer + >>> hub = SAMPHubServer() + >>> hub.start() + +You can then stop the hub by calling:: + + >>> hub.stop() + +However, this method is generally not recommended for average users because it +does not work correctly when web SAMP clients try to connect. Instead, this +should be reserved for developers who want to embed a SAMP hub in a GUI, for +example. For more information, see :doc:`advanced_embed_samp_hub`. diff --git a/docs/samp/example_table_image.rst b/docs/samp/example_table_image.rst new file mode 100644 index 000000000000..acbabff6c77c --- /dev/null +++ b/docs/samp/example_table_image.rst @@ -0,0 +1,283 @@ +.. doctest-skip-all + +.. _vo-samp-example-table-image: + +Sending and Receiving Tables and Images over SAMP +************************************************* + +In the following examples, we make use of: + +* |TOPCAT|, which is a tool to explore tabular data. +* `SAO DS9 `_, which is an image + visualization tool that can overplot catalogs. +* `Aladin Desktop `_, which is another tool that + can visualize images and catalogs. + +TOPCAT and Aladin will run a SAMP Hub if none is found, so for the following +examples you can either start up one of these applications first, or you can +start up the `astropy.samp` hub. You can start this using the following +command:: + + $ samp_hub + +Sending a Table to TOPCAT and DS9 +================================= + +The easiest way to send a VO table to TOPCAT is to make use of the +|SAMPIntegratedClient| class. Once TOPCAT is open, first instantiate a +|SAMPIntegratedClient| instance and then connect to the hub:: + + >>> from astropy.samp import SAMPIntegratedClient + >>> client = SAMPIntegratedClient() + >>> client.connect() + +Next, we have to set up a dictionary that contains details about the table to +send. This should include ``url``, which is the URL to the file, and ``name``, +which is a human-readable name for the table. The URL can be a local URL +(starting with ``file:///``):: + + >>> params = {} + >>> params["url"] = 'file:///Users/tom/Desktop/aj285677t3_votable.xml' + >>> params["name"] = "Robitaille et al. (2008), Table 3" + +.. note:: To construct a local URL, you can also make use of ``urlparse`` as + follows:: + + >>> import urlparse + >>> params["url"] = urlparse.urljoin('file:', os.path.abspath("aj285677t3_votable.xml")) + +Now we can set up the message itself. This includes the type of message (here +we use ``table.load.votable``, which indicates that a VO table should be loaded +and the details of the table that we set above):: + + >>> message = {} + >>> message["samp.mtype"] = "table.load.votable" + >>> message["samp.params"] = params + +Finally, we can broadcast this to all clients that are listening for +``table.load.votable`` messages using +:meth:`~astropy.samp.integrated_client.SAMPIntegratedClient.notify_all`:: + + >>> client.notify_all(message) + +The above message will actually be broadcast to all applications connected via +SAMP. For example, if we open `SAO DS9 `_ in +addition to TOPCAT, and we run the above command, both applications will load +the table. We can use the +:meth:`~astropy.samp.integrated_client.SAMPIntegratedClient.get_registered_clients` method to +find all of the clients connected to the hub:: + + >>> client.get_registered_clients() + ['hub', 'c1', 'c2'] + +These IDs do not mean much, but we can find out more using:: + + >>> client.get_metadata('c1') + {'author.affiliation': 'Astrophysics Group, Bristol University', + 'author.email': 'm.b.taylor@bristol.ac.uk', + 'author.name': 'Mark Taylor', + 'home.page': 'https://www.star.bristol.ac.uk/mbt/topcat/', + 'samp.description.text': 'Tool for OPerations on Catalogues And Tables', + 'samp.documentation.url': 'http://127.0.0.1:2525/doc/sun253/index.html', + 'samp.icon.url': 'http://127.0.0.1:2525/doc/images/tc_sok.gif', + 'samp.name': 'topcat', + 'topcat.version': '4.0-1'} + +We can see that ``c1`` is the TOPCAT client. We can now resend the data, but +this time only to TOPCAT, using the +:meth:`~astropy.samp.integrated_client.SAMPIntegratedClient.notify` method:: + + >>> client.notify('c1', message) + +Once finished, we should make sure we disconnect from the hub:: + + >>> client.disconnect() + +Receiving a Table from TOPCAT +============================= + +To receive a table from TOPCAT, we have to set up a client that listens for +messages from the hub. As before, we instantiate a |SAMPIntegratedClient| +instance and connect to the hub:: + + >>> from astropy.samp import SAMPIntegratedClient + >>> client = SAMPIntegratedClient() + >>> client.connect() + +We now set up a receiver class which will handle any received messages. We need +to take care to write handlers for both notifications and calls (the difference +between the two being that calls expect a reply):: + + >>> class Receiver: + ... def __init__(self, client): + ... self.client = client + ... self.received = False + ... def receive_call(self, private_key, sender_id, msg_id, mtype, params, extra): + ... self.params = params + ... self.received = True + ... self.client.reply(msg_id, {"samp.status": "samp.ok", "samp.result": {}}) + ... def receive_notification(self, private_key, sender_id, mtype, params, extra): + ... self.params = params + ... self.received = True + +And we instantiate it: + + >>> r = Receiver(client) + +We can now use the +:meth:`~astropy.samp.integrated_client.SAMPIntegratedClient.bind_receive_call` +and +:meth:`~astropy.samp.integrated_client.SAMPIntegratedClient.bind_receive_notification` +methods to tell our receiver to listen to all ``table.load.votable`` messages:: + + >>> client.bind_receive_call("table.load.votable", r.receive_call) + >>> client.bind_receive_notification("table.load.votable", r.receive_notification) + +We can now check that the message has not been received yet:: + + >>> r.received + False + +We can now broadcast the table from TOPCAT. After a few seconds, we can check +again if the message has been received:: + + >>> r.received + True + +Success! The table URL should now be available in ``r.params['url']``, so we +can do:: + + >>> from astropy.table import Table + >>> t = Table.read(r.params['url']) + Downloading http://127.0.0.1:2525/dynamic/4/t12.vot [Done] + >>> t + col1 col2 col3 col4 col5 col6 col7 col8 col9 col10 + ------------------------- -------- ------- -------- -------- ----- ---- ----- ---- ----- + SSTGLMC G000.0046+01.1431 0.0046 1.1432 265.2992 -28.3321 6.67 5.04 6.89 5.22 N + SSTGLMC G000.0106-00.7315 0.0106 -0.7314 267.1274 -29.3063 7.18 6.07 nan 5.17 Y + SSTGLMC G000.0110-01.0237 0.0110 -1.0236 267.4151 -29.4564 8.32 6.30 8.34 6.32 N + ... + +As before, we should remember to disconnect from the hub once we are done:: + + >>> client.disconnect() + +Example +======= + +.. + EXAMPLE START + Receiving and Reading a Table over SAMP + +The following is a full example of a script that can be used to receive and +read a table. It includes a loop that waits until the message is received, and +reads the table once it has:: + + import time + + from astropy.samp import SAMPIntegratedClient + from astropy.table import Table + + # Instantiate the client and connect to the hub + client=SAMPIntegratedClient() + client.connect() + + # Set up a receiver class + class Receiver: + def __init__(self, client): + self.client = client + self.received = False + def receive_call(self, private_key, sender_id, msg_id, mtype, params, extra): + self.params = params + self.received = True + self.client.reply(msg_id, {"samp.status": "samp.ok", "samp.result": {}}) + def receive_notification(self, private_key, sender_id, mtype, params, extra): + self.params = params + self.received = True + + # Instantiate the receiver + r = Receiver(client) + + # Listen for any instructions to load a table + client.bind_receive_call("table.load.votable", r.receive_call) + client.bind_receive_notification("table.load.votable", r.receive_notification) + + # We now run the loop to wait for the message in a try/finally block so that if + # the program is interrupted e.g. by control-C, the client terminates + # gracefully. + + try: + + # We test every 0.1s to see if the hub has sent a message + while True: + time.sleep(0.1) + if r.received: + t = Table.read(r.params['url']) + break + + finally: + + client.disconnect() + + # Print out table + print t + +.. + EXAMPLE END + +Sending an Image to DS9 and Aladin +================================== + +As for tables, the most convenient way to send a FITS image over SAMP is to +make use of the |SAMPIntegratedClient| class. Once Aladin or DS9 are open, +first instantiate a |SAMPIntegratedClient| instance and then connect to the hub +as before:: + + >>> from astropy.samp import SAMPIntegratedClient + >>> client = SAMPIntegratedClient() + >>> client.connect() + +Next, we have to set up a dictionary that contains details about the image to +send. This should include ``url``, which is the URL to the file, and ``name``, +which is a human-readable name for the table. The URL can be a local URL +(starting with ``file:///``):: + + >>> params = {} + >>> params["url"] = 'file:///Users/tom/Desktop/MSX_E.fits' + >>> params["name"] = "MSX Band E Image of the Galactic Center" + +See `Sending a Table to TOPCAT and DS9`_ for an example of a recommended way to +construct local URLs. Now we can set up the message itself. This includes the +type of message (here we use ``image.load.fits`` which indicates that a FITS +image should be loaded, and the details of the table that we set above):: + + >>> message = {} + >>> message["samp.mtype"] = "image.load.fits" + >>> message["samp.params"] = params + +Finally, we can broadcast this to all clients that are listening for +``table.load.votable`` messages:: + + >>> client.notify_all(message) + +As for `Sending a Table to TOPCAT and DS9`_, the +:meth:`~astropy.samp.integrated_client.SAMPIntegratedClient.notify_all` +method will broadcast the image to all listening clients, and for tables it +is possible to instead use the +:meth:`~astropy.samp.integrated_client.SAMPIntegratedClient.notify` method +to send it to a specific client. + +Once finished, we should make sure we disconnect from the hub:: + + >>> client.disconnect() + +Receiving a Table from DS9 or Aladin +==================================== + +Receiving images over SAMP is identical to `Receiving a Table from TOPCAT`_, +with the exception that the message type should be ``image.load.fits`` instead +of ``table.load.votable``. Once the URL has been received, the FITS image can +be opened with:: + + >>> from astropy.io import fits + >>> fits.open(r.params['url']) diff --git a/docs/samp/index.rst b/docs/samp/index.rst new file mode 100644 index 000000000000..d943e98d3d8b --- /dev/null +++ b/docs/samp/index.rst @@ -0,0 +1,83 @@ +.. doctest-skip-all + +.. _vo-samp: + +************************************************************* +SAMP (Simple Application Messaging Protocol) (`astropy.samp`) +************************************************************* + +`astropy.samp` is a Python implementation of the SAMP messaging system. + +Simple Application Messaging Protocol (SAMP) is an inter-process communication +system that allows different client programs, usually running on the same +computer, to communicate with each other by exchanging short messages that may +reference external data files. The protocol has been developed within the +International Virtual Observatory Alliance (IVOA) and is understood by many +desktop astronomy tools, including |TOPCAT|, `SAO DS9 `_, +and `Aladin `_. + +So by using the classes in `astropy.samp`, Python code can interact with +other running desktop clients, for instance displaying a named FITS file in DS9, +prompting Aladin to recenter on a given sky position, or receiving a message +identifying the row when a user highlights a plotted point in TOPCAT. + +The way the protocol works is that a SAMP "Hub" process must be running on the +local host, and then various client programs can connect to it. Once connected, +these clients can send messages to each other via the hub. The details are +described in the `SAMP standard `_. + +`astropy.samp` provides classes both to set up such a hub process, and to +help implement a client that can send and receive messages. It also provides a +stand-alone program ``samp_hub`` which can run a persistent hub in its own +process. Note that setting up the hub from Python is not always necessary, since +various other SAMP-aware applications may start up a hub independently; in most +cases, only one running hub is used during a SAMP session. + +The following classes are available in `astropy.samp`: + +* |SAMPHubServer|, which is used to instantiate a hub server that clients can + then connect to. +* |SAMPHubProxy|, which is used to connect to an existing hub (including hubs + started from other applications such as |TOPCAT|). +* |SAMPClient|, which is used to create a SAMP client. +* |SAMPIntegratedClient|, which is the same as |SAMPClient| except that it has + a self-contained |SAMPHubProxy| to provide a simpler user interface. + +`astropy.samp` is a full implementation of `SAMP V1.3 +`_. As well as the Standard +Profile, it supports the Web Profile, which means that it can be used to also +communicate with web SAMP clients; see the `sampjs +`_ library examples for more details. + +.. _IVOA Simple Application Messaging Protocol: http://www.ivoa.net/documents/latest/SAMP.html + +Using `astropy.samp` +==================== + +.. toctree:: + :maxdepth: 2 + + example_hub + example_table_image + example_clients + advanced_embed_samp_hub + +.. note that if this section gets too long, it should be moved to a separate + doc page - see the top of performance.inc.rst for the instructions on how to do + that +.. include:: performance.inc.rst + +Reference/API +============= + +.. toctree:: + :maxdepth: 2 + + ref_api + +Acknowledgments +=============== + +This code is adapted from the `SAMPy `__ +package written by Luigi Paioro, who has granted the Astropy Project permission +to use the code under a BSD license. diff --git a/docs/samp/performance.inc.rst b/docs/samp/performance.inc.rst new file mode 100644 index 000000000000..baef1348f042 --- /dev/null +++ b/docs/samp/performance.inc.rst @@ -0,0 +1,12 @@ +.. note that if this is changed from the default approach of using an *include* + (in index.rst) to a separate performance page, the header needs to be changed + from === to ***, the filename extension needs to be changed from .inc.rst to + .rst, and a link needs to be added in the subpackage toctree + +.. _astropy-samp-performance: + +.. Performance Tips +.. ================ +.. +.. Here we provide some tips and tricks for how to optimize performance of code +.. using `astropy.samp`. diff --git a/docs/samp/ref_api.rst b/docs/samp/ref_api.rst new file mode 100644 index 000000000000..bbf24fcf50be --- /dev/null +++ b/docs/samp/ref_api.rst @@ -0,0 +1,4 @@ +Reference/API +************* + +.. automodapi:: astropy.samp diff --git a/docs/stability.rst b/docs/stability.rst deleted file mode 100644 index d639670197d0..000000000000 --- a/docs/stability.rst +++ /dev/null @@ -1,294 +0,0 @@ -****************************** -Current status of sub-packages -****************************** - -Astropy has benefited from the addition of widely tested legacy code, as well -as new development, resulting in variations in stability across -sub-packages. This document summarizes the current status of the Astropy -sub-packages, so that users understand where they might expect changes in -future, and which sub-packages they can safely use for production code. - -The classification is as follows: - -.. raw:: html - - - -
- - - - - - - - - - - - - - - - -
Planned
Actively developed, be prepared for possible significant changes.
Reasonably stable, any significant changes/additions will generally include backwards-compatiblity.
Mature. Additions/improvements possible, but no major changes planned.
- -The current planned and existing sub-packages are: - -.. raw:: html - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Sub-Package - -   - - Comments -
- astropy.analytic_functions - - - - New in v1.0. -
- astropy.constants - - - - Constants were changed to Quantity objects in v0.2. Since then on, the package has been stable, with occasional additions of new constants. -
- astropy.convolution - - - - New top-level package in v0.3 (was previously part of - astropy.nddata). - No major changes since, likely will maintain backwards compatibility but possible future additions or improvements. -
- astropy.coordinates - - - - New in v0.2, major changes in v0.4. Subsequent versions should - maintain a stable/backwards-compatible API, following the plan of APE 5. Further major additions/enhancements likely, but with basic framework unchanged. -
- astropy.cosmology - - - - Incremental improvements since v0.1, but mostly stable API. - Pure functional interface deprecated in v0.4. -
- astropy.io.ascii - - - - Originally developed as asciitable, and has maintained a stable API. -
- astropy.io.fits - - - - Originally developed as pyfits, and retains an API consistent with the standalone version. -
- astropy.io.misc - - mature - - The functionality that is currently present is stable, but this sub-package will likely see major additions in future. -
- astropy.io.votable - - - - Originally developed as vo.table, and has a stable API. -
- astropy.modeling - - - - New in v0.3. Major changes in v1.0, signficant additions planned. Backwards-compatibility likely to be maintained, but not guaranteed. -
- astropy.nddata - - - - Significantly revised in v1.0 to implement APE 7. Major changes in the API are not anticipated, broader use may reveal flaws that require API changes. -
- astropy.photometry - - - -   -
- astropy.stats - - - - Likely to maintain backwards-compatibility, but functionality continually being expanded, so significant additions likely in the future. -
- astropy.table - - - - Incremental improvements since v0.1, but mostly stable API. -
- astropy.time - - - - Incremental improvements since v0.1, API likely to remain stable - for the foreseeable future. -
- astropy.units - - - - New in v0.2. Adapted from pnbody and integrated into Astropy. Current functionality stable with intent to maintain backwards compatibility. Significant new functionality is likely to be added in future versions. -
- astropy.utils - - - - Contains mostly utilities destined for internal use with other parts of Astropy. Existing functionality generally stable, but reglar additions and occasional changes. -
- astropy.visualization - - - - New in v1.0, and in development. -
- astropy.vo - - - - Virtual Observatory service access and validation. Currently, only Simple Cone Search and SAMP are supported. -
- astropy.wcs - - - - Originally developed as pywcs, and has a stable API for now. However, there are plans to generalize the WCS interface to accommodate non-FITS WCS transformations, and this may lead to small changes in the user interface. -
diff --git a/docs/stats/circ.rst b/docs/stats/circ.rst new file mode 100644 index 000000000000..2bcec60dc9e1 --- /dev/null +++ b/docs/stats/circ.rst @@ -0,0 +1,15 @@ +.. _stats-circular: + +******************* +Circular Statistics +******************* + +.. automodapi:: astropy.stats.circstats + + +References +---------- +.. [1] S. R. Jammalamadaka, A. SenGupta. "Topics in Circular Statistics". + Series on Multivariate Analysis, Vol. 5, 2001. +.. [2] C. Agostinelli, U. Lund. "Circular Statistics from 'Topics in + Circular Statistics (2001)'". 2015. diff --git a/docs/stats/index.rst b/docs/stats/index.rst index bee9e01b0efc..bc1170d8c571 100644 --- a/docs/stats/index.rst +++ b/docs/stats/index.rst @@ -7,32 +7,207 @@ Astrostatistics Tools (`astropy.stats`) Introduction ============ -The `astropy.stats` package holds statistical functions or algorithms used -in astronomy and astropy. +The `astropy.stats` package holds statistical functions or algorithms +used in astronomy. While the `scipy.stats` and `statsmodels +`_ packages contains a +wide range of statistical tools, they are general-purpose packages and +are missing some tools that are particularly useful or specific to +astronomy. This package is intended to provide such functionality, +but *not* to replace `scipy.stats` if its implementation satisfies +astronomers' needs. + Getting Started =============== -The current tools are fairly self-contained, and include relevant examples in +A number of different tools are contained in the stats package, and +they can be accessed by importing them:: + + >>> from astropy import stats + +A full list of the different tools are provided below. Please see the +documentation for their different usages. For example, sigma clipping, +which is a common way to estimate the background of an image, can be +performed with the :func:`~astropy.stats.sigma_clip` function. +By default, the function returns a masked array, a type of Numpy array +used for handling missing or invalid entries. Masked arrays retain the +original data but also store another boolean array of the same shape +where ``True`` indicates that the value is masked. Most Numpy ufuncs +will understand masked arrays and treat them appropriately. +For example, consider the following dataset with a clear outlier:: + + >>> import numpy as np + >>> from astropy.stats import sigma_clip + >>> x = np.array([1, 0, 0, 1, 99, 0, 0, 1, 0]) + +The mean is skewed by the outlier:: + + >>> x.mean() + np.float64(11.333333333333334) + +Sigma-clipping (3 sigma by default) returns a masked array, +and so functions like ``mean`` will ignore the outlier:: + + >>> clipped = sigma_clip(x) + >>> clipped + masked_array(data=[1, 0, 0, 1, --, 0, 0, 1, 0], + mask=[False, False, False, False, True, False, False, False, + False], + fill_value=999999) + >>> clipped.mean() + np.float64(0.375) + +If you need to access the original data directly, you can use the +``data`` property. Combined with the ``mask`` property, you can get the +original outliers, or the values that were not clipped:: + + >>> outliers = clipped.data[clipped.mask] + >>> outliers + array([99]) + >>> valid = clipped.data[~clipped.mask] + >>> valid + array([1, 0, 0, 1, 0, 0, 1, 0]) + +For more information on masked arrays, including see the +:ref:`numpy.ma module `. + +Examples +-------- + +.. + EXAMPLE START + Sigma Clipping with Astropy Stats sigma_clip Function + +To estimate the background of an image:: + + >>> data = [1, 5, 6, 8, 100, 5, 3, 2] + >>> data_clipped = stats.sigma_clip(data, sigma=2, maxiters=5) + >>> data_clipped + masked_array(data=[1, 5, 6, 8, --, 5, 3, 2], + mask=[False, False, False, False, True, False, False, False], + fill_value=999999) + >>> np.mean(data_clipped) # doctest: +FLOAT_CMP + np.float64(4.285714285714286) + +.. + EXAMPLE END + +.. + EXAMPLE START + Sigma Clipping with Astropy Stats SigmaClip Class + +Alternatively, the :class:`~astropy.stats.SigmaClip` class provides an +object-oriented interface to sigma clipping, which also returns a +masked array by default:: + + >>> sigclip = stats.SigmaClip(sigma=2, maxiters=5) + >>> sigclip(data) + masked_array(data=[1, 5, 6, 8, --, 5, 3, 2], + mask=[False, False, False, False, True, False, False, False], + fill_value=999999) + +.. + EXAMPLE END + +.. + EXAMPLE START + Calculating Sigma Clipping Statistics + +In addition, there are also several convenience functions for making +the calculation of statistics even more convenient. For example, +:func:`~astropy.stats.sigma_clipped_stats` will return the mean, +median, and standard deviation of a sigma-clipped array:: + + >>> stats.sigma_clipped_stats(data, sigma=2, maxiters=5) # doctest: +FLOAT_CMP + (np.float64(4.285714285714286), np.float64(5.0), np.float64(2.249716535431946)) + +There are also tools for calculating :ref:`robust statistics +`, sampling the data, :ref:`circular statistics +`, confidence limits, spatial statistics, and adaptive +histograms. + +.. + EXAMPLE END + +Most tools are fairly self-contained, and include relevant examples in their docstrings. +Using `astropy.stats` +===================== + +More detailed information on using the package is provided on separate pages, +listed below. + +.. toctree:: + :maxdepth: 2 + + robust.rst + circ.rst + ripley.rst + +Also see :ref:`astropy-visualization-hist`. + + +Constants +========= + +The `astropy.stats` package defines two constants useful for +converting between Gaussian sigma and full width at half maximum +(FWHM): + +.. data:: gaussian_sigma_to_fwhm + + Factor with which to multiply Gaussian 1-sigma standard deviation + to convert it to full width at half maximum (FWHM). + + >>> from astropy.stats import gaussian_sigma_to_fwhm + >>> gaussian_sigma_to_fwhm # doctest: +FLOAT_CMP + 2.3548200450309493 + +.. data:: gaussian_fwhm_to_sigma + + Factor with which to multiply Gaussian full width at half maximum + (FWHM) to convert it to 1-sigma standard deviation. + + >>> from astropy.stats import gaussian_fwhm_to_sigma + >>> gaussian_fwhm_to_sigma # doctest: +FLOAT_CMP + 0.42466090014400953 + + See Also ======== * :mod:`scipy.stats` - This scipy package contains a variety of useful statistical functions and - classes. The functionality in `astropy.stats` is intended to supplement + This SciPy package contains a variety of useful statistical functions + and classes. The functionality in `astropy.stats` is intended to supplement this, *not* replace it. +* `statsmodels `_ + The statsmodels package provides functionality for estimating + different statistical models, tests, and data exploration. + +* `astroML `_ + The astroML package is a Python module for machine learning and + data mining. Some of the tools from this package have been + migrated here, but there are still a number of tools there that + are useful for astronomy and statistical analysis. + * :func:`astropy.visualization.hist` The :func:`~astropy.stats.histogram` routine and related functionality defined here are used within the :func:`astropy.visualization.hist` function. For a discussion of these methods for determining histogram binnings, see :ref:`astropy-visualization-hist`. +.. note that if this section gets too long, it should be moved to a separate + doc page - see the top of performance.inc.rst for the instructions on how to do + that +.. include:: performance.inc.rst Reference/API ============= -.. automodapi:: astropy.stats +.. toctree:: + :maxdepth: 2 + + ref_api diff --git a/docs/stats/performance.inc.rst b/docs/stats/performance.inc.rst new file mode 100644 index 000000000000..dea575690dcc --- /dev/null +++ b/docs/stats/performance.inc.rst @@ -0,0 +1,25 @@ +.. note that if this is changed from the default approach of using an *include* + (in index.rst) to a separate performance page, the header needs to be changed + from === to ***, the filename extension needs to be changed from .inc.rst to + .rst, and a link needs to be added in the subpackage toctree + +.. _astropy-stats-performance: + +Performance Tips +================ + +If you are finding sigma clipping to be slow, and if you have not already done +so, consider installing the `bottleneck `_ +package, which will speed up some of the internal computations. In addition, if +you are using standard functions for ``cenfunc`` and/or ``stdfunc``, make sure +you specify these as strings rather than passing a NumPy function — that is, +use:: + + >>> sigma_clip(array, cenfunc='median') # doctest: +SKIP + +instead of:: + + >>> sigma_clip(array, cenfunc=np.nanmedian) # doctest: +SKIP + +Using strings will allow the sigma-clipping algorithm to pick the fastest +implementation available for finding the median. diff --git a/docs/stats/ref_api.rst b/docs/stats/ref_api.rst new file mode 100644 index 000000000000..c8291a832da3 --- /dev/null +++ b/docs/stats/ref_api.rst @@ -0,0 +1,4 @@ +Reference/API +************* + +.. automodapi:: astropy.stats diff --git a/docs/stats/ripley.rst b/docs/stats/ripley.rst new file mode 100644 index 000000000000..166aaa22be47 --- /dev/null +++ b/docs/stats/ripley.rst @@ -0,0 +1,87 @@ +.. _stats-ripley: + +****************************** +Ripley's K Function Estimators +****************************** + +Spatial correlation functions have been used in the astronomical +context to estimate the probability of finding an object (e.g., a galaxy) +within a given distance of another object [1]_. + +Ripley's K function is a type of estimator used to characterize the correlation +of such spatial point processes +[2]_, [3]_, [4]_, [5]_, [6]_. +More precisely, it describes correlation among objects in a given field. +The `~astropy.stats.RipleysKEstimator` class implements some +estimators for this function which provides several methods for +edge effects correction. + +Basic Usage +=========== + +The actual implementation of Ripley's K function estimators lie in the method +``evaluate``, which take the following arguments: ``data``, ``radii``, and +optionally, ``mode``. + +The ``data`` argument is a 2D array which represents the set of observed +points (events) in the area of study. The ``radii`` argument corresponds to a +set of distances for which the estimator will be evaluated. The ``mode`` +argument takes a value on the following linguistic set +``{none, translation, ohser, var-width, ripley}``; each keyword represents a +different method to perform correction due to edge effects. See the API +documentation and references for details about these methods. + +Instances of `~astropy.stats.RipleysKEstimator` can also be used as +callables (which is equivalent to calling the ``evaluate`` method). + +Example +------- + +.. + EXAMPLE START + Using Ripley's K Function Estimators + +To use Ripley's K Function Estimators from ``astropy``'s stats sub-package: + +.. plot:: + :include-source: + + import numpy as np + from matplotlib import pyplot as plt + from astropy.stats import RipleysKEstimator + + rng = np.random.default_rng() + z = rng.uniform(low=5, high=10, size=(100, 2)) + Kest = RipleysKEstimator(area=25, x_max=10, y_max=10, x_min=5, y_min=5) + + r = np.linspace(0, 2.5, 100) + fig, ax = plt.subplots() + ax.plot(r, Kest.poisson(r), color='green', ls=':', label=r'$K_{pois}$') + ax.plot(r, Kest(data=z, radii=r, mode='none'), color='red', ls='--', + label=r'$K_{un}$') + ax.plot(r, Kest(data=z, radii=r, mode='translation'), color='black', + label=r'$K_{trans}$') + ax.plot(r, Kest(data=z, radii=r, mode='ohser'), color='blue', ls='-.', + label=r'$K_{ohser}$') + ax.plot(r, Kest(data=z, radii=r, mode='var-width'), color='green', + label=r'$K_{var-width}$') + ax.plot(r, Kest(data=z, radii=r, mode='ripley'), color='yellow', + label=r'$K_{ripley}$') + ax.legend() + +.. + EXAMPLE END + +References +========== +.. [1] Peebles, P.J.E. *The large scale structure of the universe*. + +.. [2] Ripley, B.D. *The second-order analysis of stationary point processes*. + Journal of Applied Probability. 13: 255–266, 1976. +.. [3] *Spatial descriptive statistics*. + +.. [4] Cressie, N.A.C. *Statistics for Spatial Data*, Wiley, New York. +.. [5] Stoyan, D., Stoyan, H. *Fractals, Random Shapes and Point Fields*, + Akademie Verlag GmbH, Chichester, 1992. +.. [6] *Correlation function*. + diff --git a/docs/stats/robust.rst b/docs/stats/robust.rst new file mode 100644 index 000000000000..6e619f7737e9 --- /dev/null +++ b/docs/stats/robust.rst @@ -0,0 +1,206 @@ +.. _stats-robust: + +.. testsetup:: + + >>> import numpy as np + >>> np.random.seed(0) + +***************************** +Robust Statistical Estimators +***************************** + +Robust statistics provides reliable estimates of basic statistics for complex +distributions. The statistics package includes several robust statistical +functions that are commonly used in astronomy. This includes methods for +rejecting outliers as well as statistical description of the underlying +distributions. + +In addition to the functions mentioned here, models can be fit with outlier +rejection using :func:`~astropy.modeling.fitting.FittingWithOutlierRemoval`. + +Sigma Clipping +============== + +Sigma clipping provides a fast method for identifying outliers in a +distribution. For a distribution of points, a center and a standard +deviation are calculated. Values which are less or more than a +specified number of standard deviations from a center value are +rejected. The process can be iterated to further reject outliers. + +The `astropy.stats` package provides both a functional and +object-oriented interface for sigma clipping. The function is called +:func:`~astropy.stats.sigma_clip` and the class is called +:class:`~astropy.stats.SigmaClip`. By default, they both return a +masked array where the rejected points are masked. + +Examples +-------- + +.. + EXAMPLE START + Functional Sigma Clipping with astropy.stats.sigma_clip + +We can start by generating some data that has a mean of 0 and standard +deviation of 0.2, but with outliers: + +.. doctest-requires:: scipy + + >>> import numpy as np + >>> import scipy.stats as stats + >>> rng = np.random.default_rng(0) + >>> x = np.arange(200) + >>> y = np.zeros(200) + >>> c = stats.bernoulli.rvs(0.35, size=x.shape) + >>> y += (rng.normal(0., 0.2, x.shape) + + ... c * rng.normal(3.0, 5.0, x.shape)) + +Now we can use :func:`~astropy.stats.sigma_clip` to perform sigma +clipping on the data: + +.. doctest-requires:: scipy + + >>> from astropy.stats import sigma_clip + >>> filtered_data = sigma_clip(y, sigma=3, maxiters=10) + +The output masked array then can be used to calculate statistics on +the data, fit models to the data, or otherwise explore the data. + +.. + EXAMPLE END + +.. + EXAMPLE START + Object-Oriented Sigma Clipping with the astropy.stats.SigmaClip Class + +To perform the same sigma clipping with the +:class:`~astropy.stats.SigmaClip` class: + +.. doctest-requires:: scipy + + >>> from astropy.stats import SigmaClip + >>> sigclip = SigmaClip(sigma=3, maxiters=10) + >>> print(sigclip) # doctest: +SKIP + + sigma: 3 + sigma_lower: None + sigma_upper: None + maxiters: 10 + cenfunc: + stdfunc: + >>> filtered_data = sigclip(y) + +Note that once the ``sigclip`` instance is defined above, it can be +applied to other data using the same already defined sigma-clipping +parameters. + +.. + EXAMPLE END + +For basic statistics, :func:`~astropy.stats.sigma_clipped_stats` is a +convenience function to calculate the sigma-clipped mean, median, and +standard deviation of an array. As can be seen, rejecting the +outliers returns accurate values for the underlying distribution. + +.. + EXAMPLE START + Calculating the Sigma-Clipped Mean, Median, and Standard Deviation of an Array + +To use :func:`~astropy.stats.sigma_clipped_stats` for sigma-clipped statistics +calculation: + +.. doctest-requires:: scipy + + >>> from astropy.stats import sigma_clipped_stats + >>> y.mean(), np.median(y), y.std() # doctest: +FLOAT_CMP + (np.float64(0.7068938765410144), np.float64(0.013567387681385379), np.float64(3.599605215851649)) + >>> sigma_clipped_stats(y, sigma=3, maxiters=10) # doctest: +FLOAT_CMP + (np.float64(-0.0228473012826993), np.float64(-0.02356858871405204), np.float64(0.2079616996908159)) + + +:class:`~astropy.stats.SigmaClippedStats` is a +convenience class that extends the functionality of +:func:`~astropy.stats.sigma_clipped_stats`: + +.. doctest-requires:: scipy + + >>> from astropy.stats import SigmaClippedStats + >>> stats = SigmaClippedStats(y, sigma=3, maxiters=10) + >>> stats.mean(), stats.median(), stats.std() # doctest: +FLOAT_CMP + (np.float64(-0.0228473012826993), np.float64(-0.02356858871405204), np.float64(0.2079616996908159)) + >>> stats.mode(), stats.var(), stats.mad_std() # doctest: +FLOAT_CMP + (np.float64(-0.025011163576757534), np.float64(0.043248068538293126), np.float64(0.21277510956855722)) + >>> stats.biweight_location(), stats.biweight_scale() # doctest: +FLOAT_CMP + (np.float64(-0.0183718864859565), np.float64(0.21730062377965248)) + +:func:`~astropy.stats.sigma_clip` and +:class:`~astropy.stats.SigmaClip` can be combined with other robust +statistics to provide improved outlier rejection as well. + +.. plot:: + :include-source: + + import numpy as np + import scipy.stats as stats + from matplotlib import pyplot as plt + from astropy.stats import sigma_clip, mad_std + + # Generate fake data that has a mean of 0 and standard deviation of 0.2 with outliers + rng = np.random.default_rng(0) + x = np.arange(200) + y = np.zeros(200) + c = stats.bernoulli.rvs(0.35, size=x.shape) + y += (rng.normal(0., 0.2, x.shape) + + c * rng.normal(3.0, 5.0, x.shape)) + + filtered_data = sigma_clip(y, sigma=3, maxiters=1, stdfunc=mad_std) + + # plot the original and rejected data + fig, ax = plt.subplots(figsize=(8, 5)) + ax.plot(x, y, '+', color='#1f77b4', label="original data") + ax.plot(x[filtered_data.mask], y[filtered_data.mask], 'x', + color='#d62728', label="rejected data") + ax.set(xlabel='x', ylabel='y') + ax.legend(loc=2, numpoints=1) + +.. automodapi:: astropy.stats.sigma_clipping + +.. + EXAMPLE END + +Median Absolute Deviation +========================= + +The median absolute deviation (MAD) is a measure of the spread of a +distribution and is defined as ``median(abs(a - median(a)))``. The +MAD can be calculated using `~astropy.stats.median_absolute_deviation`. For a +normal distribution, the MAD is related to the standard deviation by a factor +of 1.4826, and a convenience function, `~astropy.stats.mad_std`, is +available to apply the conversion. + +.. note:: + + A function can be supplied to the + `~astropy.stats.median_absolute_deviation` to specify the median + function to be used in the calculation. Depending on the version + of NumPy and whether the array is masked or contains irregular + values, significant performance increases can be had by + preselecting the median function. If the median function is not + specified, `~astropy.stats.median_absolute_deviation` will attempt + to select the most relevant function according to the input data. + + +Biweight Estimators +=================== + +A set of functions are included in the `astropy.stats` package that use the +biweight formalism. These functions have long been used in astronomy, +particularly to calculate the velocity dispersion of galaxy clusters [1]_. The +following set of tasks are available for biweight measurements: + +.. automodapi:: astropy.stats.biweight + + +References +---------- + +.. [1] Beers, Flynn, and Gebhardt (1990; AJ 100, 32) (https://ui.adsabs.harvard.edu/abs/1990AJ....100...32B) diff --git a/docs/table/access_table.rst b/docs/table/access_table.rst index db166ec2e4a5..af84b6c88eea 100644 --- a/docs/table/access_table.rst +++ b/docs/table/access_table.rst @@ -1,25 +1,22 @@ .. _access_table: -.. include:: references.txt +Accessing a Table +***************** -Accessing a table ------------------ +Accessing table properties and data is generally consistent with the basic +interface for ``numpy`` `structured arrays +`_. -Accessing the table properties and data is straightforward and is generally consistent with -the basic interface for `numpy` structured arrays. +Basics +====== -Quick overview -^^^^^^^^^^^^^^ - -For the impatient, the code below shows the basics of accessing table data. -Where relevant there is a comment about what sort of object. Except where -noted, the table access returns objects that can be modified in order to -update table data or properties. -In cases where is returned and how -the data contained in that object relate to the original table data -(i.e. whether it is a copy or reference, see :ref:`copy_versus_reference`). +For a quick overview, the code below shows the basics of accessing table data. +Where relevant, there is a comment about what sort of object is returned. +Except where noted, table access returns objects that can be modified in order +to update the original table data or properties. See also the section on +:ref:`copy_versus_reference` to learn more about this topic. -**Make table** +**Make a table** :: from astropy.table import Table @@ -31,7 +28,7 @@ the data contained in that object relate to the original table data **Table properties** :: - t.columns # Dict of table columns + t.columns # Dict of table columns (access by column name, index, or slice) t.colnames # List of column names t.meta # Dict of meta-data len(t) # Number of table rows @@ -41,8 +38,9 @@ the data contained in that object relate to the original table data t['a'] # Column 'a' t['a'][1] # Row 1 of column 'a' - t[1] # Row obj for with row 1 values + t[1] # Row 1 t[1]['a'] # Column 'a' of row 1 + t[1][1:] # Row 1, columns b and c t[2:5] # Table object with rows 2:5 t[[1, 3, 4]] # Table object with rows 1, 3, 4 (copy) t[np.array([1, 3, 4])] # Table object with rows 1, 3, 4 (copy) @@ -51,43 +49,48 @@ the data contained in that object relate to the original table data dat = np.array(t) # Copy table data to numpy structured array object t['a'].quantity # an astropy.units.Quantity for Column 'a' t['a'].to('km') # an astropy.units.Quantity for Column 'a' in units of kilometers + t.columns[1] # Column 1 (which is the 'b' column) + t.columns[0:2] # New table with columns 0 and 1 .. Note:: Although they appear nearly equivalent, there is a factor of two performance difference between ``t[1]['a']`` (slower, because an intermediate |Row| - object gets created) versus ``t['a'][1]`` (faster). Always use the latter + object gets created) versus ``t['a'][1]`` (faster). Always use the latter when possible. **Print table or column** :: - print t # Print formatted version of table to the screen + print(t) # Print formatted version of table to the screen t.pprint() # Same as above t.pprint(show_unit=True) # Show column unit t.pprint(show_name=False) # Do not show column names - t.pprint(max_lines=-1, max_width=-1) # Print full table no matter how long / wide it is + t.pprint_all() # Print full table no matter how long / wide it is (same as t.pprint(max_lines=-1, max_width=-1)) t.more() # Interactively scroll through table like Unix "more" - print t['a'] # Formatted column values + print(t['a']) # Formatted column values t['a'].pprint() # Same as above, with same options as Table.pprint() - t['a'].more() # Interactively scroll through column + t['a'].more() # Interactively scroll through column + t['a', 'c'].pprint() # Print columns 'a' and 'c' of table lines = t.pformat() # Formatted table as a list of lines (same options as pprint) lines = t['a'].pformat() # Formatted column values as a list Details -^^^^^^^ +======= -For all the following examples it is assumed that the table has been created as below:: +For all of the following examples it is assumed that the table has been created +as follows:: >>> from astropy.table import Table, Column >>> import numpy as np + >>> import astropy.units as u >>> arr = np.arange(15, dtype=np.int32).reshape(5, 3) >>> t = Table(arr, names=('a', 'b', 'c'), meta={'keywords': {'key1': 'val1'}}) - >>> t['a'].format = "%6.3f" # print as a float with 3 digits after decimal point + >>> t['a'].format = "{:.3f}" # print with 3 digits after decimal point >>> t['a'].unit = 'm sec^-1' >>> t['a'].description = 'unladen swallow velocity' >>> print(t) @@ -100,12 +103,82 @@ For all the following examples it is assumed that the table has been created as 9.000 10 11 12.000 13 14 -Accessing properties -"""""""""""""""""""" +.. Note:: + + In the example above the ``format``, ``unit``, and ``description`` + attributes of the |Column| were set directly. For :ref:`mixin_columns` like + |Quantity| you must set via the ``info`` attribute, for example, + ``t['a'].info.format = "{:.3f}"``. You can use the ``info`` attribute with + |Column| objects as well, so the general solution that works with any table + column is to set via the ``info`` attribute. See :ref:`mixin_attributes` for + more information. + +.. _table-summary-information: + +Summary Information +------------------- + +You can get summary information about the table as follows:: + + >>> t.info + + name dtype unit format description + ---- ----- -------- ------ ------------------------ + a int32 m sec^-1 {:.3f} unladen swallow velocity + b int32 + c int32 + +If called as a function then you can supply an ``option`` that specifies +the type of information to return. The built-in ``option`` choices are +``'attributes'`` (column attributes, which is the default) or ``'stats'`` +(basic column statistics). The ``option`` argument can also be a list +of available options:: + + >>> t.info('stats') # doctest: +FLOAT_CMP +
+ name mean std min max + ---- ---- ------- --- --- + a 6 4.24264 0 12 + b 7 4.24264 1 13 + c 8 4.24264 2 14 + + >>> t.info(['attributes', 'stats']) # doctest: +FLOAT_CMP +
+ name dtype unit format description mean std min max + ---- ----- -------- ------ ------------------------ ---- ------- --- --- + a int32 m sec^-1 {:.3f} unladen swallow velocity 6 4.24264 0 12 + b int32 7 4.24264 1 13 + c int32 8 4.24264 2 14 + +Columns also have an ``info`` property that has the same behavior and +arguments, but provides information about a single column:: + + >>> t['a'].info + name = a + dtype = int32 + unit = m sec^-1 + format = {:.3f} + description = unladen swallow velocity + class = Column + n_bad = 0 + length = 5 + + >>> t['a'].info('stats') # doctest: +FLOAT_CMP + name = a + mean = 6 + std = 4.24264 + min = 0 + max = 12 + n_bad = 0 + length = 5 + + +Accessing Properties +-------------------- The code below shows accessing the table columns as a |TableColumns| object, -getting the column names, table meta-data, and number of table rows. The table -meta-data is simply an ordered dictionary (OrderedDict_) by default. +getting the column names, table metadata, and number of table rows. The table +metadata is a :class:`dict` by default. :: >>> t.columns @@ -121,14 +194,14 @@ meta-data is simply an ordered dictionary (OrderedDict_) by default. 5 -Accessing data -"""""""""""""" +Accessing Data +-------------- As expected you can access a table column by name and get an element from that column with a numerical index:: >>> t['a'] # Column 'a' - + 0.000 3.000 6.000 @@ -137,10 +210,10 @@ column with a numerical index:: >>> t['a'][1] # Row 1 of column 'a' - 3 + np.int32(3) When a table column is printed, it is formatted according to the ``format`` -attribute (see :ref:`table_format_string`). Note the difference between the +attribute (see :ref:`table_format_string`). Note the difference between the column representation above and how it appears via ``print()`` or ``str()``:: >>> print(t['a']) @@ -156,28 +229,31 @@ column representation above and how it appears via ``print()`` or ``str()``:: Likewise a table row and a column from that row can be selected:: >>> t[1] # Row object corresponding to row 1 - + + a b c + m sec^-1 + int32 int32 int32 + -------- ----- ----- + 3.000 4 5 >>> t[1]['a'] # Column 'a' of row 1 - 3 + np.int32(3) -A |Row| object has the same columns and meta-data as its parent table:: +A |Row| object has the same columns and metadata as its parent table:: >>> t[1].columns - >>> t[1].colnames - ['a', 'b', 'c'] + >>> t[1].meta + {'keywords': {'key1': 'val1'}} -Slicing a table returns a new table object which references to the original -data within the slice region (See :ref:`copy_versus_reference`). The table -meta-data and column definitions are copied. +Slicing a table returns a new table object with references to the original +data within the slice region (See :ref:`copy_versus_reference`). The table +metadata and column definitions are copied. :: >>> t[2:5] # Table object with rows 2:5 (reference) -
+
a b c m sec^-1 int32 int32 int32 @@ -187,7 +263,7 @@ meta-data and column definitions are copied. 12.000 13 14 It is possible to select table rows with an array of indexes or by specifying -multiple column names. This returns a copy of the original table for the +multiple column names. This returns a copy of the original table for the selected rows or columns. :: >>> print(t[[1, 3, 4]]) # Table object with rows 1, 3, 4 (copy) @@ -219,41 +295,159 @@ selected rows or columns. :: 9.000 11 12.000 14 -Finally, you can access the underlying table data as a native `numpy` -structured array by creating a copy or reference with ``np.array``:: +We can select rows from a table using conditionals to create boolean masks. A +table indexed with a boolean array will only return rows where the mask array +element is `True`. Different conditionals can be combined using the bitwise +operators. :: + + >>> mask = (t['a'] > 4) & (t['b'] > 8) # Table rows where column a > 4 + >>> print(t[mask]) # and b > 8 + ... + a b c + m sec^-1 + -------- --- --- + 9.000 10 11 + 12.000 13 14 + +Finally, you can access the underlying table data as a native ``numpy`` +structured array by creating a copy or reference with :func:`numpy.array`:: >>> data = np.array(t) # copy of data in t as a structured array >>> data = np.array(t, copy=False) # reference to data in t -Formatted printing -"""""""""""""""""" +Possibly missing columns +^^^^^^^^^^^^^^^^^^^^^^^^ + +In some cases it might not be guaranteed that a column is present in a table, +but there does exist a good default value that can be used if it is not. The +columns of a |Table| can be represented as a :class:`dict` subclass instance +through the ``columns`` attribute, which means that a replacement for missing +columns can be provided using the :meth:`dict.get` method:: + + >>> t.columns.get("b", np.zeros(len(t))) + + 1 + 4 + 7 + 10 + 13 + >>> t.columns.get("x", np.zeros(len(t))) + array([0., 0., 0., 0., 0.]) + +In case of a single |Row| it is possible to use its +:meth:`~astropy.table.Row.get` method without having to go through +``columns``:: + + >>> row = t[2] + >>> row.get("c", -1) + np.int32(8) + >>> row.get("y", -1) + -1 + + +Table Equality +-------------- + +We can check table data equality using two different methods: + +- The ``==`` comparison operators. In the general case, this returns a 1D array + with ``dtype=bool`` mapping each row to ``True`` if and only if the *entire row* + matches. For incomparable data (different ``dtype`` or unbroacastable lengths), + a boolean ``False`` is returned. + This is in contrast to the behavior of ``numpy`` where trying to compare + structured arrays might raise exceptions. +- Table :meth:`~astropy.table.Table.values_equal` to compare table values + element-wise. This returns a boolean `True` or `False` for each table + *element*, so you get a `~astropy.table.Table` of values. + +.. note:: both methods will report equality *after* broadcasting, which + matches ``numpy`` array comparison. + +Examples +^^^^^^^^ + +.. EXAMPLE START: Checking Table Equality + +To check table equality:: + + >>> t1 = Table(rows=[[1, 2, 3], + ... [4, 5, 6], + ... [7, 7, 9]], names=['a', 'b', 'c']) + >>> t2 = Table(rows=[[1, 2, -1], + ... [4, -1, 6], + ... [7, 7, 9]], names=['a', 'b', 'c']) + + >>> t1 == t2 + array([False, False, True]) + + >>> t1.values_equal(t2) # Compare to another table +
+ a b c + bool bool bool + ---- ----- ----- + True True False + True False True + True True True + + >>> t1.values_equal([2, 4, 7]) # Compare to an array column-wise +
+ a b c + bool bool bool + ----- ----- ----- + False True False + True False False + True True False + + >>> t1.values_equal(7) # Compare to a scalar column-wise +
+ a b c + bool bool bool + ----- ----- ----- + False False False + False False False + True True False + +.. EXAMPLE END + +Formatted Printing +------------------ The values in a table or column can be printed or retrieved as a formatted table using one of several methods: -- `print` statement (Python 2) or `print()` function (Python 3). -- Table :meth:`~astropy.table.Table.more` or Column - :meth:`~astropy.table.Column.more` methods to interactively scroll - through table values. -- Table :meth:`~astropy.table.Table.pprint` or Column - :func:`~astropy.table.Column.pprint` methods to print a formatted version of +- `print()` function. +- `Table.more() ` or `Column.more() + ` methods to interactively scroll through + table values. +- `Table.pprint() ` or `Column.pprint() + ` methods to print a formatted version of the table to the screen. -- Table :meth:`~astropy.table.Table.pformat` or Column - :func:`~astropy.table.Column.pformat` methods to return the formatted table - or column as a list of fixed-width strings. This could be used as a quick - way to save a table. +- `Table.pformat() ` or `Column.pformat() + ` methods to return the formatted table + or column as a list of fixed-width strings. This could be used as a quick way + to save a table. These methods use :ref:`table_format_string` if available and strive to make the output readable. By default, table and column printing will -not print the table larger than the available interactive screen size. If the +not print the table larger than the available interactive screen size. If the screen size cannot be determined (in a non-interactive environment or on -Windows) then a default size of 25 rows by 80 columns is used. If a table is -too large then rows and/or columns are cut from the middle so it fits. For example:: +Windows) then a default size of 25 rows by 80 columns is used. If a table is +too large, then rows and/or columns are cut from the middle so it fits. + +Example +^^^^^^^ + +.. EXAMPLE START: Printing Formatted Tables + +To print a formatted table:: >>> arr = np.arange(3000).reshape(100, 30) # 100 rows x 30 columns array >>> t = Table(arr) + >>> from astropy.table import conf + >>> conf.max_width = 80 + >>> conf.max_lines = 20 >>> print(t) col0 col1 col2 col3 col4 col5 col6 ... col23 col24 col25 col26 col27 col28 col29 ---- ---- ---- ---- ---- ---- ---- ... ----- ----- ----- ----- ----- ----- ----- @@ -268,7 +462,6 @@ too large then rows and/or columns are cut from the middle so it fits. For exam 240 241 242 243 244 245 246 ... 263 264 265 266 267 268 269 270 271 272 273 274 275 276 ... 293 294 295 296 297 298 299 ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... - 2670 2671 2672 2673 2674 2675 2676 ... 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 ... 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 ... 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 ... 2783 2784 2785 2786 2787 2788 2789 @@ -281,14 +474,16 @@ too large then rows and/or columns are cut from the middle so it fits. For exam 2970 2971 2972 2973 2974 2975 2976 ... 2993 2994 2995 2996 2997 2998 2999 Length = 100 rows + .. EXAMPLE END + more() method -''''''''''''' +^^^^^^^^^^^^^ -In order to browse all rows of a table or column use the Table -:meth:`~astropy.table.Table.more` or Column :func:`~astropy.table.Column.more` -methods. These let you interactively scroll through the rows much like the -linux ``more`` command. Once part of the table or column is displayed the -supported navigation keys are: +In order to browse all rows of a table or column use the `Table.more() +` or `Column.more() ` +methods. These let you interactively scroll through the rows much like the Unix +``more`` command. Once part of the table or column is displayed the supported +navigation keys are: | **f, space** : forward one page | **b** : back one page @@ -301,18 +496,17 @@ supported navigation keys are: | **h** : print this help pprint() method -''''''''''''''' +^^^^^^^^^^^^^^^ -In order to fully control the print output use the Table -:meth:`~astropy.table.Table.pprint` or Column -:func:`~astropy.table.Column.pprint` methods. These have keyword -arguments ``max_lines``, ``max_width``, ``show_name``, ``show_unit`` with -meaning as shown below:: +In order to fully control the print output use the `Table.pprint() +` or `Column.pprint() +` methods. These have keyword arguments +``max_lines``, ``max_width``, ``show_name``, ``show_unit``, and +``show_dtype``, with meanings as shown below:: >>> arr = np.arange(3000, dtype=float).reshape(100, 30) >>> t = Table(arr) >>> t['col0'].format = '%e' - >>> t['col1'].format = '%.6f' >>> t['col0'].unit = 'km**2' >>> t['col29'].unit = 'kg sec m**-2' @@ -326,14 +520,13 @@ meaning as shown below:: 2.970000e+03 ... 2999.0 Length = 100 rows - >>> t.pprint(max_lines=8, max_width=40, show_unit=True) - col0 ... col29 - km2 ... kg sec m**-2 - ------------ ... ------------ - 0.000000e+00 ... 29.0 - ... ... ... - 2.940000e+03 ... 2969.0 - 2.970000e+03 ... 2999.0 + >>> t.pprint(max_lines=8, max_width=40, show_unit=False) + col0 ... col29 + ------------ ... ------ + 0.000000e+00 ... 29.0 + ... ... ... + 2.940000e+03 ... 2969.0 + 2.970000e+03 ... 2999.0 Length = 100 rows >>> t.pprint(max_lines=8, max_width=40, show_name=False) @@ -346,11 +539,24 @@ meaning as shown below:: 2.970000e+03 ... 2999.0 Length = 100 rows + >>> t.pprint(max_lines=8, max_width=40, show_dtype=True) + col0 col1 ... col29 + km2 ... kg sec m**-2 + float64 float64 ... float64 + ------------ ------- ... ------------ + 0.000000e+00 1.0 ... 29.0 + ... ... ... ... + 2.970000e+03 2971.0 ... 2999.0 + Length = 100 rows + In order to force printing all values regardless of the output length or width -set ``max_lines`` or ``max_width`` to ``-1``, respectively. For the wide -table in this example you see 6 lines of wrapped output like the following:: +use :meth:`~astropy.table.Table.pprint_all`, which is equivalent to setting +``max_lines`` and ``max_width`` to ``-1`` in :meth:`~astropy.table.Table.pprint`. +:meth:`~astropy.table.Table.pprint_all` takes the same arguments as :meth:`~astropy.table.Table.pprint`. +For the wide table in this example you see six lines of wrapped output like the +following:: - >>> t.pprint(max_lines=8, max_width=-1) # doctest: +SKIP + >>> t.pprint_all(max_lines=8) # doctest: +SKIP col0 col1 col2 col3 col4 col5 col6 col7 col8 col9 col10 col11 col12 col13 col14 col15 col16 col17 col18 col19 col20 col21 col22 col23 col24 col25 col26 col27 col28 col29 km2 kg sec m**-2 ------------ ----------- ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------ ------------ @@ -360,9 +566,8 @@ table in this example you see 6 lines of wrapped output like the following:: 2.970000e+03 2971.000000 2972.0 2973.0 2974.0 2975.0 2976.0 2977.0 2978.0 2979.0 2980.0 2981.0 2982.0 2983.0 2984.0 2985.0 2986.0 2987.0 2988.0 2989.0 2990.0 2991.0 2992.0 2993.0 2994.0 2995.0 2996.0 2997.0 2998.0 2999.0 Length = 100 rows -For columns the syntax and behavior of -:func:`~astropy.table.Column.pprint` is the same except that there is no -``max_width`` keyword argument:: +For columns, the syntax and behavior of :func:`~astropy.table.Column.pprint` is +the same except that there is no ``max_width`` keyword argument:: >>> t['col3'].pprint(max_lines=8) col3 @@ -375,119 +580,326 @@ For columns the syntax and behavior of Length = 100 rows Column alignment -'''''''''''''''' +^^^^^^^^^^^^^^^^ Individual columns have the ability to be aligned in a number of different -ways, for an enhanced viewing experience. - - >>> t1 = Table() - >>> t1['long column name 1'] = [1,2,3] - >>> t1['long column name 2'] = [4,5,6] - >>> t1['long column name 3'] = [7,8,9] - >>> t1['long column name 4'] = [700000,800000,900000] - >>> t1['long column name 2'].format = '<' - >>> t1['long column name 3'].format = '0=' - >>> t1['long column name 4'].format = '^' - >>> t1.pprint() +ways for an enhanced viewing experience:: + + >>> t1 = Table() + >>> t1['long column name 1'] = [1, 2, 3] + >>> t1['long column name 2'] = [4, 5, 6] + >>> t1['long column name 3'] = [7, 8, 9] + >>> t1['long column name 4'] = [700000, 800000, 900000] + >>> t1['long column name 2'].info.format = '<' + >>> t1['long column name 3'].info.format = '0=' + >>> t1['long column name 4'].info.format = '^' + >>> t1.pprint() long column name 1 long column name 2 long column name 3 long column name 4 ------------------ ------------------ ------------------ ------------------ - 1 4 000000000000000007 700000 - 2 5 000000000000000008 800000 - 3 6 000000000000000009 900000 - -Conveniently, alignment can be handled another way, by passing a list to the -keyword argument ``align``. - - >>> t1 = Table() - >>> t1['column1'] = [1,2,3,4,5] - >>> t1['column2'] = [2,4,6,8,10] - >>> t1.pprint(align=['<','0=']) - column1 column2 - ------- ------- - 1 0000002 - 2 0000004 - 3 0000006 - 4 0000008 - 5 0000010 - -By default, if the length of the list does not match the number of columns -within the table, alignment defaults to right-aligned columns. This default -behavior also holds true even if a single alignment character is passed in as a -list (e.g., align=['^']), where global alignment of all columns within a table -is intended. For very large tables, this can be a nuissance, and so looping is the -recommended solution for this task. - + 1 4 000000000000000007 700000 + 2 5 000000000000000008 800000 + 3 6 000000000000000009 900000 + +Conveniently, alignment can be handled another way — by passing a list to the +keyword argument ``align``:: + + >>> t1 = Table() + >>> t1['column1'] = [1, 2, 3] + >>> t1['column2'] = [2, 4, 6] + >>> t1.pprint(align=['<', '0=']) + column1 column2 + ------- ------- + 1 0000002 + 2 0000004 + 3 0000006 + +It is also possible to set the alignment of all columns with a single +string value:: + + >>> t1.pprint(align='^') + column1 column2 + ------- ------- + 1 2 + 2 4 + 3 6 + +The fill character for justification can be set as a prefix to the +alignment character (see `Format Specification Mini-Language +`_ +for additional explanation). This can be done both in the ``align`` argument +and in the column ``format`` attribute. Note the interesting interaction below:: + + >>> t1 = Table([[1.0, 2.0], [1, 2]], names=['column1', 'column2']) + + >>> t1['column1'].format = '#^.2f' + >>> t1.pprint() + column1 column2 + ------- ------- + ##1.00# 1 + ##2.00# 2 + +Now if we set a global align, it seems like our original column format +got lost:: + + >>> t1.pprint(align='!<') + column1 column2 + ------- ------- + 1.00!!! 1!!!!!! + 2.00!!! 2!!!!!! + +The way to avoid this is to explicitly specify the alignment strings +for every column and use `None` where the column format should be +used:: + + >>> t1.pprint(align=[None, '!<']) + column1 column2 + ------- ------- + ##1.00# 1!!!!!! + ##2.00# 2!!!!!! + pformat() method -'''''''''''''''' +^^^^^^^^^^^^^^^^ In order to get the formatted output for manipulation or writing to a file use -the Table :meth:`~astropy.table.Table.pformat` or Column -:func:`~astropy.table.Column.pformat` methods. These behave just as for -:meth:`~astropy.table.Table.pprint` but return a list corresponding to each formatted line in the -:meth:`~astropy.table.Table.pprint` output. +the `Table.pformat() ` or `Column.pformat() +` methods. These behave just as for +:meth:`~astropy.table.Table.pprint` but return a list corresponding to each +formatted line in the :meth:`~astropy.table.Table.pprint` output. The +:meth:`~astropy.table.Table.pformat_all` method can be used to return a list +for all lines in the |Table|. >>> lines = t['col3'].pformat(max_lines=8) +Hiding columns +^^^^^^^^^^^^^^ + +The |Table| class has functionality to selectively show or hide certain columns +within the table when using any of the print methods. This can be useful for +columns that are very wide or else "uninteresting" for various reasons. The +specification of which columns are outputted is associated with the table itself +so that it persists through slicing, copying, and serialization (e.g. saving to +:ref:`ecsv_format`). One use case is for specialized table subclasses that +contain auxiliary columns that are not typically useful to the user. + +The specification of which columns to include when printing is handled through +two complementary |Table| attributes: + +- `~astropy.table.Table.pprint_include_names`: column names to include, where + the default value of `None` implies including all columns. +- `~astropy.table.Table.pprint_exclude_names`: column names to exclude, where + the default value of `None` implies excluding no columns. + +Typically you should use just one of the two attributes at a time. However, +both can be set at once and the set of columns that actually gets printed +is conceptually expressed in this pseudo-code:: + + include_names = (set(table.pprint_include_names() or table.colnames) + - set(table.pprint_exclude_names() or ()) + +Examples +"""""""" +Let's start with defining a simple table with one row and six columns:: + + >>> from astropy.table.table_helpers import simple_table + >>> t = simple_table(size=1, cols=6) + >>> print(t) + a b c d e f + --- --- --- --- --- --- + 1 1.0 c 4 4.0 f + +Now you can get the value of the ``pprint_include_names`` attribute by calling +it as a function, and then include some names for printing:: + + >>> print(t.pprint_include_names()) + None + >>> t.pprint_include_names = ('a', 'c', 'e') + >>> print(t.pprint_include_names()) + ('a', 'c', 'e') + >>> print(t) + a c e + --- --- --- + 1 c 4.0 + +Now you can instead exclude some columns from printing. Note that for both +include and exclude, you can add column names that do not exist in the table. +This allows pre-defining the attributes before the table has been fully +constructed. +:: + + >>> t.pprint_include_names = None # Revert to printing all columns + >>> t.pprint_exclude_names = ('a', 'c', 'e', 'does-not-exist') + >>> print(t) + b d f + --- --- --- + 1.0 4 f + +Next you can ``add`` or ``remove`` names from the attribute:: + + >>> t = simple_table(size=1, cols=6) # Start with a fresh table + >>> t.pprint_exclude_names.add('b') # Single name + >>> t.pprint_exclude_names.add(['d', 'f']) # List or tuple of names + >>> t.pprint_exclude_names.remove('f') # Single name or list/tuple of names + >>> t.pprint_exclude_names() + ('b', 'd') + +Finally, you can temporarily set the attributes within a `context manager +`_. For +example:: + + >>> t = simple_table(size=1, cols=6) + >>> t.pprint_include_names = ('a', 'b') + >>> print(t) + a b + --- --- + 1 1.0 + + >>> # Show all (for pprint_include_names the value of None => all columns) + >>> with t.pprint_include_names.set(None): + ... print(t) + a b c d e f + --- --- --- --- --- --- + 1 1.0 c 4 4.0 f + +The specification of names for these attributes can include Unix-style globs +like ``*`` and ``?``. See `fnmatch` for details (and in particular how to +escape those characters if needed). For example:: + + >>> t = Table() + >>> t.pprint_exclude_names = ['boring*'] + >>> t['a'] = [1] + >>> t['b'] = ['b'] + >>> t['boring_ra'] = [122.0] + >>> t['boring_dec'] = [89.9] + >>> print(t) + a b + --- --- + 1 b + Multidimensional columns -'''''''''''''''''''''''' +^^^^^^^^^^^^^^^^^^^^^^^^ If a column has more than one dimension then each element of the column is -itself an array. In the example below there are 3 rows, each of which is a -``2 x 2`` array. The formatted output for such a column shows only the first +itself an array. In the example below there are three rows, each of which is a +``2 x 2`` array. The formatted output for such a column shows only the first and last value of each row element and indicates the array dimensions in the column name header:: - >>> from astropy.table import Table, Column - >>> import numpy as np >>> t = Table() - >>> arr = [ np.array([[ 1, 2], - ... [10, 20]]), - ... np.array([[ 3, 4], - ... [30, 40]]), - ... np.array([[ 5, 6], - ... [50, 60]]) ] + >>> arr = [ np.array([[ 1., 2.], + ... [10., 20.]]), + ... np.array([[ 3., 4.], + ... [30., 40.]]), + ... np.array([[ 5., 6.], + ... [50., 60.]]) ] >>> t['a'] = arr >>> t['a'].shape (3, 2, 2) >>> t.pprint() - a [2,2] - ------- - 1 .. 20 - 3 .. 40 - 5 .. 60 + a + ----------- + 1.0 .. 20.0 + 3.0 .. 40.0 + 5.0 .. 60.0 + + +There are two ways to see all of the data values for a multidimensional column. First, +you can set the ``astropy.table.conf.format_size_threshold`` configuration option +to a value greater than or equal to the number of items in each column row (4 in this +case). This respects any formatting options that are defined for the column. + +.. code-block:: python -In order to see all the data values for a multidimensional column use the -column representation. This uses the standard `numpy` mechanism for printing -any array:: + >>> from astropy.table import conf + >>> with conf.set_temp("format_size_threshold", 4): + ... t.pprint() + ... + a + ----------------------- + [[1.0 2.0] [10.0 20.0]] + [[3.0 4.0] [30.0 40.0]] + [[5.0 6.0] [50.0 60.0]] + +A second option is to print the column as a ``numpy`` array which uses the +standard ``numpy`` mechanism for printing the array:: >>> t['a'].data - array([[[ 1, 2], - [10, 20]], - [[ 3, 4], - [30, 40]], - [[ 5, 6], - [50, 60]]]) - -Columns and Quantities -'''''''''''''''''''''' -Columns with units that the `astropy.units` package understands can be -converted explicitly to ``~astropy.units.Quantity`` objects via the + array([[[ 1., 2.], + [10., 20.]], + [[ 3., 4.], + [30., 40.]], + [[ 5., 6.], + [50., 60.]]]) + +.. _format_stuctured_array_columns: + +Structured array columns +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. EXAMPLE START: Creating a formatted Astropy Table with a Structured Column + +For columns which are structured arrays, the format string must be a string +that uses `"new style" format strings +`_ with +parameter substitutions corresponding to the field names in the structured +array. Consider the example below including a column of parameters values where +the value, min and max are stored in the in the column as fields named ``val``, +``min``, and ``max``. By default the field values are shown as a tuple:: + + >>> pars = np.array( + ... [(1.2345678, -20, 3), + ... (12.345678, 4.5678, 33)], + ... dtype=[('val', 'f8'), ('min', 'f8'), ('max', 'f8')] + ... ) + >>> t = Table() + >>> t['a'] = [1, 2] + >>> t['par'] = pars + >>> print(t) + a par [val, min, max] + --- ------------------------- + 1 (1.2345678, -20.0, 3.0) + 2 (12.345678, 4.5678, 33.0) + + +However, setting the format string appropriately allows formatting each of the +field values and controlling the overall output:: + + >>> t['par'].info.format = '{val:6.2f} ({min:5.1f}, {max:5.1f})' + >>> print(t) + a par [val, min, max] + --- --------------------- + 1 1.23 (-20.0, 3.0) + 2 12.35 ( 4.6, 33.0) + +.. EXAMPLE END + +.. _columns_with_units: + +Columns with Units +^^^^^^^^^^^^^^^^^^ + +.. note:: + + |Table| and |QTable| instances handle entries with units differently. The + following describes |Table|. :ref:`quantity_and_qtable` explains how a + |QTable| differs from a |Table|. + +A |Column| object with units within a standard |Table| has certain +quantity-related conveniences available. To begin with, it can be converted +explicitly to a |Quantity| object via the :attr:`~astropy.table.Column.quantity` property and the :meth:`~astropy.table.Column.to` method:: - >>> from astropy.table import Table - >>> from astropy import units as u - >>> data = [[1., 2., 3.],[40000., 50000., 60000.]] + >>> data = [[1., 2., 3.], [40000., 50000., 60000.]] >>> t = Table(data, names=('a', 'b')) >>> t['a'].unit = u.m >>> t['b'].unit = 'km/s' - >>> t['a'].quantity - + >>> t['a'].quantity # doctest: +FLOAT_CMP + >>> t['b'].to(u.kpc/u.Myr) # doctest: +FLOAT_CMP - + Note that the :attr:`~astropy.table.Column.quantity` property is actually -a *view* of the data in the column, not a copy. Hence, you can set the +a *view* of the data in the column, not a copy. Hence, you can set the values of a column in a way that respects units by making in-place changes to the :attr:`~astropy.table.Column.quantity` property:: @@ -504,27 +916,24 @@ changes to the :attr:`~astropy.table.Column.quantity` property:: 50000.0 60000.0 -Even without explicit conversion, columns with units can be treated like -like an Astropy `~astropy.units.Quantity` in *some* arithmetic -expressions (see the warning below for caveats to this):: +Even without explicit conversion, columns with units can be treated like a +|Quantity| in *some* arithmetic expressions (see the warning below for caveats +to this):: - >>> t['a'] + .005*u.km - + >>> t['a'] + .005*u.km # doctest: +FLOAT_CMP + >>> from astropy.constants import c >>> (t['b'] / c).decompose() # doctest: +FLOAT_CMP - + .. warning:: - Table columns do *not* always behave the same as - `~astropy.units.Quantity`. Table columns act more like regular numpy - arrays unless either explicitly converted to a - `~astropy.units.Quantity` or combined with an - `~astropy.units.Quantity` using an arithmetic operator.For example, - the following does not work the way you would expect:: + |Table| columns do *not* always behave the same as |Quantity|. |Table| + columns act more like regular ``numpy`` arrays unless either explicitly + converted to a |Quantity| or combined with a |Quantity| using an arithmetic + operator. For example, the following does not work in the way you would + expect:: - >>> import numpy as np - >>> from astropy.table import Table >>> data = [[30, 90]] >>> t = Table(data, names=('angle',)) >>> t['angle'].unit = 'deg' @@ -533,10 +942,84 @@ expressions (see the warning below for caveats to this):: -0.988031624093 0.893996663601 - This is wrong both in that it says the unit is degrees, *and* ``sin`` - treated the values and radians rather than degrees. If at all in - doubt that you'll get the right result, the safest choice is to - explicitly convert to `~astropy.units.Quantity`:: + This is wrong both in that it says the result is in degrees, *and* + `~numpy.sin` treated the values as radians rather than degrees. If at all in + doubt that you will get the right result, the safest choice is to either use + |QTable| or to explicitly convert to |Quantity|:: >>> np.sin(t['angle'].quantity) # doctest: +FLOAT_CMP - + + +.. _bytestring-columns-python-3: + +Bytestring Columns +^^^^^^^^^^^^^^^^^^ + +Using bytestring columns (``numpy`` ``'S'`` dtype) is possible +with ``astropy`` tables since they can be compared with the natural +Python string (``str``) type. See `The bytes/str dichotomy in Python 3 +`_ +for a very brief overview of the difference. + +The standard method of representing strings in ``numpy`` is via the +unicode ``'U'`` dtype. The problem is that this requires 4 bytes per +character, and if you have a very large number of strings this could +fill memory and impact performance. A very common use case is that these +strings are actually ASCII and can be represented with 1 byte per character. +In ``astropy`` it is possible to work directly and conveniently with +bytestring data in |Table| and |Column| operations. + +Note that the bytestring issue is a particular problem when dealing with HDF5 +files, where character data are read as bytestrings (``'S'`` dtype) when using +the :ref:`table_io`. Since HDF5 files are frequently used to store very large +datasets, the memory bloat associated with conversion to ``'U'`` dtype is +unacceptable. + + +Examples +"""""""" + +.. EXAMPLE START: Bytestring Data in Astropy Tables + +The examples below illustrate dealing with bytestring data in ``astropy``:: + + >>> t = Table([['abc', 'def']], names=['a'], dtype=['S']) + + >>> t['a'] == 'abc' # Gives expected answer + array([ True, False]) + + >>> t['a'] == b'abc' # Still gives expected answer + array([ True, False]) + + >>> t['a'][0] == 'abc' # Expected answer + True + + >>> t['a'][0] == b'abc' # Cannot compare to bytestring + False + + >>> t['a'][0] = 'bä' + >>> t +
+ a + bytes3 + ------ + bä + def + + >>> t['a'] == 'bä' + array([ True, False]) + +.. doctest-skip:: + + >>> # Round trip unicode strings through HDF5 + >>> t.write('test.hdf5', format='hdf5', path='data', overwrite=True) + >>> t2 = Table.read('test.hdf5', format='hdf5', path='data') + >>> t2 +
+ col0 + bytes3 + ------ + bä + def + +.. EXAMPLE END diff --git a/docs/table/construct_table.rst b/docs/table/construct_table.rst index 70aadc3f288e..27c6d2da47d5 100644 --- a/docs/table/construct_table.rst +++ b/docs/table/construct_table.rst @@ -1,63 +1,80 @@ -.. include:: references.txt - .. _construct_table: -Constructing a table --------------------- +Constructing a Table +******************** There is great deal of flexibility in the way that a table can be initially -constructed. Details on the inputs to the |Table| -constructor are in the `Initialization Details`_ section. However, the -easiest way to understand how to make a table is by example. +constructed. Details on the inputs to the |Table| and |QTable| +constructors are in the `Initialization Details`_ section. However, the +best way to understand how to make a table is by example. Examples -^^^^^^^^ - -Much of the flexibility lies in the types of data structures -which can be used to initialize the table data. The examples below show how to -create a table from scratch with no initial data, create a table with a list of -columns, a dictionary of columns, or from `numpy` arrays (either structured or -homogeneous). +======== Setup -""""" -For the following examples you need to import the |Table| and |Column| classes -along with the `numpy` package:: +----- + +For the following examples you need to import the |QTable|, |Table|, and +|Column| classes along with the :ref:`astropy-units` package and the ``numpy`` +package:: - >>> from astropy.table import Table, Column + >>> from astropy.table import QTable, Table, Column + >>> from astropy import units as u >>> import numpy as np -Creating from scratch -""""""""""""""""""""" -A Table can be created without any initial input data or even without any -initial columns. This is useful for building tables dynamically if the initial +Creating from Scratch +--------------------- + +.. EXAMPLE START: Creating an Astropy Table from Scratch + +A |Table| can be created without any initial input data or even without any +initial columns. This is useful for building tables dynamically if the initial size, columns, or data are not known. .. Note:: Adding rows requires making a new copy of the entire - table each time, so in the case of large tables this may be slow. - On the other hand, adding columns is quite fast. + table each time, so in the case of large tables this may be slow. + On the other hand, adding columns is fast. :: >>> t = Table() >>> t['a'] = [1, 4] - >>> t['b'] = Column([2.0, 5.0], unit='cm', description='Velocity') + >>> t['b'] = [2.0, 5.0] >>> t['c'] = ['x', 'y'] >>> t = Table(names=('a', 'b', 'c'), dtype=('f4', 'i4', 'S2')) >>> t.add_row((1, 2.0, 'x')) >>> t.add_row((4, 5.0, 'y')) + >>> t = Table(dtype=[('a', 'f4'), ('b', 'i4'), ('c', 'S2')]) + +If your data columns have physical units associated with them then we +recommend using the |QTable| class. This will allow the column to be +stored in the table as a native |Quantity| and bring the full power of +:ref:`astropy-units` to the table. See :ref:`quantity_and_qtable` for details. +:: + + >>> t = QTable() + >>> t['a'] = [1, 4] + >>> t['b'] = [2.0, 5.0] * u.cm / u.s + >>> t['c'] = ['x', 'y'] + >>> type(t['b']) + + +.. EXAMPLE END + +List of Columns +--------------- + +.. EXAMPLE START: Creating an Astropy Table from a List of Columns -List of columns -""""""""""""""" A typical case is where you have a number of data columns with the same length -defined in different variables. These might be Python lists or `numpy` arrays -or a mix of the two. These can be used to create a |Table| by putting the column -data variables into a Python list. In this case the column names are not +defined in different variables. These might be Python lists or ``numpy`` arrays +or a mix of the two. These can be used to create a |Table| by putting the column +data variables into a Python list. In this case the column names are not defined by the input data, so they must either be set using the ``names`` -keyword or they will be auto-generated as ``col``. +keyword or they will be automatically generated as ``col``. :: @@ -66,35 +83,36 @@ keyword or they will be auto-generated as ``col``. >>> c = ['x', 'y'] >>> t = Table([a, b, c], names=('a', 'b', 'c')) >>> t -
- a b c - int32 float64 string8 - ----- ------- ------- - 1 2.0 x - 4 5.0 y +
+ a b c + int32 float64 str1 + ----- ------- ---- + 1 2.0 x + 4 5.0 y +.. EXAMPLE END **Make a new table using columns from the first table** -Once you have a |Table| then you can make new table by selecting columns -and putting this into a Python list, e.g. ``[ t['c'], t['a'] ]``:: +Once you have a |Table|, then you can make a new table by selecting columns +and putting them into a Python list (e.g., ``[ t['c'], t['a'] ]``):: >>> Table([t['c'], t['a']]) -
- c a - string8 int32 - ------- ----- - x 1 - y 4 +
+ c a + str1 int32 + ---- ----- + x 1 + y 4 **Make a new table using expressions involving columns** -The |Column| object is derived from the standard `numpy` array and can be used -directly in arithmetic expressions. This allows for a compact way of making a +The |Column| object is derived from the standard |ndarray| and can be used +directly in arithmetic expressions. This allows for a compact way of making a new table with modified column values:: >>> Table([t['a']**2, t['b'] + 10]) -
+
a b int32 float64 ----- ------- @@ -107,66 +125,73 @@ new table with modified column values:: The list input method for |Table| is very flexible since you can use a mix of different data types to initialize a table:: - >>> a = (1, 4) - >>> b = np.array([[2, 3], [5, 6]]) # vector column + >>> a = (1., 4.) + >>> b = np.array([[2, 3], [5, 6]], dtype=np.int64) # vector column >>> c = Column(['x', 'y'], name='axis') - >>> arr = (a, b, c) - >>> Table(arr) # doctest: +SKIP -
- col0 col1 [2] axis - int64 int64 string8 - ----- -------- ------- - 1 2 .. 3 x - 4 5 .. 6 y + >>> d = u.Quantity([([1., 2., 3.], [.1, .2, .3]), + ... ([4., 5., 6.], [.4, .5, .6])], 'm,m/s') + >>> QTable([a, b, c, d]) + + col0 col1 axis col3 [f0, f1] + (m, m / s) + float64 int64[2] str1 (float64[3], float64[3]) + ------- -------- ---- ---------------------------------- + 1.0 2 .. 3 x ([1.0, 2.0, 3.0], [0.1, 0.2, 0.3]) + 4.0 5 .. 6 y ([4.0, 5.0, 6.0], [0.4, 0.5, 0.6]) Notice that in the third column the existing column name ``'axis'`` is used. +Dict of Columns +--------------- + +.. EXAMPLE START: Creating an Astropy Table from a Dictionary of Columns -Dict of columns -"""""""""""""""" -A dictionary of column data can be used to initialize a |Table|. +A :class:`dict` of column data can be used to initialize a |Table|:: >>> arr = {'a': np.array([1, 4], dtype=np.int32), ... 'b': [2.0, 5.0], ... 'c': ['x', 'y']} >>> - >>> Table(arr) # doctest: +SKIP -
- a c b - int32 string8 float64 - ----- ------- ------- - 1 x 2.0 - 4 y 5.0 + >>> Table(arr) +
+ a b c + int32 float64 str1 + ----- ------- ---- + 1 2.0 x + 4 5.0 y + +.. EXAMPLE END **Specify the column order and optionally the data types** :: - >>> Table(arr, names=('a', 'b', 'c'), dtype=('f8', 'i4', 'S2')) -
- a b c - float64 int32 string16 - ------- ----- -------- - 1.0 2 x - 4.0 5 y + >>> Table(arr, names=('a', 'c', 'b'), dtype=('f8', 'U2', 'i4')) +
+ a c b + float64 str2 int32 + ------- ---- ----- + 1.0 x 2 + 4.0 y 5 **Different types of column data** -The input column data can be any data type that can initialize a |Column| object:: +The input column data can be any data type that can initialize a |Column| +object:: - >>> arr = {'a': (1, 4), - ... 'b': np.array([[2, 3], [5, 6]]), + >>> arr = {'a': (1., 4.), + ... 'b': np.array([[2, 3], [5, 6]], dtype=np.int64), ... 'c': Column(['x', 'y'], name='axis')} - >>> Table(arr, names=('a', 'b', 'c')) # doctest: +SKIP -
- a b [2] c - int64 int64 string8 - ----- ------ ------- - 1 2 .. 3 x - 4 5 .. 6 y + >>> Table(arr, names=('a', 'b', 'c')) +
+ a b c + float64 int64[2] str1 + ------- -------- ---- + 1.0 2 .. 3 x + 4.0 5 .. 6 y Notice that the key ``'c'`` takes precedence over the existing column name -``'axis'`` in the third column. Also see that the ``'b'`` column is a vector -column where each row element is itself a 2-element array. +``'axis'`` in the third column. Also see that the ``'b'`` column is a vector +column where each row element is itself a two-element array. **Renaming columns is not possible** :: @@ -176,13 +201,15 @@ column where each row element is itself a 2-element array. ... KeyError: 'a_new' +.. _Row data: + +List of Rows +------------ -Row data -""""""""" Row-oriented data can be used to create a table using the ``rows`` keyword argument. -**List of data records as list or tuple** +**List or tuple of data records** If you have row-oriented input data such as a list of records, you need to use the ``rows`` keyword to create a table:: @@ -198,30 +225,52 @@ need to use the ``rows`` keyword to create a table:: 4 5.0 y 5 8.2 z -The data object passed as the ``rows`` argument can be any form which is -parsable by the ``np.rec.fromrecords()`` function. - **List of dict objects** -You can also initialize a table with row values. This is constructed as a -list of dict objects. The keys determine the column names:: +You can also initialize a table with row values. This is constructed as a +list of :class:`dict` objects. The keys determine the column names:: >>> data = [{'a': 5, 'b': 10}, ... {'a': 15, 'b': 20}] - >>> Table(rows=data) # doctest: +SKIP -
- a b - int64 int64 - ----- ----- - 5 10 - 15 20 + >>> t = Table(rows=data) + >>> print(t) + a b + --- --- + 5 10 + 15 20 -Every row must have the same set of keys or a ValueError will be thrown:: +If there are missing keys in one or more rows then the corresponding values +will be marked as missing (masked):: - >>> t = Table(rows=[{'a': 5, 'b': 10}, {'a': 15, 'b': 30, 'c': 50}]) - Traceback (most recent call last): - ... - ValueError: Row 0 has no value for column c + >>> t = Table(rows=[{'a': 5, 'b': 10}, {'a': 15, 'c': 50}]) + >>> print(t) + a b c + --- --- --- + 5 10 -- + 15 -- 50 + +You can specify the column order with the ``names`` argument:: + + >>> data = [{'a': 5, 'b': 10}, + ... {'a': 15, 'b': 20}] + >>> t = Table(rows=data, names=('b', 'a')) + >>> print(t) + b a + --- --- + 10 5 + 20 15 + +If ``names`` are not provided then column ordering will be determined by +order in which they appear as the :class:`list` of :class:`dict` is iterated over. + + >>> data = [{'b': 10, 'c': 7, }, + ... {'a': 15, 'c': 35, 'b': 20}] + >>> t = Table(rows=data) + >>> print(t) + b c a + --- --- --- + 10 7 -- + 20 35 15 **Single row** @@ -233,102 +282,128 @@ You can also make a new table from a single row of an existing table:: >>> t2 = Table(rows=t[1]) Remember that a |Row| has effectively a zero length compared to the -newly created |Table| which has a length of one. This is similar to -the difference between a scalar ``1`` (length 0) and an array like +newly created |Table| which has a length of one. This is similar to +the difference between a scalar ``1`` (length 0) and an array such as ``np.array([1])`` with length 1. .. Note:: - In the case of input data as a list of dicts or a single Table row, it is - allowed to supply the data as the ``data`` argument since these forms - are always unambiguous. For example ``Table([{'a': 1}, {'a': 2}])`` is - accepted. However, a list of records must always be provided using the + In the case of input data as a list of dicts or a single |Table| row, you + can supply the data as the ``data`` argument since these forms + are always unambiguous. For example, ``Table([{'a': 1}, {'a': 2}])`` is + accepted. However, a list of records must always be provided using the ``rows`` keyword, otherwise it will be interpreted as a list of columns. -NumPy structured array -"""""""""""""""""""""" -The structured array is the standard mechanism in `numpy` for storing -heterogeneous table data. Most scientific I/O packages that read table -files (e.g. `PyFITS -`_, `vo.table -`_, `asciitable -`_) will return the table in an -object that is based on the structured array. A structured array can be +NumPy Structured Array +---------------------- + +The `structured array `_ is +the standard mechanism in ``numpy`` for storing heterogeneous table data. Most +scientific I/O packages that read table files (e.g., `astropy.io.fits`, +`astropy.io.votable`, and `asciitable +`_) will return the table in an +object that is based on the structured array. A structured array can be created using:: >>> arr = np.array([(1, 2.0, 'x'), ... (4, 5.0, 'y')], - ... dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'S2')]) + ... dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'U2')]) -From ``arr`` it is simple to create the corresponding |Table| object:: +From ``arr`` it is possible to create the corresponding |Table| object:: >>> Table(arr) -
- a b c - int32 float64 string16 - ----- ------- -------- - 1 2.0 x - 4 5.0 y +
+ a b c + int32 float64 str2 + ----- ------- ---- + 1 2.0 x + 4 5.0 y -Note that in the above example and most the following ones we are creating a -table and immediately asking the interactive Python interpreter to print the -table to see what we made. In real code you might do something like:: +Note that in the above example and most of the following examples we are +creating a table and immediately asking the interactive Python interpreter to +print the table to see what we made. In real code you might do something like:: >>> table = Table(arr) - >>> print table + >>> print(table) a b c --- --- --- 1 2.0 x 4 5.0 y +.. _structured-array-as-a-column: + +Structured Array as a Column +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In some cases it is convenient to include a structured array as a single column +in a table. The `~astropy.coordinates.EarthLocation` class is one case in +astropy where this is done, where the structured column has three elements +``x``, ``y`` and ``z``. Another example would be a modeling parameter that has a +value, a minimum allowed value and a maximum allowed value. Here we demonstrate +including the simple structured array defined previously as a column:: + + >>> table = Table() + >>> table['name'] = ['Micah', 'Mazzy'] + >>> table['arr'] = arr + >>> print(table) + name arr [a, b, c] + ----- ------------- + Micah (1, 2.0, 'x') + Mazzy (4, 5.0, 'y') + +You can access or print a single field in the structured column as follows:: + + >>> print(table['arr']['b']) + [2. 5.] + **New column names** The column names can be changed from the original values by providing the ``names`` argument:: >>> Table(arr, names=('a_new', 'b_new', 'c_new')) -
- a_new b_new c_new - int32 float64 string16 - ----- ------- -------- - 1 2.0 x - 4 5.0 y +
+ a_new b_new c_new + int32 float64 str2 + ----- ------- ----- + 1 2.0 x + 4 5.0 y **New data types** -Likewise the data type for each column can by changed with ``dtype``:: +The data type for each column can likewise be changed with ``dtype``:: - >>> Table(arr, dtype=('f4', 'i4', 'S4')) -
- a b c - float32 int32 string32 - ------- ----- -------- - 1.0 2 x - 4.0 5 y + >>> Table(arr, dtype=('f4', 'i4', 'U4')) +
+ a b c + float32 int32 str4 + ------- ----- ---- + 1.0 2 x + 4.0 5 y - >>> Table(arr, names=('a_new', 'b_new', 'c_new'), dtype=('f4', 'i4', 'S4')) -
- a_new b_new c_new - float32 int32 string32 - ------- ----- -------- - 1.0 2 x - 4.0 5 y + >>> Table(arr, names=('a_new', 'b_new', 'c_new'), dtype=('f4', 'i4', 'U4')) +
+ a_new b_new c_new + float32 int32 str4 + ------- ----- ----- + 1.0 2 x + 4.0 5 y +NumPy Homogeneous Array +----------------------- -NumPy homogeneous array -""""""""""""""""""""""" -A `numpy` 1-d array is treated as a single row table where each element of the +A ``numpy`` 1D array is treated as a single row table where each element of the array corresponds to a column:: >>> Table(np.array([1, 2, 3]), names=['a', 'b', 'c'], dtype=('i8', 'i8', 'i8')) -
+
a b c int64 int64 int64 ----- ----- ----- 1 2 3 -A `numpy` 2-d array (where all elements have the same type) can also be -converted into a |Table|. In this case the column names are not specified by +A ``numpy`` 2D array (where all elements have the same type) can also be +converted into a |Table|. In this case the column names are not specified by the data and must either be provided by the user or will be automatically generated as ``col`` where ```` is the column number. @@ -338,7 +413,7 @@ generated as ``col`` where ```` is the column number. >>> arr = np.array([[1, 2, 3], ... [4, 5, 6]], dtype=np.int32) >>> Table(arr) -
+
col0 col1 col2 int32 int32 int32 ----- ----- ----- @@ -348,27 +423,30 @@ generated as ``col`` where ```` is the column number. **Column names and types specified** :: - >>> Table(arr, names=('a_new', 'b_new', 'c_new'), dtype=('f4', 'i4', 'S4')) -
- a_new b_new c_new - float32 int32 string32 - ------- ----- -------- - 1.0 2 3 - 4.0 5 6 + >>> Table(arr, names=('a_new', 'b_new', 'c_new'), dtype=('f4', 'i4', 'U4')) +
+ a_new b_new c_new + float32 int32 str4 + ------- ----- ----- + 1.0 2 3 + 4.0 5 6 **Referencing the original data** -It is possible to reference the original data for an homogeneous array as long -as the data types are not changed:: +It is possible to reference the original data as long as the data types are not +changed:: >>> t = Table(arr, copy=False) -**Python arrays versus `numpy` arrays as input** +See the `Copy versus Reference`_ section for more information. -There is a slightly subtle issue that is important to understand in the way -that |Table| objects are created. Any data input that looks like a Python list -(including a tuple) is considered to be a list of columns. In contrast an -homogeneous `numpy` array input is interpreted as a list of rows:: +**Python arrays versus NumPy arrays as input** + +There is a slightly subtle issue that is important to understand about the way +that |Table| objects are created. Any data input that looks like a Python +:class:`list` (including a :class:`tuple`) is considered to be a list of +columns. In contrast, a homogeneous |ndarray| input is interpreted as a list of +rows:: >>> arr = [[1, 2, 3], ... [4, 5, 6]] @@ -388,195 +466,342 @@ homogeneous `numpy` array input is interpreted as a list of rows:: 4 5 6 This dichotomy is needed to support flexible list input while retaining the -natural interpretation of 2-d `numpy` arrays where the first index corresponds -to data "rows" and the second index corresponds to data "columns". +natural interpretation of 2D ``numpy`` arrays where the first index corresponds +to data "rows" and the second index corresponds to data "columns." + +From an Existing Table +---------------------- + +.. EXAMPLE START: Creating an Astropy Table from an Existing Table -Table columns -""""""""""""" A new table can be created by selecting a subset of columns in an existing table:: >>> t = Table(names=('a', 'b', 'c')) >>> t['c', 'b', 'a'] # Makes a copy of the data -
+
c b a float64 float64 float64 ------- ------- ------- -An alternate way to use the ``columns`` attribute (explained in the -`TableColumns`_ section) to initialize a new table. This let's you choose +An alternate way is to use the ``columns`` attribute (explained in the +`TableColumns`_ section) to initialize a new table. This lets you choose columns by their numerical index or name and supports slicing syntax:: >>> Table(t.columns[0:2]) -
+
a b float64 float64 ------- ------- >>> Table([t.columns[0], t.columns['c']]) -
+
a c float64 float64 ------- ------- +To create a copy of an existing table that is empty (has no rows):: + + >>> t = Table([[1.0, 2.3], [2.1, 3]], names=['x', 'y']) + >>> t +
+ x y + float64 float64 + ------- ------- + 1.0 2.1 + 2.3 3.0 + + >>> tcopy = t[:0].copy() + >>> tcopy +
+ x y + float64 float64 + ------- ------- + +.. EXAMPLE END + +Empty Array of a Known Size +--------------------------- + +.. EXAMPLE START: Creating an Astropy Table from an Empty Array + +If you do know the size that your table will be, but do not know the values in +advance, you can create a zeroed |ndarray| and build the |Table| from it:: + + >>> N = 3 + >>> dtype = [('a', 'i4'), ('b', 'f8'), ('c', 'bool')] + >>> t = Table(data=np.zeros(N, dtype=dtype)) + >>> t +
+ a b c + int32 float64 bool + ----- ------- ----- + 0 0.0 False + 0 0.0 False + 0 0.0 False + +For example, you can then fill in this table row by row with values extracted +from another table, or generated on the fly:: + + >>> for i in range(len(t)): + ... t[i] = (i, 2.5*i, i % 2) + >>> t +
+ a b c + int32 float64 bool + ----- ------- ----- + 0 0.0 False + 1 2.5 True + 2 5.0 False + +.. EXAMPLE END + +SkyCoord +-------- + +A |SkyCoord| object can be converted to a |QTable| using its +:meth:`~astropy.coordinates.SkyCoord.to_table` method. For details and examples +see :ref:`skycoord-table-conversion`. + +Pandas DataFrame +---------------- + +The section on :ref:`pandas` gives details on how to initialize a |Table| using +a :class:`pandas.DataFrame` via the :func:`~astropy.table.Table.from_pandas` +class method. This provides a convenient way to take advantage of the many I/O +and table manipulation methods in `pandas `_. + +Comment Lines +------------- + +.. EXAMPLE START: Adding Comment Lines in an ASCII File + +Comment lines in a text file can be added via the ``'comments'`` key in the +table's metadata. The following will insert two comment lines in the output +text file unless ``comment=False`` is explicitly set in ``write()``:: + + >>> import sys + >>> from astropy.table import Table + >>> t = Table(names=('a', 'b', 'c'), dtype=('f4', 'i4', 'S2')) + >>> t.add_row((1, 2.0, 'x')) + >>> t.meta['comments'] = ['Here is my explanatory text. This is awesome.', + ... 'Second comment line.'] + >>> t.write(sys.stdout, format='ascii') + # Here is my explanatory text. This is awesome. + # Second comment line. + a b c + 1.0 2 x + +.. EXAMPLE END + Initialization Details -^^^^^^^^^^^^^^^^^^^^^^ +====================== A table object is created by initializing a |Table| class object with the following arguments, all of which are optional: -``data`` : numpy ndarray, dict, list, or Table +``data`` : |ndarray|, :class:`dict`, :class:`list`, |Table|, or table-like object, optional Data to initialize table. -``names`` : list - Specify column names -``dtype`` : list - Specify column data types -``meta`` : dict-like - Meta-Data associated with the table -``copy`` : boolean - Copy the input data (default=True). +``masked`` : :class:`bool`, optional + Specify whether the table is masked. +``names`` : :class:`list`, optional + Specify column names. +``dtype`` : :class:`list`, optional + Specify column data types. +``meta`` : :class:`dict`, optional + Metadata associated with the table. +``copy`` : :class:`bool`, optional + Copy the input data. If the input is a |Table| the ``meta`` is always + copied regardless of the ``copy`` parameter. + Default is `True`. +``rows`` : |ndarray|, :class:`list` of lists, optional + Row-oriented data for table instead of ``data`` argument. +``copy_indices`` : :class:`bool`, optional + Copy any indices in the input data. Default is `True`. +``units`` : :class:`list`, :class:`dict`, optional + List or dict of units to apply to columns. +``descriptions`` : :class:`list`, :class:`dict`, optional + List or dict of descriptions to apply to columns. +``**kwargs`` : :class:`dict`, optional + Additional keyword args when converting table-like object. The following subsections provide further detail on the values and options for each of the keyword arguments that can be used to create a new |Table| object. data -"""" +---- The |Table| object can be initialized with several different forms for the ``data`` argument. -**numpy ndarray (structured array)** +**NumPy ndarray (structured array)** The base column names are the field names of the ``data`` structured - array. The ``names`` list (optional) can be used to select - particular fields and/or reorder the base names. The ``dtype`` list + array. The ``names`` list (optional) can be used to select + particular fields and/or reorder the base names. The ``dtype`` list (optional) must match the length of ``names`` and is used to override the existing ``data`` types. -**numpy ndarray (homogeneous)** - If the ``data`` ndarray is 1-dimensional then it is treated as a single row - table where each element of the array corresponds to a column. +**NumPy ndarray (homogeneous)** + If the ``data`` is a one-dimensional |ndarray| then it is treated as a + single row table where each element of the array corresponds to a column. - If the ``data`` ndarray is at least 2-dimensional then the first + If the ``data`` is an at least two-dimensional |ndarray|, then the first (left-most) index corresponds to row number (table length) and the - second index corresponds to column number (table width). Higher + second index corresponds to column number (table width). Higher dimensions get absorbed in the shape of each table cell. - If provided the ``names`` list must match the "width" of the ``data`` - argument. The default for ``names`` is to auto-generate column names - in the form "col". If provided the ``dtype`` list overrides the + If provided, the ``names`` list must match the "width" of the ``data`` + argument. The default for ``names`` is to auto-generate column names + in the form ``col``. If provided, the ``dtype`` list overrides the base column types and must match the length of ``names``. **dict-like** - The keys of the ``data`` object define the base column names. The - corresponding values can be Column objects, numpy arrays, or list-like - objects. The ``names`` list (optional) can be used to select - particular fields and/or reorder the base names. The ``dtype`` list + The keys of the ``data`` object define the base column names. The + corresponding values can be |Column| objects, ``numpy`` arrays, or + list-like objects. The ``names`` list (optional) can be used to select + particular fields and/or reorder the base names. The ``dtype`` list (optional) must match the length of ``names`` and is used to override the existing or default data types. **list-like** Each item in the ``data`` list provides a column of data values and - can be a Column object, numpy array, or list-like object. The - ``names`` list defines the name of each column. The names will be - auto-generated if not provided (either from the ``names`` argument or - by Column objects). If provided the ``names`` argument must match the - number of items in the ``data`` list. The optional ``dtype`` list + can be a |Column| object, |ndarray|, or list-like object. The + ``names`` list defines the name of each column. The names will be + auto-generated if not provided (either with the ``names`` argument or + by |Column| objects). If provided, the ``names`` argument must match the + number of items in the ``data`` list. The optional ``dtype`` list will override the existing or default data types and must match ``names`` in length. **list-of-dicts** - Similar to Python's builtin ``csv.DictReader``, each item in the - ``data`` list provides a row of data values and must be a dict. The - key values in each dict define the column names and each row must - have identical column names. The ``names`` argument may be supplied - to specify column ordering. If it is not provided, the column order will - default to alphabetical. The ``dtype`` list may be specified, and must - correspond to the order of output columns. If any row's keys do no match - the rest of the rows, a ValueError will be thrown. - + Similar to Python's built-in :class:`csv.DictReader`, each item in the + ``data`` list provides a row of data values and must be a :class:`dict`. + The key values in each :class:`dict` define the column names. The ``names`` + argument may be supplied to specify column ordering. If ``names`` are not + provided then column ordering will be determined by the first :class:`dict` + if it contains values for all the columns, or by sorting the column names + alphabetically if it does not. The ``dtype`` list may be specified, and + must correspond to the order of output columns. + +**Table-like object** + If another table-like object has a ``__astropy_table__()`` method then + that object can be used to directly create a |Table|. See the + `table-like objects`_ section for details. **None** - Initialize a zero-length table. If ``names`` and optionally ``dtype`` - are provided then the corresponding columns are created. + Initialize a zero-length table. If ``names`` and optionally ``dtype`` + are provided, then the corresponding columns are created. names -""""" +----- The ``names`` argument provides a way to specify the table column names or -override the existing ones. By default the column names are either taken -from existing names (for ``ndarray`` or ``Table`` input) or auto-generated -as ``col``. If ``names`` is provided then it must be a list with the -same length as the number of columns. Any list elements with value -``None`` fall back to the default name. +override the existing ones. By default, the column names are either taken from +existing names (for |ndarray| or |Table| input) or auto-generated as +``col``. If ``names`` is provided, then it must be a list with the same +length as the number of columns. Any list elements with value `None` fall back +to the default name. -In the case where ``data`` is provided as dict of columns, the ``names`` -argument can be supplied to specify the order of columns. The ``names`` list -must then contain each of the keys in the ``data`` dict. If ``names`` is not -supplied then the order of columns in the output table is not determinate. +In the case where ``data`` is provided as a :class:`dict` of columns, the +``names`` argument can be supplied to specify the order of columns. The +``names`` list must then contain each of the keys in the ``data`` +:class:`dict`. dtype -""""" - -The ``dtype`` argument provides a way to specify the table column data -types or override the existing types. By default the types are either -taken from existing types (for ``ndarray`` or ``Table`` input) or -auto-generated by the ``numpy.array()`` routine. If ``dtype`` is provided -then it must be a list with the same length as the number of columns. The -values must be valid ``numpy.dtype`` initializers or ``None``. Any list -elements with value ``None`` fall back to the default type. +----- -In the case where ``data`` is provided as dict of columns, the ``dtype`` argument -must be accompanied by a corresponding ``names`` argument in order to uniquely -specify the column ordering. +The ``dtype`` argument provides a way to specify the table column data types or +override the existing types. By default, the types are either taken from +existing types (for |ndarray| or |Table| input) or auto-generated by the +:func:`numpy.array` routine. If ``dtype`` is provided then it must be a list +with the same length as the number of columns. The values must be valid +:class:`numpy.dtype` initializers or `None`. Any list elements with value +`None` fall back to the default type. meta -"""" +---- -The ``meta`` argument is simply an object that contains meta-data associated -with the table. It is recommended that this object be a dict or -OrderedDict_, but the only firm requirement is that it can be copied with -the standard library ``copy.deepcopy()`` routine. By default ``meta`` is -an empty OrderedDict_. +The ``meta`` argument is an object that contains metadata associated with the +table. It is recommended that this object be a :class:`dict`, but the +only firm requirement is that it *must be a dict-like mapping* and can +be copied with the standard library :func:`copy.deepcopy` routine. By +default, ``meta`` is an empty :class:`dict`. copy -"""" +---- -By default the input ``data`` are copied into a new internal ``np.ndarray`` -object in the Table object. In the case where ``data`` is either an -``np.ndarray`` object or an existing ``Table``, it is possible to use a -reference to the existing data by setting ``copy=False``. This has the -advantage of reducing memory use and being faster. However one should take -care because any modifications to the new Table data will also be seen in the -original input data. See the `Copy versus Reference`_ section for more -information. +In the case where ``data`` is either an |ndarray| object, a :class:`dict`, or +an existing |Table|, it is possible to use a reference to the existing data by +setting ``copy=False``. This has the advantage of reducing memory use and being +faster. However, you should take care because any modifications to the new +|Table| data will also be seen in the original input data. See the `Copy versus +Reference`_ section for more information. +rows +---- + +This argument allows for providing data as a sequence of rows, in contrast +to the ``data`` keyword, which generally assumes data are a sequence of columns. +The `Row data`_ section provides details. + +copy_indices +------------ + +If you are initializing a |Table| from another |Table| that makes use of +:ref:`table-indexing`, then this option allows copying that table *without* +copying the indices by setting ``copy_indices=False``. By default, the indices +are copied. + +units +----- + +This allows for setting the unit for one or more columns at the time of +creating the table. The input can be either a list of unit values corresponding +to each of the columns in the table (using `None` or ``''`` for no unit), or a +:class:`dict` that provides the unit for specified column names. For example:: + + >>> dat = [[1, 2], ['hello', 'world']] + >>> qt = QTable(dat, names=['a', 'b'], units=(u.m, None)) + >>> qt = QTable(dat, names=['a', 'b'], units={'a': u.m}) + +See :ref:`quantity_and_qtable` for why we used a |QTable| here instead of a +|Table|. + +descriptions +------------ + +This allows for setting the description for one or more columns at the time of +creating the table. The input can be either a list of description values +corresponding to each of the columns in the table (using `None` for no +description), or a :class:`dict` that provides the description for specified +column names. This works in the same way as the ``units`` example above. .. _copy_versus_reference: Copy versus Reference -^^^^^^^^^^^^^^^^^^^^^ +===================== -Normally when a new |Table| object is created, the input data are *copied* into -a new internal array object. This ensures that if the new table elements are -modified then the original data will not be affected. However, when creating a -table from a numpy ndarray object (structured or homogeneous), it is possible to -disable copying so that instead a memory reference to the original data is -used. This has the advantage of being faster and using less memory. However, -caution must be exercised because the new table data and original data will be -linked, as shown below:: +Normally when a new |Table| object is created, the input data are *copied*. +This ensures that if the new table elements are modified then the original data +will not be affected. However, when creating a table from an existing |Table|, +a |ndarray| object (structured or homogeneous) or a :class:`dict`, it is +possible to disable copying so that a memory reference to the original data is +used instead. This has the advantage of being faster and using less memory. +However, caution must be exercised because the new table data and original data +will be linked, as shown below:: >>> arr = np.array([(1, 2.0, 'x'), ... (4, 5.0, 'y')], ... dtype=[('a', 'i8'), ('b', 'f8'), ('c', 'S2')]) - >>> print arr['a'] # column "a" of the input array + >>> print(arr['a']) # column "a" of the input array [1 4] >>> t = Table(arr, copy=False) >>> t['a'][1] = 99 - >>> print arr['a'] # arr['a'] got changed when we modified t['a'] + >>> print(arr['a']) # arr['a'] got changed when we modified t['a'] [ 1 99] Note that when referencing the data it is not possible to change the data types -since that operation requires making a copy of the data. In this case an error +since that operation requires making a copy of the data. In this case an error occurs:: >>> t = Table(arr, copy=False, dtype=('f4', 'i4', 'S4')) @@ -584,42 +809,42 @@ occurs:: ... ValueError: Cannot specify dtype when copy=False -Another caveat in using referenced data is that you if add a new row to the -table then the reference to the original data array is lost and instead the -table will now hold a copy of the original values (in addition to the new row). +Another caveat to using referenced data is that if you add a new row to the +table, the reference to the original data array is lost and the table will now +instead hold a copy of the original values (in addition to the new row). -Column and TableColumns classes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Column and TableColumns Classes +=============================== There are two classes, |Column| and |TableColumns|, that are useful when constructing new tables. Column -"""""" +------ A |Column| object can be created as follows, where in all cases the column -``name`` should be provided as a keyword argument and one can optionally provide +``name`` should be provided as a keyword argument and you can optionally provide these values: -``data`` : list, ndarray or None - Column data values -``dtype`` : numpy.dtype compatible value - Data type for column -``description`` : str - Full description of column -``unit`` : str - Physical unit -``format`` : str or function - `Format specifier`_ for outputting column values -``meta`` : dict - Meta-data associated with the column - -Initialization options -'''''''''''''''''''''' +``data`` : :class:`list`, |ndarray| or `None` + Column data values. +``dtype`` : :class:`numpy.dtype` compatible value + Data type for column. +``description`` : :class:`str` + Full description of column. +``unit`` : :class:`str` + Physical unit. +``format`` : :class:`str` or function + `Format specifier`_ for outputting column values. +``meta`` : :class:`dict` + Metadata associated with the column. + +Initialization Options +^^^^^^^^^^^^^^^^^^^^^^ The column data values, shape, and data type are specified in one of two ways: -**Provide a ``data`` value but not a ``length`` or ``shape``** +**Provide data but not length or shape** Examples:: @@ -629,29 +854,30 @@ The column data values, shape, and data type are specified in one of two ways: col = Column(np.array([1, 2]), name='a') col = Column(['hello', 'world'], name='a') - The ``dtype`` argument can be any value which is an acceptable - fixed-size data-type initializer for the numpy.dtype() method. See - ``_. - Examples include: + The ``dtype`` argument can be any value which is an acceptable fixed-size + data type initializer for a :class:`numpy.dtype`. See the reference for + `data type objects + `_. Examples + include: - - Python non-string type (float, int, bool) - - Numpy non-string type (e.g. np.float32, np.int64, np.bool) - - Numpy.dtype array-protocol type strings (e.g. 'i4', 'f8', 'S15') + - Python non-string type (:class:`float`, :class:`int`, :class:`bool`). + - ``numpy`` non-string type (e.g., ``np.float32``, ``np.int64``). + - ``numpy.dtype`` array-protocol type strings (e.g., ``'i4'``, ``'f8'``, ``'U15'``). - If no ``dtype`` value is provided then the type is inferred using - ``np.array(data)``. When ``data`` is provided then the ``shape`` + If no ``dtype`` value is provided, then the type is inferred using + :func:`numpy.array`. When ``data`` is provided then the ``shape`` and ``length`` arguments are ignored. -**Provide ``length`` and optionally ``shape``, but not ``data``** +**Provide length and optionally shape, but not data** Examples:: col = Column(name='a', length=5) col = Column(name='a', dtype=int, length=10, shape=(3,4)) - The default ``dtype`` is ``np.float64``. The ``shape`` argument is the array shape of a - single cell in the column. The default ``shape`` is () which means a single value in - each element. + The default ``dtype`` is ``np.float64``. The ``shape`` argument is the array + shape of a single cell in the column. The default ``shape`` is ``()`` which means + a single value in each element. .. note:: @@ -661,45 +887,45 @@ The column data values, shape, and data type are specified in one of two ways: .. _table_format_string: -Format specifier -'''''''''''''''' +Format Specifier +^^^^^^^^^^^^^^^^ The format specifier controls the output of column values when a table or column -is printed or written to an ASCII table. In the simplest case, it is a string -that can be passed to python's built-in `format -`_ function. For more -complicated formatting, one can also give "old-style" or "new-style" +is printed or written to a text table. In the simplest case, it is a string +that can be passed to Python's built-in :func:`format` function. For more +complicated formatting, one can also give "old style" or "new style" format strings, or even a function: **Plain format specification** -This type of string specifies directly how the value should be formatted, +This type of string specifies directly how the value should be formatted using a `format specification mini-language -`_ that is +`_ that is quite similar to C. ``".4f"`` will give four digits after the decimal in float format, or - ``"6d"`` will give integers in 6-character fields. + ``"6d"`` will give integers in six-character fields. -**Old-style format string** +**Old style format string** This corresponds to syntax like ``"%.4f" % value`` as documented in -`String formatting operations `_. +`printf-style String Formatting +`_. ``"%.4f"`` to print four digits after the decimal in float format, or - ``"%6d"`` to print an integer in a 6-character wide field. + ``"%6d"`` to print an integer in a six-character wide field. -**New-style format string** +**New style format string** This corresponds to syntax like ``"{:.4f}".format(value)`` as documented in `format string syntax -`_. +`_. ``"{:.4f}"`` to print four digits after the decimal in float format, or - ``"{:6d}"`` to print an integer in a 6-character wide field. + ``"{:6d}"`` to print an integer in a six-character wide field. Note that in either format string case any Python string that formats exactly one value is valid, so ``{:.4f} angstroms`` or ``Value: %12.2f`` would both @@ -707,17 +933,21 @@ work. **Function** +.. EXAMPLE START: Initialization Options for Column Objects + The greatest flexibility can be achieved by setting a formatting function. This -function must accept a single argument (the value) and return a string. In the +function must accept a single argument (the value) and return a string. One +caveat is that such a format function cannot be saved to file and you will get +an exception if you attempt to do so. In the following example this is used to make a LaTeX ready output:: >>> t = Table([[1,2],[1.234e9,2.34e-12]], names = ('a','b')) >>> def latex_exp(value): - ... val = '{0:8.2}'.format(value) + ... val = f'{value:8.2}' ... mant, exp = val.split('e') ... # remove leading zeros ... exp = exp[0] + exp[1:].lstrip('0') - ... return '$ {0} \\times 10^{{ {1} }}$' .format(mant, exp) + ... return f'$ {mant} \\times 10^{{ {exp} }}$' >>> t['b'].format = latex_exp >>> t['a'].format = '.4f' >>> import sys @@ -730,22 +960,32 @@ following example this is used to make a LaTeX ready output:: \end{tabular} \end{table} +.. EXAMPLE END + +**Format string for structured array column** + +For columns which are structured arrays, the format string must be a a string +that uses `"new style" format strings +`_ with +parameter substitutions corresponding to the field names in the structured +array. See :ref:`format_stuctured_array_columns` for an example. TableColumns -"""""""""""" +------------ Each |Table| object has an attribute ``columns`` which is an ordered dictionary that stores all of the |Column| objects in the table (see also the `Column`_ -section). Technically the ``columns`` attribute is a |TableColumns| object, +section). Technically, the ``columns`` attribute is a |TableColumns| object, which is an enhanced ordered dictionary that provides easier ways to select -multiple columns. There are a few key points to remember: +multiple columns. There are a few key points to remember: -- A |Table| can be initialized from a |TableColumns| object (copy is always True). +- A |Table| can be initialized from a |TableColumns| object (``copy`` is always + `True`). - Selecting multiple columns from a |TableColumns| object returns another |TableColumns| object. -- Select one column from a |TableColumns| object returns a |Column|. +- Selecting one column from a |TableColumns| object returns a |Column|. -So now look at the ways to select columns from a |TableColumns| object: +There are a few different ways to select columns from a |TableColumns| object: **Select columns by name** :: @@ -764,31 +1004,70 @@ So now look at the ways to select columns from a |TableColumns| object: >>> t.columns[::-1] # Reverse column order -**Select column by index or name** +**Select single columns by index or name** :: - >>> t.columns[1] # Choose columns by index + >>> t.columns[1] # Choose a column by index - >>> t.columns['b'] # Choose column by name + >>> t.columns['b'] # Choose a column by name .. _subclassing_table: Subclassing Table -^^^^^^^^^^^^^^^^^ +================= For some applications it can be useful to subclass the |Table| class in order -to introduce specialized behavior. In addition to subclassing |Table| it is -frequently desirable to change the behavior of the internal class objects which -are contained or created by a Table. This includes rows, columns, formatting, -and the columns container. In order to do this the subclass needs to declare -what class to use (if it is different from the built-in version). This is done by -specifying one or more of the class attributes ``Row``, ``Column``, +to introduce specialized behavior. Here we address two particular use cases +for subclassing: adding custom table attributes and changing the behavior of +internal class objects. + +.. _table-custom-attributes: + +Adding Custom Table Attributes +------------------------------ + +One simple customization that can be useful is adding new attributes to +the table object. There is nothing preventing setting an attribute on an +existing table object, for example ``t.foo = 'hello'``. However, this attribute +would be ephemeral because it will be lost if the table is sliced, copied, or +pickled. Instead, you can add persistent attributes as shown in this example:: + + from astropy.table import Table, TableAttribute + + class MyTable(Table): + foo = TableAttribute() + bar = TableAttribute(default=[]) + baz = TableAttribute(default=1) + + t = MyTable([[1, 2]], foo='foo') + t.bar.append(2.0) + t.baz = 'baz' + +Some key points: + +- A custom attribute can be set when the table is created or using + the usual syntax for setting an object attribute. +- A custom attribute always has a default value, either explicitly set + in the class definition or `None`. +- The attribute values are stored in the table ``meta`` dictionary. This is + the mechanism by which they are persistent through copy, slice, and + serialization such as pickling or writing to an :ref:`ecsv_format` file. + +Changing Behavior of Internal Class Objects +------------------------------------------- + +It is also possible to change the behavior of the internal class objects which +are contained or created by a |Table|. This includes rows, columns, formatting, +and the columns container. In order to do this the subclass needs to declare +what class to use (if it is different from the built-in version). This is done +by specifying one or more of the class attributes ``Row``, ``Column``, ``MaskedColumn``, ``TableColumns``, or ``TableFormatter``. The following trivial example overrides all of these with do-nothing -subclasses, but in practice you would override only the necessary subcomponents:: +subclasses, but in practice you would override only the necessary +subcomponents:: >>> from astropy.table import Table, Row, Column, MaskedColumn, TableColumns, TableFormatter @@ -810,13 +1089,15 @@ subclasses, but in practice you would override only the necessary subcomponents: Example -""""""" +^^^^^^^ + +.. EXAMPLE START: Subclassing the Table Class -As a more practical example, suppose you have a table of data with a certain set of fixed -columns, but you also want to carry an arbitrary dictionary of keyword=value +As a more practical example, suppose you have a table of data with a certain +set of fixed columns, but you also want to carry an arbitrary dictionary of parameters for each row and then access those values using the same item access -syntax as if they were columns. It is assumed here that the extra parameters -are contained in a numpy object-dtype column named ``params``:: +syntax as if they were columns. It is assumed here that the extra parameters +are contained in a ``numpy`` object-dtype column named ``params``:: >>> from astropy.table import Table, Row >>> class ParamsRow(Row): @@ -826,9 +1107,9 @@ are contained in a numpy object-dtype column named ``params``:: ... """ ... def __getitem__(self, item): ... if item not in self.colnames: - ... return super(ParamsRow, self).__getitem__('params')[item] + ... return super().__getitem__('params')[item] ... else: - ... return super(ParamsRow, self).__getitem__(item) + ... return super().__getitem__(item) ... ... def keys(self): ... out = [name for name in self.colnames if name != 'params'] @@ -848,10 +1129,10 @@ First make a table and add a couple of rows:: >>> t = ParamsTable(names=['a', 'b', 'params'], dtype=['i', 'f', 'O']) >>> t.add_row((1, 2.0, {'x': 1.5, 'y': 2.5})) >>> t.add_row((2, 3.0, {'z': 'hello', 'id': 123123})) - >>> print(t) # doctest: +SKIP + >>> print(t) a b params --- --- ---------------------------- - 1 2.0 {'y': 2.5, 'x': 1.5} + 1 2.0 {'x': 1.5, 'y': 2.5} 2 3.0 {'z': 'hello', 'id': 123123} Now see what we have from our specialized ``ParamsRow`` object:: @@ -863,17 +1144,17 @@ Now see what we have from our specialized ``ParamsRow`` object:: >>> t[1].keys() ['a', 'b', 'id', 'z'] >>> t[1].values() - [2, 3.0, 123123, 'hello'] + [np.int32(2), np.float32(3.0), 123123, 'hello'] -To make this example really useful you might want to override -``Table.__getitem__`` in order to allow table-level access to the parameter -fields. This might look something like:: +To make this example really useful, you might want to override +``Table.__getitem__()`` in order to allow table-level access to the parameter +fields. This might look something like:: class ParamsTable(table.Table): Row = ParamsRow def __getitem__(self, item): - if isinstance(item, six.string_types): + if isinstance(item, str): if item in self.colnames: return self.columns[item] else: @@ -881,7 +1162,7 @@ fields. This might look something like:: # corresponding to self['params'][item] for each row. This # might not exist in some rows so mark as masked (missing) in # those cases. - mask = np.zeros(len(self), dtype=np.bool) + mask = np.zeros(len(self), dtype=np.bool_) item = item.upper() values = [params.get(item) for params in self['params']] for ii, value in enumerate(values): @@ -891,3 +1172,127 @@ fields. This might look something like:: return self.MaskedColumn(name=item, data=values, mask=mask) # ... and then the rest of the original __getitem__ ... + +.. EXAMPLE END + +Columns and Quantities +====================== + +.. EXAMPLE START: Handling Astropy Column and Quantity Objects within Tables + +``astropy`` `~astropy.units.Quantity` objects can be handled within tables in +two complementary ways. The first method stores the `~astropy.units.Quantity` +object natively within the table via the "mixin" column protocol. See the +sections on :ref:`mixin_columns` and :ref:`quantity_and_qtable` for details, +but in brief, the key difference is using the `~astropy.table.QTable` class to +indicate that a `~astropy.units.Quantity` should be stored natively within the +table:: + + >>> from astropy.table import QTable + >>> from astropy import units as u + >>> t = QTable() + >>> t['velocity'] = [3, 4] * u.m / u.s + >>> type(t['velocity']) + + +For new code that is quantity-aware we recommend using `~astropy.table.QTable`, +but this may not be possible in all situations (particularly when interfacing +with legacy code that does not handle quantities) and there are +:ref:`details_and_caveats` that apply. In this case, use the +`~astropy.table.Table` class, which will convert a `~astropy.units.Quantity` to +a `~astropy.table.Column` object with a ``unit`` attribute:: + + >>> from astropy.table import Table + >>> t = Table() + >>> t['velocity'] = [3, 4] * u.m / u.s + >>> type(t['velocity']) + + >>> t['velocity'].unit + Unit("m / s") + +To learn more about using standard `~astropy.table.Column` objects with defined +units, see the :ref:`columns_with_units` section. + +.. EXAMPLE END + +.. _Table-like Objects: + +Table-like Objects +================== + +In order to improve interoperability between different table classes, an +``astropy`` |Table| object can be created directly from any other table-like +object that provides an ``__astropy_table__()`` method. In this case the +``__astropy_table__()`` method will be called as follows:: + + >>> data = SomeOtherTableClass({'a': [1, 2], 'b': [3, 4]}) # doctest: +SKIP + >>> t = QTable(data, copy=False, mask_invalid=True) # doctest: +SKIP + +Internally the following call will be made to ask the ``data`` object +to return a representation of itself as an ``astropy`` |Table|, respecting +the ``copy`` preference of the original call to ``QTable()``:: + + data.__astropy_table__(cls, copy, **kwargs) + +Here ``cls`` is the |Table| class or subclass that is being instantiated +(|QTable| in this example), ``copy`` indicates whether a copy of the values in +``data`` should be provided, and ``**kwargs`` are any extra keyword arguments +which are not valid |Table| ``__init__()`` keyword arguments. In the example +above, ``mask_invalid=True`` would end up in ``**kwargs`` and get passed to +``__astropy_table__()``. + +The implementation might choose to allow additional keyword arguments (e.g., +``mask_invalid`` which gets passed via ``**kwargs``). + +As a concise example, imagine a dict-based table class. (Note that |Table| +already can be initialized from a dict-like object, so this is a bit contrived +but does illustrate the principles involved.) Please pay attention to the +method signature:: + + def __astropy_table__(self, cls, copy, **kwargs): + +Your class implementation of this must use the ``**kwargs`` technique for +catching keyword arguments at the end. This is to ensure future compatibility +in case additional keywords are added to the internal ``table = +data.__astropy_table__(cls, copy)`` call. Including ``**kwargs`` will prevent +breakage in this case. :: + + class DictTable(dict): + """ + Trivial "table" class that just uses a dict to hold columns. + This does not actually implement anything useful that makes + this a table. + + The non-standard ``mask_invalid=False`` keyword arg here will be passed + via the **kwargs of Table __init__(). + """ + + def __astropy_table__(self, cls, copy, mask_invalid=False, **kwargs): + """ + Return an astropy Table of type ``cls``. + + Parameters + ---------- + cls : type + Astropy ``Table`` class or subclass. + copy : bool + Copy input data (True) or return a reference (False). + mask_invalid : bool, optional + Controls whether invalid values (NaNs) should be masked. + Default is False. + **kwargs : dict, optional + Additional keyword args (ignored currently). + """ + if kwargs: + warnings.warn(f'unexpected keyword args {kwargs}') + + cols = list(self.values()) + names = list(self.keys()) + + if mask_invalid: + cols = [ + Masked(col, mask=mask) if np.any(mask := np.isnan(col)) else col + for col in cols + ] + + return cls(cols, names=names, copy=copy) diff --git a/docs/table/dataframes.rst b/docs/table/dataframes.rst new file mode 100644 index 000000000000..f01a1e01b0e6 --- /dev/null +++ b/docs/table/dataframes.rst @@ -0,0 +1,283 @@ +.. doctest-skip-all + +.. _df_narwhals: + +Interfacing with DataFrames +*************************** + +The :class:`~astropy.table.Table` class provides comprehensive support for interfacing with DataFrame libraries through two complementary approaches: + +1. **Generic narwhals-based methods**: :meth:`~astropy.table.Table.to_df` and :meth:`~astropy.table.Table.from_df` for multi-backend DataFrame support +2. **Legacy pandas-specific methods**: :meth:`~astropy.table.Table.to_pandas` and :meth:`~astropy.table.Table.from_pandas` for direct pandas integration + +The narwhals-based approach uses `Narwhals `_ as a unified translation layer, enabling seamless interoperability with multiple DataFrame libraries including `pandas `_, `polars `__, `pyarrow `__, and others. + +Generic Multi-Backend Methods +============================= + +For users working with multiple DataFrame libraries or seeking broader compatibility, the generic methods provide a unified interface through narwhals. + +Supported Backends +------------------ + +The generic methods support DataFrame libraries with eager execution, including: + +* **pandas** - A popular and pioneering DataFrame library +* **polars** - High-performance DataFrame library with different handling of multidimensional data +* **pyarrow** - In-memory columnar format with good performance +* **modin** - Distributed pandas-compatible DataFrames +* **cudf** - GPU-accelerated DataFrames + +.. note:: + **Testing and Support**: **pandas**, **polars**, and **pyarrow** are directly tested in the Astropy test suite. While other narwhals-compatible backends should work in principle, they may exhibit unexpected behavior or incompatibilities. If you encounter issues with any backend, please file a bug report on the `Astropy GitHub repository `_. + +.. warning:: + **Backend Differences**: Different DataFrame libraries implement varying data models, type systems, and computational paradigms. These fundamental differences can lead to inconsistent behavior across backends, particularly with respect to data type handling, missing value representation, and memory layout. Users should verify that round-trip conversions preserve the expected data integrity for their specific use case and chosen backend. + +Basic Multi-Backend Example +--------------------------- + +.. EXAMPLE START: Using Generic Multi-Backend Methods + +Create a table and convert to different DataFrame backends:: + + >>> from astropy.table import Table + >>> t = Table() + >>> t['a'] = [1, 2, 3, 4] + >>> t['b'] = ['a', 'b', 'c', 'd'] + + # Convert to pandas DataFrame + >>> df_pandas = t.to_df("pandas") + >>> type(df_pandas) + + + # Convert to polars DataFrame + >>> df_polars = t.to_df("polars") + >>> type(df_polars) + + +Create a table from any supported DataFrame:: + + >>> t2 = Table.from_df(df_pandas) # From pandas + >>> t3 = Table.from_df(df_polars) # From polars + +.. EXAMPLE END + +Known Backend-Specific Differences +---------------------------------- + +Different DataFrame backends handle data differently: + +**Multidimensional Columns:** + - Pandas: Not supported, raises an error + - Polars: Supported as Array type for arbitrary dimensions + - PyArrow: Limited support for 1D arrays, currently unavailable. + +**Index Support:** + - Pandas: Full index support with :meth:`~astropy.table.Table.to_df` + - Other backends: Index parameter raises an error (not supported) + +**Missing Value Handling:** + - All backends use sentinel values (NaN, null) rather than Astropy's mask arrays + +Pandas-Specific Methods +======================= + +For pandas users, Astropy provides dedicated methods that offer the most direct and feature-complete integration with pandas DataFrames. + +Basic Pandas Example +-------------------- + +.. EXAMPLE START: Using Pandas-Specific Methods + +To demonstrate, we can create a minimal table:: + + >>> from astropy.table import Table + >>> t = Table() + >>> t['a'] = [1, 2, 3, 4] + >>> t['b'] = ['a', 'b', 'c', 'd'] + +Convert to a pandas DataFrame using the pandas-specific method:: + + >>> df = t.to_pandas() + >>> df + a b + 0 1 a + 1 2 b + 2 3 c + 3 4 d + >>> type(df) + + +Create a table from a pandas DataFrame:: + + >>> t2 = Table.from_pandas(df) + >>> t2 +
+ a b + int64 string8 + ----- ------- + 1 a + 2 b + 3 c + 4 d + +.. EXAMPLE END + +Pandas Index Support +-------------------- + +The pandas-specific methods provide full support for DataFrame indexing, which is a unique pandas feature:: + + >>> from astropy.time import Time + >>> tm = Time([1998, 2002], format="jyear") + >>> x = [1, 2] + >>> t = Table([tm, x], names=["tm", "x"]) + + # Use a column as the DataFrame index + >>> df = t.to_pandas(index="tm") + >>> df.index.name + 'tm' + + # Convert back including the index as a column + >>> t_back = Table.from_pandas(df, index=True) + >>> t_back.colnames + ['tm', 'x'] + +Pandas Excel Support +-------------------- + +Read an Excel file into a table by utilizing the pandas backend:: + + >>> t = Table.from_pandas(pandas.read_excel("myexceltable.xlsx")) + +When to Use Which Method +======================== + +**Use pandas-specific methods** (:meth:`~astropy.table.Table.to_pandas`, :meth:`~astropy.table.Table.from_pandas`) when: + +* Working exclusively with pandas +* Need DataFrame index support +* Want the most battle-tested and feature-complete pandas integration +* Require the best performance for pandas-specific workflows + +**Use generic methods** (:meth:`~astropy.table.Table.to_df`, :meth:`~astropy.table.Table.from_df`) when: + +* Working with multiple DataFrame backends +* Need to support polars, pyarrow, or other backends +* Building library code that should work with various DataFrame types +* Want forward compatibility as new backends are added to narwhals + +Conversion Details and Limitations +=================================== + +Both approaches share common limitations when converting between Tables and DataFrames: + +Data Type Limitations +--------------------- + +* **Multidimensional columns**: Support varies by backend. At the time of writing, Pandas does not support them, while Polars can handle them as Array types. + +* **Masked tables**: DataFrames typically use sentinel values (e.g., `numpy.nan` or `None`) for missing data, while Astropy preserves the original value under the mask. The original values under the mask are lost during conversion. + +* **Mixed-type columns**: Object dtype columns have varied support. Pandas preserves them while other backends may fail to import such data. + +Mixin Column Limitations +------------------------ + +Tables with :ref:`mixin_columns` such as `~astropy.time.Time`, `~astropy.coordinates.SkyCoord`, and |Quantity| can be converted, but **with loss of information**: + +* **Time columns**: Converted to native datetime types with reduced precision and loss of astronomical time scale information +* **SkyCoord columns**: Split into separate coordinate component columns (e.g., ``ra``, ``dec``) with loss of units and coordinate frame information +* **Quantity columns**: Converted to plain numeric columns with complete loss of unit information + +Complex Example with Both Methods +================================= + +.. EXAMPLE START: Complex DataFrame Conversion Example + +Create a table with masked and mixin columns:: + + >>> import numpy as np + >>> from astropy.table import MaskedColumn, QTable + >>> from astropy.time import Time + >>> from astropy.coordinates import SkyCoord + >>> import astropy.units as u + >>> t = QTable() + >>> t['a'] = MaskedColumn([1, 2, 3], mask=[False, True, False]) + >>> t['b'] = MaskedColumn([1.0, 2.0, 3.0], mask=[False, False, True]) + >>> t['c'] = MaskedColumn(["a", "b", "c"], mask=[True, False, False]) + >>> t['tm'] = Time(["2021-01-01", "2021-01-02", "2021-01-03"]) + >>> t['sc'] = SkyCoord(ra=[1, 2, 3] * u.deg, dec=[4, 5, 6] * u.deg) + >>> t['q'] = [1, 2, 3] * u.m + + >>> t + + a b c tm sc q + deg,deg m + int64 float64 str1 Time SkyCoord float64 + ----- ------- ---- ----------------------- -------- ------- + 1 1.0 -- 2021-01-01 00:00:00.000 1.0,4.0 1.0 + -- 2.0 b 2021-01-02 00:00:00.000 2.0,5.0 2.0 + 3 -- c 2021-01-03 00:00:00.000 3.0,6.0 3.0 + +Convert using the pandas-specific method:: + + >>> df_pandas = t.to_pandas() + >>> df_pandas + a b c tm sc.ra sc.dec q + 0 1 1.0 NaN 2021-01-01 1.0 4.0 1.0 + 1 2.0 b 2021-01-02 2.0 5.0 2.0 + 2 3 NaN c 2021-01-03 3.0 6.0 3.0 + +Convert using the generic method to pandas:: + + >>> df_generic = t.to_df("pandas") + >>> # Results are identical to df_pandas + +Convert to polars using the generic method:: + + >>> df_polars = t.to_df("polars") + >>> df_polars + shape: (3, 7) + ┌──────â”Ŧ──────â”Ŧ──────â”Ŧ─────────────────────â”Ŧ───────â”Ŧ────────â”Ŧ─────┐ + │ a ┆ b ┆ c ┆ tm ┆ sc.ra ┆ sc.dec ┆ q │ + │ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │ + │ i64 ┆ f64 ┆ str ┆ datetime[ns] ┆ f64 ┆ f64 ┆ f64 │ + ╞══════â•Ē══════â•Ē══════â•Ē═════════════════════â•Ē═══════â•Ē════════â•Ē═════╡ + │ 1 ┆ 1.0 ┆ null ┆ 2021-01-01 00:00:00 ┆ 1.0 ┆ 4.0 ┆ 1.0 │ + │ null ┆ 2.0 ┆ b ┆ 2021-01-02 00:00:00 ┆ 2.0 ┆ 5.0 ┆ 2.0 │ + │ 3 ┆ null ┆ c ┆ 2021-01-03 00:00:00 ┆ 3.0 ┆ 6.0 ┆ 3.0 │ + └──────┴──────┴──────┴─────────────────────┴───────┴────────┴─────┘ + +Convert back to tables:: + + >>> t_from_pandas = QTable.from_pandas(df_pandas) # Using pandas-specific method + >>> t_from_generic = QTable.from_df(df_pandas) # Using generic method + >>> t_from_polars = QTable.from_df(df_polars) # From polars DataFrame + +Note the data transformations that occurred: + +1. **Masked values**: Original values under the mask are lost and replaced with sentinel values +2. **Time columns**: Converted to basic datetime objects, preserving temporal information but losing astropy Time features +3. **SkyCoord columns**: Split into separate ``ra`` and ``dec`` columns, losing coordinate frame and unit information +4. **Quantity columns**: Converted to plain float columns, completely losing unit information + +.. EXAMPLE END + +Method Reference +================ + +Pandas-Specific Methods +----------------------- + +- :meth:`~astropy.table.Table.to_pandas` - Convert Table to pandas DataFrame +- :meth:`~astropy.table.Table.from_pandas` - Create Table from pandas DataFrame + +Generic Multi-Backend Methods +----------------------------- + +- :meth:`~astropy.table.Table.to_df` - Convert Table to DataFrame using specified backend +- :meth:`~astropy.table.Table.from_df` - Create Table from any narwhals-compatible DataFrame + +See the `Narwhals documentation `_ for more information about supported backends and their capabilities. diff --git a/docs/table/implementation_change_1.0.rst b/docs/table/implementation_change_1.0.rst deleted file mode 100644 index 65a10551dfaa..000000000000 --- a/docs/table/implementation_change_1.0.rst +++ /dev/null @@ -1,188 +0,0 @@ -.. include:: references.txt - -.. |table_before| image:: table_before_1.0.png - :width: 45% - -.. |table_after| image:: table_after_1.0.png - :width: 45% - -.. |column_before| image:: table_column_before_1.0.png - :width: 45% - -.. |column_after| image:: table_column_after_1.0.png - :width: 45% - -.. |row_before| image:: table_row_before_1.0.png - :width: 73% - -.. |row_after| image:: table_row_after_1.0.png - :width: 83% - - -.. _table_implementation_change: - -Table implementation change in 1.0 ----------------------------------- - -This page discusses the change in the internal implementation of the |Table| -class which took place starting from version 1.0 of astropy. The motivation -for making this change is discussed in the `Benefits`_ section. - -Architecture -^^^^^^^^^^^^^^ - -Data container -"""""""""""""" - -The images below illustrate the basic architecture of the |Table| class for -astropy versions 0.4.x and earlier (left) and after version 1.0 (right). - -On the left side (before 1.0) the fundamental data container is a numpy -structured array referenced as an internal attribute ``_data``. All public -methods and operations (e.g. column access, row indexing) are done via this -internal `~numpy.ndarray` object. The ``columns`` attribute is used to manage -table columns and provide access. It is a |TableColumns| object which is -essentially an ordered dictionary of |Column| or |MaskedColumn| objects which -provide views of the ``_data`` array. - -On the right side (after 1.0) the fundamental data container is now the -collection of individual column objects and there is no longer a structured -array associated with the table. Each |Column| object is the sole owner of its -data. As before, the ``columns`` attribute is used to manage columns and -provide access. - -|table_before| |table_after| - -Columns -"""""""" - -For versions before 1.0 the |Column| object is an `~numpy.ndarray` subclass with -a *memory view* of the corresponding column in the ``_data`` array. This means -that the physical memory for the |Column| object data is exactly the same as -the memory storing the ``_data`` array. Therefore updating an element in the -column results in the corresponding update in the ``_data`` value. This model -is convenient in many ways, but also has drawbacks. In particular, astropy -tables are easily mutable (e.g. you can add or remove columns) while numpy -structured arrays are not. This means that key operations require -regenerating the entire ``_data`` structured array and likewise regenerating -all the |Column| view objects. This is relatively slow and results in -additional code complexity to always ensure correspondence. - -Starting with version 1.0 the |Column| object is the same `~numpy.ndarray` -subclass but it is sole owner of the data. This simplifies table management -considerably along with making operations like adding or removing columns -*much* faster because there is no structured array to regenerate. - -|column_before| |column_after| - -Rows -""""" - -A |Row| object corresponds to a single row in the table. For versions before -1.0, when a |Row| object is requested it uses numpy indexing into the table -``_data`` array to generate a ``numpy.void`` or ``numpy.ma.mvoid`` object as the -``data`` attribute [#]_. This delegates most of the row access functionality like -``row['a']`` to the numpy void classes. For unmasked tables this ``data`` -attribute is a memory view of the parent table row, though for masked tables -(due to the implementation of numpy masked arrays), the ``data`` attribute is -*not* a view. - -|row_before| - -For version 1.0 and later, the |Row| object does not create a view of the full -row at any point. Instead it manages access (like ``row['a']``) dynamically in -a way that maintains the same interface. Due to improved implementation this -is actually faster. - -The row ``data`` attribute is part of the public API before 1.0, therefore it -is still available in 1.0 but as a *deprecated* property. In this case -accessing ``data`` runs the `~astropy.table.Row.as_void()` method to dynamically -create and return a ``numpy.void`` or ``numpy.ma.mvoid`` object. This provides a -copy of the original data, not a view. Code which was relying on the row -``data`` attribute as a *view* into the parent table will need to be modified. - -|row_after| - -.. [#] ``numpy.void`` is a ``dtype`` that can be used to represent structures - of arbitrary byte width. - -Differences -^^^^^^^^^^^ - -``Row.data`` -"""""""""""" - -The ``data`` property of the |Row| object is deprecated in version 1.0 and -may be removed in a later version. Code which requires access to a -``numpy.void`` or ``numpy.ma.mvoid`` object corresponding to a table row -can now use the `~astropy.table.Row.as_void()` method. This is public and -stable, with the caveat that it is relatively slow and returns a copy of the -row data, not a view. - -``Table._data`` -""""""""""""""" - -While the ``_data`` property of the |Table| object is not part of the public -API in any astropy release, some users may have let this creep into their -code as back-door access to the numpy object. In version 1.0 this attribute is -formally deprecated and will generate a warning. - -From 1.0 the public method for getting the corresponding numpy structured array -or masked array version of a table is the |Table| method -`~astropy.table.Table.as_array()`. This dynamically generates the requested -object, making a copy of the table data. Be aware that the ``_data`` property -calls `~astropy.table.Table.as_array()`, so accessing ``_data`` will -effectively double the memory usage of the table. - -An alternative is to use `~numpy.array` to do the conversion, e.g. for an -astropy |Table| object named ``dat`` use ``np_dat = np.array(dat)``. Be aware that -for a masked table this operation always returns a pure `~numpy.ndarray` -with data corresponding to the unmasked values. - -High-level operations -""""""""""""""""""""" - -In version 1.0 the operations described in :ref:`table_operations` rely on -`~astropy.table.Table.as_array()` to create numpy structured arrays which -are used in the actual array manipulations. This creates temporary copies of -the tables. - -Performance regressions -""""""""""""""""""""""" - -From version 1.0 most common operations run at the same speed or are faster -(sometimes significantly faster). The only operations which are noticeably -slower are adding a row in a masked table (~2 times slower) and setting a -column like ``dat['a'][:] = 10`` in a masked table (~6 times slower). - - -Benefits -^^^^^^^^ - -The key benefits of the version 1.0 change are as follows: - -- Allows for much faster addition or removal of columns. A common idiom is - creating a table and then adding columns:: - - >>> from astropy.table import Table - >>> import numpy as np - >>> t = Table() - >>> t['a'] = np.arange(100) - >>> t['b'] = np.random.uniform(size=100) - >>> t['c'] = t['a'] + t['b'] - - Prior to 1.0 this idiom was extremely inefficient because the underlying - structured array was being entirely regenerated with each column addition. - From 1.0 forward this is fast and a good way to write code. - -- Provides the infrastructure to allow for Tables to easily hold column types - beyond just |Column| and |MaskedColumn|. This includes - `~astropy.units.Quantity`, `~astropy.time.Time`, and - `~astropy.coordinates.SkyCoord` objects. Other ideas like nested |Table| - objects are also possible. - -- Generally faster because of improved implementation in key areas. - Column-based access is faster because the column data are held in contiguous - memory instead of being strided within the numpy structure array. - -- Reduces code complexity in a number of core table routines. diff --git a/docs/table/implementation_details.rst b/docs/table/implementation_details.rst index 98694b64589c..02d63083b15c 100644 --- a/docs/table/implementation_details.rst +++ b/docs/table/implementation_details.rst @@ -1,47 +1,38 @@ -.. include:: references.txt - -.. |column_after| image:: table_column_after_1.0.png - :width: 45% - - .. _table_implementation_details: -Table implementation details ------------------------------ +Table Implementation Details +***************************** This page provides a brief overview of the |Table| class implementation, in -particular highlighting the internal data storage architecture. This is aimed +particular highlighting the internal data storage architecture. This is aimed at developers and/or users who are interested in optimal use of the |Table| -class. Note that this applies to astropy version 1.0 and later. For -differences between version 1.0 and 0.4.x see the -:ref:`table_implementation_change` page. +class. The image below illustrates the basic architecture of the |Table| class. The fundamental data container is an ordered dictionary of individual column -objects maintained as the ``columns`` attribute. It is via this container +objects maintained as the ``columns`` attribute. It is via this container that columns are managed and accessed. .. image:: table_architecture.png :width: 45% -Each |Column| (or |MaskedColumn|) object is an `~numpy.ndarray` subclass and is -the sole owner of its data. Maintaining the table as separate columns -simplifies table management considerably. It also makes operations like adding -or removing columns much faster in comparison to implementations using a numpy -structured array container. +Each |Column| (or |MaskedColumn|) object is an |ndarray| (or +:class:`numpy.ma.MaskedArray`) subclass and is the sole owner of its data. +Maintaining the table as separate columns simplifies table management +considerably. It also makes operations like adding or removing columns much +faster in comparison to implementations using a ``numpy`` structured array +container. -As shown below, a |Row| object corresponds to a single row in the table. The -|Row| object does not create a view of the full row at any point. Instead it -manages access (e.g. ``row['a']``) dynamically by referencing the appropriate +As shown below, a |Row| object corresponds to a single row in the table. The +|Row| object does not create a view of the full row at any point. Instead it +manages access (e.g., ``row['a']``) dynamically by referencing the appropriate elements of the parent table. .. image:: table_row.png :width: 83% -In some cases it is desirable to have a static copy of the full row. This is +In some cases it is desirable to have a static copy of the full row. This is available via the `~astropy.table.Row.as_void()` method, which creates and -returns a ``numpy.void`` or ``numpy.ma.mvoid`` object with a copy of the -original data. For backward compatibility the row ``data`` attribute is -available but as a *deprecated* property. - +returns a :class:`numpy.void` or ``numpy.ma.mvoid`` object with a copy of the +original data. diff --git a/docs/table/index.rst b/docs/table/index.rst index 9e9ec5dcc117..c986d12eb02d 100644 --- a/docs/table/index.rst +++ b/docs/table/index.rst @@ -1,5 +1,3 @@ -.. include:: references.txt - .. _astropy-table: ***************************** @@ -9,124 +7,136 @@ Data Tables (`astropy.table`) Introduction ============ -`astropy.table` provides functionality for storing and manipulating -heterogeneous tables of data in a way that is familiar to `numpy` users. A few -notable features of this package are: - -* Initialize a table from a wide variety of input data structures and types. -* Modify a table by adding or removing columns, changing column names, - or adding new rows of data. -* Handle tables containing missing values. -* Include table and column metadata as flexible data structures. -* Specify a description, units and output formatting for columns. -* Interactively scroll through long tables similar to using ``more``. -* Create a new table by selecting rows or columns from a table. -* Perform :ref:`table_operations` like database joins and concatenation. -* Manipulate multidimensional columns. -* Methods for :ref:`read_write_tables` to files -* Hooks for :ref:`subclassing_table` and its component classes - -Currently `astropy.table` is used when reading an ASCII table using -`astropy.io.ascii`. Future releases of AstroPy are expected to use -the |Table| class for other subpackages such as `astropy.io.votable` and `astropy.io.fits` . - -.. Note:: +`astropy.table` provides a flexible and easy-to-use set of tools for working with +tabular data using an interface based on `numpy`. In addition to basic table creation, +access, and modification operations, key features include: - Starting with version 1.0 of astropy the internal implementation of the - |Table| class changed so that it no longer uses numpy structured arrays as - the core table data container. Instead the table is stored as a collection - of individual column objects. *For most users there is NO CHANGE to the - interface and behavior of |Table| objects.* +* Support columns of astropy :ref:`time `, :ref:`coordinates `, and :ref:`quantities `. +* Support multidimensional and :ref:`structured array columns `. +* Maintain the units, description, and format of columns. +* Provide flexible metadata structures for the table and individual columns. +* Perform :ref:`table_operations` like database joins, concatenation, and binning. +* Maintain a table index for fast retrieval of table items or ranges. +* Support a general :ref:`mixin protocol ` for flexible data containers in tables. +* :ref:`Read and write ` to files via the :ref:`Unified File Read/Write Interface `. +* Convert to and from `pandas.DataFrame` or ``polars.DataFrame``. - The page on :ref:`table_implementation_change` provides details about the - change. This includes discussion of the table architecture, key differences, - and benefits of the change. +The :ref:`astropy-table-and-dataframes` page provides the rationale for maintaining +and using the dedicated `astropy.table` package instead of relying on `pandas` or ``polars``. Getting Started =============== The basic workflow for creating a table, accessing table elements, -and modifying the table is shown below. These examples show a very simple +and modifying the table is shown below. These examples demonstrate a concise case, while the full `astropy.table` documentation is available from the :ref:`using_astropy_table` section. -First create a simple table with three columns of data named ``a``, ``b``, -and ``c``. These columns have integer, float, and string values respectively:: +First create a simple table with columns of data named ``a``, ``b``, ``c``, and +``d``. These columns have integer, float, string, and |Quantity| values +respectively:: - >>> from astropy.table import Table - >>> a = [1, 4, 5] - >>> b = [2.0, 5.0, 8.2] + >>> from astropy.table import QTable + >>> import astropy.units as u + >>> import numpy as np + + >>> a = np.array([1, 4, 5], dtype=np.int32) + >>> b = [2.0, 5.0, 8.5] >>> c = ['x', 'y', 'z'] - >>> t = Table([a, b, c], names=('a', 'b', 'c'), meta={'name': 'first table'}) + >>> d = [10, 20, 30] * u.m / u.s -If you have row-oriented input data such as a list of records, use the ``rows`` -keyword. In this example we also explicitly set the data types for each column:: + >>> t = QTable([a, b, c, d], + ... names=('a', 'b', 'c', 'd'), + ... meta={'name': 'first table'}) - >>> data_rows = [(1, 2.0, 'x'), - ... (4, 5.0, 'y'), - ... (5, 8.2, 'z')] - >>> t = Table(rows=data_rows, names=('a', 'b', 'c'), meta={'name': 'first table'}, - ... dtype=('i4', 'f8', 'S1')) +Comments: -There are a few ways to examine the table. You can get detailed information -about the table values and column definitions as follows:: +- Column ``a`` is a |ndarray| with a specified ``dtype`` of ``int32``. If the + data type is not provided, the default type for integers is ``int64`` on Mac + and Linux and ``int32`` on Windows. +- Column ``b`` is a list of ``float`` values, represented as ``float64``. +- Column ``c`` is a list of ``str`` values, represented as unicode. + See :ref:`bytestring-columns-python-3` for more information. +- Column ``d`` is a |Quantity| array. Since we used |QTable|, this stores a + native |Quantity| within the table and brings the full power of + :ref:`astropy-units` to this column in the table. - >>> t -
- a b c - int32 float64 string8 - ----- ------- ------- - 1 2.0 x - 4 5.0 y - 5 8.2 z - -You can also assign a unit to the columns. If any column has a unit -assigned, all units would be shown as follows:: - - >>> t['b'].unit = 's' - >>> t -
- a b c - s - int32 float64 string8 - ----- ------- ------- - 1 2.0 x - 4 5.0 y - 5 8.2 z +.. Note:: -A column with a unit works with and can be easily converted to an -`~astropy.units.Quantity` object:: + If the table data have no units or you prefer to not use |Quantity|, then you + can use the |Table| class to create tables. The **only** difference between + |QTable| and |Table| is the behavior when adding a column that has units. + See :ref:`quantity_and_qtable` and :ref:`columns_with_units` for details on + the differences and use cases. - >>> t['b'].quantity - - >>> t['b'].to('min') # doctest: +FLOAT_CMP - +There are many other ways of :ref:`construct_table`, including from a list of +rows (either tuples or dicts), from a ``numpy`` structured or 2D array, by +adding columns or rows incrementally, or even converting from a |SkyCoord|, a +:class:`pandas.DataFrame`, or a ``polars.DataFrame``. -From within the IPython notebook, the table is displayed as a formatted HTML table: +There are a few ways of :ref:`access_table`. You can get detailed information +about the table values and column definitions as follows:: + + >>> t + + a b c d + m / s + int32 float64 str1 float64 + ----- ------- ---- ------- + 1 2.0 x 10.0 + 4 5.0 y 20.0 + 5 8.5 z 30.0 + +You can get summary information about the table as follows:: + + >>> t.info + + name dtype unit class + ---- ------- ----- -------- + a int32 Column + b float64 Column + c str1 Column + d float64 m / s Quantity + +From within a `Jupyter notebook `_, the table is +displayed as a formatted HTML table (details of how it appears can be changed +by altering the `astropy.table.conf.default_notebook_table_class +` item in the +:ref:`astropy_config`: .. image:: table_repr_html.png + :width: 450px + +Or you can get a fancier notebook interface with :meth:`~astropy.table.Table.show_in_notebook`, +e.g., when used with ``backend="ipydatagrid"``, it comes with in-browser filtering and sort: + +.. image:: https://raw.githubusercontent.com/jupyter-widgets/ipydatagrid/main/static/ipydatagrid_1.gif + :width: 450px + :alt: Animated DataGrid usage example from ipydatagrid. If you print the table (either from the notebook or in a text console session) then a formatted version appears:: >>> print(t) - a b c - s - --- --- --- - 1 2.0 x - 4 5.0 y - 5 8.2 z + a b c d + m / s + --- --- --- ----- + 1 2.0 x 10.0 + 4 5.0 y 20.0 + 5 8.5 z 30.0 + -If you do not like the format of a particular column, you can change it:: +If you do not like the format of a particular column, you can change it through +:ref:`the 'info' property `:: - >>> t['b'].format = '7.3f' + >>> t['b'].info.format = '7.3f' >>> print(t) - a b c - s - --- ------- --- - 1 2.000 x - 4 5.000 y - 5 8.200 z + a b c d + m / s + --- ------- --- ----- + 1 2.000 x 10.0 + 4 5.000 y 20.0 + 5 8.500 z 30.0 For a long table you can scroll up and down through the table one page at time:: @@ -137,20 +147,21 @@ You can also display it as an HTML-formatted table in the browser:: >>> t.show_in_browser() # doctest: +SKIP -or as an interactive (searchable & sortable) javascript table:: +Or as an interactive (searchable and sortable) javascript table:: >>> t.show_in_browser(jsviewer=True) # doctest: +SKIP Now examine some high-level information about the table:: >>> t.colnames - ['a', 'b', 'c'] + ['a', 'b', 'c', 'd'] >>> len(t) 3 >>> t.meta {'name': 'first table'} -Access the data by column or row using familiar `numpy` structured array syntax:: +Access the data by column or row using familiar ``numpy`` structured array +syntax:: >>> t['a'] # Column 'a' @@ -159,75 +170,119 @@ Access the data by column or row using familiar `numpy` structured array syntax: 5 >>> t['a'][1] # Row 1 of column 'a' - 4 + np.int32(4) + + >>> t[1] # Row 1 of the table + + a b c d + m / s + int32 float64 str1 float64 + ----- ------- ---- ------- + 4 5.000 y 20.0 - >>> t[1] # Row obj for with row 1 values - >>> t[1]['a'] # Column 'a' of row 1 - 4 + np.int32(4) -You can retrieve a subset of a table by rows (using a slice) or +You can retrieve a subset of a table by rows (using a :class:`slice`) or by columns (using column names), where the subset is returned as a new table:: >>> print(t[0:2]) # Table object with rows 0 and 1 - a b c - s - --- ------- --- - 1 2.000 x - 4 5.000 y + a b c d + m / s + --- ------- --- ----- + 1 2.000 x 10.0 + 4 5.000 y 20.0 + - >>> print(t['a', 'c']) # Table with cols 'a', 'c' + >>> print(t['a', 'c']) # Table with cols 'a' and 'c' a c --- --- 1 x 4 y 5 z -Modifying table values in place is flexible and works as one would expect:: +:ref:`modify_table` in place is flexible and works as you would expect:: - >>> t['a'] = [-1, -2, -3] # Set all column values + >>> t['a'][:] = [-1, -2, -3] # Set all column values in place >>> t['a'][2] = 30 # Set row 2 of column 'a' - >>> t[1] = (8, 9.0, "W") # Set all row values + >>> t[1] = (8, 9.0, "W", 4 * u.m / u.s) # Set all values of row 1 >>> t[1]['b'] = -9 # Set column 'b' of row 1 >>> t[0:2]['b'] = 100.0 # Set column 'b' of rows 0 and 1 >>> print(t) - a b c - s - --- ------- --- - -1 100.000 x - 8 100.000 W - 30 8.200 z - -Add, remove, and rename columns with the following:: - - >>> t['d'] = [1, 2, 3] - >>> del t['c'] - >>> t.rename_column('a', 'A') + a b c d + m / s + --- ------- --- ----- + -1 100.000 x 10.0 + 8 100.000 W 4.0 + 30 8.500 z 30.0 + +Replace, add, remove, and rename columns with the following:: + + >>> t['b'] = ['a', 'new', 'dtype'] # Replace column 'b' (different from in-place) + >>> t['e'] = [1, 2, 3] # Add column 'e' + >>> del t['c'] # Delete column 'c' + >>> t.rename_column('a', 'A') # Rename column 'a' to 'A' >>> t.colnames - ['A', 'b', 'd'] + ['A', 'b', 'd', 'e'] -Adding a new row of data to the table is as follows:: +Adding a new row of data to the table is as follows. Note that the unit +value is given in ``cm / s`` but will be added to the table as ``0.1 m / s`` in +accord with the existing unit. - >>> t.add_row([-8, -9, 10]) - >>> len(t) - 4 + >>> t.add_row([-8, 'string', 10 * u.cm / u.s, 10]) + >>> t['d'] + -Lastly, you can create a table with support for missing values, for example by setting -``masked=True``:: +Tables can be used for data with missing values:: - >>> t = Table([a, b, c], names=('a', 'b', 'c'), masked=True, dtype=('i4', 'f8', 'S1')) - >>> t['a'].mask = [True, True, False] + >>> from astropy.table import MaskedColumn + >>> a_masked = MaskedColumn(a, mask=[True, True, False]) + >>> t = QTable([a_masked, b, c], names=('a', 'b', 'c'), + ... dtype=('i4', 'f8', 'U1')) + >>> t + + a b c + int32 float64 str1 + ----- ------- ---- + -- 2.0 x + -- 5.0 y + 5 8.5 z + +In addition to |Quantity|, you can include certain object types like +`~astropy.time.Time`, `~astropy.coordinates.SkyCoord`, and +`~astropy.table.NdarrayMixin` in your table. These "mixin" columns behave like +a hybrid of a regular `~astropy.table.Column` and the native object type (see +:ref:`mixin_columns`). For example:: + + >>> from astropy.time import Time + >>> from astropy.coordinates import SkyCoord + >>> tm = Time(['2000:002', '2002:345']) + >>> sc = SkyCoord([10, 20], [-45, +40], unit='deg') + >>> t = QTable([tm, sc], names=['time', 'skycoord']) >>> t -
- a b c - int32 float64 string8 - ----- ------- ------- - -- 2.0 x - -- 5.0 y - 5 8.2 z + + time skycoord + deg,deg + Time SkyCoord + --------------------- ---------- + 2000:002:00:00:00.000 10.0,-45.0 + 2002:345:00:00:00.000 20.0,40.0 + +Now let us compute the interval since the launch of the `Chandra X-ray Observatory +`_ aboard `STS-93 +`_ and store this in our table as a +|Quantity| in days:: + + >>> dt = t['time'] - Time('1999-07-23 04:30:59.984') + >>> t['dt_cxo'] = dt.to(u.d) + >>> t['dt_cxo'].info.format = '.3f' + >>> print(t) + time skycoord dt_cxo + deg,deg d + --------------------- ---------- -------- + 2000:002:00:00:00.000 10.0,-45.0 162.812 + 2002:345:00:00:00.000 20.0,40.0 1236.812 .. _using_astropy_table: @@ -236,7 +291,7 @@ Using ``table`` The details of using `astropy.table` are provided in the following sections: -Construct table +Construct Table --------------- .. toctree:: @@ -244,65 +299,81 @@ Construct table construct_table.rst -Access table ---------------- +Access Table +------------ .. toctree:: :maxdepth: 2 access_table.rst -Modify table ---------------- +Modify Table +------------ .. toctree:: :maxdepth: 2 modify_table.rst -Table operations ------------------ +Table Operations +---------------- .. toctree:: :maxdepth: 2 operations.rst +Indexing +-------- + +.. toctree:: + :maxdepth: 2 + + indexing.rst + Masking ---------------- +------- .. toctree:: :maxdepth: 2 masking.rst -I/O with tables ----------------- +Mixin Columns +------------- .. toctree:: :maxdepth: 2 - io.rst - pandas.rst + mixin_columns.rst -Mixin columns ----------------- +Astropy Table and DataFrames +---------------------------- .. toctree:: :maxdepth: 2 - mixin_columns.rst + dataframes.rst + pandas.rst + table_and_dataframes.rst Implementation ----------------- +-------------- .. toctree:: :maxdepth: 2 implementation_details.rst - implementation_change_1.0.rst + +.. note that if this section gets too long, it should be moved to a separate + doc page - see the top of performance.inc.rst for the instructions on how to do + that +.. include:: performance.inc.rst Reference/API ============= -.. automodapi:: astropy.table +.. toctree:: + :maxdepth: 2 + + ref_api diff --git a/docs/table/indexing.rst b/docs/table/indexing.rst new file mode 100644 index 000000000000..c86220f12735 --- /dev/null +++ b/docs/table/indexing.rst @@ -0,0 +1,364 @@ +.. |add_index| replace:: :func:`~astropy.table.Table.add_index` +.. |index_mode| replace:: :func:`~astropy.table.Table.index_mode` + +.. _table-indexing: + +Table Indexing +************** + +Once a |Table| has been created, it is possible to create indices on one or +more columns of the table. An index internally sorts the rows of a table based +on the index column(s), allowing for element retrieval by column value(s) and +improved performance for certain table operations. + +Creating an Index +================= + +.. EXAMPLE START: Creating Indices on Table Columns + +To create an index on a table, use the |add_index| method:: + + >>> from astropy.table import Table + >>> t = Table([(2, 3, 2, 1), (8, 7, 6, 5)], names=('a', 'b')) + >>> t.add_index('a') + +The optional argument ``unique`` may be specified to create an index with +uniquely valued elements. + +To create a composite index on multiple columns, pass a list of columns +instead:: + + >>> t.add_index(['a', 'b']) + +In particular, the first index created using the +|add_index| method is considered the default index or the "primary key." To +retrieve an index from a table, use the `~astropy.table.Table.indices` +property:: + + >>> t.indices['a'] + + a rows + --- ---- + 1 3 + 2 0 + 2 2 + 3 1>> + >>> t.indices['a', 'b'] + + a b rows + --- --- ---- + 1 5 3 + 2 6 2 + 2 8 0 + 3 7 1>> + +.. EXAMPLE END + +Row Retrieval using Indices +=========================== + +.. EXAMPLE START: Retrieving Table Rows using Indices + +Row retrieval can be accomplished using two table properties: +`~astropy.table.Table.loc` and `~astropy.table.Table.iloc`. The +`~astropy.table.Table.loc` property can be indexed either by column value, +range of column values (*including* the bounds), or a :class:`list` or +|ndarray| of column values:: + + >>> t = Table([(1, 2, 3, 4), (10, 1, 9, 9)], names=('a', 'b'), dtype=['i8', 'i8']) + >>> t.add_index('a') + >>> t.loc[2] # the row(s) where a == 2 + + a b + int64 int64 + ----- ----- + 2 1 + >>> t.loc[[1, 4]] # the row(s) where a in [1, 4] +
+ a b + int64 int64 + ----- ----- + 1 10 + 4 9 + >>> t.loc[1:3] # the row(s) where a in [1, 2, 3] +
+ a b + int64 int64 + ----- ----- + 1 10 + 2 1 + 3 9 + >>> t.loc[:] +
+ a b + int64 int64 + ----- ----- + 1 10 + 2 1 + 3 9 + 4 9 + +Using multiple indices +---------------------- +By default, `~astropy.table.Table.loc` uses the primary index, which +here is column ``'a'``. You can use a different index with the ``with_index`` method as shown below:: + + >>> t.add_index('b') + >>> t.loc.with_index('b')[8:10] +
+ a b + int64 int64 + ----- ----- + 3 9 + 4 9 + 1 10 + +The ``with_index`` method takes an index identifier as input, where the format is +flexible as shown in these examples:: + + >>> t.add_index(['a', 'b']) + >>> t.loc # defaults to primary key # doctest: +IGNORE_OUTPUT + >>> t.loc.with_index('b')[10] # doctest: +IGNORE_OUTPUT + >>> t.loc.with_index(['b'])[[10, 9]] # doctest: +IGNORE_OUTPUT + >>> t.loc.with_index('a', 'b')[1, 10] # doctest: +IGNORE_OUTPUT + >>> t.loc.with_index(['a', 'b'])[1, 10] # doctest: +IGNORE_OUTPUT + +Using a multi-column index +-------------------------- +You can create an index on multiple table columns and select table rows that match all +values of the indexed columns:: + + >>> t.add_index(["a", "b"]) + >>> t.loc.with_index("a", "b")[3, 9] + + a b + int64 int64 + ----- ----- + 3 9 + >>> t.loc.with_index("a", "b")[[(3, 9), (4, 9)]] +
+ a b + int64 int64 + ----- ----- + 3 9 + 4 9 + +The property `~astropy.table.Table.iloc` works similarly, except that the +retrieval information must be either an integer or a :class:`slice`, and +relates to the sorted order of the index rather than column values. For +example:: + + >>> t.iloc[0] # smallest row by value 'a' + + a b + int64 int64 + ----- ----- + 1 10 + >>> t.iloc.with_index('b')[1:] # all but smallest value of 'b' +
+ a b + int64 int64 + ----- ----- + 3 9 + 4 9 + 1 10 + +.. EXAMPLE END + +Effects on Performance +====================== + +Table operations change somewhat when indices are present, and there are a +number of factors to consider when deciding whether the use of indices will +improve performance. In general, indexing offers the following advantages: + +* Table grouping and sorting based on indexed column(s) both become faster. +* Retrieving values by index is faster than custom searching. + +There are certain caveats, however: + +* Creating an index requires time and memory. +* Table modifications become slower due to automatic index updates. +* Slicing a table becomes slower due to index relabeling. + +See `here +`_ +for an IPython notebook profiling various aspects of table indexing. + +Index Modes +=========== + +The |index_mode| method allows for some flexibility in the behavior of table +indexing by allowing the user to enter a specific indexing mode via a context +manager. There are currently three indexing modes: ``'freeze'``, +``'copy_on_getitem'``, and ``'discard_on_copy'``. + +.. EXAMPLE START: Table Indexing with the "freeze" Index Mode + +The ``'freeze'`` mode prevents automatic index updates whenever a column of the +index is modified, and all indices refresh themselves after the context ends:: + + >>> t = Table([(1, 2, 3, 4), (10, 1, 9, 9)], names=('a', 'b'), dtype=['i8', 'i8']) + >>> t.add_index('a') + >>> with t.index_mode('freeze'): + ... t['a'][0] = 0 + ... print(t.indices['a']) # unmodified + + a rows + --- ---- + 1 0 + 2 1 + 3 2 + 4 3>> + >>> print(t.indices['a']) # modified + + a rows + --- ---- + 0 0 + 2 1 + 3 2 + 4 3>> + +.. EXAMPLE END + +.. EXAMPLE START: Table Indexing with the "copy_on_getitem" Index Mode + +The ``'copy_on_getitem'`` mode forces columns to copy and relabel their indices +upon slicing. In the absence of this mode, table slices will preserve +indices while column slices will not:: + + >>> ca = t['a'][[1, 3]] + >>> ca.info.indices + [] + >>> with t.index_mode('copy_on_getitem'): + ... ca = t['a'][[1, 3]] + ... print(ca.info.indices) + [ + a rows + --- ---- + 2 0 + 4 1>>] + +.. EXAMPLE END + +.. EXAMPLE START: Table Indexing with the "discard_on_copy" Index Mode + +The ``'discard_on_copy'`` mode prevents indices from being copied whenever a +column or table is copied:: + + >>> t2 = Table(t) + >>> t2.indices['a'] + + a rows + --- ---- + 0 0 + 2 1 + 3 2 + 4 3>> + >>> with t.index_mode('discard_on_copy'): + ... t2 = Table(t) + ... print(t2.indices) + [] + +.. EXAMPLE END + +Updating Rows using Indices +=========================== + +.. EXAMPLE START: Updating Table Rows using Indices + +Row updates can be accomplished by assigning the table property +`~astropy.table.Table.loc` a complete row or a list of rows:: + + >>> t = Table([('w', 'x', 'y', 'z'), (10, 1, 9, 9)], names=('a', 'b'), dtype=['str', 'i8']) + >>> t.add_index('a') + >>> t.loc['x'] + + a b + str1 int64 + ---- ----- + x 1 + >>> t.loc['x'] = ['a', 12] + >>> t +
+ a b + str1 int64 + ---- ----- + w 10 + a 12 + y 9 + z 9 + >>> t.loc[['w', 'y']] +
+ a b + str1 int64 + ---- ----- + w 10 + y 9 + >>> t.loc[['w', 'z']] = [['b', 23], ['c', 56]] + >>> t +
+ a b + str1 int64 + ---- ----- + b 23 + a 12 + y 9 + c 56 + +.. EXAMPLE END + +Retrieving the Location of Rows using Indices +============================================= + +.. EXAMPLE START: Retrieving the Location of Table Rows using Indices + +Retrieval of the location of rows can be accomplished using a table property: +`~astropy.table.Table.loc_indices`. The `~astropy.table.Table.loc_indices` +property can be indexed either by column value, range of column values +(*including* the bounds), or a :class:`list` or |ndarray| of column values:: + + >>> t = Table([('w', 'x', 'y', 'z'), (10, 1, 9, 9)], names=('a', 'b'), dtype=['str', 'i8']) + >>> t.add_index('a') + >>> t.loc_indices['x'] + np.int64(1) + +.. EXAMPLE END + +Storing the Table Indices to File +================================= + +You can write a table with indices to FITS, ECVS, or HDF5 formats by supplying +``write_indices=True`` in the call to the table `~astropy.table.Table.write` method. In +this case the row index values are included in the table column data and column metadata +to describe the indices is stored. This allows efficiently restoring the table index or +indices when the data file is read back in. For an indexed table ``t``, you can do the +following:: + + >>> t.write("data.fits", write_indices=True) # doctest: +SKIP + >>> t.write("data.ecsv", write_indices=True) # doctest: +SKIP + >>> t.write("data.hdf5", write_indices=True, path="root", serialize_meta=True) # doctest: +SKIP + >>> t_fits = Table.read("data.fits", astropy_native=True) # doctest: +SKIP + >>> t_ecsv = Table.read("data.ecsv") # doctest: +SKIP + >>> t_hdf5 = Table.read("data.hdf5", path="root") # doctest: +SKIP + +Engines +======= + +When creating an index via |add_index|, the keyword argument ``engine`` may be +specified to use a particular indexing engine. The available engines are: + +* `~astropy.table.SortedArray`, a sorted array engine using an underlying + sorted |Table|. +* `~astropy.table.SCEngine`, a sorted list engine using the `Sorted Containers + `_ package. +* `~astropy.table.BST`, a Python-based binary search tree engine (not recommended). + +The SCEngine depends on the ``sortedcontainers`` dependency. The most important takeaway is that +`~astropy.table.SortedArray` (the default engine) is usually best, although +`~astropy.table.SCEngine` may be more appropriate for an index created on an +empty column since adding new values is quicker. + +The `~astropy.table.BST` engine demonstrates a simple pure Python implementation +of a search tree engine, but the performance is poor for larger tables. This +is available in the code largely as an implementation reference. diff --git a/docs/table/io.rst b/docs/table/io.rst deleted file mode 100644 index 801eff14dc58..000000000000 --- a/docs/table/io.rst +++ /dev/null @@ -1,65 +0,0 @@ -.. doctest-skip-all - -.. _read_write_tables: - -Reading and writing Table objects -=================================== - -Astropy provides a unified interface for reading and writing data -in different formats. For many common cases this will -simplify the process of file I/O and reduce the need to master -the separate details of all the I/O packages within Astropy. For details and -examples of using this interface see the :ref:`table_io` -section. - -Getting started ----------------- - -The :class:`~astropy.table.Table` class includes two methods, -:meth:`~astropy.table.Table.read` and -:meth:`~astropy.table.Table.write`, that make it possible to read from -and write to files. A number of formats are automatically supported (see -:ref:`built_in_readers_writers`) and new file formats and extensions can be -registered with the :class:`~astropy.table.Table` class (see -:ref:`io_registry`). - -To use this interface, first import the :class:`~astropy.table.Table` class, then -simply call the :class:`~astropy.table.Table` -:meth:`~astropy.table.Table.read` method with the name of the file and -the file format, for instance ``'ascii.daophot'``:: - - >>> from astropy.table import Table - >>> t = Table.read('photometry.dat', format='ascii.daophot') - -It is possible to load tables directly from the Internet using URLs. For example, -download tables from Vizier catalogues in CDS format (``'ascii.cds'``):: - - >>> t = Table.read("ftp://cdsarc.u-strasbg.fr/pub/cats/VII/253/snrs.dat", - ... readme="ftp://cdsarc.u-strasbg.fr/pub/cats/VII/253/ReadMe", - ... format="ascii.cds") - -For certain file formats, the format can be automatically detected, for -example from the filename extension:: - - >>> t = Table.read('table.tex') - -Similarly, for writing, the format can be explicitly specified:: - - >>> t.write(filename, format='latex') - -As for the :meth:`~astropy.table.Table.read` method, the format may -be automatically identified in some cases. - -Any additional arguments specified will depend on the format. For examples of this see the -section :ref:`built_in_readers_writers`. This section also provides the full list of -choices for the ``format`` argument. - -Supported formats ------------------- - -The :ref:`table_io` has built-in support for the following data file formats: - -* :ref:`table_io_ascii` -* :ref:`table_io_hdf5` -* :ref:`table_io_fits` -* :ref:`table_io_votable` diff --git a/docs/table/masking.rst b/docs/table/masking.rst index e1c5f2b56a35..bcd2ec32b84b 100644 --- a/docs/table/masking.rst +++ b/docs/table/masking.rst @@ -1,84 +1,96 @@ -.. include:: references.txt +.. _masking_and_missing_values: -Masking and missing values --------------------------- +Masking and Missing Values +************************** -The `astropy.table` package provides support for masking and missing -values in a table by wrapping the ``numpy.ma`` masked array package. -This allows handling tables with missing or invalid entries in much -the same manner as for standard (unmasked) tables. It -is useful to be familiar with the `masked array -`_ -documentation when using masked tables within `astropy.table`. +The `astropy.table` package provides support for masking and missing values in +a table by using the ``numpy.ma`` `masked array +`_ package to define +masked columns and by supporting :ref:`mixin_columns` that provide masking. +This allows handling tables with missing or invalid entries in much the same +manner as for standard (unmasked) tables. It is useful to be familiar with the +`masked array documentation +`_ +when using masked tables within `astropy.table`. In a nutshell, the concept is to define a boolean mask that mirrors -the structure of the table data array. Wherever a mask value is +the structure of a column data array. Wherever a mask value is `True`, the corresponding entry is considered to be missing or invalid. Operations involving column or row access and slicing are unchanged. The key difference is that arithmetic or reduction operations involving columns or column slices follow the rules for `operations on masked arrays -`_. +`_. .. Note:: - Reduction operations like `numpy.sum` or `numpy.mean` follow the - convention of ignoring masked (invalid) values. This differs from + Reduction operations like :func:`numpy.sum` or :func:`numpy.mean` follow the + convention of ignoring masked (invalid) values. This differs from the behavior of the floating point ``NaN``, for which the sum of an array including one or more ``NaN's`` will result in ``NaN``. - See ``_ for a very - interesting discussion of different strategies for handling - missing data in the context of `numpy`. -Table creation -^^^^^^^^^^^^^^^ + For more information see NumPy Enhancement Proposals `24 + `_, `25 + `_, and `26 + `_. + +Table Creation +============== A masked table can be created in several ways: -**Create a new table object and specify masked=True** :: +**Create a table with one or more columns as a MaskedColumn object** >>> from astropy.table import Table, Column, MaskedColumn - >>> t = Table([(1, 2), (3, 4)], names=('a', 'b'), masked=True, dtype=('i4', 'i8')) - >>> t -
+ >>> a = MaskedColumn([1, 2], name='a', mask=[False, True], dtype='i4') + >>> b = Column([3, 4], name='b', dtype='i8') + >>> Table([a, b]) +
a b int32 int64 ----- ----- 1 3 - 2 4 - - -Notice the table attributes ``mask`` and ``fill_value`` that are -available for a masked table. + -- 4 -**Create a table with one or more columns as a MaskedColumn object** +The |MaskedColumn| is the masked analog of the |Column| class and provides the +interface for creating and manipulating a column of masked data. The +|MaskedColumn| class inherits from :class:`numpy.ma.MaskedArray`, in contrast +to |Column| which inherits from |ndarray|. This distinction is the main reason +there are different classes for these two cases. - >>> a = MaskedColumn([1, 2], name='a') - >>> b = Column([3, 4], name='b') - >>> t = Table([a, b]) +Notice that masked entries in the table output are shown as ``--``. -The |MaskedColumn| is the masked analog of the |Column| class and -provides the interface for creating and manipulating a column of -masked data. The |MaskedColumn| class inherits from -`numpy.ma.MaskedArray`, in contrast to |Column| which inherits from -`numpy.ndarray`. This distinction is the main reason there are -different classes for these two cases. +**Create a table with one or more columns as a NumPy MaskedArray** -**Create a table with one or more columns as a numpy MaskedArray** - - >>> from numpy import ma # masked array package - >>> a = ma.array([1, 2]) + >>> import numpy as np + >>> a = np.ma.array([1, 2]) >>> b = [3, 4] >>> t = Table([a, b], names=('a', 'b')) +**Create a table from list data containing numpy.ma.masked** + +You can use the `numpy.ma.masked` constant to indicate masked or invalid data:: + + >>> a = [1.0, np.ma.masked] + >>> b = [np.ma.masked, 'val'] + >>> Table([a, b], names=('a', 'b')) +
+ a b + float64 str3 + ------- ---- + 1.0 -- + -- val + +Initializing from lists with embedded `numpy.ma.masked` elements is +considerably slower than using :func:`numpy.ma.array` or |MaskedColumn| +directly, so if performance is a concern you should use the latter methods if +possible. + **Add a MaskedColumn object to an existing table** >>> t = Table([[1, 2]], names=['a']) >>> b = MaskedColumn([3, 4], mask=[True, False]) >>> t['b'] = b - INFO: Upgrading Table to masked Table. Use Table.filled() to convert to unmasked table. [astropy.table.table] - -Note the INFO message because the underlying type of the table is modified in this operation. **Add a new row to an existing table and specify a mask argument** @@ -86,44 +98,57 @@ Note the INFO message because the underlying type of the table is modified in th >>> b = Column([3, 4], name='b') >>> t = Table([a, b]) >>> t.add_row([3, 6], mask=[True, False]) - INFO: Upgrading Table to masked Table. Use Table.filled() to convert to unmasked table. [astropy.table.table] + +**Create a new table object and specify masked=True** + +If ``masked=True`` is provided when creating the table then every column will +be created as a |MaskedColumn|, and new columns will always be added as a +|MaskedColumn|. + + >>> Table([(1, 2), (3, 4)], names=('a', 'b'), masked=True, dtype=('i4', 'i8')) +
+ a b + int32 int64 + ----- ----- + 1 3 + 2 4 **Convert an existing table to a masked table** >>> t = Table([[1, 2], ['x', 'y']]) # standard (unmasked) table - >>> t = Table(t, masked=True) # convert to masked table + >>> t = Table(t, masked=True, copy=False) # convert to masked table -Table access -^^^^^^^^^^^^ +This operation will convert every |Column| to |MaskedColumn| and ensure that any +subsequently added columns are masked. -Nearly all the of standard methods for accessing and modifying data -columns, rows, and individual elements also apply to masked tables. +Table Access +============ -There are two minor differences for the |Row| object that is obtained by -indexing a single row of a table: +Nearly all of the standard methods for accessing and modifying data +columns, rows, and individual elements also apply to masked tables. -- For standard tables, two such rows can be compared for equality, but - in masked tables this comparison will produce an exception. -- For standard tables a |Row| object provides a view of the underlying - table data so that it is possible to modify a table by modifying the - row values. In masked tables this is a copy so that modifying the - |Row| object has no effect on the original table data. +There is a difference however regarding the |Row| objects that are obtained by +indexing a single row of a table. For standard tables, two such rows can be +compared for equality, but for masked tables this comparison will produce an +exception:: -Both of these differences are due to issues in the underlying -`numpy.ma.MaskedArray` implementation. + >>> t[0] == t[1] + Traceback (most recent call last): + ... + ValueError: Unable to compare rows for masked table due to numpy.ma bug -Masking and filling -^^^^^^^^^^^^^^^^^^^^ +Masking and Filling +=================== -Both the |Table| and |MaskedColumn| classes provide -attributes and methods to support manipulating tables with missing or -invalid data. +Both the |Table| and |MaskedColumn| classes provide attributes and methods to +support manipulating tables with missing or invalid data. Mask -"""" +---- -The actual mask for the table as a whole or a single column can be -viewed and modified via the ``mask`` attribute:: +.. EXAMPLE START: Manipulating Tables with Missing Data using Masks + +The mask for a column can be viewed and modified via the ``mask`` attribute:: >>> t = Table([(1, 2), (3, 4)], names=('a', 'b'), masked=True) >>> t['a'].mask = [False, True] # Modify column mask (boolean array) @@ -134,44 +159,66 @@ viewed and modified via the ``mask`` attribute:: 1 -- -- 4 -Masked entries are shown as ``--`` when the table is printed. +Masked entries are shown as ``--`` when the table is printed. You can +view the mask directly, either at the column or table level:: + + >>> t['a'].mask + array([False, True]...) + + >>> t.mask +
+ a b + bool bool + ----- ----- + False True + True False + +To get the indices of masked elements, use an expression like:: + + >>> t['a'].mask.nonzero()[0] # doctest: +SKIP + array([1]) + +.. EXAMPLE END Filling -""""""" - -The entries which are masked (i.e. missing or invalid) can be replaced -with specified fill values. In this case the |MaskedColumn| or masked -|Table| will be converted to a standard |Column| or table. Each column -in a masked table has a ``fill_value`` attribute that specifies the -default fill value for that column. To perform the actual replacement -operation the ``filled()`` method is called. This takes an optional -argument which can override the default column ``fill_value`` +------- + +.. EXAMPLE START: Manipulating Tables with Missing Data by Filling Masked Values + +The entries which are masked (i.e., missing or invalid) can be replaced with +specified fill values. Filling a |MaskedColumn| produces a |Column|. Each +column in a masked table has a ``fill_value`` attribute that specifies the +default fill value for that column. To perform the actual replacement operation +the :meth:`~astropy.table.Table.filled` method is called. This takes an +optional argument which can override the default column ``fill_value`` attribute. :: >>> t['a'].fill_value = -99 >>> t['b'].fill_value = 33 - >>> print t.filled() + >>> print(t.filled()) a b --- --- 1 33 -99 4 - >>> print t['a'].filled() + >>> print(t['a'].filled()) a --- 1 -99 - >>> print t['a'].filled(999) + >>> print(t['a'].filled(999)) a --- 1 999 - >>> print t.filled(1000) + >>> print(t.filled(1000)) a b ---- ---- 1 1000 1000 4 + +.. EXAMPLE END diff --git a/docs/table/mixin_columns.rst b/docs/table/mixin_columns.rst index ba0793e05137..e215ca344a8a 100644 --- a/docs/table/mixin_columns.rst +++ b/docs/table/mixin_columns.rst @@ -1,34 +1,29 @@ -.. include:: references.txt .. |join| replace:: :func:`~astropy.table.join` -.. |Quantity| replace:: :class:`~astropy.units.Quantity` -.. |Time| replace:: :class:`~astropy.time.Time` -.. |SkyCoord| replace:: :class:`~astropy.coordinates.SkyCoord` .. _mixin_columns: -Mixin columns ---------------- +Mixin Columns +************* -Version 1.0 of astropy introduces a new concept of the "Mixin -Column" in tables which allows integration of appropriate non-|Column| based -class objects within a |Table| object. These mixin column objects are not -converted in any way but are used natively. +``astropy`` tables support the concept of "mixin columns", which +allows integration of appropriate non-|Column| based class objects within a +|Table| object. These mixin column objects are not converted in any way but are +used natively. The available built-in mixin column classes are: -- |Quantity| -- |SkyCoord| -- |Time| +- |Quantity| and subclasses +- |SkyCoord| and coordinate frame classes +- |Time| and :class:`~astropy.time.TimeDelta` +- :class:`~astropy.coordinates.EarthLocation` +- `~astropy.table.NdarrayMixin` -.. Warning:: +Basic Example +============= - The interface for using mixin columns is experimental at this point and it - is not recommended to use this feature in production code. There are known - limitations and some table functionality which is not yet implemented for - mixin columns. API changes are likely and since the code is all new there - may be some bugs. +.. EXAMPLE START: Using Mixin Columns in Tables -As a first example we can create a table and add a time column:: +As an example we can create a table and add a time column:: >>> from astropy.table import Table >>> from astropy.time import Time @@ -41,37 +36,44 @@ As a first example we can create a table and add a time column:: 1 2001-01-02T12:34:56.000 2 2001-02-03T00:01:02.000 -The important point here is that the ``time`` column is a bona fide |Time| object:: +The important point here is that the ``time`` column is a bona fide |Time| +object:: >>> t['time']
+ index data + int64 object + ----- -------------------------------------...- + 0 <__main__.ExampleDataClass object at ...> + 1 <__main__.ExampleDataClass object at ...> + 2 <__main__.ExampleDataClass object at ...> + 3 <__main__.ExampleDataClass object at ...> + +What happened is that the instance is seen as a scalar object, and a +|Column| with ``dtype=object`` is created, which has the same entry for +each row. The same would happen if, e.g., you set ``t['data'] = None``. + +However, you can create a function (or 'handler') which takes +an instance of the data class you want to have automatically +handled and turns it into a mixin column:: + + >>> from astropy.table.table_helpers import ArrayWrapper + >>> def handle_example_data_class(obj): + ... return ArrayWrapper(obj._data) + +You can then register this by providing the fully qualified name +of the class and the handler function:: + + >>> from astropy.table.mixins.registry import register_mixin_handler + >>> register_mixin_handler('__main__.ExampleDataClass', handle_example_data_class) + >>> t['data'] = ExampleDataClass() + >>> t +
+ index data + int64 float64 + ----- ------- + 0 0.0 + 1 1.0 + 2 3.0 + 3 4.0 + +.. testcleanup:: + + >>> from astropy.table.mixins.registry import _handlers + >>> del _handlers['__main__.ExampleDataClass'] + +Because we defined the data class as part of the example +above, the fully qualified name starts with ``__main__``, +but for a class in a third-party package, this might look +like ``package.Class`` for example. diff --git a/docs/table/modify_table.rst b/docs/table/modify_table.rst index 1a9948a61908..2f5d46ccf9c0 100644 --- a/docs/table/modify_table.rst +++ b/docs/table/modify_table.rst @@ -1,23 +1,29 @@ .. _modify_table: -.. include:: references.txt - -Modifying a table ------------------ +Modifying a Table +***************** The data values within a |Table| object can be modified in much the same manner -as for `numpy` structured arrays by accessing columns or rows of data and -assigning values appropriately. A key enhancement provided by the |Table| class -is the ability to easily modify the structure of the table: one can add or -remove columns, and add new rows of data. +as for ``numpy`` `structured arrays +`_ by accessing columns or +rows of data and assigning values appropriately. A key enhancement provided by +the |Table| class is the ability to modify the structure of the table: you can +add or remove columns, and add new rows of data. -Quick overview -^^^^^^^^^^^^^^ +Quick Overview +============== The code below shows the basics of modifying a table and its data. +Examples +-------- + +.. EXAMPLE START: Making a Table and Modifying Data -**Make a table** +.. _table-mod-make-a-table: + +Make a table +^^^^^^^^^^^^ :: >>> from astropy.table import Table @@ -25,20 +31,22 @@ The code below shows the basics of modifying a table and its data. >>> arr = np.arange(15).reshape(5, 3) >>> t = Table(arr, names=('a', 'b', 'c'), meta={'keywords': {'key1': 'val1'}}) -**Modify data values** +.. _table-mod-modify-data-values: + +Modify data values +^^^^^^^^^^^^^^^^^^ :: - >>> t['a'] = [1, -2, 3, -4, 5] # Set all column values - >>> t['a'][2] = 30 # Set row 2 of column 'a' - >>> t[1] = (8, 9, 10) # Set all row values - >>> t[1]['b'] = -9 # Set column 'b' of row 1 - >>> t[0:3]['c'] = 100 # Set column 'c' of rows 0, 1, 2 + >>> t['a'][:] = [1, -2, 3, -4, 5] # Set all values of column 'a' + >>> t['a'][2] = 30 # Set row 2 of column 'a' + >>> t[1] = (8, 9, 10) # Set all values of row 1 + >>> t[1]['b'] = -9 # Set column 'b' of row 1 + >>> t[0:3]['c'] = 100 # Set column 'c' of rows 0, 1, 2 -Note that ``table[row][column]`` assignments will not work with -`numpy` "fancy" ``row`` indexing (in that case ``table[row]`` would be -a *copy* instead of a *view*). "Fancy" `numpy` indices include a -`list`, `numpy.ndarray`, or `tuple` of `numpy.ndarray` (e.g. the -return from `numpy.where`):: +Note that ``table[row][column]`` assignments will not work with ``numpy`` +"fancy" ``row`` indexing (in that case ``table[row]`` would be a *copy* instead +of a *view*). "Fancy" ``numpy`` indices include a :class:`list`, |ndarray|, or +:class:`tuple` of |ndarray| (e.g., the return from :func:`numpy.where`):: >>> t[[1, 2]]['a'] = [3., 5.] # doesn't change table t >>> t[np.array([1, 2])]['a'] = [3., 5.] # doesn't change table t @@ -63,32 +71,46 @@ the conventions of `~astropy.units.Quantity` by using the 1000.0 2000.0 -**Add a column or columns** +.. note:: + + The best way to combine the functionality of the |Table| and |Quantity| + classes is to use a |QTable|. See :ref:`quantity_and_qtable` for more + information. + +.. EXAMPLE END + +.. _table-mod-add-a-column-or-columns: + +Add a column or columns +^^^^^^^^^^^^^^^^^^^^^^^ -A single column can be added to a table using syntax like adding a dict value. -The value on the right hand side can be a list or array -of the correct size, or a scalar value that will be broadcast:: +.. EXAMPLE START: Adding Columns to Tables + +A single column can be added to a table using syntax like adding a key-value +pair to a :class:`dict`. The value on the right hand side can be a +:class:`list` or |ndarray| of the correct size, or a scalar value that will be +`broadcast `_:: >>> t['d1'] = np.arange(5) >>> t['d2'] = [1, 2, 3, 4, 5] >>> t['d3'] = 6 # all 5 rows set to 6 -For more explicit control the :meth:`~astropy.table.Table.add_column` and -:meth:`~astropy.table.Table.add_columns` methods can be used to add one or multiple -columns to a table. In both cases the new columns must be specified as |Column| or -|MaskedColumn| objects with the ``name`` defined:: +For more explicit control, the :meth:`~astropy.table.Table.add_column` and +:meth:`~astropy.table.Table.add_columns` methods can be used to add one or +multiple columns to a table. In both cases the new column(s) can be specified as +a :class:`list`, |ndarray|, |Column|, |MaskedColumn|, or a scalar:: >>> from astropy.table import Column - >>> aa = Column(np.arange(5), name='aa') - >>> t.add_column(aa, index=0) # Insert before the first table column - - # Make a new table with the same number of rows and add columns to original table - >>> t2 = Table(np.arange(25).reshape(5, 5), names=('e', 'f', 'g', 'h', 'i')) - >>> t.add_columns(t2.columns.values()) + >>> t.add_column(np.arange(5), name='aa', index=0) # Insert before first table column + >>> t.add_column(1.0, name='bb') # Add column of all 1.0 to end of table + >>> c = Column(np.arange(5), name='e') + >>> t.add_column(c, index=0) # Add Column using the existing column name 'e' + >>> t.add_columns([[1, 2, 3, 4, 5], ['v', 'w', 'x', 'y', 'z']], names=['h', 'i']) -Finally, columns can also be added from -:class:`~astropy.units.Quantity` objects, which automatically sets the -``.unit`` attribute on the column: +Finally, columns can also be added from |Quantity| objects, which automatically +sets the ``unit`` attribute on the column (but you might find it more +convenient to add a |Quantity| to a |QTable| instead, see +:ref:`quantity_and_qtable` for details):: >>> from astropy import units as u >>> t['d'] = np.arange(1., 6.) * u.m @@ -100,50 +122,235 @@ Finally, columns can also be added from 4.0 5.0 -**Remove columns** -:: +.. EXAMPLE END - >>> t.remove_column('f') - >>> t.remove_columns(['aa', 'd1', 'd2', 'd3', 'e']) - >>> del t['g'] +.. _table-mod-remove-columns: + +Remove columns +^^^^^^^^^^^^^^ + +.. EXAMPLE START: Removing Columns from Tables + +To remove a column from a table:: + + >>> t.remove_column('d1') + >>> t.remove_columns(['aa', 'd2', 'e']) + >>> del t['d3'] >>> del t['h', 'i'] >>> t.keep_columns(['a', 'b']) -**Rename columns** -:: +.. EXAMPLE END + +.. _table-mod-replace-a-column: + +Replace a column +^^^^^^^^^^^^^^^^ + +.. EXAMPLE START: Replacing Columns in Tables + +You can entirely replace an existing column with a new column by setting the +column to any object that could be used to initialize a table column (e.g., a +:class:`list` or |ndarray|). For example, you could change the data type of the +``a`` column from ``int`` to ``float`` using:: + + >>> t['a'] = t['a'].astype(float) + +If the right-hand side value is not column-like, then an in-place update using +`broadcasting `_ +will be done, for example:: + + >>> t['a'] = 1 # Internally does t['a'][:] = 1 + +.. EXAMPLE END + +.. _table-mod-perform-a-dictionary-style-update: + +Perform a dictionary-style update +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It is possible to perform a dictionary-style update, which adds new columns to +the table and replaces existing ones:: + + >>> t1 = Table({'name': ['foo', 'bar'], 'val': [0., 0.]}, meta={'n': 2}) + >>> t2 = Table({'val': [1., 2.], 'val2': [10., 10.]}, meta={'id': 0}) + >>> t1 |= t2 + >>> t1 +
+ name val val2 + str3 float64 float64 + ---- ------- ------- + foo 1.0 10.0 + bar 2.0 10.0 + +When using ``|=``, the other object does not need to be a |Table|, it can be +anything that can be used for :ref:`construct_table` with a compatible number +of rows:: + + >>> t1 = Table({'name': ['foo', 'bar'], 'val': [0., 0.]}, meta={'n': 2}) + >>> d = dict({'val': [1., 2.], 'val2': [10., 10.]}) + >>> t1 |= d + >>> t1 +
+ name val val2 + str3 float64 float64 + ---- ------- ------- + foo 1.0 10.0 + bar 2.0 10.0 + +It is also possible to use the ``|`` operator to merge multiple |Table| instances +into a new table:: + + >>> from astropy.table import QTable + >>> t1 = Table({'name': ['foo', 'bar'], 'val': [0., 0.]}, meta={'n': 2}) + >>> t2 = QTable({'val': [1., 2.], 'val2': [10., 10.]}, meta={'id': 0}) + >>> t3 = t1 | t2 # Create a new table as result of update + >>> t3 +
+ name val val2 + str3 float64 float64 + ---- ------- ------- + foo 1.0 10.0 + bar 2.0 10.0 + +``|`` and ``|=`` also take care of silently :ref:`merging_metadata`:: + + >>> t3.meta + {'n': 2, 'id': 0} + +The columns in the updated |Table| are going to be copies of the originals. If +you need them to be references you can use the +:meth:`~astropy.table.Table.update` method with ``copy=False``, see :ref:`copy_versus_reference` +for details. + +.. _table-mod-ensure-the-existence-of-a-column: + +Ensure the existence of a column +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +|Table| has a :meth:`~astropy.table.Table.setdefault` method, which is +analogous to :meth:`dict.setdefault`. +It adds a column with a given name to the table if such a column is not in the +table already. +The default value passed to the method will be validated and, if necessary, +converted. +Either way the (possibly just inserted) column in the table is returned:: + + >>> t0 = Table({"a": ["Ham", "Spam"]}) + >>> t0 +
+ a + str4 + ---- + Ham + Spam + >>> t0.setdefault("a", ["Breakfast"]) # Existing column + + Ham + Spam + >>> t0.setdefault("approved", False) # New column + + False + False + >>> t0 +
+ a approved + str4 bool + ---- -------- + Ham False + Spam False + +.. _table-mod-rename-columns: + +Rename columns +^^^^^^^^^^^^^^ + +.. EXAMPLE START: Renaming Columns in Tables + +To rename a column:: >>> t.rename_column('a', 'a_new') >>> t['b'].name = 'b_new' -**Add a row of data** -:: +To rename multiple columns at once:: + + >>> t.rename_columns(['a_new', 'b_new'], ['a', 'b']) + +.. EXAMPLE END + +.. _table-mod-add-a-row-of-data: + +Add a row of data +^^^^^^^^^^^^^^^^^ + +.. EXAMPLE START: Adding a Row of Data to a Table + +To add a row:: >>> t.add_row([-8, -9]) -**Remove rows** -:: +.. EXAMPLE END + +.. _table-mod-remove-rows: + +Remove rows +^^^^^^^^^^^ + +.. EXAMPLE START: Removing Rows of Data from Tables + +To remove a row:: >>> t.remove_row(0) >>> t.remove_rows(slice(4, 5)) >>> t.remove_rows([1, 2]) -**Sort by one more more columns** -:: +.. EXAMPLE END - >>> t.sort('b_new') - >>> t.sort(['a_new', 'b_new']) +.. _table-mod-sort-by-one-or-more-columns: -**Reverse table rows** -:: +Sort by one or more columns +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. EXAMPLE START: Sorting Columns in Tables + +To sort columns:: + + >>> t.sort('b') + >>> t.sort(['a', 'b']) + +.. EXAMPLE END + +.. _table-mod-reverse-table-rows: + +Reverse table rows +^^^^^^^^^^^^^^^^^^ + +.. EXAMPLE START: Reversing Table Rows + +To reverse the order of table rows:: >>> t.reverse() -**Modify meta-data** -:: +.. EXAMPLE END + +.. _table-mod-modify-metadata: + +Modify metadata +^^^^^^^^^^^^^^^ + +.. EXAMPLE START: Modifying Metadata in Tables + +To modify metadata:: >>> t.meta['key'] = 'value' -**Select or reorder columns** +.. EXAMPLE END + +.. _table-mod-select-or-reorder-columns: + +Select or reorder columns +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. EXAMPLE START: Selecting or Reordering Columns in Tables A new table with a subset or reordered list of columns can be created as shown in the following example:: @@ -152,43 +359,100 @@ created as shown in the following example:: >>> t_acb = t['a', 'c', 'b'] Another way to do the same thing is to provide a list or tuple -as the item as shown below:: +as the item, as shown below:: >>> new_order = ['a', 'c', 'b'] # List or tuple >>> t_acb = t[new_order] -Caveats -^^^^^^^ +.. EXAMPLE END -Modifying the table data and properties is fairly straightforward. There are -only a few things to keep in mind: +Caveats +======= -- The data type for a column cannot be changed in place. In order to do this - you must make a copy of the table with the column type changed appropriately. -- Adding or removing a column will generate a new copy - in memory of all the data. If the table is very large this may be slow. -- Adding a row *may* require a new copy in memory of the table data. This - depends on the detailed layout of Python objects in memory and cannot be - reliably controlled. In some cases it may be possible to build a table - row by row in less than O(N**2) time but you cannot count on it. +Modifying the table data and properties is fairly clear-cut, but one thing +to keep in mind is that adding a row *may* require a new copy in memory of the +table data. This depends on the detailed layout of Python objects in memory +and cannot be reliably controlled. In some cases it may be possible to build a +table row by row in less than O(N**2) time but you cannot count on it. -Another subtlety to keep in mind are cases where the return value of an -operation results in a new table in memory versus a view of the existing -table data. As an example, imagine trying to set two table elements -using column selection with ``t['a', 'c']`` in combination with row index selection:: +Another subtlety to keep in mind is that in some cases the return value of an +operation results in a new table in memory while in other cases it results in a +view of the existing table data. As an example, imagine trying to set two table +elements using column selection with ``t['a', 'c']`` in combination with row +index selection:: >>> t = Table([[1, 2], [3, 4], [5, 6]], names=('a', 'b', 'c')) >>> t['a', 'c'][1] = (100, 100) - >>> print t + >>> print(t) a b c --- --- --- 1 3 5 2 4 6 This might be surprising because the data values did not change and there -was no error. In fact what happened is that ``t['a', 'c']`` created a -new temporary table in memory as a *copy* of the original and then updated -row 1 of the copy. The original ``t`` table was unaffected and the new -temporary table disappeared once the statement was complete. The takeaway +was no error. In fact, what happened is that ``t['a', 'c']`` created a +new temporary table in memory as a *copy* of the original and then updated the +first row of the copy. The original ``t`` table was unaffected and the new +temporary table disappeared once the statement was complete. The takeaway is to pay attention to how certain operations are performed one step at a time. + +.. _table-replace-1_3: + +In-Place Versus Replace Column Update +===================================== + +Consider this code snippet:: + + >>> t = Table([[1, 2, 3]], names=['a']) + >>> t['a'] = [10.5, 20.5, 30.5] + +There are a couple of ways this could be handled. It could update the existing +array values in-place (truncating to integer), or it could replace the entire +column with a new column based on the supplied data values. + +The answer for ``astropy`` is that the operation shown above does a *complete +replacement* of the column object. In this case it makes a new column object +with float values by internally calling ``t.replace_column('a', [10.5, 20.5, +30.5])``. In general this behavior is more consistent with Python and `pandas +`_ behavior. + +.. _table-mod-forcing-in-place-update: + +Forcing in-place update +----------------------- + +It is possible to force an in-place update of a column as follows:: + + t[colname][:] = value + +.. _table-mod-finding-the-source-of-problems: + +Finding the source of problems +------------------------------ + +In order to find potential problems related to replacing columns, there is the +option `astropy.table.conf.replace_warnings +` in the :ref:`astropy_config`. This +controls a set of warnings that are emitted under certain circumstances when a +table column is replaced. This option must be set to a list that includes zero +or more of the following string values: + +``always`` : + Print a warning every time a column gets replaced via the + ``__setitem__()`` syntax (i.e., ``t['a'] = new_col``). + +``slice`` : + Print a warning when a column that appears to be a :class:`slice` of + a parent column is replaced. + +``refcount`` : + Print a warning when the Python reference count for the + column changes. This indicates that a stale object exists that might + be used elsewhere in the code and give unexpected results. + +``attributes`` : + Print a warning if any of the standard column attributes changed. + +The default value for the ``table.conf.replace_warnings`` option is +``[]`` (no warnings). diff --git a/docs/table/operations.rst b/docs/table/operations.rst index 610bb13abbf2..f335c7711b0e 100644 --- a/docs/table/operations.rst +++ b/docs/table/operations.rst @@ -1,13 +1,12 @@ -.. include:: references.txt .. |join| replace:: :func:`~astropy.table.join` .. _table_operations: -Table operations ------------------ +Table Operations +**************** -In this section we describe higher-level operations that can be used to generate a new -table from one or more input tables. This includes: +In this section we describe high-level operations that can be used to generate +a new table from one or more input tables. This includes: ======================= @@ -20,29 +19,40 @@ table from one or more input tables. This includes: - Function * - `Grouped operations`_ - Group tables and columns by keys - - `~astropy.table.Table.group_by` + - :func:`~astropy.table.Table.group_by` + * - `Binning`_ + - Binning tables + - :func:`~astropy.table.Table.group_by` * - `Stack vertically`_ - Concatenate input tables along rows - - `~astropy.table.vstack` + - :func:`~astropy.table.vstack` * - `Stack horizontally`_ - Concatenate input tables along columns - - `~astropy.table.hstack` + - :func:`~astropy.table.hstack` * - `Join`_ - Database-style join of two tables - - `~astropy.table.join` + - |join| * - `Unique rows`_ - Unique table rows by keys - - `~astropy.table.unique` + - :func:`~astropy.table.unique` + * - `Set difference`_ + - Set difference of two tables + - :func:`~astropy.table.setdiff` + * - `Table diff`_ + - Generic difference of two simple tables + - :func:`~astropy.utils.diff.report_diff_values` .. _grouped-operations: -Grouped operations -^^^^^^^^^^^^^^^^^^ +Grouped Operations +------------------ -Sometimes in a table or table column there are natural groups within the dataset for which -it makes sense to compute some derived values. A simple example is a list of objects with -photometry from various observing runs:: +.. EXAMPLE START: Grouped Operations in Tables + +Sometimes in a table or table column there are natural groups within the dataset +for which it makes sense to compute some derived values. A minimal example is a +list of objects with photometry from various observing runs:: >>> from astropy.table import Table >>> obs = Table.read("""name obs_date mag_b mag_v @@ -57,14 +67,19 @@ photometry from various observing runs:: ... M101 2012-03-26 15.1 13.5 ... M101 2012-03-26 14.8 14.3 ... """, format='ascii') + >>> # Make sure magnitudes are printed with one digit after the decimal point + >>> obs['mag_b'].info.format = '{:.1f}' + >>> obs['mag_v'].info.format = '{:.1f}' + +.. EXAMPLE END -Table groups -~~~~~~~~~~~~~~ +Table Groups +^^^^^^^^^^^^ -Now suppose we want the mean magnitudes for each object. We first group the data by the -``name`` column with the :func:`~astropy.table.Table.group_by` method. This returns -a new table sorted by ``name`` which has a ``groups`` property specifying the unique -values of ``name`` and the corresponding table rows:: +Now suppose we want the mean magnitudes for each object. We first group the data +by the ``name`` column with the :func:`~astropy.table.Table.group_by` method. +This returns a new table sorted by ``name`` which has a ``groups`` property +specifying the unique values of ``name`` and the corresponding table rows:: >>> obs_by_name = obs.group_by('name') >>> print(obs_by_name) # doctest: +SKIP @@ -90,26 +105,35 @@ values of ``name`` and the corresponding table rows:: >>> print(obs_by_name.groups.indices) [ 0 4 7 10] -The ``groups`` property is the portal to all grouped operations with tables and columns. -It defines how the table is grouped via an array of the unique row key values and the -indices of the group boundaries for those key values. The groups here correspond to the -row slices ``0:4``, ``4:7``, and ``7:10`` in the ``obs_by_name`` table. +The ``groups`` property is the portal to all grouped operations with tables and +columns. It defines how the table is grouped via an array of the unique row key +values and the indices of the group boundaries for those key values. The groups +here correspond to the row slices ``0:4``, ``4:7``, and ``7:10`` in the +``obs_by_name`` table. + +The output grouped table has two important properties: -The initial argument (``keys``) for the `~astropy.table.Table.group_by` function -can take a number of input data types: +- The groups in the order of the lexically sorted key values (``M101``, ``M31``, + ``M82`` in our example). +- The rows within each group are in the same order as they appear in the + original table. + +The initial argument (``keys``) for the :func:`~astropy.table.Table.group_by` +function can take a number of input data types: - Single string value with a table column name (as shown above) - List of string values with table column names - Another |Table| or |Column| with same length as table -- Numpy structured array with same length as table -- Numpy homogeneous array with same length as table +- ``numpy`` structured array with same length as table +- ``numpy`` homogeneous array with same length as table -In all cases the corresponding row elements are considered as a tuple of values which -form a key value that is used to sort the original table and generate -the required groups. +In all cases the corresponding row elements are considered as a :class:`tuple` +of values which form a key value that is used to sort the original table and +generate the required groups. As an example, to get the average magnitudes for each object on each observing -night, we would first group the table on both ``name`` and ``obs_date`` as follows:: +night, we would first group the table on both ``name`` and ``obs_date`` as +follows:: >>> print(obs.group_by(['name', 'obs_date']).groups.keys) name obs_date @@ -123,13 +147,15 @@ night, we would first group the table on both ``name`` and ``obs_date`` as follo M82 2012-03-26 -Manipulating groups -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Manipulating Groups +^^^^^^^^^^^^^^^^^^^ -Once you have applied grouping to a table then you can easily access the individual -groups or subsets of groups. In all cases this returns a new grouped table. -For instance to get the sub-table which corresponds to the second group (index=1) -do:: +.. EXAMPLE START: Manipulating Groups in Tables + +Once you have applied grouping to a table then you can access the individual +groups or subsets of groups. In all cases this returns a new grouped table. +For instance, to get the subtable which corresponds to the second group +(index=1) do:: >>> print(obs_by_name.groups[1]) name obs_date mag_b mag_v @@ -138,7 +164,7 @@ do:: M31 2012-01-02 17.1 17.4 M31 2012-02-14 16.9 17.3 -To get the first and second groups together use a slice:: +To get the first and second groups together use a :class:`slice`:: >>> groups01 = obs_by_name.groups[0:2] >>> print(groups01) @@ -157,8 +183,8 @@ To get the first and second groups together use a slice:: M101 M31 -You can also supply a numpy array of indices or a boolean mask to select particular -groups, e.g.:: +You can also supply a ``numpy`` array of indices or a boolean mask to select +particular groups, for example:: >>> mask = obs_by_name.groups.keys['name'] == 'M101' >>> print(obs_by_name.groups[mask]) @@ -169,13 +195,12 @@ groups, e.g.:: M101 2012-03-26 15.1 13.5 M101 2012-03-26 14.8 14.3 -One can iterate over the group sub-tables and corresponding keys with:: +You can iterate over the group subtables and corresponding keys with:: - >>> from itertools import izip - >>> for key, group in izip(obs_by_name.groups.keys, obs_by_name.groups): - ... print('****** {0} *******'.format(key['name'])) + >>> for key, group in zip(obs_by_name.groups.keys, obs_by_name.groups): + ... print(f'****** {key["name"]} *******') ... print(group) - ... print + ... print('') ... ****** M101 ******* name obs_date mag_b mag_v @@ -197,19 +222,26 @@ One can iterate over the group sub-tables and corresponding keys with:: M82 2012-02-14 15.2 15.5 M82 2012-03-26 15.7 16.5 +.. EXAMPLE END + Column Groups -~~~~~~~~~~~~~~ +^^^^^^^^^^^^^ Like |Table| objects, |Column| objects can also be grouped for subsequent -manipulation with grouped operations. This can apply both to columns within a +manipulation with grouped operations. This can apply both to columns within a |Table| or bare |Column| objects. As for |Table|, the grouping is generated with the -`~astropy.table.Table.group_by` method. The difference here is that +:func:`~astropy.table.Table.group_by` method. The difference here is that there is no option of providing one or more column names since that -doesn't make sense for a |Column|. +does not make sense for a |Column|. + +Examples +~~~~~~~~ + +.. EXAMPLE START: Grouping Column Objects in Tables -Examples:: +To generate grouping in columns:: >>> from astropy.table import Column >>> import numpy as np @@ -217,10 +249,10 @@ Examples:: >>> key_vals = np.array(['foo', 'bar', 'foo', 'foo', 'qux', 'qux']) >>> cg = c.group_by(key_vals) - >>> for key, group in izip(cg.groups.keys, cg.groups): - ... print('****** {0} *******'.format(key)) + >>> for key, group in zip(cg.groups.keys, cg.groups): + ... print(f'****** {key} *******') ... print(group) - ... print + ... print('') ... ****** bar ******* a @@ -238,36 +270,39 @@ Examples:: 5 6 +.. EXAMPLE END Aggregation -~~~~~~~~~~~~~~ +^^^^^^^^^^^ -Aggregation is the process of applying a -specified reduction function to the values within each group for each -non-key column. This function must accept a numpy array as the first -argument and return a single scalar value. Common function examples are -`numpy.sum`, `numpy.mean`, and `numpy.std`. +Aggregation is the process of applying a specified reduction function to the +values within each group for each non-key column. This function must accept a +|ndarray| as the first argument and return a single scalar value. Common +function examples are :func:`numpy.sum`, :func:`numpy.mean`, and +:func:`numpy.std`. -For the example grouped table ``obs_by_name`` from above we compute the group means with -the `~astropy.table.groups.TableGroups.aggregate` method:: +For the example grouped table ``obs_by_name`` from above, we compute the group +means with the :meth:`~astropy.table.groups.TableGroups.aggregate` method:: - >>> obs_mean = obs_by_name.groups.aggregate(np.mean) # doctest: +SKIP - WARNING: Cannot aggregate column 'obs_date' [astropy.table.groups] - >>> print(obs_mean) # doctest: +SKIP + >>> obs_mean = obs_by_name.groups.aggregate(np.mean) # doctest: +SHOW_WARNINGS + AstropyUserWarning: Cannot aggregate column 'obs_date' with type '>> print(obs_mean) name mag_b mag_v - ---- ----- ------ - M101 15.0 13.725 - M31 17.0 17.4 - M82 15.7 15.5 - -It seems the magnitude values were successfully averaged, but what -about the WARNING? Since the ``obs_date`` column is a string-type -array, the `numpy.mean` function failed and raised an exception. -Any time this happens then `~astropy.table.groups.TableGroups.aggregate` -will issue a warning and then -drop that column from the output result. Note that the ``name`` -column is one of the ``keys`` used to determine the grouping so -it is automatically ignored from aggregation. + ---- ----- ----- + M101 15.0 13.7 + M31 17.0 17.4 + M82 15.7 15.5 + +It seems the magnitude values were successfully averaged, but what about the +:class:`~astropy.utils.exceptions.AstropyUserWarning`? Since the ``obs_date`` +column is a string-type array, the :func:`numpy.mean` function failed and +raised an exception ``cannot perform reduceat with flexible type``. Any time this happens +:meth:`~astropy.table.groups.TableGroups.aggregate` will issue a warning and +then drop that column from the output result. Note that the ``name`` column is +one of the ``keys`` used to determine the grouping so it is automatically +ignored from aggregation. + +.. EXAMPLE START: Performing Aggregation on Grouped Tables From a grouped table it is possible to select one or more columns on which to perform the aggregation:: @@ -279,12 +314,15 @@ to perform the aggregation:: 17.0 15.7 +The order of the columns can be specified too:: + >>> print(obs_by_name['name', 'mag_v', 'mag_b'].groups.aggregate(np.mean)) - name mag_v mag_b - ---- ------ ----- - M101 13.725 15.0 - M31 17.4 17.0 - M82 15.5 15.7 + name mag_v mag_b + ---- ----- ----- + M101 13.7 15.0 + M31 17.4 17.0 + M82 15.5 15.7 + A single column of data can be aggregated as well:: @@ -292,62 +330,97 @@ A single column of data can be aggregated as well:: >>> key_vals = np.array(['foo', 'bar', 'foo', 'foo', 'qux', 'qux']) >>> cg = c.group_by(key_vals) >>> cg_sums = cg.groups.aggregate(np.sum) - >>> for key, cg_sum in izip(cg.groups.keys, cg_sums): - ... print('Sum for {0} = {1}'.format(key, cg_sum)) + >>> for key, cg_sum in zip(cg.groups.keys, cg_sums): + ... print(f'Sum for {key} = {cg_sum}') ... Sum for bar = 2 Sum for foo = 8 Sum for qux = 11 -If the specified function has a `numpy.ufunc.reduceat` method, this will be called instead. -This can improve the performance by a factor of 10 to 100 (or more) for large unmasked -tables or columns with many relatively small groups. It also allows for the use of -certain numpy functions which normally take more than one input array but also work as -reduction functions, like `numpy.add`. The numpy functions which should take advantage of -using `numpy.ufunc.reduceat` include: - -`numpy.add`, `numpy.arctan2`, `numpy.bitwise_and`, `numpy.bitwise_or`, `numpy.bitwise_xor`, -`numpy.copysign`, `numpy.divide`, `numpy.equal`, `numpy.floor_divide`, `numpy.fmax`, -`numpy.fmin`, `numpy.fmod`, `numpy.greater_equal`, `numpy.greater`, `numpy.hypot`, -`numpy.left_shift`, `numpy.less_equal`, `numpy.less`, `numpy.logaddexp2`, -`numpy.logaddexp`, `numpy.logical_and`, `numpy.logical_or`, `numpy.logical_xor`, -`numpy.maximum`, `numpy.minimum`, `numpy.mod`, `numpy.multiply`, `numpy.not_equal`, -`numpy.power`, `numpy.remainder`, `numpy.right_shift`, `numpy.subtract` and `numpy.true_divide`. - -As special cases `numpy.sum` and `numpy.mean` are substituted with their -respective reduceat methods. - +.. EXAMPLE END + +If the specified function has a :meth:`numpy.ufunc.reduceat` method, this will +be called instead. This can improve the performance by a factor of 10 to 100 +(or more) for large unmasked tables or columns with many relatively small +groups. It also allows for the use of certain ``numpy`` functions which +normally take more than one input array but also work as reduction functions, +like `numpy.add`. The ``numpy`` functions which should take advantage of using +:meth:`numpy.ufunc.reduceat` include: + +- `numpy.add` +- `numpy.arctan2` +- `numpy.bitwise_and` +- `numpy.bitwise_or` +- `numpy.bitwise_xor` +- `numpy.copysign` +- `numpy.divide` +- `numpy.equal` +- `numpy.floor_divide` +- `numpy.fmax` +- `numpy.fmin` +- `numpy.fmod` +- `numpy.greater_equal` +- `numpy.greater` +- `numpy.hypot` +- `numpy.left_shift` +- `numpy.less_equal` +- `numpy.less` +- `numpy.logaddexp2` +- `numpy.logaddexp` +- `numpy.logical_and` +- `numpy.logical_or` +- `numpy.logical_xor` +- `numpy.maximum` +- `numpy.minimum` +- `numpy.mod` +- `numpy.multiply` +- `numpy.not_equal` +- `numpy.power` +- `numpy.remainder` +- `numpy.right_shift` +- `numpy.subtract` +- `numpy.true_divide` + +In special cases, :func:`numpy.sum` and :func:`numpy.mean` are substituted with +their respective ``reduceat`` methods. Filtering -~~~~~~~~~~ +^^^^^^^^^ Table groups can be filtered by means of the -`~astropy.table.groups.TableGroups.filter` method. This is done by -supplying a function which is called for each group. The function +:meth:`~astropy.table.groups.TableGroups.filter` method. This is done by +supplying a function which is called for each group. The function which is passed to this method must accept two arguments: - ``table`` : |Table| object - ``key_colnames`` : list of columns in ``table`` used as keys for grouping -It must then return either `True` or `False`. As an example, the following -will select all table groups with only positive values in the non-key columns:: +It must then return either `True` or `False`. + +Example +~~~~~~~ + +.. EXAMPLE START: Filtering Table Groups + +The following will select all table groups with only positive values in the +non-key columns:: >>> def all_positive(table, key_colnames): ... colnames = [name for name in table.colnames if name not in key_colnames] ... for colname in colnames: - ... if np.any(table[colname] < 0): + ... if np.any(table[colname] <= 0): ... return False ... return True An example of using this function is:: >>> t = Table.read(""" a b c - ... -2 7.0 0 + ... -2 7.0 2 ... -2 5.0 1 ... 1 3.0 -5 ... 1 -2.0 -6 ... 1 1.0 7 - ... 0 0.0 4 + ... 0 4.0 4 ... 3 3.0 5 ... 3 -2.0 6 ... 3 1.0 7""", format='ascii') @@ -355,42 +428,111 @@ An example of using this function is:: >>> t_positive = tg.groups.filter(all_positive) >>> for group in t_positive.groups: ... print(group) - ... print + ... print('') ... a b c --- --- --- - -2 7.0 0 + -2 7.0 2 -2 5.0 1 a b c --- --- --- - 0 0.0 4 + 0 4.0 4 -As can be seen only the groups with ``a == -2`` and ``a == 0`` have all positive values -in the non-key columns, so those are the ones that are selected. +As can be seen only the groups with ``a == -2`` and ``a == 0`` have all +positive values in the non-key columns, so those are the ones that are selected. Likewise a grouped column can be filtered with the -`~astropy.table.groups.ColumnGroups.filter`, method but in this case the filtering -function takes only a single argument which is the column group. It still must return -either `True` or `False`. For example:: +:meth:`~astropy.table.groups.ColumnGroups.filter`, method but in this case the +filtering function takes only a single argument which is the column group. It +still must return either `True` or `False`. For example:: def all_positive(column): - if np.any(column < 0): - return False - return True + return np.all(column > 0) + +.. EXAMPLE END + +.. _table_binning: + +Binning +------- + +A common tool in analysis is to bin a table based on some reference value. +Examples: + +- Photometry of a binary star in several bands taken over a + span of time which should be binned by orbital phase. +- Reducing the sampling density for a table by combining + 100 rows at a time. +- Unevenly sampled historical data which should binned to + four points per year. + +All of these examples of binning a table can be accomplished using +`grouped operations`_. The examples in that section are focused on the +case of discrete key values such as the name of a source. In this +section we show a concise yet powerful way of applying grouped operations to +accomplish binning on key values such as time, phase, or row number. + +The common theme in all of these cases is to convert the key value array into +a new float- or int-valued array whose values are identical for rows in the same +output bin. + +Example +^^^^^^^ + +.. EXAMPLE START: Binning a Table using Grouped Operations + +As an example, we generate a fake light curve:: + + >>> year = np.linspace(2000.0, 2010.0, 200) # 200 observations over 10 years + >>> period = 1.811 + >>> y0 = 2005.2 + >>> mag = 14.0 + 1.2 * np.sin(2 * np.pi * (year - y0) / period) + >>> phase = ((year - y0) / period) % 1.0 + >>> dat = Table([year, phase, mag], names=['year', 'phase', 'mag']) + +Now we make an array that will be used for binning the data by 0.25 year +intervals:: + + >>> year_bin = np.trunc(year / 0.25) + +This has the property that all samples in each 0.25 year bin have the same +value of ``year_bin``. Think of ``year_bin`` as the bin number for ``year``. +Then do the binning by grouping and immediately aggregating with +:func:`numpy.mean`. + + >>> dat_grouped = dat.group_by(year_bin) + >>> dat_binned = dat_grouped.groups.aggregate(np.mean) + +We can plot the results with ``plt.plot(dat_binned['year'], dat_binned['mag'], +'.')``. Alternately, we could bin into 10 phase bins:: + + >>> phase_bin = np.trunc(phase / 0.1) + >>> dat_grouped = dat.group_by(phase_bin) + >>> dat_binned = dat_grouped.groups.aggregate(np.mean) + +This time, try plotting with ``plt.plot(dat_binned['phase'], +dat_binned['mag'])``. + +.. EXAMPLE END .. _stack-vertically: -Stack vertically -^^^^^^^^^^^^^^^^^^^^ +Stack Vertically +---------------- The |Table| class supports stacking tables vertically with the -`~astropy.table.vstack` function. This process is also commonly known as -concatenating or appending tables in the row direction. It corresponds roughly -to the `numpy.vstack` function. +:func:`~astropy.table.vstack` function. This process is also commonly known as +concatenating or appending tables in the row direction. It corresponds roughly +to the :func:`numpy.vstack` function. + +Examples +^^^^^^^^ -For example, suppose one has two tables of observations with several -column names in common:: +.. EXAMPLE START: Stacking (or Concatenating) Tables Vertically + +Suppose we have two tables of observations with several column names in +common:: >>> from astropy.table import Table, vstack >>> obs1 = Table.read("""name obs_date mag_b logLx @@ -415,10 +557,10 @@ Now we can stack these two tables:: M31 1999-01-05 -- 43.1 M82 2012-10-30 -- 45.0 -Notice that the ``obs2`` table is missing the ``mag_b`` column, so in the stacked output -table those values are marked as missing. This is the default behavior and corresponds to -``join_type='outer'``. There are two other allowed values for the ``join_type`` argument, -``'inner'`` and ``'exact'``:: +Notice that the ``obs2`` table is missing the ``mag_b`` column, so in the +stacked output table those values are marked as missing. This is the default +behavior and corresponds to ``join_type='outer'``. There are two other allowed +values for the ``join_type`` argument, ``'inner'`` and ``'exact'``:: >>> print(vstack([obs1, obs2], join_type='inner')) name obs_date logLx @@ -430,18 +572,18 @@ table those values are marked as missing. This is the default behavior and corr M31 1999-01-05 43.1 M82 2012-10-30 45.0 - >>> print(vstack([obs1, obs2], join_type='exact')) + >>> print(vstack([obs1, obs2], join_type='exact')) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... TableMergeError: Inconsistent columns in input arrays (use 'inner' or 'outer' join_type to allow non-matching columns) -In the case of ``join_type='inner'``, only the common columns (the intersection) are -present in the output table. When ``join_type='exact'`` is specified then -`~astropy.table.vstack` requires that all the input tables -have exactly the same column names. +In the case of ``join_type='inner'``, only the common columns (the intersection) +are present in the output table. When ``join_type='exact'`` is specified, then +:func:`~astropy.table.vstack` requires that all of the input tables have +exactly the same column names. -More than two tables can be stacked by supplying a list of table objects:: +More than two tables can be stacked by supplying a longer list of tables:: >>> obs3 = Table.read("""name obs_date mag_b logLx ... M45 2012-02-03 15.0 40.5""", format='ascii') @@ -456,21 +598,28 @@ More than two tables can be stacked by supplying a list of table objects:: M82 2012-10-30 -- 45.0 M45 2012-02-03 15.0 40.5 -See also the sections on `Merging metadata`_ and `Merging column -attributes`_ for details on how these characteristics of the input tables are merged in -the single output table. Note also that you can use a single table row instead of a -full table as one of the inputs. +See also the sections on `Merging metadata`_ and `Merging column attributes`_ +for details on how these characteristics of the input tables are merged in the +single output table. Note also that you can use a single table |Row| instead of +a full table as one of the inputs. + +.. EXAMPLE END .. _stack-horizontally: -Stack horizontally -^^^^^^^^^^^^^^^^^^^^^ +Stack Horizontally +------------------ -The |Table| class supports stacking tables horizontally (in the column-wise direction) with the -`~astropy.table.hstack` function. It corresponds roughly -to the `numpy.hstack` function. +The |Table| class supports stacking tables horizontally (in the column-wise +direction) with the :func:`~astropy.table.hstack` function. It corresponds +roughly to the :func:`numpy.hstack` function. -For example, suppose one has the following two tables:: +Examples +^^^^^^^^ + +.. EXAMPLE START: Stacking (or Concatenating) Tables Horizontally + +Suppose we have the following two tables:: >>> from astropy.table import Table, hstack >>> t1 = Table.read("""a b c @@ -490,12 +639,12 @@ Now we can stack these two tables horizontally:: 2 bar 2.1 spam toast 3 baz 2.8 -- -- -As with `~astropy.table.vstack`, there is an optional ``join_type`` argument -that can take values ``'inner'``, ``'exact'``, and ``'outer'``. The default is -``'outer'``, which effectively takes the union of available rows and masks out any missing -values. This is illustrated in the example above. The other options give the -intersection of rows, where ``'exact'`` requires that all tables have exactly the same -number of rows:: +As with :func:`~astropy.table.vstack`, there is an optional ``join_type`` +argument that can take values ``'inner'``, ``'exact'``, and ``'outer'``. The +default is ``'outer'``, which effectively takes the union of available rows and +masks out any missing values. This is illustrated in the example above. The +other options give the intersection of rows, where ``'exact'`` requires that +all tables have exactly the same number of rows:: >>> print(hstack([t1, t2], join_type='inner')) a b c d e @@ -503,15 +652,15 @@ number of rows:: 1 foo 1.4 ham eggs 2 bar 2.1 spam toast - >>> print(hstack([t1, t2], join_type='exact')) + >>> print(hstack([t1, t2], join_type='exact')) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... TableMergeError: Inconsistent number of rows in input arrays (use 'inner' or 'outer' join_type to allow non-matching rows) -More than two tables can be stacked by supplying a list of table objects. The example -below also illustrates the behavior when there is a conflict in the input column names -(see the section on `Column renaming`_ for details):: +More than two tables can be stacked by supplying a longer list of tables. The +example below also illustrates the behavior when there is a conflict in the +input column names (see the section on `Column renaming`_ for details):: >>> t3 = Table.read("""a b ... M45 2012-02-03""", format='ascii') @@ -522,22 +671,87 @@ below also illustrates the behavior when there is a conflict in the input column 2 bar 2.1 spam toast -- -- 3 baz 2.8 -- -- -- -- +The metadata from the input tables is merged by the process described in the +`Merging metadata`_ section. Note also that you can use a single table |Row| +instead of a full table as one of the inputs. + +.. EXAMPLE END + +.. _stack-depthwise: + +Stack Depth-Wise +---------------- + +The |Table| class supports stacking columns within tables depth-wise using the +:func:`~astropy.table.dstack` function. It corresponds roughly to running the +:func:`numpy.dstack` function on the individual columns matched by name. + +Examples +^^^^^^^^ + +.. EXAMPLE START: Stacking (or Concatenating) Tables Depth-Wise + +Suppose we have tables of data for sources giving information on the enclosed +source counts for different PSF fractions:: + + >>> from astropy.table import Table, dstack + >>> src1 = Table.read("""psf_frac counts + ... 0.10 45. + ... 0.50 90. + ... 0.90 120. + ... """, format='ascii') + + >>> src2 = Table.read("""psf_frac counts + ... 0.10 200. + ... 0.50 300. + ... 0.90 350. + ... """, format='ascii') + +Now we can stack these two tables depth-wise to get a single table with the +characteristics of both sources:: + + >>> srcs = dstack([src1, src2]) + >>> print(srcs) + psf_frac counts + ---------- -------------- + 0.1 .. 0.1 45.0 .. 200.0 + 0.5 .. 0.5 90.0 .. 300.0 + 0.9 .. 0.9 120.0 .. 350.0 -The metadata from the input tables is merged by the process described in the `Merging -metadata`_ section. Note also that you can use a single table row instead of a -full table as one of the inputs. +In this case the counts for the first source are accessible as +``srcs['counts'][:, 0]``, and likewise the second source counts are +``srcs['counts'][:, 1]``. + +For this function the length of all input tables must be the same. This +function can accept ``join_type`` and ``metadata_conflicts`` just like the +:func:`~astropy.table.vstack` function. The ``join_type`` argument controls how +to handle mismatches in the columns of the input table. + +See also the sections on `Merging metadata`_ and `Merging column attributes`_ +for details on how these characteristics of the input tables are merged in the +single output table. Note also that you can use a single table |Row| instead of +a full table as one of the inputs. + +.. EXAMPLE END .. _table-join: Join -^^^^^^^^^^^^^^ +---- + +The |Table| class supports the `database join +`_ operation. This provides a flexible +and powerful way to combine tables based on the values in one or more key +columns. + +Examples +^^^^^^^^ -The |Table| class supports the `database join `_ -operation. This provides a flexible and powerful way to combine tables based on the -values in one or more key columns. +.. EXAMPLE START: Combining Tables using the Database Join Operation -For example, suppose one has two tables of observations, the first with B and V magnitudes -and the second with X-ray luminosities of an overlapping (but not identical) sample:: +Suppose we have two tables of observations, the first with B and V magnitudes +and the second with X-ray luminosities of an overlapping (but not identical) +sample:: >>> from astropy.table import Table, join >>> optical = Table.read("""name obs_date mag_b mag_v @@ -549,11 +763,11 @@ and the second with X-ray luminosities of an overlapping (but not identical) sam ... M31 1999-01-05 43.1 ... M82 2012-10-29 45.0""", format='ascii') -The |join| method allows one to merge these two tables into a single table based on -matching values in the "key columns". By default the key columns are the set of columns -that are common to both tables. In this case the key columns are ``name`` and -``obs_date``. We can find all the observations of the same object on the same date as -follows:: +The |join| method allows you to merge these two tables into a single table based +on matching values in the "key columns". By default, the key columns are the set +of columns that are common to both tables. In this case the key columns are +``name`` and ``obs_date``. We can find all of the observations of the same +object on the same date as follows:: >>> opt_xray = join(optical, xray) >>> print(opt_xray) @@ -561,8 +775,8 @@ follows:: ---- ---------- ----- ----- ----- M82 2012-10-29 16.2 15.2 45.0 -We can perform the match only by ``name`` by providing the ``keys`` argument, which can be -either a single column name or a list of column names:: +We can perform the match by ``name`` only by providing the ``keys`` argument, +which can be either a single column name or a list of column names:: >>> print(join(optical, xray, keys='name')) name obs_date_1 mag_b mag_v obs_date_2 logLx @@ -570,16 +784,25 @@ either a single column name or a list of column names:: M31 2012-01-02 17.0 16.0 1999-01-05 43.1 M82 2012-10-29 16.2 15.2 2012-10-29 45.0 -This output table has all observations that have both optical and X-ray data for an object -(M31 and M82). Notice that since the ``obs_date`` column occurs in both tables it has -been split into two columns, ``obs_date_1`` and ``obs_date_2``. The values are taken from -the "left" (``optical``) and "right" (``xray``) tables, respectively. +This output table has all of the observations that have both optical and X-ray +data for an object (M31 and M82). Notice that since the ``obs_date`` column +occurs in both tables, it has been split into two columns, ``obs_date_1`` and +``obs_date_2``. The values are taken from the "left" (``optical``) and "right" +(``xray``) tables, respectively. + +.. EXAMPLE END + +Different Join Options +^^^^^^^^^^^^^^^^^^^^^^ + +The table joins so far are known as "inner" joins and represent the strict +intersection of the two tables on the key columns. -The table joins so far are known as "inner" joins and represent the strict intersection of -the two tables on the key columns. +.. EXAMPLE START: Table Join Options -If one wants to make a new table which has *every* row from the left table and includes -matching values from the right table when available, this is known as a left join:: +If you want to make a new table which has *every* row from the left table and +includes matching values from the right table when available, this is known as a +left join:: >>> print(join(optical, xray, join_type='left')) name obs_date mag_b mag_v logLx @@ -588,11 +811,10 @@ matching values from the right table when available, this is known as a left joi M31 2012-01-02 17.0 16.0 -- M82 2012-10-29 16.2 15.2 45.0 -Two of the observations do not have X-ray data, as indicated by the "--" in the table. -When there are any missing values the output will be a masked table. You might be -surprised that there is no X-ray data for M31 in the output. Remember that the default -matching key includes both ``name`` and ``obs_date``. Specifying the key as only the -``name`` column gives:: +Two of the observations do not have X-ray data, as indicated by the ``--`` in +the table. You might be surprised that there is no X-ray data for M31 in the +output. Remember that the default matching key includes both ``name`` and +``obs_date``. Specifying the key as only the ``name`` column gives:: >>> print(join(optical, xray, join_type='left', keys='name')) name obs_date_1 mag_b mag_v obs_date_2 logLx @@ -601,10 +823,10 @@ matching key includes both ``name`` and ``obs_date``. Specifying the key as onl M31 2012-01-02 17.0 16.0 1999-01-05 43.1 M82 2012-10-29 16.2 15.2 2012-10-29 45.0 -Likewise one can construct a new table with every row of the right table and matching left -values (when available) using ``join_type='right'``. +Likewise you can construct a new table with every row of the right table and +matching left values (when available) using ``join_type='right'``. -Finally, to make a table with the union of rows from both tables do an "outer" join:: +To make a table with the union of rows from both tables do an "outer" join:: >>> print(join(optical, xray, join_type='outer')) name obs_date mag_b mag_v logLx @@ -615,12 +837,69 @@ Finally, to make a table with the union of rows from both tables do an "outer" j M82 2012-10-29 16.2 15.2 45.0 NGC3516 2011-11-11 -- -- 42.1 +In all the above cases the output join table will be sorted by the key +column(s) and in general will not preserve the row order of the input tables. -Identical keys -~~~~~~~~~~~~~~ +Finally, you can do a "Cartesian" join, which is the Cartesian product of all +available rows. In this case there are no key columns (and supplying the +``keys`` argument is an error):: -The |Table| join operation works even if there are multiple rows with identical key -values. For example the following tables have multiple rows for the key column ``x``:: + >>> print(join(optical, xray, join_type='cartesian')) + name_1 obs_date_1 mag_b mag_v name_2 obs_date_2 logLx + ------ ---------- ----- ----- ------- ---------- ----- + M31 2012-01-02 17.0 16.0 NGC3516 2011-11-11 42.1 + M31 2012-01-02 17.0 16.0 M31 1999-01-05 43.1 + M31 2012-01-02 17.0 16.0 M82 2012-10-29 45.0 + M82 2012-10-29 16.2 15.2 NGC3516 2011-11-11 42.1 + M82 2012-10-29 16.2 15.2 M31 1999-01-05 43.1 + M82 2012-10-29 16.2 15.2 M82 2012-10-29 45.0 + M101 2012-10-31 15.1 15.5 NGC3516 2011-11-11 42.1 + M101 2012-10-31 15.1 15.5 M31 1999-01-05 43.1 + M101 2012-10-31 15.1 15.5 M82 2012-10-29 45.0 + +.. EXAMPLE END + +Non-Identical Key Column Names +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. EXAMPLE START: Joining Tables with Unique Key Column Names + +To use the |join| function with non-identical key column names, use the +``keys_left`` and ``keys_right`` arguments. In the following example one table +has a ``'name'`` column while the other has an ``'obj_id'`` column:: + + >>> optical = Table.read("""name obs_date mag_b mag_v + ... M31 2012-01-02 17.0 16.0 + ... M82 2012-10-29 16.2 15.2 + ... M101 2012-10-31 15.1 15.5""", format='ascii') + >>> xray_1 = Table.read("""obj_id obs_date logLx + ... NGC3516 2011-11-11 42.1 + ... M31 1999-01-05 43.1 + ... M82 2012-10-29 45.0""", format='ascii') + +In order to perform a match based on the names of the objects, do the +following:: + + >>> print(join(optical, xray_1, keys_left='name', keys_right='obj_id')) + name obs_date_1 mag_b mag_v obj_id obs_date_2 logLx + ---- ---------- ----- ----- ------ ---------- ----- + M31 2012-01-02 17.0 16.0 M31 1999-01-05 43.1 + M82 2012-10-29 16.2 15.2 M82 2012-10-29 45.0 + +The ``keys_left`` and ``keys_right`` arguments can also take a list of column +names or even a list of column-like objects. The latter case allows specifying +the matching key column values independent of the tables being joined. + +.. EXAMPLE END + +Identical Key Values +^^^^^^^^^^^^^^^^^^^^ + +.. EXAMPLE START: Joining Tables with Identical Key Values + +The |Table| join operation works even if there are multiple rows with identical +key values. For example, the following tables have multiple rows for the column +``'key'``:: >>> from astropy.table import Table, join >>> left = Table([[0, 1, 1, 2], ['L1', 'L2', 'L3', 'L4']], names=('key', 'L')) @@ -640,12 +919,11 @@ values. For example the following tables have multiple rows for the key column 2 R3 4 R4 -Doing an outer join on these tables shows that what is really happening is a `Cartesian -product `_. For each matching key, every -combination of the left and right tables is represented. When there is no match in either -the left or right table, the corresponding column values are designated as missing. - -.. doctest-skip:: win32 +Doing an outer join on these tables shows that what is really happening is a +`Cartesian product `_. For +each matching key, every combination of the left and right tables is +represented. When there is no match in either the left or right table, the +corresponding column values are designated as missing:: >>> print(join(left, right, join_type='outer')) key L R @@ -658,17 +936,8 @@ the left or right table, the corresponding column values are designated as missi 2 L4 R3 4 -- R4 -.. note:: - - The output table is sorted on the key columns, but when there are rows with identical - keys the output order in the non-key columns is not guaranteed to be identical across - installations. In the example above the order within the four rows with ``key == 1`` - can vary. - -An inner join is the same but only returns rows where there is a key match in both the -left and right tables: - -.. doctest-skip:: win32 +An inner join is the same but only returns rows where there is a key match in +both the left and right tables:: >>> print(join(left, right, join_type='inner')) key L R @@ -679,35 +948,36 @@ left and right tables: 1 L3 R2 2 L4 R3 -Conflicts in the input table names are handled by the process described in the section on -`Column renaming`_. See also the sections on `Merging metadata`_ and `Merging column -attributes`_ for details on how these characteristics of the input tables are merged in -the single output table. +Conflicts in the input table names are handled by the process described in the +section on `Column renaming`_. See also the sections on `Merging metadata`_ and +`Merging column attributes`_ for details on how these characteristics of the +input tables are merged in the single output table. -Merging details -^^^^^^^^^^^^^^^^^^^^ +.. EXAMPLE END + +Merging Details +--------------- When combining two or more tables there is the need to merge certain -characteristics in the inputs and potentially resolve conflicts. This +characteristics in the inputs and potentially resolve conflicts. This section describes the process. -Column renaming -~~~~~~~~~~~~~~~~~ - +Column Renaming +^^^^^^^^^^^^^^^ In cases where the input tables have conflicting column names, there -is a mechanism to generate unique output column names. There are two +is a mechanism to generate unique output column names. There are two keyword arguments that control the renaming behavior: ``table_names`` - Two-element list of strings that provide a name for the tables being joined. + List of strings that provide names for the tables being joined. By default this is ``['1', '2', ...]``, where the numbers correspond to the input tables. ``uniq_col_name`` String format specifier with a default value of ``'{col_name}_{table_name}'``. -This is most easily understood by example using the ``optical`` and ``xray`` tables +This is best understood by example using the ``optical`` and ``xray`` tables in the |join| example defined previously:: >>> print(join(optical, xray, keys='name', @@ -718,27 +988,33 @@ in the |join| example defined previously:: M31 2012-01-02 17.0 16.0 1999-01-05 43.1 M82 2012-10-29 16.2 15.2 2012-10-29 45.0 +.. _merging_metadata: -Merging metadata -~~~~~~~~~~~~~~~~~~~ +Merging Metadata +^^^^^^^^^^^^^^^^ |Table| objects can have associated metadata: - ``Table.meta``: table-level metadata as an ordered dictionary - ``Column.meta``: per-column metadata as an ordered dictionary -The table operations described here handle the task of merging the metadata in the input -tables into a single output structure. Because the metadata can be arbitrarily complex -there is no unique way to do the merge. The current implementation uses a simple -recursive algorithm with four rules: +The table operations described here handle the task of merging the metadata in +the input tables into a single output structure. Because the metadata can be +arbitrarily complex there is no unique way to do the merge. The current +implementation uses a recursive algorithm with four rules: + +- :class:`dict` elements are merged by keys. +- Conflicting :class:`list` or :class:`tuple` elements are concatenated. +- Conflicting :class:`dict` elements are merged by recursively calling the + merge function. +- Conflicting elements that are not :class:`list`, :class:`tuple`, or + :class:`dict` will follow the following rules: -- `dict` elements are merged by keys -- Conflicting `list` or `tuple` elements are concatenated -- Conflicting `dict` elements are merged by recursively calling the merge function -- Conflicting elements that are not both `list`, `tuple`, or `dict` will follow the following rules: - - If both metadata values are identical, the output is set to this value - - If one of the conflicting metadata values is `None`, the other value is picked - - If both metadata values are different and neither is `None`, the one for the last table in the list is picked + - If both metadata values are identical, the output is set to this value. + - If one of the conflicting metadata values is `None`, the other value is + picked. + - If both metadata values are different and neither is `None`, the one for + the last table in the list is picked. By default, a warning is emitted in the last case (both metadata values are not `None`). The warning can be silenced or made into an exception using the @@ -746,16 +1022,31 @@ By default, a warning is emitted in the last case (both metadata values are not :func:`~astropy.table.vstack`, or :func:`~astropy.table.join`. The ``metadata_conflicts`` option can be set to: -- ``'silent'`` - no warning is emitted, the value for the last table is silently picked -- ``'warn'`` - a warning is emitted, the value for the last table is picked -- ``'error'`` - an exception is raised +- ``'silent'`` – no warning is emitted, the value for the last table is silently + picked. +- ``'warn'`` – a warning is emitted, the value for the last table is picked. +- ``'error'`` – an exception is raised. + +The default strategies for merging metadata can be augmented or customized by +defining subclasses of the `~astropy.utils.metadata.MergeStrategy` base class. +In most cases you will also use +:func:`~astropy.utils.metadata.enable_merge_strategies` for enabling the custom +strategies. The linked documentation strings provide details. + +Merging Column Attributes +^^^^^^^^^^^^^^^^^^^^^^^^^ -Merging column attributes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +In addition to the table and column ``meta`` attributes, the column attributes +``unit``, ``format``, and ``description`` are merged by going through the input +tables in order and taking the last value which is defined (i.e., is not +`None`). -In addition to the table and column ``meta`` attributes, the column attributes ``unit``, -``format``, and ``description`` are merged by going through the input tables in -order and taking the first value which is defined (i.e. is not None). For example:: +Example +~~~~~~~ + +.. EXAMPLE START: Merging Column Attributes in a Table + +To merge column attributes ``unit``, ``format``, or ``description``:: >>> from astropy.table import Column, Table, vstack >>> col1 = Column([1], name='a') @@ -764,30 +1055,161 @@ order and taking the first value which is defined (i.e. is not None). For examp >>> t1 = Table([col1]) >>> t2 = Table([col2]) >>> t3 = Table([col3]) - >>> out = vstack([t1, t2, t3]) # doctest: +SKIP - WARNING: MergeConflictWarning: In merged column 'a' the 'unit' attribute does - not match (cm != m). Using m for merged output [astropy.table.operations] - >>> out['a'].unit # doctest: +SKIP + >>> out = vstack([t1, t2, t3]) # doctest: +SHOW_WARNINGS + MergeConflictWarning: In merged column 'a' the 'unit' attribute does + not match (cm != m). Using m for merged output + >>> out['a'].unit Unit("m") -The rules for merging are as for `Merging metadata`_, and the +The rules for merging are the same as for `Merging metadata`_, and the ``metadata_conflicts`` option also controls the merging of column attributes. +.. EXAMPLE END + +.. _astropy-table-join-functions: + +Joining Coordinates and Custom Join Functions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Source catalogs that have |SkyCoord| coordinate columns can be joined using +cross-matching of the coordinates with a specified distance threshold. This is +a special case of a more general problem of "fuzzy" matching of key column +values, where instead of an exact match we require only an approximate match. +This is supported using the ``join_funcs`` argument. + +.. warning:: + + The coordinate and distance table joins discussed in this section are most + applicable in the case where the relevant entries in at least one of the + tables are all separated from one another by more than twice the join + distance. If this is not satisfied then the join results may be unexpected. + + This is a consequence of the algorithm which effectively finds clusters of + nearby points (an "equivalence class") and assigns a unique cluster + identifier to each entry in both tables. This assumes the join matching + function is a transitive relation where ``join_func(A, B)`` and + ``join_func(B, C)`` implies ``join_func(A, C)``. With multiple matches on + both left and right sides it is possible for the cluster of points having a + single cluster identifier to expand in size beyond the distance threshold. + + Users should be especially aware of this issue if additional join keys + are provided beyond the ``join_funcs``. The code does not do a "pre-join" + on the other keys, so the possibility of having overlaps within the distance + in both tables is higher. + +Example +~~~~~~~ + +.. EXAMPLE START: Joining a Table on Coordinates + +To join two tables on a |SkyCoord| key column we use the ``join_funcs`` keyword +to supply a :class:`dict` of functions that specify how to match a particular +key column by name. In the example below we are joining on the ``sc`` column, +so we provide the following argument:: + + join_funcs={'sc': join_skycoord(0.2 * u.deg)} + +This tells |join| to match the ``sc`` key column using the join function +:func:`~astropy.table.join_skycoord` with a matching distance threshold of 0.2 +deg. Under the hood this calls +:meth:`~astropy.coordinates.SkyCoord.search_around_sky` or +:meth:`~astropy.coordinates.SkyCoord.search_around_3d` to do the +cross-matching. The default is to use +:meth:`~astropy.coordinates.SkyCoord.search_around_sky` (angle) matching, but +:meth:`~astropy.coordinates.SkyCoord.search_around_3d` (length or +dimensionless) is also available. This is specified using the ``distance_func`` +argument of :func:`~astropy.table.join_skycoord`, which can also be a function +that matches the input and output API of +:meth:`~astropy.coordinates.SkyCoord.search_around_sky`. + +Now we show the whole process: + +.. doctest-requires:: scipy + + >>> from astropy.coordinates import SkyCoord + >>> import astropy.units as u + >>> from astropy.table import Table, join, join_skycoord + +.. doctest-requires:: scipy + + >>> sc1 = SkyCoord([0, 1, 1.1, 2], [0, 0, 0, 0], unit='deg') + >>> sc2 = SkyCoord([1.05, 0.5, 2.1], [0, 0, 0], unit='deg') + +.. doctest-requires:: scipy + + >>> t1 = Table([sc1, [0, 1, 2, 3]], names=['sc', 'idx']) + >>> t2 = Table([sc2, [0, 1, 2]], names=['sc', 'idx']) + +.. doctest-requires:: scipy + + >>> t12 = join(t1, t2, keys='sc', join_funcs={'sc': join_skycoord(0.2 * u.deg)}) + >>> print(t12) + sc_id sc_1 idx_1 sc_2 idx_2 + deg,deg deg,deg + ----- ------- ----- -------- ----- + 1 1.0,0.0 1 1.05,0.0 0 + 1 1.1,0.0 2 1.05,0.0 0 + 2 2.0,0.0 3 2.1,0.0 2 + +The joined table has matched the sources within 0.2 deg and created a new +column ``sc_id`` with a unique identifier for each source. + +.. EXAMPLE END + +You might be wondering what is happening in the join function defined above, +especially if you are interested in defining your own such function. This could +be done in order to allow fuzzy word matching of tables, for example joining +tables of people by name where the names do not always match exactly. + +The first thing to note here is that the :func:`~astropy.table.join_skycoord` +function actually returns a function itself. This allows specifying a variable +match distance via a function enclosure. The requirement of the join function +is that it accepts two arguments corresponding to the two key columns, and +returns a tuple of ``(ids1, ids2)``. These identifiers correspond to the +identification of each column entry with a unique matched source. + +.. doctest-requires:: scipy + + >>> join_func = join_skycoord(0.2 * u.deg) + >>> join_func(sc1, sc2) # Associate each coordinate with unique source ID + (array([3, 1, 1, 2]), array([1, 4, 2])) + +If you would like to write your own fuzzy matching function, we suggest starting +from the source code for :func:`~astropy.table.join_skycoord` or +:func:`~astropy.table.join_distance`. + +Join on Distance +~~~~~~~~~~~~~~~~ + +The example above focused on joining on a |SkyCoord|, but you can also join on +a generic distance between column values using the +:func:`~astropy.table.join_distance` join function. This can apply to 1D or 2D +(vector) columns. This will look very similar to the coordinates example, but +here there is a bit more flexibility. The matching is done using +:class:`scipy.spatial.KDTree` and +:meth:`scipy.spatial.KDTree.query_ball_tree`, and the behavior of these can be +controlled via the ``kdtree_args`` and ``query_args`` arguments, respectively. .. _unique-rows: -Unique rows -^^^^^^^^^^^ +Unique Rows +----------- Sometimes it makes sense to use only rows with unique key columns or even fully unique rows from a table. This can be done using the above described -:func:`~astropy.table.Table.group_by` method and ``groups`` attribute, or -with the `~astropy.table.unique` convenience method. The -`~astropy.table.unique` method returns with a sorted table containing the -first row for each unique ``keys`` column value. If no ``keys`` is provided -it returns with a sorted table containing all the fully unique rows. +:meth:`~astropy.table.Table.group_by` method and ``groups`` attribute, or with +the :func:`~astropy.table.unique` convenience function. The +:func:`~astropy.table.unique` function returns a sorted table containing the +first row for each unique ``keys`` column value. If no ``keys`` is provided, it +returns a sorted table containing all of the fully unique rows. + +Example +^^^^^^^ -A simple example is a list of objects with photometry from various observing +.. EXAMPLE START: Grouping Unique Rows in Tables + +An example of a situation where you might want to use rows with unique key +columns is a list of objects with photometry from various observing runs. Using ``'name'`` as the only ``keys``, it returns with the first occurrence of each of the three targets:: @@ -827,3 +1249,99 @@ Using multiple columns as ``keys``:: M31 2012-02-14 16.9 17.3 M82 2012-02-14 16.2 14.5 M82 2012-03-26 15.7 16.5 + +.. EXAMPLE END + +.. _set-difference: + +Set Difference +-------------- + +A set difference will tell you the elements that are contained in the first set +but not in the other. This concept can be applied to rows of a table by using +the :func:`~astropy.table.setdiff` function. You provide the function with two +input tables and it will return all rows in the first table which do not occur +in the second table. + +The optional ``keys`` parameter specifies the names of columns that are used to +match table rows. This can be a subset of the full list of columns, but both +the first and second tables must contain all columns specified by ``keys``. +If not provided, then ``keys`` defaults to all column names in the first table. + +If no different rows are found, the :func:`~astropy.table.setdiff` function +will return an empty table. + +Example +^^^^^^^ + +.. EXAMPLE START: Using Set Difference in Tables + +The example below illustrates finding the set difference of two observation +lists using a common subset of the columns in two tables.:: + + >>> from astropy.table import Table, setdiff + >>> cat_1 = Table.read("""name obs_date mag_b mag_v + ... M31 2012-01-02 17.0 16.0 + ... M82 2012-10-29 16.2 15.2 + ... M101 2012-10-31 15.1 15.5""", format='ascii') + >>> cat_2 = Table.read(""" name obs_date logLx + ... NGC3516 2011-11-11 42.1 + ... M31 2012-01-02 43.1 + ... M82 2012-10-29 45.0""", format='ascii') + >>> sdiff = setdiff(cat_1, cat_2, keys=['name', 'obs_date']) + >>> print(sdiff) + name obs_date mag_b mag_v + ---- ---------- ----- ----- + M101 2012-10-31 15.1 15.5 + +In this example there is a column in the first table that is not +present in the second table, so the ``keys`` parameter must be used to specify +the desired column names. + +.. EXAMPLE END + +.. _table-diff: + +Table Diff +---------- + +To compare two tables, you can use +:func:`~astropy.utils.diff.report_diff_values`, which would produce a report +identical to :ref:`FITS diff `. + +Example +^^^^^^^ + +.. EXAMPLE START: Using Table Diff to Compare Tables + +The example below illustrates finding the difference between two tables:: + + >>> from astropy.table import Table + >>> from astropy.utils.diff import report_diff_values + >>> import sys + >>> cat_1 = Table.read("""name obs_date mag_b mag_v + ... M31 2012-01-02 17.0 16.0 + ... M82 2012-10-29 16.2 15.2 + ... M101 2012-10-31 15.1 15.5""", format='ascii') + >>> cat_2 = Table.read("""name obs_date mag_b mag_v + ... M31 2012-01-02 17.0 16.5 + ... M82 2012-10-29 16.2 15.2 + ... M101 2012-10-30 15.1 15.5 + ... NEW 2018-05-08 nan 9.0""", format='ascii') + >>> identical = report_diff_values(cat_1, cat_2, fileobj=sys.stdout) + name obs_date mag_b mag_v + ---- ---------- ----- ----- + a> M31 2012-01-02 17.0 16.0 + ? ^ + b> M31 2012-01-02 17.0 16.5 + ? ^ + M82 2012-10-29 16.2 15.2 + a> M101 2012-10-31 15.1 15.5 + ? ^ + b> M101 2012-10-30 15.1 15.5 + ? ^ + b> NEW 2018-05-08 nan 9.0 + >>> identical + False + +.. EXAMPLE END diff --git a/docs/table/pandas.rst b/docs/table/pandas.rst index f5802296eb96..3d3c6d1eb2c1 100644 --- a/docs/table/pandas.rst +++ b/docs/table/pandas.rst @@ -2,63 +2,10 @@ .. _pandas: -Interfacing with the pandas package -=================================== +Interfacing with the Pandas Package +*********************************** -The `pandas `__ package is a package for high -performance data analysis of table-like structures that is complementary to the -:class:`~astropy.table.Table` class in Astropy. - -In order to be able to easily exchange data between the :class:`~astropy.table.Table` class and the pandas `DataFrame`_ class (the main data structure in pandas), the :class:`~astropy.table.Table` class includes two methods, :meth:`~astropy.table.Table.to_pandas` and :meth:`~astropy.table.Table.from_pandas`. - -To demonstrate these, we can create a simple table:: - - >>> from astropy.table import Table - >>> t = Table() - >>> t['a'] = [1, 2, 3, 4] - >>> t['b'] = ['a', 'b', 'c', 'd'] - -which we can then convert to a pandas `DataFrame`_:: - - >>> df = t.to_pandas() - >>> df - a b - 0 1 a - 1 2 b - 2 3 c - 3 4 d - >>> type(df) - - -It is also possible to create a table from a `DataFrame`_:: - - >>> t2 = Table.from_pandas(df) - >>> t2 -
- a b - int64 string8 - ----- ------- - 1 a - 2 b - 3 c - 4 d - -The conversions to/from pandas are subject to the following caveats: - -* The pandas `DataFrame`_ structure does not support multi-dimensional - columns, so :class:`~astropy.table.Table` objects with multi-dimensional - columns cannot be converted to `DataFrame`_. - -* Masked tables can be converted, but `DataFrame`_ uses ``numpy.nan`` to - indicate masked values, so all numerical columns (integer or float) are - converted to ``numpy.float`` columns in `DataFrame`_, and string columns with - missing values are converted to object columns with ``numpy.nan`` values to - indicate missing values. For numerical columns, the conversion therefore does - not necessarily round-trip if converting back to an Astropy table, because the - distinction between ``numpy.nan`` and masked values is lost, and the different - for example integer columns will be converted to floating-point. - -* Tables with mixin columns can currently not be converted, but this may be - implemented in the future. - -.. _DataFrame: http://pandas.pydata.org/pandas-docs/dev/generated/pandas.DataFrame.html +.. note:: + Since Astropy 7.2, this documentation has been expanded and moved to :ref:`df_narwhals`. + The linked paragraph covers both pandas-specific methods and generic multi-backend + DataFrame support through narwhals. diff --git a/docs/table/performance.inc.rst b/docs/table/performance.inc.rst new file mode 100644 index 000000000000..58ee1d4b43b4 --- /dev/null +++ b/docs/table/performance.inc.rst @@ -0,0 +1,78 @@ +.. note that if this is changed from the default approach of using an *include* + (in index.rst) to a separate performance page, the header needs to be changed + from === to ***, the filename extension needs to be changed from .inc.rst to + .rst, and a link needs to be added in the subpackage toctree + +.. doctest-skip-all + +.. _astropy-table-performance: + +Performance Tips +================ + +Constructing |Table| objects row by row using +:meth:`~astropy.table.Table.add_row` can be very slow:: + + >>> from astropy.table import Table + >>> t = Table(names=['a', 'b']) + >>> for i in range(100): + ... t.add_row((1, 2)) + +If you do need to loop in your code to create the rows, a much faster approach +is to construct a list of rows and then create the |Table| object at the very +end:: + + >>> rows = [] + >>> for i in range(100): + ... rows.append((1, 2)) + >>> t = Table(rows=rows, names=['a', 'b']) + +Writing a |Table| with |MaskedColumn| to ``.ecsv`` using +:meth:`~astropy.table.Table.write` can be very slow:: + + >>> from astropy.table import Table + >>> import numpy as np + >>> x = np.arange(10000, dtype=float) + >>> tm = Table([x], masked=True) + >>> tm.write('tm.ecsv', overwrite=True) + +If you want to write ``.ecsv`` using :meth:`~astropy.table.Table.write`, +then use ``serialize_method='data_mask'``. +This uses the non-masked version of data and it is faster:: + + >>> tm.write('tm.ecsv', overwrite=True, serialize_method='data_mask') + +Read FITS with memmap=True +-------------------------- + +By default :meth:`~astropy.table.Table.read` will read the whole table into +memory, which can take a lot of memory and can take a lot of time, depending on +the table size and file format. In some cases, it is possible to only read a +subset of the table by choosing the option ``memmap=True``. + +For FITS binary tables, the data is stored row by row, and it is possible to +read only a subset of rows, but reading a full column loads the whole table data +into memory:: + + >>> import numpy as np + >>> from astropy.table import Table + >>> tbl = Table({'a': np.arange(1e7), + ... 'b': np.arange(1e7, dtype=float), + ... 'c': np.arange(1e7, dtype=float)}) + >>> tbl.write('test.fits', overwrite=True) + >>> table = Table.read('test.fits', memmap=True) # Very fast, doesn't actually load data + >>> table2 = tbl[:100] # Fast, will read only first 100 rows + >>> print(table2) # Accessing column data triggers the read + a b c + ---- ---- ---- + 0.0 0.0 0.0 + 1.0 1.0 1.0 + 2.0 2.0 2.0 + ... ... ... + 98.0 98.0 98.0 + 99.0 99.0 99.0 + Length = 100 rows + >>> col = table['my_column'] # Will load all table into memory + +:meth:`~astropy.table.Table.read` does not support ``memmap=True`` +for the HDF5 and text file formats. diff --git a/docs/table/ref_api.rst b/docs/table/ref_api.rst new file mode 100644 index 000000000000..bd4440fe2488 --- /dev/null +++ b/docs/table/ref_api.rst @@ -0,0 +1,12 @@ +Reference/API +************* + +Capabilities +============ + +.. automodapi:: astropy.table + +Notebook Backends +================= + +.. automodapi:: astropy.table.notebook_backends diff --git a/docs/table/references.txt b/docs/table/references.txt deleted file mode 100644 index 884557a8e023..000000000000 --- a/docs/table/references.txt +++ /dev/null @@ -1,7 +0,0 @@ -.. |Row| replace:: :class:`~astropy.table.Row` -.. |Table| replace:: :class:`~astropy.table.Table` -.. |QTable| replace:: :class:`~astropy.table.QTable` -.. |Column| replace:: :class:`~astropy.table.Column` -.. |MaskedColumn| replace:: :class:`~astropy.table.MaskedColumn` -.. |TableColumns| replace:: :class:`~astropy.table.TableColumns` -.. _OrderedDict: http://docs.python.org/library/collections.html#collections.OrderedDict diff --git a/docs/table/table_after_1.0.png b/docs/table/table_after_1.0.png deleted file mode 100644 index 817a983ace1d..000000000000 Binary files a/docs/table/table_after_1.0.png and /dev/null differ diff --git a/docs/table/table_and_dataframes.rst b/docs/table/table_and_dataframes.rst new file mode 100644 index 000000000000..5c8794a15cf3 --- /dev/null +++ b/docs/table/table_and_dataframes.rst @@ -0,0 +1,90 @@ +.. _astropy-table-and-dataframes: + +Astropy Table and DataFrames +============================ + +`Pandas `_ is a popular data manipulation library for Python +that provides a `~pandas.DataFrame` object which is similar to `astropy.table`. A common +question is why Astropy does not use `~pandas.DataFrame` as the base table object, or a DataFrame-like object from another library such as ``polars``. +The answer stems from a number of domain-specific requirements related to astronomical data and analysis. + +Units and Quantities +-------------------- + +Astronomy is a physical science, and the data often have units associated with +them. The `astropy.table` package natively supports |Quantity| columns, which are a +powerful way to attach units to array data and perform unit-aware operations. In +addition, the base `~astropy.table.Column` class holds a ``unit`` attribute as +metadata to allow tracking of the units of the data for applications not using +|Quantity|. + +*At the time of writing, neither pandas nor polars provide support for units.* + +Multi-dimensional and Structured Columns +---------------------------------------- + +Astronomers deal with images, spectra, and other multi-dimensional data that are +commonly stored in a table. An example is a source catalog with an image thumbnail and a +spectrum for each source. Structured columns are less common, but are useful for storing +vectorized data like an `~astropy.coordinates.EarthLocation` in a table. + +*Pandas is not able to natively store multi-dimensional or structured columns.* +*There is limited support for multi-dimensional columns in Polars.* + +Lossless representation of FITS and VOTable data via metadata +------------------------------------------------------------- + +The `astropy.table` package strives to provide lossless representation of FITS and +VOTable data. This means that when you read a FITS or VOTable file into a table and then +write it back out, the data will be effectively identical. This is made possible by +robust support for table and column metadata which allows storing and propagating common +column information such as the unit, description, and format. For VOTable data, more +information like the UCD is maintained. + +*Pandas provides limited support for metadata, but as of late-2024 it is highlighted as +"experimental" in the documentation.* +*Polars has limited support for metadata.* + +Time and Coordinates +-------------------- + +Time and coordinates are fundamental to astronomy, and astropy provides robust support +for them with the `~astropy.time.Time` and `~astropy.coordinates.SkyCoord` classes. +Arrays of times and coordinates can be natively stored in `astropy.table`, meaning that +the full power of these objects is available when working with them as columns within a +table. + +*Pandas supports* `timeseries +`_ *data, but with key +limitations*: + +- Leap seconds are not supported. In many circumstances (for instance planning an + observation) this limitation is not acceptable. +- Pandas times are stored with 64-bit precision, which is not sufficient for some + astronomical applications. Astropy uses 128-bit precision for time to allow + sub-nanosecond precision over the age of the universe. +- Different :ref:`time scales ` common in astronomy (e.g., TAI, UT1) are + not supported. +- :ref:`Time formats ` used in astronomy such as the FITS time format are + not supported. + +*Pandas does not support sky coordinate columns.* +*Limitations are similar for Polars as well.* + +Responsiveness to Community Needs +--------------------------------- + +The `astropy.table` package is developed by the Astropy community, which is focused on +the needs of astronomers and astrophysicists. This means that the development of the +package can be responsive to the needs of this community and we can develop features +without being constrained by the potential impact to the far broader user base of +Pandas or Polars. + +Interoperability +---------------- + +We recognize that Pandas is a popular library and that there are many users who are +familiar with it. For this reason, we have made it easy to convert between +`astropy.table` and DataFrame-like objects, such as `~pandas.DataFrame` and ``polars.DataFrame`` through the `narwhals `_ translation layer as documented in :ref:`df_narwhals`. +This allows users to take advantage of all supported features of both packages as needed, +within the limitations stated above. diff --git a/docs/table/table_before_1.0.png b/docs/table/table_before_1.0.png deleted file mode 100644 index bad178e6c65d..000000000000 Binary files a/docs/table/table_before_1.0.png and /dev/null differ diff --git a/docs/table/table_column_after_1.0.png b/docs/table/table_column_after_1.0.png deleted file mode 100644 index 817a983ace1d..000000000000 Binary files a/docs/table/table_column_after_1.0.png and /dev/null differ diff --git a/docs/table/table_column_before_1.0.png b/docs/table/table_column_before_1.0.png deleted file mode 100644 index f2896272788b..000000000000 Binary files a/docs/table/table_column_before_1.0.png and /dev/null differ diff --git a/docs/table/table_repr_html.png b/docs/table/table_repr_html.png index 4b39d63e8145..82d27e19012b 100644 Binary files a/docs/table/table_repr_html.png and b/docs/table/table_repr_html.png differ diff --git a/docs/table/table_row_after_1.0.png b/docs/table/table_row_after_1.0.png deleted file mode 100644 index 4d9c154a1187..000000000000 Binary files a/docs/table/table_row_after_1.0.png and /dev/null differ diff --git a/docs/table/table_row_before_1.0.png b/docs/table/table_row_before_1.0.png deleted file mode 100644 index 276382ee466f..000000000000 Binary files a/docs/table/table_row_before_1.0.png and /dev/null differ diff --git a/docs/time/index.rst b/docs/time/index.rst index 0f68dc776dca..a68af35c78bc 100644 --- a/docs/time/index.rst +++ b/docs/time/index.rst @@ -1,43 +1,44 @@ -.. include:: references.txt - .. _astropy-time: -**************************************************** +******************************* Time and Dates (`astropy.time`) -**************************************************** - -.. |Quantity| replace:: :class:`~astropy.units.Quantity` -.. |Longitude| replace:: :class:`~astropy.coordinates.Longitude` -.. |EarthLocation| replace:: :class:`~astropy.coordinates.EarthLocation` +******************************* Introduction ============ The `astropy.time` package provides functionality for manipulating times and -dates. Specific emphasis is placed on supporting time scales (e.g. UTC, TAI, -UT1, TDB) and time representations (e.g. JD, MJD, ISO 8601) that are used in -astronomy and required to calculate, e.g., sidereal times and barycentric -corrections. -It uses Cython to wrap the C language `ERFA`_ time and calendar -routines, using a fast and memory efficient vectorization scheme. +dates. Specific emphasis is placed on supporting time scales (e.g., UTC, TAI, +UT1, TDB) and time representations (e.g., JD, MJD, ISO 8601) that are used in +astronomy and required to calculate, for example, sidereal times and barycentric +corrections. The `astropy.time` package is based on fast and memory efficient +|PyERFA| wrappers around the |ERFA| time and calendar routines. All time manipulations and arithmetic operations are done internally using two -64-bit floats to represent time. Floating point algorithms from [#]_ are used so +64-bit floats to represent time. Floating point algorithms from [#]_ are used so that the |Time| object maintains sub-nanosecond precision over times spanning the age of the universe. -.. [#] `Shewchuk, 1997, Discrete & Computational Geometry 18(3):305-363 - `_ +.. [#] Shewchuk, 1997, Discrete & Computational Geometry 18(3):305-363 Getting Started =============== -The basic way to use `astropy.time` is to create a |Time| -object by supplying one or more input time values as well as the `time format`_ and -`time scale`_ of those values. The input time(s) can either be a single scalar like -``"2010-01-01 00:00:00"`` or a list or a `numpy` array of values as shown below. -In general any output values have the same shape (scalar or array) as the input. +The usual way to use `astropy.time` is to create a |Time| object by +supplying one or more input time values as well as the `time format`_ and `time +scale`_ of those values. The input time(s) can either be a single scalar like +``"2010-01-01 00:00:00"`` or a list or a ``numpy`` array of values as shown +below. In general, any output values have the same shape (scalar or array) as +the input. + +Examples +-------- +.. EXAMPLE START: Creating a Time Object with astropy.time + +To create a |Time| object: + + >>> import numpy as np >>> from astropy.time import Time >>> times = ['1999-01-01T00:00:00.123456789', '2010-01-01T00:00:00'] >>> t = Time(times, format='isot', scale='utc') @@ -46,66 +47,99 @@ In general any output values have the same shape (scalar or array) as the input. >>> t[1]
- -To repeat the above and suppress *all* the screen outputs (not recommended): - ->>> import warnings ->>> with warnings.catch_warnings(): -... warnings.simplefilter('ignore') -... result = vos_catalog.call_vo_service( -... 'conesearch_good', -... kwargs={'RA': c.ra.degree, 'DEC': c.dec.degree, 'SR': sr.value}, -... catalog_db='The PMM USNO-A1.0 Catalogue (Monet 1997) 1', -... verbose=False) - -You can also use custom VO database, say, ``'my_vo_database.json'`` from -:ref:`VO database examples `: - ->>> import os ->>> from astropy.vo.client.vos_catalog import BASEURL ->>> with BASEURL.set_temp(os.curdir): -... try: -... result = vos_catalog.call_vo_service( -... 'my_vo_database', -... kwargs={'RA': c.ra.degree, 'DEC': c.dec.degree, -... 'SR': sr.value}) -... except Exception as e: -... print(e) -Trying http://ex.org/cgi-bin/cs.pl? -Downloading http://ex.org/cgi-bin/cs.pl?SR=0.5&DEC=-72.0814444&RA=6.0223292 -|===========================================| 1.8k/1.8k (100.00%) 00s -None of the available catalogs returned valid results. - - -.. _vo-sec-client-scs: - -Simple Cone Search ------------------- - -`astropy.vo.client.conesearch` supports VO Simple Cone Search capabilities. - -Available databases are generated on the server-side hosted by STScI -using :ref:`vo-sec-validator-validate`. The database used is -controlled by `astropy.vo.Conf.conesearch_dbname`, which can be -changed in :ref:`vo-sec-scs-config` below. Here are the available -options: - -#. ``'conesearch_good'`` - Default. Passed validation without critical warnings and exceptions. -#. ``'conesearch_warn'`` - Has critical warnings but no exceptions. Use at your own risk. -#. ``'conesearch_exception'`` - Has some exceptions. *Never* use this. -#. ``'conesearch_error'`` - Has network connection error. *Never* use this. - -In the default setting, it searches the good Cone Search services one by one, -stops at the first one that gives non-zero match(es), and returns the result. -Since the list of services are extracted from a Python dictionary, the search -order might differ from call to call. - -There are also functions, both synchronously and asynchronously, available to -return *all* the Cone Search query results. However, this is not recommended -unless one knows what one is getting into, as it could potentially take up -significant run time and computing resources. - -:ref:`vo-sec-scs-examples` below show how to use non-default search behaviors, -where the user has more control of which catalog(s) to search, et cetera. - -.. note:: - - Most services currently fail to parse when ``pedantic=True``. - -.. warning:: - - When Cone Search returns warnings, you should decide - whether the results are reliable by inspecting the - warning codes in `astropy.io.votable.exceptions`. - -.. _vo-sec-scs-config: - -Configurable Items -^^^^^^^^^^^^^^^^^^ - -These parameters are set via :ref:`astropy_config`: - -* `astropy.vo.Conf.conesearch_dbname` - Cone Search database name to query. - -Also depends on -:ref:`General VO Services Access Configurable Items `. - -.. _vo-sec-scs-examples: - -Examples -^^^^^^^^ - ->>> from astropy.vo.client import conesearch - -Shows a sorted list of Cone Search services to be searched: - ->>> conesearch.list_catalogs() -[u'Guide Star Catalog 2.3 1', - u'SDSS DR7 - Sloan Digital Sky Survey Data Release 7 1', - u'SDSS DR7 - Sloan Digital Sky Survey Data Release 7 2', - u'SDSS DR7 - Sloan Digital Sky Survey Data Release 7 3', - u'SDSS DR7 - Sloan Digital Sky Survey Data Release 7 4', - u'SDSS DR8 - Sloan Digital Sky Survey Data Release 8 1', - u'SDSS DR8 - Sloan Digital Sky Survey Data Release 8 2', - u'The HST Guide Star Catalog, Version 1.1 (Lasker+ 1992) 1', - u'The HST Guide Star Catalog, Version 1.2 (Lasker+ 1996) 1', - u'The HST Guide Star Catalog, Version GSC-ACT (Lasker+ 1996-99) 1', - u'The PMM USNO-A1.0 Catalogue (Monet 1997) 1', - u'The USNO-A2.0 Catalogue (Monet+ 1998) 1', - u'Two Micron All Sky Survey (2MASS) 1', - u'Two Micron All Sky Survey (2MASS) 2', - u'USNO-A2 Catalogue 1', - u'USNO-A2.0 1'] - -To inspect them in detail, do the following and then refer to the examples in -:ref:`vo-sec-client-db-manip`: - ->>> from astropy.vo.client import vos_catalog ->>> good_db = vos_catalog.get_remote_catalog_db('conesearch_good') - -Select a catalog to search: - ->>> my_catname = 'The PMM USNO-A1.0 Catalogue (Monet 1997) 1' - -By default, pedantic is ``False``: - ->>> from astropy.io.votable import conf ->>> conf.pedantic -False - -Perform Cone Search in the selected catalog above for 0.5 degree radius -around 47 Tucanae with minimum verbosity, if supported. -The ``catalog_db`` keyword gives control over which catalog(s) to use. -If running this for the first time, a copy of the catalogs database will be -downloaded to local cache. To run this again without -using cached data, set ``cache=False``: - ->>> from astropy import coordinates as coord ->>> from astropy import units as u ->>> c = coord.SkyCoord.from_name('47 Tuc') # doctest: +REMOTE_DATA ->>> c - ->>> sr = 0.5 * u.degree ->>> sr - ->>> result = conesearch.conesearch(c, sr, catalog_db=my_catname) -Trying http://vizier.u-strasbg.fr/viz-bin/votable/-A?-source=I/243/out& -Downloading ... -WARNING: W22: ... The DEFINITIONS element is deprecated in VOTable 1.1... - -To run the command above using custom timeout of -30 seconds for each Cone Search service query: - ->>> from astropy.utils import data ->>> with data.conf.set_temp('remote_timeout', 30): -... result = conesearch.conesearch(c, sr, catalog_db=my_catname) - -To suppress *all* the screen outputs (not recommended): - ->>> import warnings ->>> with warnings.catch_warnings(): -... warnings.simplefilter('ignore') -... result = conesearch.conesearch(c, sr, catalog_db=my_catname, -... verbose=False) - -Extract Numpy array containing the matched objects. See -`numpy` for available operations: - ->>> cone_arr = result.array.data ->>> cone_arr -array([(0.499298, 4.403473, -72.124045, '0150-00088188'), - (0.499075, 4.403906, -72.122762, '0150-00088198'), - (0.499528, 4.404531, -72.045198, '0150-00088210'), ..., - (0.4988, 7.641731, -72.113156, '0150-00225965'), - (0.499554, 7.645489, -72.103167, '0150-00226134'), - (0.499917, 7.6474, -72.0876, '0150-00226223')], - dtype=[('_r', '>> cone_arr.dtype.names -('_r', '_RAJ2000', '_DEJ2000', 'USNO-A1.0') ->>> cone_arr.size -36184 ->>> ra_list = cone_arr['_RAJ2000'] ->>> ra_list -array([ 4.403473, 4.403906, 4.404531, ..., 7.641731, 7.645489, 7.6474 ]) ->>> cone_arr[0] # First row -(0.499298, 4.403473, -72.124045, '0150-00088188') ->>> cone_arr[-1] # Last row -(0.499917, 7.6474, -72.0876, '0150-00226223') ->>> cone_arr[:10] # First 10 rows -array([(0.499298, 4.403473, -72.124045, '0150-00088188'), - (0.499075, 4.403906, -72.122762, '0150-00088198'), - (0.499528, 4.404531, -72.045198, '0150-00088210'), - (0.497252, 4.406078, -72.095045, '0150-00088245'), - (0.499739, 4.406462, -72.139545, '0150-00088254'), - (0.496312, 4.410623, -72.110492, '0150-00088372'), - (0.49473, 4.415053, -72.071217, '0150-00088494'), - (0.494171, 4.415939, -72.087512, '0150-00088517'), - (0.493722, 4.417678, -72.0972, '0150-00088572'), - (0.495147, 4.418262, -72.047142, '0150-00088595')], - dtype=[('_r', '>> import numpy as np ->>> sep = cone_arr['_r'] ->>> i_sorted = np.argsort(sep) ->>> cone_arr[i_sorted] -array([(0.081971, 5.917787, -72.006075, '0150-00145335'), - (0.083181, 6.020339, -72.164623, '0150-00149799'), - (0.089166, 5.732798, -72.077698, '0150-00137181'), ..., - (0.499981, 7.024962, -72.477503, '0150-00198745'), - (0.499987, 6.423773, -71.597364, '0150-00168596'), - (0.499989, 6.899589, -72.5043, '0150-00192872')], - dtype=[('_r', '>> from astropy import units as u ->>> ra_field = result.get_field_by_id('_RAJ2000') ->>> ra_field.title -u'Right ascension (FK5, Equinox=J2000.0) (computed by VizieR, ...)' ->>> ra_field.unit -Unit("deg") ->>> ra_field.unit.to(u.arcsec) * ra_list -array([ 15852.5028, 15854.0616, 15856.3116, ..., 27510.2316, - 27523.7604, 27530.64 ]) - -Perform the same Cone Search as above but asynchronously using -`~astropy.vo.client.conesearch.AsyncConeSearch`. Queries to -individual Cone Search services are still governed by -`astropy.utils.data.Conf.remote_timeout`. Cone Search is forced to run -in silent mode asynchronously, but warnings are still controlled by -:py:mod:`warnings`: - ->>> async_search = conesearch.AsyncConeSearch(c, sr, catalog_db=my_catname) - -Check asynchronous search status: - ->>> async_search.running() -True ->>> async_search.done() -False - -Get search results after a 30-second wait (not to be confused with -`astropy.utils.data.Conf.remote_timeout` that governs individual Cone -Search queries). If search is still not done after 30 seconds, -``TimeoutError`` is raised. Otherwise, Cone Search result is returned -and can be manipulated as above. If no ``timeout`` keyword given, it -waits until completion: - ->>> async_result = async_search.get(timeout=30) ->>> cone_arr = async_result.array.data ->>> cone_arr.size -36184 - -Estimate the execution time and the number of objects for -the Cone Search service URL from above. The prediction naively -assumes a linear model, which might not be accurate for some cases. -It also uses the normal :func:`~astropy.vo.client.conesearch.conesearch`, -not the asynchronous version. This example uses a custom -timeout of 30 seconds and runs silently (except for warnings): - ->>> result.url -u'http://vizier.u-strasbg.fr/viz-bin/votable/-A?-source=I/243/out&' ->>> with data.conf.set_temp('remote_timeout', 30): -... t_est, n_est = conesearch.predict_search( -... result.url, c, sr, verbose=False, plot=True) -WARNING: W22: ... The DEFINITIONS element is deprecated in VOTable 1.1... -# ... ->>> t_est # Predicted execution time -10.757875269998323 ->>> n_est # Predicted number of objects -37340 - -.. image:: images/client_predict_search_t.png - :width: 450px - :alt: Example plot from conesearch.predict_search() for t_est - -.. image:: images/client_predict_search_n.png - :width: 450px - :alt: Example plot from conesearch.predict_search() for n_est - -For debugging purpose, one can obtain the actual execution time -and number of objects, and compare them with the predicted values -above. The INFO message shown in controlled by `astropy.logger`. -Keep in mind that running this for every prediction -would defeat the purpose of the prediction itself: - ->>> t_real, tab = conesearch.conesearch_timer( -... c, sr, catalog_db=result.url, verbose=False) -INFO: conesearch_timer took 11.5103080273 s on AVERAGE for 1 call(s). [...] ->>> t_real # Actual execution time -9.33926796913147 ->>> tab.array.size # Actual number of objects -36184 - -One can also search in a list of catalogs instead of a single one. -In this example, we look for all catalogs containing ``'guide*star'`` in their -titles and only perform Cone Search using those services. -The first catalog in the list to successfully return non-zero result is used. -Therefore, the order of catalog names given in ``catalog_db`` is important: - ->>> gsc_cats = conesearch.list_catalogs(pattern='guide*star') ->>> gsc_cats -[u'Guide Star Catalog 2.3 1', - u'The HST Guide Star Catalog, Version 1.1 (Lasker+ 1992) 1', - u'The HST Guide Star Catalog, Version 1.2 (Lasker+ 1996) 1', - u'The HST Guide Star Catalog, Version GSC-ACT (Lasker+ 1996-99) 1'] ->>> gsc_result = conesearch.conesearch(c, sr, catalog_db=gsc_cats) -Trying http://gsss.stsci.edu/webservices/vo/ConeSearch.aspx?CAT=GSC23& -WARNING: W50: ... Invalid unit string 'pixel' [...] -WARNING: W48: ... Unknown attribute 'nrows' on TABLEDATA [...] ->>> gsc_result.array.size -74276 ->>> gsc_result.url -u'http://gsss.stsci.edu/webservices/vo/ConeSearch.aspx?CAT=GSC23&' - -To repeat the Cone Search above with the services listed in a -different order: - ->>> gsc_cats_reordered = [gsc_cats[i] for i in (3, 1, 2, 0)] ->>> gsc_cats_reordered -[u'The HST Guide Star Catalog, Version GSC-ACT (Lasker+ 1996-99) 1', - u'The HST Guide Star Catalog, Version 1.1 (Lasker+ 1992) 1', - u'The HST Guide Star Catalog, Version 1.2 (Lasker+ 1996) 1', - u'Guide Star Catalog 2.3 1'] ->>> gsc_result = conesearch.conesearch(c, sr, catalog_db=gsc_cats_reordered) -Trying http://vizier.u-strasbg.fr/viz-bin/votable/-A?-source=I/255/out& -Downloading ... -WARNING: W22: ... The DEFINITIONS element is deprecated in VOTable 1.1... ->>> gsc_result.array.size -2997 ->>> gsc_result.url -u'http://vizier.u-strasbg.fr/viz-bin/votable/-A?-source=I/255/out&' - -To obtain results from *all* the services above: - ->>> all_gsc_results = conesearch.search_all(c, sr, catalog_db=gsc_cats) -Trying http://gsss.stsci.edu/webservices/vo/ConeSearch.aspx?CAT=GSC23& -Downloading ... -Trying http://vizier.u-strasbg.fr/viz-bin/votable/-A?-source=I/220/out& -Downloading ... -WARNING: W22: ... The DEFINITIONS element is deprecated in VOTable 1.1... -Trying http://vizier.u-strasbg.fr/viz-bin/votable/-A?-source=I/254/out& -Downloading ... -WARNING: W22: ... The DEFINITIONS element is deprecated in VOTable 1.1... -Trying http://vizier.u-strasbg.fr/viz-bin/votable/-A?-source=I/255/out& -Downloading ... ->>> len(all_gsc_results) -4 ->>> for url, tab in all_gsc_results.items(): -... print('{0} has {1} results'.format(url, tab.array.size)) -http://vizier.u-strasbg.fr/viz-bin/votable/-A?-source=I/254/out& has 2998 results -http://vizier.u-strasbg.fr/viz-bin/votable/-A?-source=I/255/out& has 2997 results -http://gsss.stsci.edu/webservices/vo/ConeSearch.aspx?CAT=GSC23& has 74276 results -http://vizier.u-strasbg.fr/viz-bin/votable/-A?-source=I/220/out& has 2997 results - -To repeat the above asynchronously: - ->>> async_search_all = conesearch.AsyncSearchAll(c, sr, catalog_db=gsc_cats) ->>> async_search_all.running() -True ->>> async_search_all.done() -False ->>> all_gsc_results = async_search_all.get() - -If one is unable to obtain any results using the default -Cone Search database, ``'conesearch_good'``, that only contains -sites that cleanly passed validation, one can use :ref:`astropy_config` -to use another database, ``'conesearch_warn'``, containing sites with -validation warnings. One should use these sites with caution: - ->>> from astropy.vo import conf ->>> conf.conesearch_dbname = 'conesearch_warn' ->>> conesearch.list_catalogs() -Downloading http://stsdas.stsci.edu/astrolib/vo_databases/conesearch_warn.json -|===========================================| 87k/ 87k (100.00%) 00s -[u'2MASS All-Sky Catalog of Point Sources (Cutri+ 2003) 1', - u'2MASS All-Sky Point Source Catalog 1', - u'Data release 7 of Sloan Digital Sky Survey catalogs 1', - u'Data release 7 of Sloan Digital Sky Survey catalogs 2', - u'Data release 7 of Sloan Digital Sky Survey catalogs 3', - u'Data release 7 of Sloan Digital Sky Survey catalogs 4', - u'Data release 7 of Sloan Digital Sky Survey catalogs 5', - u'Data release 7 of Sloan Digital Sky Survey catalogs 6', - u'The 2MASS All-Sky Catalog 1', - u'The 2MASS All-Sky Catalog 2', - u'The USNO-B1.0 Catalog (Monet+ 2003) 1', - u'The USNO-B1.0 Catalog 1', - u'USNO-A V2.0, A Catalog of Astrometric Standards 1', - u'USNO-B1 Catalogue 1'] ->>> result = conesearch.conesearch(c, sr) -Trying http://vizier.u-strasbg.fr/viz-bin/votable/-A?-source=I/284/out& -Downloading ... -WARNING: W22: ... The DEFINITIONS element is deprecated in VOTable 1.1... ->>> result.array.data.size -50000 - -You can also use custom Cone Search database, say, ``'my_vo_database.json'`` -from :ref:`VO database examples `: - ->>> import os ->>> from astropy.vo.client.vos_catalog import BASEURL ->>> BASEURL.set(os.curdir) ->>> conesearch.CONESEARCH_DBNAME.set('my_vo_database') ->>> conesearch.list_catalogs() -[u'My Catalog 1'] ->>> result = conesearch.conesearch(c, sr) -Trying http://ex.org/cgi-bin/cs.pl? -Downloading ... -|===========================================| 1.8k/1.8k (100.00%) 00s -# ... -VOSError: None of the available catalogs returned valid results. diff --git a/docs/vo/conesearch/images/astropy_vo_flowchart.png b/docs/vo/conesearch/images/astropy_vo_flowchart.png deleted file mode 100644 index dea5962babea..000000000000 Binary files a/docs/vo/conesearch/images/astropy_vo_flowchart.png and /dev/null differ diff --git a/docs/vo/conesearch/images/client_predict_search_n.png b/docs/vo/conesearch/images/client_predict_search_n.png deleted file mode 100644 index e24e98d0356f..000000000000 Binary files a/docs/vo/conesearch/images/client_predict_search_n.png and /dev/null differ diff --git a/docs/vo/conesearch/images/client_predict_search_t.png b/docs/vo/conesearch/images/client_predict_search_t.png deleted file mode 100644 index 97fb8b6822ea..000000000000 Binary files a/docs/vo/conesearch/images/client_predict_search_t.png and /dev/null differ diff --git a/docs/vo/conesearch/images/validator_html_1.png b/docs/vo/conesearch/images/validator_html_1.png deleted file mode 100644 index 9f92d06fb9a8..000000000000 Binary files a/docs/vo/conesearch/images/validator_html_1.png and /dev/null differ diff --git a/docs/vo/conesearch/images/validator_html_2.png b/docs/vo/conesearch/images/validator_html_2.png deleted file mode 100644 index a10a56b508ec..000000000000 Binary files a/docs/vo/conesearch/images/validator_html_2.png and /dev/null differ diff --git a/docs/vo/conesearch/images/validator_html_3.png b/docs/vo/conesearch/images/validator_html_3.png deleted file mode 100644 index aa3ac256d955..000000000000 Binary files a/docs/vo/conesearch/images/validator_html_3.png and /dev/null differ diff --git a/docs/vo/conesearch/images/validator_html_4.png b/docs/vo/conesearch/images/validator_html_4.png deleted file mode 100644 index d34a44aca09a..000000000000 Binary files a/docs/vo/conesearch/images/validator_html_4.png and /dev/null differ diff --git a/docs/vo/conesearch/index.rst b/docs/vo/conesearch/index.rst deleted file mode 100644 index 8b9918e27a91..000000000000 --- a/docs/vo/conesearch/index.rst +++ /dev/null @@ -1,191 +0,0 @@ -.. doctest-skip-all - -.. _astropy_conesearch: - -VO Simple Cone Search -===================== - -Astropy offers Simple Cone Search Version 1.03 as defined in IVOA -Recommendation (February 22, 2008). Cone Search queries an -area encompassed by a given radius centered on a given RA and DEC and returns -all the objects found within the area in the given catalog. - -.. _vo-sec-default-scs-services: - -Default Cone Search Services ----------------------------- - -Currently, the default Cone Search services used are a subset of those found in -the STScI VAO Registry. They were hand-picked to represent commonly used -catalogs below: - -* 2MASS All-Sky -* HST Guide Star Catalog -* SDSS Data Release 7 -* SDSS-III Data Release 8 -* USNO A1 -* USNO A2 -* USNO B1 - -This subset undergoes daily validations hosted by STScI using -:ref:`vo-sec-validator-validate`. Those that pass without critical -warnings or exceptions are used by :ref:`vo-sec-client-scs` by -default. They are controlled by `astropy.vo.Conf.conesearch_dbname`: - -#. ``'conesearch_good'`` - Default. Passed validation without critical warnings and exceptions. -#. ``'conesearch_warn'`` - Has critical warnings but no exceptions. Use at your own risk. -#. ``'conesearch_exception'`` - Has some exceptions. *Never* use this. -#. ``'conesearch_error'`` - Has network connection error. *Never* use this. - -If you are a Cone Search service provider and would like to include your -service in the list above, please open a -`GitHub issue on Astropy `_. - - -Caching -------- - -Caching of downloaded contents is controlled by `astropy.utils.data`. -To use cached data, some functions in this package have a ``cache`` -keyword that can be set to ``True``. - - -Getting Started ---------------- - -This section only contains minimal examples showing how to perform -basic Cone Search. - ->>> from astropy.vo.client import conesearch - -List the available Cone Search catalogs: - ->>> conesearch.list_catalogs() -[u'Guide Star Catalog 2.3 1', - u'SDSS DR7 - Sloan Digital Sky Survey Data Release 7 1', - u'SDSS DR7 - Sloan Digital Sky Survey Data Release 7 2', - u'SDSS DR7 - Sloan Digital Sky Survey Data Release 7 3', - u'SDSS DR7 - Sloan Digital Sky Survey Data Release 7 4', - u'SDSS DR8 - Sloan Digital Sky Survey Data Release 8 1', - u'SDSS DR8 - Sloan Digital Sky Survey Data Release 8 2', - u'The HST Guide Star Catalog, Version 1.1 (Lasker+ 1992) 1', - u'The HST Guide Star Catalog, Version 1.2 (Lasker+ 1996) 1', - u'The HST Guide Star Catalog, Version GSC-ACT (Lasker+ 1996-99) 1', - u'The PMM USNO-A1.0 Catalogue (Monet 1997) 1', - u'The USNO-A2.0 Catalogue (Monet+ 1998) 1', - u'Two Micron All Sky Survey (2MASS) 1', - u'Two Micron All Sky Survey (2MASS) 2', - u'USNO-A2 Catalogue 1', - u'USNO-A2.0 1'] - -Select a 2MASS catalog from the list above that is to be searched: - ->>> my_catname = 'Two Micron All Sky Survey (2MASS) 1' - -Query the selected 2MASS catalog around M31 with a 0.1-degree search radius: - ->>> from astropy.coordinates import SkyCoord ->>> from astropy import units as u ->>> c = SkyCoord.from_name('M31') # doctest: +REMOTE_DATA ->>> c.ra, c.dec -(, ) ->>> result = conesearch.conesearch(c, 0.1 * u.degree, catalog_db=my_catname) -Trying http://wfaudata.roe.ac.uk/twomass-dsa/DirectCone?DSACAT=TWOMASS&... -Downloading ... -WARNING: W06: ... UCD has invalid character '?' in '??' [...] -WARNING: W50: ... Invalid unit string 'yyyy-mm-dd' [...] -WARNING: W50: ... Invalid unit string 'Julian days' [...] ->>> result -
->>> result.url -u'http://wfaudata.roe.ac.uk/twomass-dsa/DirectCone?DSACAT=TWOMASS&DSATAB=twomass_psc&' - -Get the number of matches and returned column names: - ->>> result.array.size -2008 ->>> result.array.dtype.names -('cx', - 'cy', - 'cz', - 'htmID', - 'ra', - 'dec', ..., - 'coadd_key', - 'coadd') - -Extract RA and DEC of the matches: - ->>> result.array['ra'] -masked_array(data = [10.620983 10.672264 10.651166 ..., 10.805599], - mask = [False False False ..., False], - fill_value = 1e+20) ->>> result.array['dec'] -masked_array(data = [41.192303 41.19426 41.19445 ..., 41.262123], - mask = [False False False ..., False], - fill_value = 1e+20) - - -Using `astropy.vo` ------------------- - -This package has four main components across two subpackages: - -.. toctree:: - :maxdepth: 2 - - client - validator - -They are designed to be used in a work flow as illustrated below: - -.. image:: images/astropy_vo_flowchart.png - :width: 500px - :alt: VO work flow - -The one that a typical user needs is the :ref:`vo-sec-client-scs` component -(see :ref:`Cone Search Examples `). - - -See Also --------- - -- `NVO Directory `_ - -- `Simple Cone Search Version 1.03, IVOA Recommendation (22 February 2008) `_ - -- `STScI VAO Registry `_ - -- `STScI VO Databases `_ - - -Reference/API -------------- - -.. automodapi:: astropy.vo - :no-inheritance-diagram: - -.. automodapi:: astropy.vo.client.vos_catalog - :no-inheritance-diagram: - -.. automodapi:: astropy.vo.client.conesearch - :no-inheritance-diagram: - -.. automodapi:: astropy.vo.client.async - :no-inheritance-diagram: - -.. automodapi:: astropy.vo.client.exceptions - -.. automodapi:: astropy.vo.validator - -.. automodapi:: astropy.vo.validator.validate - :no-inheritance-diagram: - -.. automodapi:: astropy.vo.validator.inspect - :no-inheritance-diagram: - -.. automodapi:: astropy.vo.validator.exceptions diff --git a/docs/vo/conesearch/validator.rst b/docs/vo/conesearch/validator.rst deleted file mode 100644 index a4cabefc1960..000000000000 --- a/docs/vo/conesearch/validator.rst +++ /dev/null @@ -1,362 +0,0 @@ -.. doctest-skip-all - -Using `astropy.vo.validator` -============================ - -VO services validator is used by STScI to support :ref:`vo-sec-client-scs`. -Currently, only Cone Search services are supported. -A typical user should not need the validator. However, this could be used by -VO service providers to validate their services. Currently, any service -to be validated has to be registered in STScI VAO Registry. - -.. _vo-sec-validator-validate: - -Validation for Simple Cone Search ---------------------------------- - -`astropy.vo.validator.validate` validates VO services. -Currently, only Cone Search validation is done using -:func:`~astropy.vo.validator.validate.check_conesearch_sites`, -which utilizes underlying `astropy.io.votable.validator` library. - -A master list of all available Cone Search services is obtained from -`astropy.vo.validator.Conf.conesearch_master_list`, which is a URL -query to STScI VAO Registry by default. However, by default, only the -ones in `astropy.vo.validator.Conf.conesearch_urls` are validated -(also see :ref:`vo-sec-default-scs-services`), while the rest are -skipped. There are also options to validate a user-defined list of -services or all of them. - -All Cone Search queries are done using RA, DEC, and SR given by -```` XML tag in the registry, and maximum verbosity. -In an uncommon case where ```` is not defined for a service, -it uses a default search for ``RA=0&DEC=0&SR=0.1``. - -The results are separated into 4 groups below. Each group -is stored as a JSON file of `~astropy.vo.client.vos_catalog.VOSDatabase`: - -#. ``conesearch_good.json`` Passed validation without critical - warnings and exceptions. This database residing in - `astropy.vo.Conf.vos_baseurl` is the one used by - :ref:`vo-sec-client-scs` by default. -#. ``conesearch_warn.json`` Has critical warnings but no - exceptions. Users can manually set - `astropy.vo.Conf.conesearch_dbname` to use this at their own - risk. -#. ``conesearch_exception.json`` - Has some exceptions. *Never* use this. - For informational purpose only. -#. ``conesearch_error.json`` - Has network connection error. *Never* use this. - For informational purpose only. - -HTML pages summarizing the validation results are stored in -``'results'`` sub-directory, which also contains downloaded XML -files from individual Cone Search queries. - -Warnings and Exceptions -^^^^^^^^^^^^^^^^^^^^^^^ - -A subset of `astropy.io.votable.exceptions` that is considered -non-critical is defined by -`astropy.vo.validator.Conf.noncritical_warnings`, which will not be -flagged as bad by the validator. However, this does not change the -behavior of `astropy.io.votable.Conf.pedantic`, which still needs to -be set to ``False`` for them not to be thrown out by -:func:`~astropy.vo.client.conesearch.conesearch`. Despite being -listed as non-critical, user is responsible to check whether the -results are reliable; They should not be used blindly. - -Some `units recognized by VizieR `_ -are considered invalid by Cone Search standards. As a result, -they will give the warning ``'W50'``, which is non-critical by default. - -User can also modify `astropy.vo.validator.Conf.noncritical_warnings` -to include or exclude any warnings or exceptions, as desired. -However, this should be done with caution. Adding exceptions to -non-critical list is not recommended. - -.. _vo-sec-validator-build-db: - -Building the Database from Registry -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Each Cone Search service is a `~astropy.vo.client.vos_catalog.VOSCatalog` in -a `~astropy.vo.client.vos_catalog.VOSDatabase` (see -:ref:`vo-sec-client-cat-manip` and :ref:`vo-sec-client-db-manip`). - -In the master registry, there are duplicate catalog titles with -different access URLs, duplicate access URLs with different titles, -duplicate catalogs with slightly different descriptions, etc. - -A Cone Search service is really defined by its access URL -regardless of title, description, etc. By default, -:func:`~astropy.vo.client.vos_catalog.VOSDatabase.from_registry` ensures -each access URL is unique across the database. -However, for user-friendly catalog listing, its title will be -the catalog key, not the access URL. - -In the case of two different access URLs sharing the same title, -each URL will have its own database entry, with a sequence number -appended to their titles (e.g., 'Title 1' and 'Title 2'). For -consistency, even if the title does not repeat, it will still be -renamed to 'Title 1'. - -In the case of the same access URL appearing multiple times in -the registry, the validator will store the first catalog with -that access URL and throw out the rest. However, it will keep -count of the number of duplicates thrown out in the -``'duplicatesIgnored'`` dictionary key of the catalog kept in the -database. - -All the existing catalog tags will be copied over as dictionary -keys, except ``'accessURL'`` that is renamed to ``'url'`` for simplicity. -In addition, new keys from validation are added: - -* ``validate_expected`` - Expected validation result category, e.g., "good". -* ``validate_network_error`` - Indication for connection error. -* ``validate_nexceptions`` - Number of exceptions found. -* ``validate_nwarnings`` - Number of warnings found. -* ``validate_out_db_name`` - Cone Search database name this entry belongs to. -* ``validate_version`` - Version of validation software. -* ``validate_warning_types`` - List of warning codes. -* ``validate_warnings`` - Descriptions of the warnings. -* ``validate_xmllint`` - Indication of whether ``xmllint`` passed. -* ``validate_xmllint_content`` - Output from ``xmllint``. - -Configurable Items -^^^^^^^^^^^^^^^^^^ - -These parameters are set via :ref:`astropy_config`: - -* `astropy.vo.validator.Conf.conesearch_master_list` - VO registry query URL that should return a VO table with all the desired - VO services. -* `astropy.vo.validator.Conf.conesearch_urls` - Subset of Cone Search access URLs to validate. -* `astropy.vo.validator.Conf.noncritical_warnings` - List of VO table parser warning codes that are considered non-critical. - -Also depends on properties in -:ref:`Simple Cone Search Configurable Items `. - -.. _vo-sec-validate-examples: - -Examples -^^^^^^^^ - ->>> from astropy.vo.validator import validate - -Validate default Cone Search sites with multiprocessing and write results -in the current directory. Reading the master registry can be slow, so the -default timeout is internally set to 60 seconds for it. However, -``astropy.utils.data.REMOTE_TIMEOUT`` should still be set to account for -accessing the individual services (at least 30 seconds is recommended). -In addition, all VO table warnings from the registry are suppressed because -we are not trying to validate the registry itself but the services it contains: - ->>> from astropy.utils import data ->>> with data.conf.set_temp('remote_timeout', 30): -... validate.check_conesearch_sites() -Downloading http://vao.stsci.edu/directory/NVORegInt.asmx/... -|===========================================| 25M/ 25M (100.00%) 00s -INFO: Only 30/11938 site(s) are validated [astropy.vo.validator.validate] -# ... -INFO: good: 14 catalog(s) [astropy.vo.validator.validate] -INFO: warn: 12 catalog(s) [astropy.vo.validator.validate] -INFO: excp: 0 catalog(s) [astropy.vo.validator.validate] -INFO: nerr: 4 catalog(s) [astropy.vo.validator.validate] -INFO: total: 30 out of 30 catalog(s) [astropy.vo.validator.validate] -INFO: check_conesearch_sites took 451.05685997 s on AVERAGE... - -Validate only Cone Search access URLs hosted by ``'stsci.edu'`` without verbose -outputs (except warnings that are controlled by :py:mod:`warnings`) or -multiprocessing, and write results in ``'subset'`` sub-directory instead of the -current directory. For this example, we use ``registry_db`` from -:ref:`VO database examples `: - ->>> urls = registry_db.list_catalogs_by_url(pattern='stsci.edu') ->>> urls -['http://archive.stsci.edu/befs/search.php?', - 'http://archive.stsci.edu/copernicus/search.php?', ..., - 'http://galex.stsci.edu/gxWS/ConeSearch/gxConeSearch.aspx?', - 'http://gsss.stsci.edu/webservices/vo/ConeSearch.aspx?CAT=GSC23&'] ->>> with data.conf.set_temp('remote_timeout', 30): -... validate.check_conesearch_sites( -... destdir='./subset', verbose=False, parallel=False, url_list=urls) -INFO: check_conesearch_sites took 84.7241549492 s on AVERAGE... - -Add ``'W24'`` from `astropy.io.votable.exceptions` to the list of -non-critical warnings to be ignored and re-run default validation. -This is *not* recommended unless you know exactly what you are doing: - ->>> from astropy.vo.validator.validate import conf ->>> with conf.set_temp('noncritical_warnings', conf.noncritical_warnings + ['W24']): -... with data.conf.set_temp('remote_timeout', 30): -... validate.check_conesearch_sites() - -Validate *all* Cone Search services in the master registry -(this will take a while) and write results in ``'all'`` sub-directory: - ->>> with data.conf.set_temp('remote_timeout', 30): -... validate.check_conesearch_sites(destdir='./all', url_list=None) - -To look at the HTML pages of the validation results in the current -directory using Firefox browser (images shown are from STScI server -but your own results should look similar):: - - firefox results/index.html - -.. image:: images/validator_html_1.png - :width: 600px - :alt: Main HTML page of validation results - -When you click on 'All tests' from the page above, you will see all the -Cone Search services validated with a summary of validation results: - -.. image:: images/validator_html_2.png - :width: 600px - :alt: All tests HTML page - -When you click on any of the listed URLs from above, you will see -detailed validation warnings and exceptions for the selected URL: - -.. image:: images/validator_html_3.png - :width: 600px - :alt: Detailed validation warnings HTML page - -When you click on the URL on top of the page above, you will see -the actual VO Table returned by the Cone Search query: - -.. image:: images/validator_html_4.png - :width: 600px - :alt: VOTABLE XML page - - -.. _vo-sec-validator-inspect: - -Inspection of Validation Results --------------------------------- - -`astropy.vo.validator.inspect` inspects results from -:ref:`vo-sec-validator-validate`. It reads in JSON files of -`~astropy.vo.client.vos_catalog.VOSDatabase` -residing in ``astropy.vo.Conf.vos_baseurl``, which -can be changed to point to a different location. - -Configurable Items -^^^^^^^^^^^^^^^^^^ - -This parameter is set via :ref:`astropy_config`: - -* `astropy.vo.Conf.vos_baseurl` - -Examples -^^^^^^^^ - ->>> from astropy.vo.validator import inspect - -Load Cone Search validation results from -``astropy.vo.Conf.vos_baseurl`` (by default, the one used by -:ref:`vo-sec-client-scs`): - ->>> r = inspect.ConeSearchResults() -Downloading http://.../conesearch_good.json -|===========================================| 48k/ 48k (100.00%) 00s -Downloading http://.../conesearch_warn.json -|===========================================| 85k/ 85k (100.00%) 00s -Downloading http://.../conesearch_exception.json -|===========================================| 3.0k/3.0k (100.00%) 00s -Downloading http://.../conesearch_error.json -|===========================================| 4.0k/4.0k (100.00%) 00s - -Print tally. In this example, there are 13 Cone Search services that -passed validation with non-critical warnings, 14 with critical warnings, -1 with exceptions, and 2 with network error: - ->>> r.tally() -good: 13 catalog(s) -warn: 14 catalog(s) -exception: 1 catalog(s) -error: 2 catalog(s) -total: 30 catalog(s) - -Print a list of good Cone Search catalogs, each with title, access URL, -warning codes collected, and individual warnings: - ->>> r.list_cats('good') -Guide Star Catalog 2.3 1 -http://gsss.stsci.edu/webservices/vo/ConeSearch.aspx?CAT=GSC23& -W48,W50 -.../vo.xml:136:0: W50: Invalid unit string 'pixel' -.../vo.xml:155:0: W48: Unknown attribute 'nrows' on TABLEDATA -# ... -USNO-A2 Catalogue 1 -http://www.nofs.navy.mil/cgi-bin/vo_cone.cgi?CAT=USNO-A2& -W17,W42,W21 -.../vo.xml:4:0: W21: vo.table is designed for VOTable version 1.1 and 1.2... -.../vo.xml:4:0: W42: No XML namespace specified -.../vo.xml:15:15: W17: VOTABLE element contains more than one DESCRIPTION... - -List Cone Search catalogs with warnings, excluding warnings that were -ignored in `astropy.vo.validator.Conf.noncritical_warnings`, and -writes the output to a file named ``'warn_cats.txt'`` in the current -directory. This is useful to see why the services failed validations: - ->>> with open('warn_cats.txt', 'w') as fout: -... r.list_cats('warn', fout=fout, ignore_noncrit=True) - -List the titles of all good Cone Search catalogs: - ->>> r.catkeys['good'] -[u'Guide Star Catalog 2.3 1', - u'SDSS DR7 - Sloan Digital Sky Survey Data Release 7 1', - u'SDSS DR7 - Sloan Digital Sky Survey Data Release 7 2', - u'SDSS DR7 - Sloan Digital Sky Survey Data Release 7 3', ..., - u'USNO-A2 Catalogue 1'] - -Print the details of catalog titled ``'USNO-A2 Catalogue 1'``: - ->>> r.print_cat('USNO-A2 Catalogue 1') -{ - "capabilityClass": "ConeSearch", - "capabilityStandardID": "ivo://ivoa.net/std/ConeSearch", - "capabilityValidationLevel": "", - "contentLevel": "#University#Research#Amateur#", - # ... - "version": "", - "waveband": "#Optical#" -} -Found in good - -Load Cone Search validation results from a local directory named ``'subset'``. -This is useful if you ran your own :ref:`vo-sec-validator-validate` -and wish to inspect the output databases. This example reads in -validation of STScI Cone Search services done in -:ref:`Validation for Simple Cone Search Examples `: - ->>> from astropy.vo import conf ->>> with conf.set_temp('vos_baseurl', './subset/'): ->>> r = inspect.ConeSearchResults() ->>> r.tally() -good: 19 catalog(s) -warn: 7 catalog(s) -exception: 2 catalog(s) -error: 0 catalog(s) -total: 28 catalog(s) ->>> r.catkeys['good'] -[u'Advanced Camera for Surveys 1', - u'Berkeley Extreme and Far-UV Spectrometer 1', - u'Copernicus Satellite 1', - u'Extreme Ultraviolet Explorer 1', ..., - u'Wisconsin Ultraviolet Photo-Polarimeter Experiment 1'] diff --git a/docs/vo/index.rst b/docs/vo/index.rst deleted file mode 100644 index 49579bf1665b..000000000000 --- a/docs/vo/index.rst +++ /dev/null @@ -1,38 +0,0 @@ -.. _astropy_vo: - -******************************************* -Virtual Observatory Access (``astropy.vo``) -******************************************* - -.. module:: astropy.vo - -Introduction -============ - -The ``astropy.vo`` subpackage handles simple access for Virtual Observatory -(VO) services. - -Current services include: - -.. toctree:: - :maxdepth: 1 - - conesearch/index - samp/index - -Other third-party Python packages and tools related to ``astropy.vo``: - -* `PyVO `__ - provides further functionality to discover - and query VO services. Its user guide contains a - `good introduction `__ - to how the VO works. - -* `Astroquery `__ - is an Astropy affiliated package that provides simply access to specific astronomical - web services, many of which do not support the VO protocol. - -* `Simple-Cone-Search-Creator `_ - shows how to ingest a catalog into a cone search service and serve it in VO - standard format using Python - (using CSV files and `healpy `__). diff --git a/docs/vo/samp/advanced_embed_samp_hub.rst b/docs/vo/samp/advanced_embed_samp_hub.rst deleted file mode 100644 index 6f110ea9ef09..000000000000 --- a/docs/vo/samp/advanced_embed_samp_hub.rst +++ /dev/null @@ -1,134 +0,0 @@ -.. include:: references.txt - -.. doctest-skip-all - -Embedding a SAMP hub in a GUI ------------------------------ - -Overview -^^^^^^^^ - -If you wish to embed a SAMP hub in your Python GUI tool, you will need to start -the hub programmatically using:: - - from astropy.vo.samp import SAMPHubServer - hub = SAMPHubServer() - hub.start() - -This launches the hub in a thread and is non-blocking. If you are not -interested in connections from web SAMP clients, then you can simply use:: - - from astropy.vo.samp import SAMPHubServer - hub = SAMPHubServer(web_profile=False) - hub.start() - -and this should be all you need to do. However, if you want to keep the web -profile active, there is an additional consideration, which is that when a web -SAMP client connects, you will need to ask the user whether they accept the -connection (for security reasons). By default, the confirmation message is a -text-based message in the terminal, but if you have a GUI tool, you will -instead likely want to open a GUI dialog. - -To do this, you will need to define a class that handles the dialog, -and you should then pass an **instance** of the class to -|SAMPHubServer| (not the class itself). This class should inherit -from `astropy.vo.samp.WebProfileDialog` and add the following: - - 1) It should have a GUI timer callback that periodically calls - ``WebProfileDialog.handle_queue`` (available as - ``self.handle_queue``). - - 2) Implement a ``show_dialog`` method to display a consent dialog. - It should take the following arguments: - - - ``samp_name``: The name of the application making the request. - - - ``details``: A dictionary of details about the client - making the request. The only key in this dictionary required by - the SAMP standard is ``samp.name`` which gives the name of the - client making the request. - - - ``client``: A hostname, port pair containing the client - address. - - - ``origin``: A string containing the origin of the - request. - - 3) Based on the user response, the ``show_dialog`` should call - ``WebProfileDialog.consent`` or ``WebProfileDialog.reject``. - This may, in some cases, be the result of another GUI callback. - -Example of embedding a SAMP hub in a Tk application -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The following code is a full example of a simple Tk application that watches -for web SAMP connections and opens the appropriate dialog:: - - - import Tkinter as tk - import tkMessageBox - - from astropy.vo.samp import SAMPHubServer - from astropy.vo.samp.hub import WebProfileDialog - - MESSAGE = """ - A Web application which declares to be - - Name: {name} - Origin: {origin} - - is requesting to be registered with the SAMP Hub. Pay attention - that if you permit its registration, such application will acquire - all current user privileges, like file read/write. - - Do you give your consent? - """ - - class TkWebProfileDialog(WebProfileDialog): - def __init__(self, root): - self.root = root - self.wait_for_dialog() - - def wait_for_dialog(self): - self.handle_queue() - self.root.after(100, self.wait_for_dialog) - - def show_dialog(self, samp_name, details, client, origin): - text = MESSAGE.format(name=samp_name, origin=origin) - - response = tkMessageBox.askyesno( - 'SAMP Hub', text, - default=tkMessageBox.NO) - - if response: - self.consent() - else: - self.reject() - - # Start up Tk application - root = tk.Tk() - tk.Label(root, text="Example SAMP Tk application", - font=("Helvetica", 36), justify=tk.CENTER).pack(pady=200) - root.geometry("500x500") - root.update() - - # Start up SAMP hub - h = SAMPHubServer(web_profile_dialog=TkWebProfileDialog(root)) - h.start() - - try: - # Main GUI loop - root.mainloop() - except KeyboardInterrupt: - pass - - h.stop() - -If you run the above script, a window will open saying "Example SAMP Tk -application". If you then go to the following page for example: - -http://astrojs.github.io/sampjs/examples/pinger.html - -and click on the Ping button, you will see the dialog open in the Tk -application. Once you click on 'CONFIRM', future 'Ping' calls will no longer -bring up the dialog. diff --git a/docs/vo/samp/example_hub.rst b/docs/vo/samp/example_hub.rst deleted file mode 100644 index c8d197ae8652..000000000000 --- a/docs/vo/samp/example_hub.rst +++ /dev/null @@ -1,51 +0,0 @@ -.. include:: references.txt - -.. doctest-skip-all - -.. _vo-samp-example_hub: - -Starting and stopping a SAMP hub server ---------------------------------------- - -There are several ways you can start up a SAMP hub: - -Using an existing hub -^^^^^^^^^^^^^^^^^^^^^ - -You can start up another application that includes a hub, such as -`TOPCAT `_, -`SAO Ds9 `_, or -`Aladin Desktop `_. - -Using the command-line hub utility -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can make use of the ``samp_hub`` command-line utility, which is included in -Astropy:: - - $ samp_hub - -To get more help on available options for ``samp_hub``:: - - $ samp_hub -h - -To stop the server, you can simply press control-C. - -Starting a hub programmatically (advanced) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can start up a hub by creating a |SAMPHubServer| instance and starting it, -either from the interactive Python prompt, or from a Python script:: - - >>> from astropy.vo.samp import SAMPHubServer - >>> hub = SAMPHubServer() - >>> hub.start() - -You can then stop the hub by calling:: - - >>> hub.stop() - -However, this method is generally not recommended for average users because it -does not work correctly when web SAMP clients try and connect. Instead, this -should be reserved for developers who want to embed a SAMP hub in a GUI for -example. For more information, see :doc:`advanced_embed_samp_hub`. diff --git a/docs/vo/samp/example_table_image.rst b/docs/vo/samp/example_table_image.rst deleted file mode 100644 index e6290eddb154..000000000000 --- a/docs/vo/samp/example_table_image.rst +++ /dev/null @@ -1,274 +0,0 @@ -.. include:: references.txt - -.. doctest-skip-all - -.. _vo-samp-example-table-image: - -Sending/receiving tables and images over SAMP ---------------------------------------------- - -In the following examples, we make use of: - -* `TOPCAT `_, which is a tool to - explore tabular data. -* `SAO Ds9 `_, which is an image - visualization tool, which can also overplot catalogs. -* `Aladin Desktop `_, which is another tool that - can visualize images and catalogs. - -TOPCAT and Aladin will run a SAMP Hub is none is found, so for the following -examples you can either start up one of these applications first, or you can -start up the `astropy.vo.samp` hub. You can start this using the following -command:: - - $ samp_hub - -Sending a table to TOPCAT and Ds9 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The easiest way to send a VO table to TOPCAT is to make use of the -|SAMPIntegratedClient| class. Once TOPCAT is open, then first instantiate a -|SAMPIntegratedClient| instance and connect to the hub:: - - >>> from astropy.vo.samp import SAMPIntegratedClient - >>> client = SAMPIntegratedClient() - >>> client.connect() - -Next, we have to set up a dictionary that contains details about the table to -send. This should include ``url``, which is the URL to the file, and ``name``, -which is a human-readable name for the table. The URL can be a local URL -(starting with ``file:///``):: - - >>> params = {} - >>> params["url"] = 'file:///Users/tom/Desktop/aj285677t3_votable.xml' - >>> params["name"] = "Robitaille et al. (2008), Table 3" - -.. note:: To construct a local URL, you can also make use of ``urlparse`` as - follows:: - - >>> import urlparse - >>> params["url"] = urlparse.urljoin('file:', os.path.abspath("aj285677t3_votable.xml")) - -Now we can set up the message itself. This includes the type of message (here -we use ``table.load.votable`` which indicates that a VO table should be loaded, -and the details of the table that we set above:: - - >>> message = {} - >>> message["samp.mtype"] = "table.load.votable" - >>> message["samp.params"] = params - -Finally, we can broadcast this to all clients that are listening for -``table.load.votable`` messages using -:meth:`~astropy.vo.samp.integrated_client.SAMPIntegratedClient.notify_all`:: - - >>> client.notify_all(message) - -The above message will actually be broadcast to all applications connected via -SAMP. For example, if we open `SAO Ds9 `_ in -addition to TOPCAT, and we run the above command, both applications will load -the table. We can use the -:meth:`~astropy.vo.samp.integrated_client.SAMPIntegratedClient.get_registered_clients` method to -find all the clients connected to the hub:: - - >>> client.get_registered_clients() - ['hub', 'c1', 'c2'] - -These IDs don't mean much, but we can find out more using:: - - >>> client.get_metadata('c1') - {'author.affiliation': 'Astrophysics Group, Bristol University', - 'author.email': 'm.b.taylor@bristol.ac.uk', - 'author.name': 'Mark Taylor', - 'home.page': 'http://www.starlink.ac.uk/topcat/', - 'samp.description.text': 'Tool for OPerations on Catalogues And Tables', - 'samp.documentation.url': 'http://127.0.0.1:2525/doc/sun253/index.html', - 'samp.icon.url': 'http://127.0.0.1:2525/doc/images/tc_sok.gif', - 'samp.name': 'topcat', - 'topcat.version': '4.0-1'} - -We can see that ``c1`` is the TOPCAT client. We can now re-send the data, but -this time only to TOPCAT, using the -:meth:`~astropy.vo.samp.integrated_client.SAMPIntegratedClient.notify` method:: - - >>> client.notify('c1', message) - -Once finished, we should make sure we disconnect from the hub:: - - >>> client.disconnect() - -Receiving a table from TOPCAT -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To receive a table from TOPCAT, we have to set up a client that listens for -messages from the hub. As before, we instantiate a |SAMPIntegratedClient| -instance and connect to the hub:: - - >>> from astropy.vo.samp import SAMPIntegratedClient - >>> client = SAMPIntegratedClient() - >>> client.connect() - -We now set up a receiver class which will handle any received message. We need -to take care to write handlers for both notifications and calls (the difference -between the two being that calls expect a reply):: - - >>> class Receiver(object): - ... def __init__(self, client): - ... self.client = client - ... self.received = False - ... def receive_call(self, private_key, sender_id, msg_id, mtype, params, extra): - ... self.params = params - ... self.received = True - ... self.client.reply(msg_id, {"samp.status": "samp.ok", "samp.result": {}}) - ... def receive_notification(self, private_key, sender_id, mtype, params, extra): - ... self.params = params - ... self.received = True - -and we instantiate it: - - >>> r = Receiver(client) - -We can now use the -:meth:`~astropy.vo.samp.integrated_client.SAMPIntegratedClient.bind_receive_call` and -:meth:`~astropy.vo.samp.integrated_client.SAMPIntegratedClient.bind_receive_notification` methods -to tell our receiver to listen to all ``table.load.votable`` messages:: - - >>> client.bind_receive_call("table.load.votable", r.receive_call) - >>> client.bind_receive_notification("table.load.votable", r.receive_notification) - -We can now check that the message has not been received yet:: - - >>> r.received - False - -Let's now broadcast the table from TOPCAT. After a few seconds, we can try and -check again if the message has been received:: - - >>> r.received - True - -Success! The table URL should now be available in ``r.params['url']``, so we can do:: - - >>> from astropy.table import Table - >>> t = Table.read(r.params['url']) - Downloading http://127.0.0.1:2525/dynamic/4/t12.vot [Done] - >>> t - col1 col2 col3 col4 col5 col6 col7 col8 col9 col10 - ------------------------- -------- ------- -------- -------- ----- ---- ----- ---- ----- - SSTGLMC G000.0046+01.1431 0.0046 1.1432 265.2992 -28.3321 6.67 5.04 6.89 5.22 N - SSTGLMC G000.0106-00.7315 0.0106 -0.7314 267.1274 -29.3063 7.18 6.07 nan 5.17 Y - SSTGLMC G000.0110-01.0237 0.0110 -1.0236 267.4151 -29.4564 8.32 6.30 8.34 6.32 N - ... - -As before, we should remember to disconnect from the hub once we are done:: - - >>> client.disconnect() - -The following is a full example of a script that can be used to receive and -read a table. It includes a loop that waits until the message is received, and -reads the table once it has:: - - import time - - from astropy.vo.samp import SAMPIntegratedClient - from astropy.table import Table - - # Instantiate the client and connect to the hub - client=SAMPIntegratedClient() - client.connect() - - # Set up a receiver class - class Receiver(object): - def __init__(self, client): - self.client = client - self.received = False - def receive_call(self, private_key, sender_id, msg_id, mtype, params, extra): - self.params = params - self.received = True - self.client.reply(msg_id, {"samp.status": "samp.ok", "samp.result": {}}) - def receive_notification(self, private_key, sender_id, mtype, params, extra): - self.params = params - self.received = True - - # Instantiate the receiver - r = Receiver(client) - - # Listen for any instructions to load a table - client.bind_receive_call("table.load.votable", r.receive_call) - client.bind_receive_notification("table.load.votable", r.receive_notification) - - # We now run the loop to wait for the message in a try/finally block so that if - # the program is interrupted e.g. by control-C, the client terminates - # gracefully. - - try: - - # We test every 0.1s to see if the hub has sent a message - while True: - time.sleep(0.1) - if r.received: - t = Table.read(r.params['url']) - break - - finally: - - client.disconnect() - - # Print out table - print t - -Sending an image to Ds9 and Aladin -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -As for tables, the easiest way to send a FITS image over SAMP is to make use of -the |SAMPIntegratedClient| class. Once Aladin or Ds9 are open, then first -instantiate a |SAMPIntegratedClient| instance and connect to the hub as before:: - - >>> from astropy.vo.samp import SAMPIntegratedClient - >>> client = SAMPIntegratedClient() - >>> client.connect() - -Next, we have to set up a dictionary that contains details about the image to -send. This should include ``url``, which is the URL to the file, and ``name``, -which is a human-readable name for the table. The URL can be a local URL -(starting with ``file:///``):: - - >>> params = {} - >>> params["url"] = 'file:///Users/tom/Desktop/MSX_E.fits' - >>> params["name"] = "MSX Band E Image of the Galactic Center" - -See `Sending a table to TOPCAT and Ds9`_ for an example of how to construct local URLs -more easily. Now we can set up the message itself. This includes the type of -message (here we use ``image.load.fits`` which indicates that a FITS image -should be loaded, and the details of the table that we set above:: - - >>> message = {} - >>> message["samp.mtype"] = "image.load.fits" - >>> message["samp.params"] = params - -Finally, we can broadcast this to all clients that are listening for -``table.load.votable`` messages:: - - >>> client.notify_all(message) - -As for `Sending a table to TOPCAT and Ds9`_, the -:meth:`~astropy.vo.samp.integrated_client.SAMPIntegratedClient.notify_all` -method will broadcast the image to all listening clients, and as for tables it -is possible to instead use the -:meth:`~astropy.vo.samp.integrated_client.SAMPIntegratedClient.notify` method -to send it to a specific client. - -Once finished, we should make sure we disconnect from the hub:: - - >>> client.disconnect() - -Receiving a table from Ds9 or Aladin -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Receiving images over SAMP is identical to `Receiving a table from TOPCAT`_, -with the exception that the message type should be ``image.load.fits`` instead -of ``table.load.votable``. Once the URL has been received, the FITS image can -be opened with:: - - >>> from astropy.io import fits - >>> fits.open(r.params['url']) - diff --git a/docs/vo/samp/index.rst b/docs/vo/samp/index.rst deleted file mode 100644 index 79717f911a5d..000000000000 --- a/docs/vo/samp/index.rst +++ /dev/null @@ -1,67 +0,0 @@ -.. include:: references.txt - -.. doctest-skip-all - -.. _vo-samp: - -*************************************************************** -SAMP (Simple Application Messaging Protocol (`astropy.vo.samp`) -*************************************************************** - -Introduction -============ - -`astropy.vo.samp` is an IVOA SAMP (Simple Application Messaging Protocol) -messaging system implementation in Python. It provides classes to easily: - -1. instantiate one or multiple Hubs; -2. interface an application or script to a running Hub; -3. create and manage a SAMP client. - -`astropy.vo.samp` provides also a stand-alone program ``samp_hub`` capable to -instantiate a persistent hub. - -SAMP is a protocol that is used by a number of other tools such as -`TOPCAT `_, -`SAO Ds9 `_, -and `Aladin `_, which means that it is possible to -send and receive data to and from these tools. The `astropy.vo.samp` package -also supports the 'web profile' for SAMP, which means that it can be used to -communicate with web SAMP clients. See the `sampjs -`_ library examples for more details. - -The following classes are available in `astropy.vo.samp`: - -* |SAMPHubServer|, which is used to instantiate a hub server that clients can - then connect to. -* |SAMPHubProxy|, which is used to connect to an existing hub (including hubs - started from other applications such as - `TOPCAT `_). -* |SAMPClient|, which is used to create a SAMP client -* |SAMPIntegratedClient|, which is the same as |SAMPClient| except that it has - a self-contained |SAMPHubProxy| to provide a simpler user interface. - -.. _IVOA Simple Application Messaging Protocol: http://www.ivoa.net/Documents/latest/SAMP.html - -Using `astropy.vo.samp` -======================= - -.. toctree:: - :maxdepth: 2 - - example_hub - example_table_image - example_clients - advanced_embed_samp_hub - -Reference/API -============= - -.. automodapi:: astropy.vo.samp - -Acknowledgments -=============== - -This code is adapted from the `SAMPy `__ -package written by Luigi Paioro, who has granted the Astropy project permission -to use the code under a BSD license. diff --git a/docs/vo/samp/references.txt b/docs/vo/samp/references.txt deleted file mode 100644 index bfc9e627172b..000000000000 --- a/docs/vo/samp/references.txt +++ /dev/null @@ -1,5 +0,0 @@ -.. |SAMPClient| replace:: :class:`~astropy.vo.samp.SAMPClient` -.. |SAMPIntegratedClient| replace:: :class:`~astropy.vo.samp.SAMPIntegratedClient` -.. |SAMPHubServer| replace:: :class:`~astropy.vo.samp.SAMPHubServer` -.. |SAMPHubProxy| replace:: :class:`~astropy.vo.samp.SAMPHubProxy` -.. |SAMPMsgReplierWrapper| replace:: :class:`~astropy.vo.samp.SAMPMsgReplierWrapper` diff --git a/docs/warnings.rst b/docs/warnings.rst index 1e16df015c04..0b02313de2e9 100644 --- a/docs/warnings.rst +++ b/docs/warnings.rst @@ -27,28 +27,41 @@ script using the `warnings.filterwarnings` function as follows:: >>> import warnings >>> from astropy.io import fits >>> warnings.filterwarnings('ignore', category=UserWarning, append=True) - >>> fits.writeto(filename, data, clobber=True) + >>> fits.writeto(filename, data, overwrite=True) An equivalent way to insert an entry into the list of warning filter specifications for simple call `warnings.simplefilter`:: >>> warnings.simplefilter('ignore', UserWarning) -Astropy includes its own warning class, -`~astropy.utils.exceptions.AstropyUserWarning`, on which all warnings from -Astropy are based. So one can also ignore warnings from Astropy (while still -allowing through warnings from other libraries like Numpy) by using something -like:: +Astropy includes its own warning classes, +`~astropy.utils.exceptions.AstropyWarning` and +`~astropy.utils.exceptions.AstropyUserWarning`. All warnings from Astropy are +based on these warning classes (see below for the distinction between them). One +can thus ignore all warnings from Astropy (while still allowing through +warnings from other libraries like Numpy) by using something like:: - >>> from astropy.utils.exceptions import AstropyUserWarning - >>> warnings.simplefilter('ignore', category=AstropyUserWarning) + >>> from astropy.utils.exceptions import AstropyWarning + >>> warnings.simplefilter('ignore', category=AstropyWarning) -However, warning filters may also be modified just within a certain context -using the `warnings.catch_warnings` context manager:: +Warning filters may also be modified just within a certain context using the +`warnings.catch_warnings` context manager:: >>> with warnings.catch_warnings(): - ... warnings.simplefilter('ignore', AstropyUserWarning) - ... fits.writeto(filename, data, clobber=True) + ... warnings.simplefilter('ignore', AstropyWarning) + ... fits.writeto(filename, data, overwrite=True) + +As mentioned above, there are actually *two* base classes for Astropy warnings. +The main distinction is that `~astropy.utils.exceptions.AstropyUserWarning` is +for warnings that are *intended* for typical users (e.g. "Warning: Ambiguous +unit", something that might be because of improper input). In contrast, +`~astropy.utils.exceptions.AstropyWarning` warnings that are *not* +`~astropy.utils.exceptions.AstropyUserWarning` may be for lower-level warnings +more useful for developers writing code that *uses* Astropy (e.g., the +deprecation warnings discussed below). So if you're a user that just wants to +silence everything, the code above will suffice, but if you are a developer and +want to hide development-related warnings from your users, you may wish to still +allow through `~astropy.utils.exceptions.AstropyUserWarning`. Astropy also issues warnings when deprecated API features are used. If you wish to *squelch* deprecation warnings, you can start Python with @@ -57,5 +70,5 @@ also an Astropy-specific `~astropy.utils.exceptions.AstropyDeprecationWarning` which can be used to disable deprecation warnings from Astropy only. See `the CPython documentation -`__ for more +`__ for more information on the -W argument. diff --git a/docs/wcs/creating_planetary_wcs.rst b/docs/wcs/creating_planetary_wcs.rst new file mode 100644 index 000000000000..b16fb797f15e --- /dev/null +++ b/docs/wcs/creating_planetary_wcs.rst @@ -0,0 +1,11 @@ +.. _creating_planetary_wcs: + +Creating a planetary WCS structure +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To create a WCS planetary object, a planetary frame needs to be defined, +with a planetary body representation (see :ref:`astropy-coordinates-representations` +for more about representations) + +.. literalinclude:: examples/planetary_wcs.py + :language: python diff --git a/docs/wcs/example_create_imaging.rst b/docs/wcs/example_create_imaging.rst new file mode 100644 index 000000000000..e4596de3bb70 --- /dev/null +++ b/docs/wcs/example_create_imaging.rst @@ -0,0 +1,19 @@ +.. _example_create_imaging: + +First Example +^^^^^^^^^^^^^ + +This example, rather than starting from a FITS header, sets WCS values +programmatically, uses those settings to transform some points, and then +saves those settings to a new FITS header. + +.. literalinclude:: examples/programmatic.py + :language: python + +.. note:: + The members of the WCS object correspond roughly to the key/value + pairs in the FITS header. However, they are adjusted and + normalized in a number of ways that make performing the WCS + transformation easier. Therefore, they can not be relied upon to + get the original values in the header. To build up a FITS header + directly and specifically, use `astropy.io.fits.Header` directly. diff --git a/docs/wcs/example_cube_wcs.rst b/docs/wcs/example_cube_wcs.rst new file mode 100644 index 000000000000..fa0ee691a7df --- /dev/null +++ b/docs/wcs/example_cube_wcs.rst @@ -0,0 +1,12 @@ +.. _example_cube_wcs: + +Second Example +^^^^^^^^^^^^^^ + +Another way of creating a WCS object is via the use of a Python +dictionary. This affords us more control over the ``NAXISn`` +FITS header keyword which is otherwise automatically default to zero +as in the case of the First Example shown above. + +.. literalinclude:: examples/cube_wcs.py + :language: python diff --git a/docs/wcs/examples/cube_wcs.py b/docs/wcs/examples/cube_wcs.py new file mode 100644 index 000000000000..c251015c1abf --- /dev/null +++ b/docs/wcs/examples/cube_wcs.py @@ -0,0 +1,25 @@ +# Define the astropy.wcs.WCS object using a Python dictionary as input + +import astropy.wcs + +wcs_dict = { + "CTYPE1": "WAVE ", + "CUNIT1": "Angstrom", + "CDELT1": 0.2, + "CRPIX1": 0, + "CRVAL1": 10, + "NAXIS1": 5, + "CTYPE2": "HPLT-TAN", + "CUNIT2": "deg", + "CDELT2": 0.5, + "CRPIX2": 2, + "CRVAL2": 0.5, + "NAXIS2": 4, + "CTYPE3": "HPLN-TAN", + "CUNIT3": "deg", + "CDELT3": 0.4, + "CRPIX3": 2, + "CRVAL3": 1, + "NAXIS3": 3, +} +input_wcs = astropy.wcs.WCS(wcs_dict) diff --git a/docs/wcs/examples/from_file.py b/docs/wcs/examples/from_file.py index 86f3077c3f60..442a1592b15e 100644 --- a/docs/wcs/examples/from_file.py +++ b/docs/wcs/examples/from_file.py @@ -1,12 +1,13 @@ # Load the WCS information from a fits header, and use it # to convert pixel coordinates to world coordinates. -from __future__ import division, print_function +import sys + +import numpy as np -import numpy from astropy import wcs from astropy.io import fits -import sys + def load_wcs_from_file(filename): # Load the FITS hdulist using astropy.io.fits @@ -21,23 +22,37 @@ def load_wcs_from_file(filename): # Print out all of the settings that were parsed from the header w.wcs.print_contents() - # Some pixel coordinates of interest. - pixcrd = numpy.array([[0, 0], [24, 38], [45, 98]], numpy.float_) + # Three pixel coordinates of interest. + # Note we've silently assumed an NAXIS=2 image here. + # The pixel coordinates are pairs of [X, Y]. + # The "origin" argument indicates whether the input coordinates + # are 0-based (as in Numpy arrays) or + # 1-based (as in the FITS convention, for example coordinates + # coming from DS9). + pixcrd = np.array([[0, 0], [24, 38], [45, 98]], dtype=np.float64) # Convert pixel coordinates to world coordinates # The second argument is "origin" -- in this case we're declaring we - # have 1-based (Fortran-like) coordinates. - world = w.wcs_pix2world(pixcrd, 1) + # have 0-based (Numpy-like) coordinates. + world = w.wcs_pix2world(pixcrd, 0) print(world) # Convert the same coordinates back to pixel coordinates. - pixcrd2 = w.wcs_world2pix(world, 1) + pixcrd2 = w.wcs_world2pix(world, 0) print(pixcrd2) # These should be the same as the original pixel coordinates, modulo # some floating-point error. - assert numpy.max(numpy.abs(pixcrd - pixcrd2)) < 1e-6 + assert np.max(np.abs(pixcrd - pixcrd2)) < 1e-6 + + # The example below illustrates the use of "origin" to convert between + # 0- and 1- based coordinates when executing the forward and backward + # WCS transform. + x = 0 + y = 0 + origin = 0 + assert w.wcs_pix2world(x, y, origin) == w.wcs_pix2world(x + 1, y + 1, origin + 1) -if __name__ == '__main__': +if __name__ == "__main__": load_wcs_from_file(sys.argv[-1]) diff --git a/docs/wcs/examples/planetary_wcs.py b/docs/wcs/examples/planetary_wcs.py new file mode 100644 index 000000000000..fa02e0443463 --- /dev/null +++ b/docs/wcs/examples/planetary_wcs.py @@ -0,0 +1,23 @@ +# Create a planetary WCS structure + +from astropy import units as u +from astropy.coordinates import BaseBodycentricRepresentation, BaseCoordinateFrame +from astropy.wcs.utils import celestial_frame_to_wcs + + +class MARSCustomBodycentricRepresentation(BaseBodycentricRepresentation): + _equatorial_radius = 3399190.0 * u.m + _flattening = 0.5886 * u.percent + + +class MARSCustomBodyFrame(BaseCoordinateFrame): + name = "Mars" + + +frame = MARSCustomBodyFrame() +frame.representation_type = MARSCustomBodycentricRepresentation +mywcs = celestial_frame_to_wcs(frame, projection="CAR") +print(mywcs.wcs.ctype) +print(mywcs.wcs.name) +print(mywcs.wcs.aux.a_radius) +print(mywcs.wcs.aux.c_radius) diff --git a/docs/wcs/examples/programmatic.py b/docs/wcs/examples/programmatic.py index 07725188f6cd..ac98ec5e81c2 100644 --- a/docs/wcs/examples/programmatic.py +++ b/docs/wcs/examples/programmatic.py @@ -1,9 +1,8 @@ # Set the WCS information manually by setting properties of the WCS # object. -from __future__ import division, print_function +import numpy as np -import numpy from astropy import wcs from astropy.io import fits @@ -14,25 +13,40 @@ # Set up an "Airy's zenithal" projection # Vector properties may be set with Python lists, or Numpy arrays w.wcs.crpix = [-234.75, 8.3393] -w.wcs.cdelt = numpy.array([-0.066667, 0.066667]) +w.wcs.cdelt = np.array([-0.066667, 0.066667]) w.wcs.crval = [0, -90] w.wcs.ctype = ["RA---AIR", "DEC--AIR"] w.wcs.set_pv([(2, 1, 45.0)]) -# Some pixel coordinates of interest. -pixcrd = numpy.array([[0, 0], [24, 38], [45, 98]], numpy.float_) - -# Convert pixel coordinates to world coordinates -world = w.wcs_pix2world(pixcrd, 1) +# Three pixel coordinates of interest. +# The pixel coordinates are pairs of [X, Y]. +# The "origin" argument indicates whether the input coordinates +# are 0-based (as in Numpy arrays) or +# 1-based (as in the FITS convention, for example coordinates +# coming from DS9). +pixcrd = np.array([[0, 0], [24, 38], [45, 98]], dtype=np.float64) + +# Convert pixel coordinates to world coordinates. +# The second argument is "origin" -- in this case we're declaring we +# have 0-based (Numpy-like) coordinates. +world = w.wcs_pix2world(pixcrd, 0) print(world) # Convert the same coordinates back to pixel coordinates. -pixcrd2 = w.wcs_world2pix(world, 1) +pixcrd2 = w.wcs_world2pix(world, 0) print(pixcrd2) # These should be the same as the original pixel coordinates, modulo # some floating-point error. -assert numpy.max(numpy.abs(pixcrd - pixcrd2)) < 1e-6 +assert np.max(np.abs(pixcrd - pixcrd2)) < 1e-6 + +# The example below illustrates the use of "origin" to convert between +# 0- and 1- based coordinates when executing the forward and backward +# WCS transform. +x = 0 +y = 0 +origin = 0 +assert w.wcs_pix2world(x, y, origin) == w.wcs_pix2world(x + 1, y + 1, origin + 1) # Now, write out the WCS object as a FITS header header = w.to_header() diff --git a/docs/wcs/history.rst b/docs/wcs/history.rst index 8883d8364bca..1b85eaf1b0b1 100644 --- a/docs/wcs/history.rst +++ b/docs/wcs/history.rst @@ -1,11 +1,11 @@ astropy.wcs History -=================== +******************* `astropy.wcs` began life as ``pywcs``. Earlier version numbers refer to that package. pywcs Version 1.11 ------------------- +================== - Updated to wcslib version 4.8, which gives much more detailed error messages. @@ -34,7 +34,7 @@ pywcs Version 1.11 are valid unit strings. pywcs Version 1.10 ------------------- +================== - Adds a ``UnitConversion`` class, which gives access to wcslib's unit conversion functionality. Given two convertible unit strings, pywcs @@ -45,7 +45,7 @@ pywcs Version 1.10 - Changes to some wcs values would not always calculate secondary values. pywcs Version 1.9 ------------------ +================= - Support binary image arrays and pixel list format WCS by presenting a way to call wcslib's ``wcsbth()`` @@ -61,7 +61,7 @@ pywcs Version 1.9 was not properly handled. Bugs -```` +---- - The `~astropy.wcs.Wcsprm.pc` member is now available with a default raw `~astropy.wcs.Wcsprm` object. @@ -73,7 +73,7 @@ Bugs - `float` properties can now be set using `int` values pywcs Version 1.3a1 -------------------- +=================== Earlier versions of pywcs had two versions of every conversion method:: @@ -87,4 +87,3 @@ conversion, with an 'origin' argument: - 1: places the origin at (1, 1), which is the Fortran/FITS convention. - diff --git a/docs/wcs/index.rst b/docs/wcs/index.rst index f6e5811a4cb2..0ea51a293a37 100644 --- a/docs/wcs/index.rst +++ b/docs/wcs/index.rst @@ -1,250 +1,169 @@ -.. doctest-skip-all +.. include:: references.txt .. _astropy-wcs: *************************************** World Coordinate System (`astropy.wcs`) *************************************** -.. _wcslib: http://www.atnf.csiro.au/~mcalabre/WCS/ -.. _FITS WCS standard: http://fits.gsfc.nasa.gov/fits_wcs.html -.. _distortion paper: http://www.atnf.csiro.au/people/mcalabre/WCS/dcs_20040422.pdf -.. _SIP: http://irsa.ipac.caltech.edu/data/SPITZER/docs/files/spitzer/shupeADASS.pdf - Introduction ============ -`astropy.wcs` contains utilities for managing World Coordinate System -(WCS) transformations in FITS files. These transformations map the -pixel locations in an image to their real-world units, such as their -position on the sky sphere. These transformations can work both -forward (from pixel to sky) and backward (from sky to pixel). +World Coordinate Systems (WCSs) describe the geometric transformations +between one set of coordinates and another. A common application is to +map the pixels in an image onto the celestial sphere. Another common +application is to map pixels to wavelength in a spectrum. -It performs three separate classes of WCS transformations: +`astropy.wcs` contains utilities for managing World Coordinate System +(WCS) transformations defined in several elaborate `FITS WCS standard`_ conventions. +These transformations work both forward (from pixel to world) and backward +(from world to pixel). + +For historical reasons and to support legacy software, `astropy.wcs` maintains +two separate application interfaces. The ``High-Level API`` should be used by +most applications. It abstracts out the underlying object and works transparently +with other packages which support the +`Common Python Interface for WCS `_, +allowing for a more flexible approach to the problem and avoiding the `limitations +of the FITS WCS standard `_. + +The ``Low Level API`` is the original `astropy.wcs` API and originally developed as ``pywcs``. +It ties applications to the `astropy.wcs` package and limits the transformations to the three distinct +types supported by it: - Core WCS, as defined in the `FITS WCS standard`_, based on Mark - Calabretta's `wcslib`_. + Calabretta's `wcslib`_. (Also includes ``TPV`` and ``TPD`` + distortion, but not ``SIP``). -- Simple Imaging Polynomial (`SIP`_) convention. +- Simple Imaging Polynomial (`SIP`_) convention. (See :doc:`note about SIP in headers `.) -- table lookup distortions as defined in the FITS WCS `distortion +- Table lookup distortions as defined in the FITS WCS `distortion paper`_. -Each of these transformations can be used independently or together in -a standard pipeline. +.. _pixel_conventions: -Getting Started -=============== +Pixel Conventions and Definitions +--------------------------------- -The basic workflow is as follows: +Both APIs assume that integer pixel values fall at the center of pixels (as assumed in +the `FITS WCS standard`_, see Section 2.1.4 of `Greisen et al., 2002, +A&A 446, 747 `_). - 1. ``from astropy import wcs`` +However, there’s a difference in what is considered to be the first pixel. The +``High Level API`` follows the Python and C convention that the first pixel is +the 0-th one, i.e. the first pixel spans pixel values -0.5 to + 0.5. The +``Low Level API`` takes an additional ``origin`` argument with values of 0 or 1 +indicating whether the input arrays are 0- or 1-based. +The Low-level interface assumes Cartesian order (x, y) of the input coordinates, +however the Common Interface for World Coordinate System accepts both conventions. +The order of the pixel coordinates ((x, y) vs (row, column)) in the Common API +depends on the method or property used, and this can normally be determined from +the property or method name. Properties and methods containing “pixel” assume (x, y) +ordering, while properties and methods containing “array” assume (row, column) ordering. - 2. Call the `~astropy.wcs.WCS` constructor with an - `astropy.io.fits` `~astropy.io.fits.Header` and/or - `~astropy.io.fits.HDUList` object. +A Simple Example +================ - 3. Optionally, if the FITS file uses any deprecated or - non-standard features, you may need to call one of the - `~astropy.wcs.wcs.WCS.fix` methods on the object. +One example of the use of the high-level WCS API is to use the +`~astropy.wcs.wcs.WCS.pixel_to_world` to yield the simplest WCS +with default values, converting from pixel to world coordinates:: - 4. Use one of the following transformation methods: + >>> from astropy.io import fits + >>> from astropy.wcs import WCS + >>> from astropy.utils.data import get_pkg_data_filename + >>> fn = get_pkg_data_filename('data/j94f05bgq_flt.fits', package='astropy.wcs.tests') + >>> f = fits.open(fn) + >>> w = WCS(f[1].header) + >>> sky = w.pixel_to_world(30, 40) + >>> print(sky) # doctest: +FLOAT_CMP + + >>> f.close() + +Similarly, another use of the high-level API is to use the +`~astropy.wcs.wcs.WCS.world_to_pixel` to yield another simple WCS, while +converting from world to pixel coordinates:: + + >>> from astropy.io import fits + >>> from astropy.wcs import WCS + >>> from astropy.utils.data import get_pkg_data_filename + >>> fn = get_pkg_data_filename('data/j94f05bgq_flt.fits', package='astropy.wcs.tests') + >>> f = fits.open(fn) + >>> w = WCS(f[1].header) + >>> x, y = w.world_to_pixel(sky) + >>> print(x, y) # doctest: +FLOAT_CMP + 30.00000214673885 39.999999958235094 + >>> f.close() - - From pixels to world coordinates: +Using `astropy.wcs` +=================== - - `~astropy.wcs.wcs.WCS.all_pix2world`: Perform all three - transformations in series (core WCS, SIP and table lookup - distortions) from pixel to world coordinates. Use this one - if you're not sure which to use. +.. toctree:: + :maxdepth: 2 - - `~astropy.wcs.wcs.WCS.wcs_pix2world`: Perform just the core - WCS transformation from pixel to world coordinates. + Shared Python Interface for World Coordinate Systems + Legacy Interface + Supported Projections - - From world to pixel coordinates: +Examples creating a WCS programmatically +======================================== - - `~astropy.wcs.wcs.WCS.all_world2pix`: Perform all three - transformations (core WCS, SIP and table lookup - distortions) from world to pixel coordinates, using an - iterative method if necessary. +.. toctree:: + :maxdepth: 2 + + Example of Imaging WCS + Example of Cube WCS + Example of a Planetary WCS + Loading From a FITS File - - `~astropy.wcs.wcs.WCS.wcs_world2pix`: Perform just the core - WCS transformation from world to pixel coordinates. +WCS Tools +========= + +.. toctree:: + :maxdepth: 1 - - Performing `SIP`_ transformations only: + wcstools.rst - - `~astropy.wcs.wcs.WCS.sip_pix2foc`: Convert from pixel to - focal plane coordinates using the `SIP`_ polynomial - coefficients. +Relax Constants +=============== - - `~astropy.wcs.wcs.WCS.sip_foc2pix`: Convert from focal - plane to pixel coordinates using the `SIP`_ polynomial - coefficients. +.. toctree:: + :maxdepth: 1 - - Performing `distortion paper`_ transformations only: + relax - - `~astropy.wcs.wcs.WCS.p4_pix2foc`: Convert from pixel to - focal plane coordinates using the table lookup distortion - method described in the FITS WCS `distortion paper`_. +Other Information +================= - - `~astropy.wcs.wcs.WCS.det2im`: Convert from detector - coordinates to image coordinates. Commonly used for narrow - column correction. +.. toctree:: + :maxdepth: 1 -For example, to convert pixel coordinates to world coordinates:: + history + validation - >>> from astropy.wcs import WCS - >>> w = WCS('image.fits') - >>> lon, lat = w.all_pix2world(30, 40, 0) - >>> print(lon, lat) +.. note that if this section gets too long, it should be moved to a separate + doc page - see the top of performance.inc.rst for the instructions on how to do + that +.. include:: performance.inc.rst +.. _wcs-reference-api: -Using `astropy.wcs` -=================== -Loading WCS information from a FITS file ----------------------------------------- - -This example loads a FITS file (supplied on the commandline) and uses -the WCS cards in its primary header to transform. - -.. literalinclude:: examples/from_file.py - :language: python - -Building a WCS structure programmatically ------------------------------------------ - -This example, rather than starting from a FITS header, sets WCS values -programmatically, uses those settings to transform some points, and then -saves those settings to a new FITS header. - -.. literalinclude:: examples/programmatic.py - :language: python - -.. note:: - The members of the WCS object correspond roughly to the key/value - pairs in the FITS header. However, they are adjusted and - normalized in a number of ways that make performing the WCS - transformation easier. Therefore, they can not be relied upon to - get the original values in the header. To build up a FITS header - directly and specifically, use `astropy.io.fits.Header` directly. - -.. _wcslint: - -Validating the WCS keywords in a FITS file ------------------------------------------- - -Astropy includes a commandline tool, ``wcslint`` to check the WCS -keywords in a FITS file:: - - > wcslint invalid.fits - HDU 1: - WCS key ' ': - - RADECSYS= 'ICRS ' / Astrometric system - RADECSYS is non-standard, use RADESYSa. - - The WCS transformation has more axes (2) than the image it is - associated with (0) - - 'celfix' made the change 'PV1_5 : Unrecognized coordinate - transformation parameter'. - - HDU 2: - WCS key ' ': - - The WCS transformation has more axes (3) than the image it is - associated with (0) - - 'celfix' made the change 'In CUNIT2 : Mismatched units type - 'length': have 'Hz', want 'm''. - - 'unitfix' made the change 'Changed units: 'HZ ' -> 'Hz''. - -Bounds checking ---------------- - -Bounds checking is enabled by default, and any computed world -coordinates outside of [-180°, 180°] for longitude and [-90°, 90°] in -latitude are marked as invalid. To disable this behavior, use -`astropy.wcs.Wcsprm.bounds_check`. - -Supported projections -===================== - -As `astropy.wcs` is based on `wcslib`_, it supports the standard -projections defined in the `FITS WCS standard`_. These projection -codes are specified in the second part of the ``CTYPEn`` keywords -(accessible through `Wcsprm.ctype `), for -example, ``RA---TAN-SIP``. The supported projection codes are: - -- ``AZP``: zenithal/azimuthal perspective -- ``SZP``: slant zenithal perspective -- ``TAN``: gnomonic -- ``STG``: stereographic -- ``SIN``: orthographic/synthesis -- ``ARC``: zenithal/azimuthal equidistant -- ``ZPN``: zenithal/azimuthal polynomial -- ``ZEA``: zenithal/azimuthal equal area -- ``AIR``: Airy's projection -- ``CYP``: cylindrical perspective -- ``CEA``: cylindrical equal area -- ``CAR``: plate carrÊe -- ``MER``: Mercator's projection -- ``COP``: conic perspective -- ``COE``: conic equal area -- ``COD``: conic equidistant -- ``COO``: conic orthomorphic -- ``SFL``: Sanson-Flamsteed ("global sinusoid") -- ``PAR``: parabolic -- ``MOL``: Mollweide's projection -- ``AIT``: Hammer-Aitoff -- ``BON``: Bonne's projection -- ``PCO``: polyconic -- ``TSC``: tangential spherical cube -- ``CSC``: COBE quadrilateralized spherical cube -- ``QSC``: quadrilateralized spherical cube -- ``HPX``: HEALPix -- ``XPH``: HEALPix polar, aka "butterfly" - -Subsetting and Pixel Scales -=========================== - -WCS objects can be broken apart into their constituent axes using the -`~astropy.wcs.WCS.sub` function. There is also a `~astropy.wcs.WCS.celestial` -convenience function that will return a WCS object with only the celestial axes -included. - -The pixel scales of a celestial image or the pixel dimensions of a non-celestial -image can be extracted with the utility functions -`~astropy.wcs.utils.proj_plane_pixel_scales` and -`~astropy.wcs.utils.non_celestial_pixel_scales`. Likewise, celestial pixel -area can be extracted with the utility function -`~astropy.wcs.utils.proj_plane_pixel_area`. - -Matplotlib plots with correct WCS projection -============================================ - -The `WCSAxes `_ affiliated package adds the -ability to use the :class:`~astropy.wcs.WCS` to define projections in -Matplotlib. More information on installing and using WCSAxes can be found `here -`__. - -Other information -================= +Reference/API +============= .. toctree:: - :maxdepth: 1 + :maxdepth: 2 - relax - history + reference_api See Also ======== - `wcslib`_ -Reference/API -============= - -.. automodapi:: astropy.wcs -.. automodapi:: astropy.wcs.utils Acknowledgments and Licenses ============================ -`wcslib`_ is licenced under the `GNU Lesser General Public License -`_. +`wcslib`_ is licenced under the GNU Lesser General Public License. diff --git a/docs/wcs/legacy_interface.rst b/docs/wcs/legacy_interface.rst new file mode 100644 index 000000000000..b0eaf3731708 --- /dev/null +++ b/docs/wcs/legacy_interface.rst @@ -0,0 +1,112 @@ +.. include:: references.txt +.. _legacy_interface: + +Legacy Interface +**************** + +astropy.wcs API +^^^^^^^^^^^^^^^ + +The ``Low Level API`` or ``Legacy Interface`` is the original `astropy.wcs` API. +It supports three types of transforms: + +- Core WCS, as defined in the `FITS WCS standard`_, based on Mark + Calabretta's `wcslib`_. (Also includes ``TPV`` and ``TPD`` + distortion, but not ``SIP``). + +- Simple Imaging Polynomial (`SIP`_) convention. (See :doc:`note about SIP in headers `.) + +- Table lookup distortions as defined in the FITS WCS `distortion paper`_. + +Each of these transformations can be used independently or together in a standard pipeline. +All methods support scalar and array inputs. Note, that all methods require an additional +positional argument which is the ``origin`` of the inputs. It has two possible values - ``0`` - +for zero-based coordinates like numpy arrays or ``1`` - for 1-based coordinates, like +the FITS standard, or those coming from ds9. + +The basic workflow is to create a WCS object calling the WCS constructor with an +`~astropy.io.fits.Header` and/or `~astropy.io.fits.HDUList` object and calling +one of the methods below:: + + >>> from astropy import wcs + >>> from astropy.io import fits + >>> from astropy.utils.data import get_pkg_data_filename + >>> fn = get_pkg_data_filename('data/j94f05bgq_flt.fits', package='astropy.wcs.tests') + >>> f = fits.open(fn) + >>> wcsobj = wcs.WCS(f[1].header) + >>> f.close() + +Optionally, if the FITS file uses any deprecated or non-standard features, you may need +to call one of the `~astropy.wcs.wcs.WCS.fix` methods on the object. + +Use one of the following transformation methods. + +1. Between pixels and world coordinates using all distortions: + + - `~astropy.wcs.wcs.WCS.all_pix2world`: Perform all three + transformations in series (core WCS, SIP and table lookup + distortions) from pixel to world coordinates. Use this one + if you're not sure which to use. + + >>> lon, lat = wcsobj.all_pix2world(30, 40, 0) + >>> print(lon, lat) # doctest: +FLOAT_CMP + 5.528442425094046 -72.05207808966726 + + - `~astropy.wcs.wcs.WCS.all_world2pix`: Perform all three + transformations (core WCS, SIP and table lookup + distortions) from world to pixel coordinates, using an + iterative method if necessary. + + >>> x, y = wcsobj.all_world2pix(lon, lat, 0) + >>> print(x, y) # # doctest: +FLOAT_CMP + 30.00000214673885 39.999999958235094 + + 2. Performing `SIP`_ transformations only: + + - `~astropy.wcs.wcs.WCS.sip_pix2foc`: Convert from pixel to + focal plane coordinates using the `SIP`_ polynomial + coefficients. + + >>> xsip, ysip = wcsobj.sip_pix2foc(30, 40, 0) + >>> print(xsip, ysip) # doctest: +FLOAT_CMP + -1985.8600487630586 -984.4223711273145 + + - `~astropy.wcs.wcs.WCS.sip_foc2pix`: Convert from focal + plane to pixel coordinates using the `SIP`_ polynomial + coefficients. Note that this method only works if the + inverse SIP distortion is specified in the header. + + 3. Performing `distortion paper`_ transformations only: + + - `~astropy.wcs.wcs.WCS.p4_pix2foc`: Convert from pixel to + focal plane coordinates using the table lookup distortion + method described in the FITS WCS `distortion paper`_. + + - `~astropy.wcs.wcs.WCS.det2im`: Convert from detector + coordinates to image coordinates. Commonly used for narrow + column correction. + +Core wcslib API +^^^^^^^^^^^^^^^ + +The core wcslib API supports the FITS WCS standard defined in WCS +papers, I, II, III, IV. Note that distortions are not applied if +the functions in the core library are used. + +1. From pixels to world coordinates: + + - `~astropy.wcs.wcs.WCS.wcs_pix2world`: Perform just the core WCS + transformation from pixel to world coordinates. + + >>> lon, lat = wcsobj.wcs_pix2world(30, 40, 0) + >>> print(lon, lat) # doctest: +FLOAT_CMP + 5.527103615238458 -72.0522441352217 + +2. From world to pixel coordinates: + + - `~astropy.wcs.wcs.WCS.wcs_world2pix`: Perform the core WCS transformation + from world to pixel coordinates. + + >>> x, y = wcsobj.wcs_world2pix(lon, lat, 0) + >>> print(x, y) # doctest: +FLOAT_CMP + 30.000000000223267 40.0000000003696 diff --git a/docs/wcs/loading_from_fits.rst b/docs/wcs/loading_from_fits.rst new file mode 100644 index 000000000000..d3e3bd90207b --- /dev/null +++ b/docs/wcs/loading_from_fits.rst @@ -0,0 +1,8 @@ +Loading WCS Information from a FITS File +---------------------------------------- + +This example loads a FITS file (supplied on the command line) and uses +the FITS keywords in its primary header to create a WCS and transform. + +.. literalinclude:: examples/from_file.py + :language: python diff --git a/docs/wcs/note_sip.rst b/docs/wcs/note_sip.rst new file mode 100644 index 000000000000..d5bb6ee42bb9 --- /dev/null +++ b/docs/wcs/note_sip.rst @@ -0,0 +1,84 @@ +.. include:: references.rst +.. doctest-skip-all +.. _note_sip: :orphan: + + +Note about SIP and WCS +********************** + +`astropy.wcs` supports the Simple Imaging Polynomial (`SIP`_) convention. +The SIP distortion is defined in FITS headers by the presence of the +SIP specific keywords **and** a ``-SIP`` suffix in ``CTYPE``, for example +``RA---TAN-SIP``, ``DEC--TAN-SIP``. + +This has not been a strict convention in the past and the default in +`astropy.wcs` is to always include the SIP distortion if the SIP coefficients +are present, even if ``-SIP`` is not included in CTYPE. +The presence of a ``-SIP`` suffix in CTYPE is not used as a trigger +to initialize the SIP distortion. + +It is important that headers implement correctly the SIP convention. +If the intention is to use the SIP distortion, a header should have +the SIP coefficients and the ``-SIP`` suffix in CTYPE. + +`astropy.wcs` prints INFO messages when inconsistent headers are detected, +for example when SIP coefficients are present but CTYPE is missing a ``-SIP`` suffix, +see examples below. +`astropy.wcs` will print a message about the inconsistent header +but will create and use the SIP distortion and it will be used in +calls to `~astropy.wcs.wcs.WCS.all_pix2world`. If this was not the intended use +(e.g. it's a drizzled image and has no distortions) it is best to remove the SIP +coefficients from the header. They can be removed temporarily from a WCS object by + +>>> wcsobj.sip = None + +In addition, if SIP is the only distortion in the header, the two methods, +`~astropy.wcs.wcs.WCS.wcs_pix2world` and `~astropy.wcs.wcs.WCS.wcs_world2pix`, +may be used to transform from pixels to world coordinate system while omitting distortions. + +Another consequence of the inconsistent header is that if +`~astropy.wcs.wcs.WCS.to_header()` is called with ``relax=True`` it will return a header +with SIP coefficients and a ``-SIP`` suffix in CTYPE and will not reproduce the original header. + +**In conclusion, when astropy.wcs detects inconsistent headers, the recommendation +is that the header is inspected and corrected to match the data.** + +Below is an example of a header with SIP coefficients when ``-SIP`` is missing from CTYPE. +The data is drizzled, i.e. distortion free, so the intention is **not** to include the +SIP distortion. + +>>> wcsobj = wcs.WCS(header) +INFO: + + Inconsistent SIP distortion information is present in the FITS header and the WCS object: + SIP coefficients were detected, but CTYPE is missing a "-SIP" suffix. + astropy.wcs is using the SIP distortion coefficients, + therefore the coordinates calculated here might be incorrect. + + If you do not want to apply the SIP distortion coefficients, + please remove the SIP coefficients from the FITS header or the + WCS object. As an example, if the image is already distortion-corrected + (e.g., drizzled) then distortion components should not apply and the SIP + coefficients should be removed. + + While the SIP distortion coefficients are being applied here, if that was indeed the intent, + for consistency please append "-SIP" to the CTYPE in the FITS header or the WCS object. + + +>>> hdr = wcsobj.to_header(relax=True) +INFO: + + Inconsistent SIP distortion information is present in the current WCS: + SIP coefficients were detected, but CTYPE is missing "-SIP" suffix, + therefore the current WCS is internally inconsistent. + + Because relax has been set to True, the resulting output WCS will have + "-SIP" appended to CTYPE in order to make the header internally consistent. + + However, this may produce incorrect astrometry in the output WCS, if + in fact the current WCS is already distortion-corrected. + + Therefore, if current WCS is already distortion-corrected (eg, drizzled) + then SIP distortion components should not apply. In that case, for a WCS + that is already distortion-corrected, please remove the SIP coefficients + from the header. diff --git a/docs/wcs/performance.inc.rst b/docs/wcs/performance.inc.rst new file mode 100644 index 000000000000..621f2c2f4c5c --- /dev/null +++ b/docs/wcs/performance.inc.rst @@ -0,0 +1,12 @@ +.. note that if this is changed from the default approach of using an *include* + (in index.rst) to a separate performance page, the header needs to be changed + from === to ***, the filename extension needs to be changed from .inc.rst to + .rst, and a link needs to be added in the subpackage toctree + +.. _astropy-wcs-performance: + +.. Performance Tips +.. ================ +.. +.. Here we provide some tips and tricks for how to optimize performance of code +.. using `astropy.wcs`. diff --git a/docs/wcs/reference_api.rst b/docs/wcs/reference_api.rst new file mode 100644 index 000000000000..3d872d08dc70 --- /dev/null +++ b/docs/wcs/reference_api.rst @@ -0,0 +1,11 @@ +Reference/API +************* + +.. automodapi:: astropy.wcs + :inherited-members: + +.. automodapi:: astropy.wcs.utils + :inherited-members: + +.. automodapi:: astropy.wcs.wcsapi + :inherited-members: diff --git a/docs/wcs/references.rst b/docs/wcs/references.rst new file mode 100644 index 000000000000..10656c195b1a --- /dev/null +++ b/docs/wcs/references.rst @@ -0,0 +1,11 @@ +:orphan: + +.. _wcslib: https://www.atnf.csiro.au/people/mcalabre/WCS/wcslib/index.html +.. _distortion paper: https://www.atnf.csiro.au/people/mcalabre/WCS/dcs_20040422.pdf +.. _SIP: https://irsa.ipac.caltech.edu/data/SPITZER/docs/files/spitzer/shupeADASS.pdf +.. _ds9: http://hea-www.harvard.edu/RD/ds9/ +.. _FITS WCS standard: http://fits.gsfc.nasa.gov/fits_wcs.html +.. _paper_I: https://arxiv.org/pdf/astro-ph/0207407.pdf +.. _paper_II: https://arxiv.org/pdf/astro-ph/0207413.pdf +.. _paper_III: https://arxiv.org/pdf/astro-ph/0507293.pdf +.. _paper_IV: https://arxiv.org/pdf/1409.7583.pdf diff --git a/docs/wcs/references.txt b/docs/wcs/references.txt index aa0aa29655f0..d7838684c9e9 100644 --- a/docs/wcs/references.txt +++ b/docs/wcs/references.txt @@ -1,4 +1,9 @@ -.. _wcslib: http://www.atnf.csiro.au/~mcalabre/WCS/ -.. _distortion paper: http://www.atnf.csiro.au/people/mcalabre/WCS/dcs_20040422.pdf -.. _SIP: http://irsa.ipac.caltech.edu/data/SPITZER/docs/files/spitzer/shupeADASS.pdf -.. _ds9: http://hea-www.harvard.edu/RD/ds9/ +.. _wcslib: https://www.atnf.csiro.au/people/mcalabre/WCS/wcslib/index.html +.. _distortion paper: https://www.atnf.csiro.au/people/mcalabre/WCS/dcs_20040422.pdf +.. _SIP: https://irsa.ipac.caltech.edu/data/SPITZER/docs/files/spitzer/shupeADASS.pdf +.. _ds9: http://ds9.si.edu/ +.. _FITS WCS standard: https://fits.gsfc.nasa.gov/fits_wcs.html +.. _paper_I: https://arxiv.org/pdf/astro-ph/0207407.pdf +.. _paper_II: https://arxiv.org/pdf/astro-ph/0207413.pdf +.. _paper_III: https://arxiv.org/pdf/astro-ph/0507293.pdf +.. _paper_IV: https://arxiv.org/pdf/1409.7583.pdf diff --git a/docs/wcs/relax.rst b/docs/wcs/relax.rst index aa641c648820..3294de2ea986 100644 --- a/docs/wcs/relax.rst +++ b/docs/wcs/relax.rst @@ -1,10 +1,5 @@ -.. include:: references.txt - .. _relax: -Relax constants -=============== - The ``relax`` keyword argument controls the handling of non-standard FITS WCS keywords. @@ -18,7 +13,7 @@ out only standard keywords), in accordance with `Postel's prescription .. _relaxread: Header-reading relaxation constants ------------------------------------ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `~astropy.wcs.WCS`, `~astropy.wcs.Wcsprm` and `~astropy.wcs.find_all_wcs` have a *relax* argument, which may be @@ -58,6 +53,32 @@ The flag bits are: - ``WCSHDR_all``: Accept all extensions recognized by the parser. (This is equivalent to the default behavior or passing `True`). +- ``WCSHDR_reject``: Reject non-standard keyrecords (that are not + otherwise explicitly accepted by one of the flags below). A warning + will be displayed by default. + + This flag may be used to signal the presence of non-standard + keywords, otherwise they are simply passed over as though they did + not exist in the header. It is mainly intended for testing + conformance of a FITS header to the WCS standard. + + Keyrecords may be non-standard in several ways: + + - The keyword may be syntactically valid but with keyvalue of + incorrect type or invalid syntax, or the keycomment may be + malformed. + + - The keyword may strongly resemble a WCS keyword but not, in fact, + be one because it does not conform to the standard. For example, + ``CRPIX01`` looks like a ``CRPIXja`` keyword, but in fact the + leading zero on the axis number violates the basic FITS standard. + Likewise, ``LONPOLE2`` is not a valid ``LONPOLEa`` keyword in the + WCS standard, and indeed there is nothing the parser can sensibly + do with it. + + - Use of the keyword may be deprecated by the standard. Such will + be rejected if not explicitly accepted via one of the flags below. + - ``WCSHDR_CROTAia``: Accept ``CROTAia``, ``iCROTna``, ``TCROTna`` - ``WCSHDR_EPOCHa``: Accept ``EPOCHa``. - ``WCSHDR_VELREFa``: Accept ``VELREFa``. @@ -80,14 +101,31 @@ The flag bits are: ``PROJPn`` is equivalent to ``PVi_ma`` with ``m`` = ``n`` <= 9, and is associated exclusively with the latitude axis. + +- ``WCSHDR_CD0i_0ja``: Accept ``CD0i_0ja`` (wcspih()). +- ``WCSHDR_PC0i_0ja``: Accept ``PC0i_0ja`` (wcspih()). +- ``WCSHDR_PV0i_0ma``: Accept ``PV0i_0ja`` (wcspih()). +- ``WCSHDR_PS0i_0ma``: Accept ``PS0i_0ja`` (wcspih()). + + Allow the numerical index to have a leading zero in + doubly-parameterized keywords, for example, ``PC01_01``. WCS Paper I + (Sects 2.1.2 & 2.1.4) explicitly disallows leading zeroes. + The FITS 3.0 standard document (Sect. 4.1.2.1) states that the + index in singly-parameterized keywords (e.g. ``CTYPEia``) "shall + not have leading zeroes", and later in Sect. 8.1 that "leading + zeroes must not be used" on ``PVi_ma`` and ``PSi_ma``. However, by an + oversight, it is silent on ``PCi_ja`` and ``CDi_ja``. + + Only available if built with wcslib 5.0 or later. + - ``WCSHDR_RADECSYS``: Accept ``RADECSYS``. This appeared in early drafts of WCS Paper I+II and was subsequently replaced by - ``RADESYSa``. The construtor accepts ``RADECSYS`` only if + ``RADESYSa``. The constructor accepts ``RADECSYS`` only if ``WCSHDR_AUXIMG`` is also enabled. - ``WCSHDR_VSOURCE``: Accept ``VSOURCEa`` or ``VSOUna``. This appeared in early drafts of WCS Paper III and was subsequently dropped in - favour of ``ZSOURCEa`` and ``ZSOUna``. The constructor accepts + favor of ``ZSOURCEa`` and ``ZSOUna``. The constructor accepts ``VSOURCEa`` only if ``WCSHDR_AUXIMG`` is also enabled. - ``WCSHDR_DOBSn``: Allow ``DOBSn``, the column-specific analogue of @@ -111,8 +149,8 @@ The flag bits are: where the primary and standard alternate forms together with the image-header equivalent are shown rightwards of the colon. - The long form of these keywords could be described as quasi- - standard. ``TPCn_ka``, ``iPVn_ma``, and ``TPVn_ma`` appeared by + The long form of these keywords could be described as + quasi-standard. ``TPCn_ka``, ``iPVn_ma``, and ``TPVn_ma`` appeared by mistake in the examples in WCS Paper II and subsequently these and also ``TCDn_ka``, ``iPSn_ma`` and ``TPSn_ma`` were legitimized by the errata to the WCS papers. @@ -135,7 +173,7 @@ The flag bits are: - ``WCSHDR_CNAMn``: Accept ``iCNAMn``, ``iCRDEn``, ``iCSYEn``, ``TCNAMn``, ``TCRDEn``, and ``TCSYEn``, i.e. with ``a`` blank. - While non-standard, these are the obvious analogues of ``iCTYPn``, + While non-standard, these are the analogues of ``iCTYPn``, ``TCTYPn``, etc. - ``WCSHDR_AUXIMG``: Allow the image-header form of an auxiliary WCS @@ -269,7 +307,7 @@ The flag bits are: .. _relaxwrite: Header-writing relaxation constants ------------------------------------ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `~astropy.wcs.wcs.WCS.to_header` and `~astropy.wcs.wcs.WCS.to_header_string` has a *relax* argument which may be either `True`, `False` or an @@ -298,7 +336,7 @@ The flag bits are: - ``WCSHDO_DOBSn``: Write ``DOBSn``, the column-specific analogue of ``DATE-OBS`` for use in binary tables and pixel lists. WCS Paper III introduced ``DATE-AVG`` and ``DAVGn`` but by an oversight - ``DOBSn`` (the obvious analogy) was never formally defined by the + ``DOBSn`` was never formally defined by the standard. The alternative to using ``DOBSn`` is to write ``DATE-OBS`` which applies to the whole table. This usage is considered to be safe and is recommended. @@ -349,7 +387,7 @@ The flag bits are: pixel lists for use with an alternate version specifier (the ``a``). Like the - ``PC``, ``CD``, ``PV``, and ``PS`` keywords there is an obvious + ``PC``, ``CD``, ``PV``, and ``PS`` keywords there is a tendency to confuse these two forms for column numbers up to 99. It is very unlikely that any parser would reject keywords in the first set with a non-blank alternate version specifier so this @@ -375,3 +413,15 @@ The flag bits are: potentially unsafe and is not recommended at this time. - ``WCSHDO_SIP``: Write out Simple Imaging Polynomial (SIP) keywords. + +- ``WCSHDO_P12``, ``WCSHDO_P13``, ``WCSHDO_P14``, ``WCSHDO_P15``, ``WCSHDO_P16``, ``WCSHDO_P17``, ``WCSHDO_EFMT`` + + These constants control the precision of the WCS keywords returned by `~astropy.wcs.WCS.to_header`. + + - ``WCSHDO_P12`` : Use "%20.12G" format for all floating-point keyvalues (12 significant digits) + - ``WCSHDO_P13`` : Use "%21.13G" format for all floating-point keyvalues (13 significant digits) + - ``WCSHDO_P14`` : Use "%22.14G" format for all floating-point keyvalues (14 significant digits) + - ``WCSHDO_P15`` : Use "%23.15G" format for all floating-point keyvalues (15 significant digits) + - ``WCSHDO_P16`` : Use "%24.16G" format for all floating-point keyvalues (16 significant digits) + - ``WCSHDO_P17`` : Use "%25.17G" format for all floating-point keyvalues (17 significant digits) + - ``WCSHDO_EFMT`` : Use "%E" format instead of the default "%G" format above diff --git a/docs/wcs/supported_projections.rst b/docs/wcs/supported_projections.rst new file mode 100644 index 000000000000..1c8787e5a956 --- /dev/null +++ b/docs/wcs/supported_projections.rst @@ -0,0 +1,57 @@ +.. include:: references.txt + +.. supported_projections: + +Supported projections +--------------------- + +As `astropy.wcs` is based on `wcslib`_, it supports the standard +projections defined in the `FITS WCS standard`_. These projection +codes are three letter strings specified in the second part of the ``CTYPEn`` keywords +(accessible through `Wcsprm.ctype `). For +example, a tangent projection with RA, DEC coordinates is defined by +``CTYPE1 = RA---TAN`` and ``CTYPE2 = DEC--TAN``. If a SIP distortion is present the +keywords become ``CTYPE1 = RA---TAN-SIP`` and ``CTYPE2 = DEC--TAN-SIP``. + +The supported projection codes are: + +- ``AZP``: zenithal/azimuthal perspective +- ``SZP``: slant zenithal perspective +- ``TAN``: gnomonic +- ``STG``: stereographic +- ``SIN``: orthographic/synthesis +- ``ARC``: zenithal/azimuthal equidistant +- ``ZPN``: zenithal/azimuthal polynomial +- ``ZEA``: zenithal/azimuthal equal area +- ``AIR``: Airy's projection +- ``CYP``: cylindrical perspective +- ``CEA``: cylindrical equal area +- ``CAR``: plate carrÊe +- ``MER``: Mercator's projection +- ``COP``: conic perspective +- ``COE``: conic equal area +- ``COD``: conic equidistant +- ``COO``: conic orthomorphic +- ``SFL``: Sanson-Flamsteed ("global sinusoid") +- ``PAR``: parabolic +- ``MOL``: Mollweide's projection +- ``AIT``: Hammer-Aitoff +- ``BON``: Bonne's projection +- ``PCO``: polyconic +- ``TSC``: tangential spherical cube +- ``CSC``: COBE quadrilateralized spherical cube +- ``QSC``: quadrilateralized spherical cube +- ``HPX``: HEALPix +- ``XPH``: HEALPix polar, aka "butterfly" + +And, if built with wcslib 5.0 or later, the following polynomial +distortions are supported: + +- ``TPV``: Polynomial distortion +- ``TUV``: Polynomial distortion + +.. note:: + + Though wcslib 5.4 and later handles ``SIP`` polynomial distortion, + for backward compatibility, ``SIP`` is handled by astropy itself + and methods exist to handle it specially. diff --git a/docs/wcs/validation.rst b/docs/wcs/validation.rst new file mode 100644 index 000000000000..2b1bdd03eab7 --- /dev/null +++ b/docs/wcs/validation.rst @@ -0,0 +1,41 @@ +.. _validation: + +Validation and Bounds checking +****************************** + +.. _wcslint: + +Validating the WCS keywords in a FITS file +------------------------------------------ + +``astropy`` includes a command-line tool, ``wcslint``, to check the WCS +keywords in a FITS file. The example below shows it reporting back +results for a problematic file named ``invalid.fits``:: + + > wcslint invalid.fits + HDU 1: + WCS key ' ': + - RADECSYS= 'ICRS ' / Astrometric system + RADECSYS is non-standard, use RADESYSa. + - The WCS transformation has more axes (2) than the image it is + associated with (0) + - 'celfix' made the change 'PV1_5 : Unrecognized coordinate + transformation parameter'. + + HDU 2: + WCS key ' ': + - The WCS transformation has more axes (3) than the image it is + associated with (0) + - 'celfix' made the change 'In CUNIT2 : Mismatched units type + 'length': have 'Hz', want 'm''. + - 'unitfix' made the change 'Changed units: 'HZ ' -> 'Hz''. + +.. _wcs-bounds-check: + +Bounds checking +--------------- + +Bounds checking is enabled by default, and any computed world +coordinates outside of [-180°, 180°] for longitude and [-90°, 90°] in +latitude are marked as invalid. To disable this behavior, use +`astropy.wcs.Wcsprm.bounds_check`. diff --git a/docs/wcs/wcsapi.rst b/docs/wcs/wcsapi.rst new file mode 100644 index 000000000000..5c5d51221213 --- /dev/null +++ b/docs/wcs/wcsapi.rst @@ -0,0 +1,378 @@ +.. _wcsapi: + +Shared Python Interface for World Coordinate Systems +**************************************************** + +Background +^^^^^^^^^^ + +The :class:`~astropy.wcs.WCS` class implements what is considered the +most common 'standard' for representing world coordinate systems in +FITS files, but it cannot represent arbitrarily complex transformations +and there is no agreement on how to use the standard beyond FITS files. +Therefore, other world coordinate system transformation approaches exist, +such as the `gwcs `_ package being developed +for the James Webb Space Telescope (which is also applicable to other data). + +Since one of the goals of the Astropy Project is to improve interoperability +between packages, we have collaboratively defined a standardized application +programming interface (API) for world coordinate system objects to be used +in Python. This API is described in the Astropy Proposal for Enhancements (APE) 14: +`A shared Python interface for World Coordinate Systems +`_. + +The core astropy package provides base classes that define the low- and +high-level APIs described in APE 14 in the :mod:`astropy.wcs.wcsapi` module, and +these are listed in the :ref:`wcs-reference-api` section below. + +Overview +^^^^^^^^ + +While the full details and motivation for the API are detailed in APE 14, this +documentation summarizes the elements that are implemented directly in the +astropy core package. The high-level interface is likely of most interest to +the average user. In particular, the most important methods are the +:meth:`~astropy.wcs.wcsapi.BaseHighLevelWCS.pixel_to_world` and +:meth:`~astropy.wcs.wcsapi.BaseHighLevelWCS.world_to_pixel` methods. These +provide the essential elements of WCS: mapping to and from world coordinates. +The remainder generally provide information about the *kind* of world +coordinates or similar information about the structure of the WCS. + +In a bit more detail, the key classes implemented here are a high-level that +provides the main user interface (:class:`~astropy.wcs.wcsapi.BaseHighLevelWCS` and +subclasses), and a lower-level interface (:class:`~astropy.wcs.wcsapi.BaseLowLevelWCS` +and subclasses). These can be distinct objects *or* the same one. For +FITS-WCS, the `~astropy.wcs.WCS` object meant for FITS-WCS follows both +interfaces, allowing immediate use of this API with files that already contain +FITS-WCS. More concrete examples are outlined below. + +Basic usage +^^^^^^^^^^^ + +Let's start off by looking at the shared Python interface for WCS by using a +simple image with two celestial axes (Right Ascension and Declination):: + + >>> from astropy.wcs import WCS + >>> from astropy.utils.data import get_pkg_data_filename + >>> from astropy.io import fits + >>> filename = get_pkg_data_filename('galactic_center/gc_2mass_k.fits') # doctest: +REMOTE_DATA + >>> hdulist = fits.open(filename) # doctest: +REMOTE_DATA + >>> hdu = hdulist[0] # doctest: +REMOTE_DATA + >>> wcs = WCS(hdu.header) # doctest: +REMOTE_DATA + >>> wcs # doctest: +REMOTE_DATA + WCS Keywords + + Number of WCS axes: 2 + CTYPE : 'RA---TAN' 'DEC--TAN' + CUNIT : 'deg' 'deg' + CRVAL : 266.4 -28.93333 + CRPIX : 361.0 360.5 + NAXIS : 721 720 + +We can check how many pixel and world axes are in the transformation as well +as the shape of the data the WCS applies to:: + + >>> wcs.pixel_n_dim # doctest: +REMOTE_DATA + 2 + >>> wcs.world_n_dim # doctest: +REMOTE_DATA + 2 + >>> wcs.array_shape # doctest: +REMOTE_DATA + (720, 721) + +Note that the array shape should match that of the data:: + + >>> hdu.data.shape # doctest: +REMOTE_DATA + (720, 721) + +As mentioned in :ref:`pixel_conventions`, what would normally be +considered the 'y-axis' of the image (when looking at it visually) is the first +dimension, while the 'x-axis' of the image is the second dimension. Thus +:attr:`~astropy.wcs.WCS.array_shape` returns the shape in the *opposite* order +to the NAXIS keywords in the FITS header (in the case of FITS-WCS). If you are +interested in the data shape in the reverse order (which would match the NAXIS +order in the case of FITS-WCS), then you can use +:attr:`~astropy.wcs.WCS.pixel_shape`:: + + >>> wcs.pixel_shape # doctest: +REMOTE_DATA + (721, 720) + +Let's now check what the physical type of each axis is:: + + >>> wcs.world_axis_physical_types # doctest: +REMOTE_DATA + ['pos.eq.ra', 'pos.eq.dec'] + +This is indeed an image with two celestial axes. + +The main part of the new interface defines standard methods for transforming +coordinates. The most convenient way is to use the high-level methods +:meth:`~astropy.wcs.wcsapi.BaseHighLevelWCS.pixel_to_world` and +:meth:`~astropy.wcs.wcsapi.BaseHighLevelWCS.world_to_pixel`, which can +transform directly to astropy objects:: + + >>> coord = wcs.pixel_to_world([1, 2], [4, 3]) # doctest: +REMOTE_DATA + >>> coord # doctest: +REMOTE_DATA + + +Similarly, we can transform astropy objects back - we can test this by creating +Galactic coordinates and these will automatically be converted:: + + >>> from astropy.coordinates import SkyCoord + >>> coord = SkyCoord('00h00m00s +00d00m00s', frame='galactic') + >>> pixels = wcs.world_to_pixel(coord) # doctest: +REMOTE_DATA + >>> pixels # doctest: +REMOTE_DATA + (array(356.85179997), array(357.45340331)) + +If you are looking to index the original data using these pixel coordinates, +be sure to instead use +:meth:`~astropy.wcs.wcsapi.BaseHighLevelWCS.world_to_array_index` which returns +the coordinates in the correct order to index Numpy arrays, and also rounds to +the nearest integer values:: + + >>> index = wcs.world_to_array_index(coord) # doctest: +REMOTE_DATA + >>> index # doctest: +REMOTE_DATA + (array(357), array(357)) + >>> hdu.data[index] # doctest: +REMOTE_DATA +FLOAT_CMP + np.float32(563.7532) + >>> hdulist.close() # doctest: +REMOTE_DATA + +Advanced usage +^^^^^^^^^^^^^^ + +Let's now take a look at a WCS for a spectral cube (two celestial axes and one +spectral axis):: + + >>> filename = get_pkg_data_filename('l1448/l1448_13co.fits') # doctest: +REMOTE_DATA + >>> hdulist = fits.open(filename) # doctest: +REMOTE_DATA + >>> hdu = hdulist[0] # doctest: +REMOTE_DATA + >>> wcs = WCS(hdu.header) # doctest: +REMOTE_DATA + >>> wcs # doctest: +REMOTE_DATA + WCS Keywords + Number of WCS axes: 3 + CTYPE : 'RA---SFL' 'DEC--SFL' 'VOPT' + CUNIT : 'deg' 'deg' 'm / s' + CRVAL : 57.6599999999 0.0 -9959.44378305 + CRPIX : -799.0 -4741.913 -187.0 + PC1_1 PC1_2 PC1_3 : 1.0 0.0 0.0 + PC2_1 PC2_2 PC2_3 : 0.0 1.0 0.0 + PC3_1 PC3_2 PC3_3 : 0.0 0.0 1.0 + CDELT : -0.006388889 0.006388889 66.42361 + NAXIS : 105 105 53 + +As before we can check how many pixel and world axes are in the transformation +as well as the shape of the data the WCS applies to, as well as the physical +types of each axis:: + + >>> wcs.pixel_n_dim # doctest: +REMOTE_DATA + 3 + >>> wcs.world_n_dim # doctest: +REMOTE_DATA + 3 + >>> wcs.array_shape # doctest: +REMOTE_DATA + (53, 105, 105) + >>> wcs.world_axis_physical_types # doctest: +REMOTE_DATA + ['pos.eq.ra', 'pos.eq.dec', 'spect.dopplerVeloc.opt'] + +This is indeed a spectral cube, with RA/Dec and a velocity axis. + +As before, we can convert between pixels and high-level Astropy objects:: + + >>> celestial, spectral = wcs.pixel_to_world([1, 2], [4, 3], [2, 3]) # doctest: +REMOTE_DATA + >>> celestial # doctest: +REMOTE_DATA + + >>> spectral # doctest: +REMOTE_DATA + ) + [2661.04211695, 2727.46572695] m / s> + +and back:: + + >>> from astropy import units as u + >>> coord = SkyCoord('03h26m36.4901s +30d45m22.2012s') + >>> pixels = wcs.world_to_pixel(coord, 3000 * u.m / u.s) # doctest: +REMOTE_DATA +IGNORE_WARNINGS + >>> pixels # doctest: +REMOTE_DATA + (array(8.11341207), array(71.0956641), array(7.10297292)) + +And as before we can index array values using:: + + >>> index = wcs.world_to_array_index(coord, 3000 * u.m / u.s) # doctest: +REMOTE_DATA +IGNORE_WARNINGS + >>> index # doctest: +REMOTE_DATA + (array(7), array(71), array(8)) + >>> hdu.data[index] # doctest: +REMOTE_DATA +FLOAT_CMP + np.float32(0.22262384) + >>> hdulist.close() # doctest: +REMOTE_DATA + +If you are interested in converting to/from world values as simple Python scalars +or Numpy arrays without using high-level astropy objects, there are methods +such as :meth:`~astropy.wcs.wcsapi.BaseLowLevelWCS.pixel_to_world_values` to +do this - see :ref:`wcs-reference-api` section for more details. + +Extending the physical types in FITS-WCS +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As shown above, the :attr:`~astropy.wcs.WCS.world_axis_physical_types` property +returns the list of physical types for each axis. For FITS-WCS, this is +determined from the CTYPE values in the header. In cases where the physical +type is not known, `None` is returned. However, it is possible to override the +physical types returned by using the +:class:`~astropy.wcs.wcsapi.fitswcs.custom_ctype_to_ucd_mapping` context +manager. Consider a WCS with the following CTYPE:: + + >>> from astropy.wcs import WCS + >>> wcs = WCS(naxis=1) + >>> wcs.wcs.ctype = ['SPAM'] + >>> wcs.world_axis_physical_types + [None] + +We can specify that for this CTYPE, the physical type should be +``'food.spam'``:: + + >>> from astropy.wcs.wcsapi.fitswcs import custom_ctype_to_ucd_mapping + >>> with custom_ctype_to_ucd_mapping({'SPAM': 'food.spam'}): + ... wcs.world_axis_physical_types + ['food.spam'] + +Preserving units in FITS-WCS +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, the :class:`~astropy.wcs.WCS` class will convert units into degrees +for angles, and SI units for other physical types:: + + >>> header = """ + ... CTYPE1 = 'GLON-CAR' + ... CTYPE2 = 'GLAT-CAR' + ... CTYPE3 = 'FREQ' + ... CUNIT1 = 'arcsec' + ... CUNIT2 = 'arcsec' + ... CUNIT3 = 'GHz' + ... CRVAL1 = 10 + ... CRVAL2 = 20 + ... CRVAL3 = 50 + ... """.strip() + >>> wcs = WCS(fits.Header.fromstring(header, sep='\n')) + >>> wcs # doctest: +FLOAT_CMP + WCS Keywords + + Number of WCS axes: 3 + CTYPE : 'GLON-CAR' 'GLAT-CAR' 'FREQ' + CUNIT : 'deg' 'deg' 'Hz' + CRVAL : 0.002777777777777778 0.005555555555555556 50000000000.0 + CRPIX : 0.0 0.0 0.0 + PC1_1 PC1_2 PC1_3 : 1.0 0.0 0.0 + PC2_1 PC2_2 PC2_3 : 0.0 1.0 0.0 + PC3_1 PC3_2 PC3_3 : 0.0 0.0 1.0 + CDELT : 0.0002777777777777778 0.0002777777777777778 1000000000.0 + NAXIS : 0 0 0 + +However, it is possible to preserve the original units by specifying +``preserve_units=True`` when initializing the :class:`~astropy.wcs.WCS` +object:: + + >>> wcs = WCS(fits.Header.fromstring(header, sep='\n'), preserve_units=True) + >>> wcs # doctest: +FLOAT_CMP + WCS Keywords + + Number of WCS axes: 3 + CTYPE : 'GLON-CAR' 'GLAT-CAR' 'FREQ' + CUNIT : 'arcsec' 'arcsec' 'GHz' + CRVAL : 10.0 20.0 50.0 + CRPIX : 0.0 0.0 0.0 + PC1_1 PC1_2 PC1_3 : 1.0 0.0 0.0 + PC2_1 PC2_2 PC2_3 : 0.0 1.0 0.0 + PC3_1 PC3_2 PC3_3 : 0.0 0.0 1.0 + CDELT : 1.0 1.0 1.0 + NAXIS : 0 0 0 + +When using this, any input/output world coordinates will now be in these +units, and accessing any of the parameters such as ``wcs.wcs.crval`` will +return values in the original header units. + +Slicing of WCS objects +^^^^^^^^^^^^^^^^^^^^^^ + +A common operation when dealing with data with WCS information attached is to +slice the WCS - this can be either to extract the WCS for a sub-region of the +data, preserving the overall number of dimensions (e.g. a cutout from an image) +or it can be reducing the dimensionality of the data and associated WCS (e.g. +extracting a slice from a spectral cube). + +The :class:`~astropy.wcs.wcsapi.SlicedLowLevelWCS` class can be used to slice +any WCS object that conforms to the :class:`~astropy.wcs.wcsapi.BaseLowLevelWCS` +API. To demonstrate this, let's start off by reading in a spectral cube file:: + + >>> filename = get_pkg_data_filename('l1448/l1448_13co.fits') # doctest: +REMOTE_DATA + >>> wcs = WCS(fits.getheader(filename, ext=0)) # doctest: +REMOTE_DATA + +The ``wcs`` object is an instance of :class:`~astropy.wcs.WCS` which conforms to the +:class:`~astropy.wcs.wcsapi.BaseLowLevelWCS` API. We can then use the +:class:`~astropy.wcs.wcsapi.SlicedLowLevelWCS` class to slice the cube:: + + >>> from astropy.wcs.wcsapi import SlicedLowLevelWCS + >>> slices = [10, slice(30, 100), slice(30, 100)] # doctest: +REMOTE_DATA + >>> subwcs = SlicedLowLevelWCS(wcs, slices=slices) # doctest: +REMOTE_DATA + +The ``slices`` argument takes any combination of slices, integer values, and +ellipsis which would normally slice a Numpy array. In the above case, we are +extracting a spectral slice, and in that slice we are extracting a sub-region +on the sky. + +If you are implementing your own WCS class, you could choose to implement +``__getitem__`` and have it internally use +:class:`~astropy.wcs.wcsapi.SlicedLowLevelWCS`. In fact, the +:class:`~astropy.wcs.WCS` class does this - the example above can be written +more succinctly as:: + + >>> wcs[10, 30:100, 30:100] # doctest: +REMOTE_DATA +ELLIPSIS + <...> + SlicedFITSWCS Transformation + + This transformation has 2 pixel and 2 world dimensions + + Array shape (Numpy order): (70, 70) + + Pixel Dim Axis Name Data size Bounds + 0 None 70 None + 1 None 70 None + + World Dim Axis Name Physical Type Units + 0 None pos.eq.ra deg + 1 None pos.eq.dec deg + + Correlation between pixel and world axes: + + Pixel Dim + World Dim 0 1 + 0 yes yes + 1 yes yes + +This slicing infrastructure is able to deal with slicing of WCS objects which +have correlated axes - in this case, you may end up with a WCS that has a +different number of pixel and world coordinates. For example, if we slice +a spectral cube to extract a 1D dataset corresponding to a row in the +image plane of a spectral slice, the final WCS will have one pixel dimension +and two world dimensions (since both RA/Dec vary over the extracted 1D slice):: + + >>> wcs[10, 40, :] # doctest: +REMOTE_DATA +ELLIPSIS + <...> + SlicedFITSWCS Transformation + + This transformation has 1 pixel and 2 world dimensions + + Array shape (Numpy order): (105,) + + Pixel Dim Axis Name Data size Bounds + 0 None 105 None + + World Dim Axis Name Physical Type Units + 0 None pos.eq.ra deg + 1 None pos.eq.dec deg + + Correlation between pixel and world axes: + + Pixel Dim + World Dim 0 + 0 yes + 1 yes diff --git a/docs/wcs/wcstools.rst b/docs/wcs/wcstools.rst new file mode 100644 index 000000000000..415e998b3763 --- /dev/null +++ b/docs/wcs/wcstools.rst @@ -0,0 +1,47 @@ +.. _wcstools: + +Subsetting and Pixel Scales +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +WCS objects can be broken apart into their constituent axes using the +`~astropy.wcs.WCS.sub` function. There is also a `~astropy.wcs.WCS.celestial` +convenience function that will return a WCS object with only the celestial axes +included. + +The pixel scales of a celestial image or the pixel dimensions of a non-celestial +image can be extracted with the utility functions +`~astropy.wcs.utils.proj_plane_pixel_scales` and +`~astropy.wcs.utils.non_celestial_pixel_scales`. Likewise, celestial pixel +area can be extracted with the utility function +`~astropy.wcs.utils.proj_plane_pixel_area`. + +Matplotlib plots with correct WCS projection +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The :ref:`WCSAxes ` framework, previously a standalone package, allows +the :class:`~astropy.wcs.WCS` to be used to define projections in Matplotlib. +More information on using WCSAxes can be found :ref:`here `. + +.. plot:: + :context: reset + :include-source: + :align: center + + import warnings + from matplotlib import pyplot as plt + from astropy.io import fits + from astropy.wcs import WCS, FITSFixedWarning + from astropy.utils.data import get_pkg_data_filename + + filename = get_pkg_data_filename('tutorials/FITS-images/HorseHead.fits') + + hdu = fits.open(filename)[0] + with warnings.catch_warnings(): + # Ignore a warning on using DATE-OBS in place of MJD-OBS + warnings.filterwarnings('ignore', message="'datfix' made the change", + category=FITSFixedWarning) + wcs = WCS(hdu.header) + + fig, ax = plt.subplots(subplot_kw=dict(projection=wcs)) + ax.imshow(hdu.data, origin='lower', cmap='viridis') + ax.set(xlabel='RA', ylabel='Dec') diff --git a/docs/whatsnew/0.1.rst b/docs/whatsnew/0.1.rst index c0b7f607c203..53c997b24e20 100644 --- a/docs/whatsnew/0.1.rst +++ b/docs/whatsnew/0.1.rst @@ -1,26 +1,3 @@ -========================= -What's New in Astropy 0.1 -========================= +:orphan: -This was the initial version of Astropy, released on June 19, 2012. It was -released primarily as a "developer preview" for developers interested in -working directly on Astropy, on affiliated packages, or on other software that -might integrate with Astropy. - -Astropy 0.1 integrated several existing packages under a single ``astropy`` -package with a unified installer, including: - - * asciitable as `astropy.io.ascii` - * PyFITS as `astropy.io.fits` - * votable as ``astropy.io.vo`` - * PyWCS as `astropy.wcs` - -It also added the beginnings of the :mod:`astropy.cosmology` package, and new -common data structures for science data in the :mod:`astropy.nddata` and -:mod:`astropy.table` packages. - -It also laid much of the groundwork for Astropy's installation and -documentation frameworks, as well as tools for managing configuration and data -management. These facilities are designed to be shared by Astropy's affiliated -packages in the hopes of providing a framework on which other Astronomy-related -Python packages can build. +`What's New in Astropy 0.1? `__ diff --git a/docs/whatsnew/0.2.rst b/docs/whatsnew/0.2.rst index 7b7800504dc5..c12fd385369f 100644 --- a/docs/whatsnew/0.2.rst +++ b/docs/whatsnew/0.2.rst @@ -1,9 +1,3 @@ -.. _whatsnew-0.2: +:orphan: -========================= -What's New in Astropy 0.2 -========================= - -See this page in the `Astropy v0.2 documentation`__. - -__ http://docs.astropy.org/en/v0.2.5/whatsnew/0.2.html +`What's New in Astropy 0.2? `__ diff --git a/docs/whatsnew/0.3.rst b/docs/whatsnew/0.3.rst index 9f44112c7755..bd8987c05883 100644 --- a/docs/whatsnew/0.3.rst +++ b/docs/whatsnew/0.3.rst @@ -1,9 +1,3 @@ -.. _whatsnew-0.3: +:orphan: -========================== -What's New in Astropy 0.3? -========================== - -See this page in the `Astropy v0.3 documentation`__. - -__ http://docs.astropy.org/en/v0.3.2/whatsnew/0.3.html +`What's New in Astropy 0.3? `__ diff --git a/docs/whatsnew/0.4.rst b/docs/whatsnew/0.4.rst index e07161c0ec23..3c6392272923 100644 --- a/docs/whatsnew/0.4.rst +++ b/docs/whatsnew/0.4.rst @@ -1,225 +1,3 @@ -.. doctest-skip-all - -.. _whatsnew-0.4: - -========================== -What's New in Astropy 0.4? -========================== - -Overview --------- - -Astropy 0.4 is a major release that adds new functionality since the -0.3.x series of releases. A new sub-package is included (see `SAMP`_), -a major overhaul of the :ref:`Coordinates ` -sub-package has been completed (see `Coordinates`_), -and many new features and improvements have been implemented for the -existing sub-packages. In addition to usability improvements, we have -made a number of changes in the infrastructure for setting up/installing -the package (see `astropy-helpers package`_), as well as reworking the -configuration system (see `Configuration`_). - -In addition to these major changes, a large number of smaller -improvements have occurred. Since v0.3, by the numbers: - -* 819 issues have been closed -* 511 pull requests have been merged -* 57 distinct people have contributed code - - -Coordinates ------------ - -The :ref:`astropy-coordinates` sub-package has been largely re-designed based -on broad community discussion and experience with v0.2 and v0.3. The key -motivation was to implement coordinates within an extensible framework that -cleanly separates the distinct aspects of data representation, coordinate -frame representation and transformation, and user interface. This is described -in the `APE5 `_ -document. Details of the new usage are given in the :ref:`astropy-coordinates` -section of the documentation. - -*An important point is that this sub-package is now considered stable and we do -not expect any further major interface changes.* - -For most users the major change is that the recommended user interface to -coordinate functionality is the `~astropy.coordinates.SkyCoord` class -instead of classes like `~astropy.coordinates.ICRS` or -`~astropy.coordinates.Galactic` (which are now -called "frame" classes). For example:: - - >>> from astropy import units as u - >>> from astropy.coordinates import SkyCoord - >>> coordinate = SkyCoord(123.4*u.deg, 56.7*u.deg, frame='icrs') - -The frame classes can still be used to create coordinate objects as before, but -they are now more powerful because they can represent abstract coordinate -frames without underlying data. The more typical use for frame classes is now:: - - >>> from astropy.coordinates import FK4 # Or ICRS, Galactic, or similar - >>> fk4_frame = FK4(equinox='J1980.0', obstime='2011-06-12T01:12:34') - >>> coordinate.transform_to(fk4_frame) - - -At the lowest level of the framework are the representation classes which -describe how to represent a point in a frame as a tuple of quantities, for -instance as spherical, cylindrical, or cartesian coordinates. Any coordinate -object can now be created using values in a number of common representations -and be displayed using those representations. For example:: - - >>> coordinate = SkyCoord(1*u.pc, 2*u.pc, 3*u.pc, representation='cartesian') - >>> coordinate - - - >>> coordinate.representation = 'physicsspherical' - >>> coordinate - - -SAMP ----- - -The :ref:`vo-samp` sub-package is a new sub-package (adapted from the `SAMPy -package `_) that contains an -implementation of the `Simple Application Messaging Protocol (SAMP) -`_ standard that allows communication -with any SAMP-enabled application (such as `TOPCAT -`_, `SAO Ds9 -`_, and `Aladin -`_). This sub-package includes both classes for a -hub and a client, as well as an *integrated client* which automatically -connects to any running SAMP hub and acts as a client:: - - >>> from astropy.vo.samp import SAMPIntegratedClient - >>> client = SAMPIntegratedClient() - >>> client.connect() - -We can then use the client to communicate with other clients:: - - >>> client.get_registered_clients() - ['hub', 'c1', 'c2'] - >>> client.get_metadata('c1') - {'author.affiliation': 'Astrophysics Group, Bristol University', - 'author.email': 'm.b.taylor@bristol.ac.uk', - 'author.name': 'Mark Taylor', - 'home.page': 'http://www.starlink.ac.uk/topcat/', - 'samp.description.text': 'Tool for OPerations on Catalogues And Tables', - 'samp.documentation.url': 'http://127.0.0.1:2525/doc/sun253/index.html', - 'samp.icon.url': 'http://127.0.0.1:2525/doc/images/tc_sok.gif', - 'samp.name': 'topcat', - 'topcat.version': '4.0-1'} - -and we can then send for example tables and images over SAMP to other -applications (see :ref:`vo-samp` for examples of how to do this). - -Quantity --------- -The `~astropy.units.Quantity` class has seen a series of optimizations -and is now substantially faster. Additionally, the `~astropy.time`, -`~astropy.coordinates`, and `~astropy.table` subpackages integrate -better with `~astropy.units.Quantity`, with further improvements on the -way for `~astropy.table`. See :doc:`/units/quantity` and the other -subpackage documentation sections for more details. - -Inspecting FITS headers from the command line ---------------------------------------------- - -The :ref:`astropy-io-fits` sub-package now provides a command line script for -inspecting the header(s) of a FITS file. With Astropy 0.4 installed, run -``fitsheader file.fits`` in your terminal to print the header information to -the screen in a human-readable format. Run ``fitsheader --help`` to see the -full usage documentation. - -Reading and writing HTML tables -------------------------------- - -The :ref:`io-ascii` sub-package now provides the capability to read a table -within an HTML file or web URL into an astropy `~astropy.table.Table` object. -This requires the `BeautifulSoup4 -`_ package to be installed. -Conversely a `~astropy.table.Table` object can now be written out as an HTML -table. - -Documentation URL changes -------------------------- - -Starting in v0.4, the astropy documentation (and any package that uses -``astropy-helpers``) will show the full name of functions and classes -prefixed by the intended user-facing location. This is in contrast to -previous versions, which pointed to the actual implementation module, -rather than the intended public API location. - -This will affect URLs pointing to specific documentation pages. For -example, this URL points to the v0.3 location of the -``astropy.cosmology.luminosity_distance`` function: - -* http://docs.astropy.org/en/v0.3/api/astropy.cosmology.funcs.luminosity_distance.html - -while the appropriate URL for v0.4 and later is: - -* http://docs.astropy.org/en/v0.4/api/astropy.cosmology.luminosity_distance.html - -astropy-helpers package ------------------------ - -We have now extracted our set-up and documentation utilities into a separate -package, `astropy-helpers `_. In -practice, this does not change anything from a user point of view, but it is -a big internal change that will allow any other packages to benefit from the -set-up utilies developed for the core package without having to first install -astropy. - -Configuration -------------- - -The configuration framework has been re-factored based on the design -described in -`APE3 `_. -If you have previously edited the astropy configuration file (typically -located at ``~/.astropy/config/astropy.cfg``) then you should read over -:ref:`config-0-4-transition` in order to understand how to update it -to the new mechanism. - -Deprecation and backward-incompatible changes ---------------------------------------------- - -- ``Quantity`` comparisons with ``==`` or ``!=`` now always return ``True`` - or ``False``, even if units do not match (for which case a ``UnitsError`` - used to be raised). [#2328] - -- The functional interface for `astropy.cosmology` (e.g. - ``cosmology.H(z=0.5)`` is now deprecated in favor of the - objected-oriented approach (``WMAP9.H(z=0.5)``). [#2343] - -- The `astropy.coordinates` sub-package has undergone major changes for - implementing the - `APE5 `_ plan - for the package. A compatibility layer has been added that will allow - common use cases of pre-v0.4 coordinates to work, but this layer will be - removed in the next major version. Hence, any use of the coordinates - package should be adapted to the new framework. Additionally, the - compatibility layer cannot be used for convenience functions (like the - ``match_catalog_*()`` functions), as these have been moved to - `~astropy.coordinates.SkyCoord`. From this point on, major changes to the - coordinates classes are not expected. [#2422] - -- The configuration framework has been re-designed to the scheme of - `APE3 `_. - The previous framework based on `~astropy.config.ConfigurationItem` is - deprecated, and will be removed in a future release. Affiliated - packages should update to the new configuration system, and any users - who have customized their configuration file should migrate to the new - configuration approach. Until they do, warnings will appear prompting - them to do so. - -Full change log ---------------- - -To see a detailed list of all changes in version 0.4 and prior, please see the -:ref:`changelog`. - -Note on future versions ------------------------ - -While the current release supports Python 2.6, 2.7, and 3.1 to 3.4, the next -release (1.0) will drop support for Python 3.1 and 3.2. +:orphan: +`What's New in Astropy 0.4? `__ diff --git a/docs/whatsnew/1.0.rst b/docs/whatsnew/1.0.rst index 578c096d57f2..440344acad18 100644 --- a/docs/whatsnew/1.0.rst +++ b/docs/whatsnew/1.0.rst @@ -1,367 +1,3 @@ -.. doctest-skip-all - -.. _whatsnew-1.0: - -========================== -What's New in Astropy 1.0? -========================== - -Overview --------- - -Astropy 1.0 is a major release that adds significant new functionality since the 0.4.x series of releases. - -In particular, coordinate conversions to/from Altitude/Azimuth are now -supported (see `Support for Alt/Az coordinates`_), a new package to help with -data visualization has been added (see :ref:`whatsnew_viz`), and a new package -for common analytic functions is now also included (see -:ref:`whatsnew_analytical_functions`). - -The :ref:`io-ascii` sub-package now includes fast C-based -readers/writers for common formats, and also supports a new ASCII format that -better preserves meta-data (see :ref:`whatsnew_io_ascii`), the modeling package -has been significantly improved and now supports composite models (see -:ref:`whatsnew_modeling`), and the :class:`~astropy.table.Table` class can now -include :class:`~astropy.coordinates.SkyCoord` and :class:`~astropy.time.Time` -objects containing arrays (see :ref:`whatsnew_table`). - -In addition to these major changes, Astropy 1.0 includes a large number of -smaller improvements and bug fixes, which are described in the :ref:`changelog`. -By the numbers: - -* 681 issues have been closed since v0.4 -* 419 pull requests have been merged since v0.4 -* 122 distinct people have contributed code - -About Long-term support ------------------------ - -Astropy v1.0 is a long-term support (LTS) release. This means v1.0 will -be supported with bug fixes for 2 years from its release, rather than 6 -months like the non-LTS releases. More details about this, including a -wider rationale for Astropy's version numbering scheme, can be found in -`Astropy Proposal for Enhancement 2 `_. - -Note that different sub-packages in Astropy have different stability levels. See -the :doc:`/stability` page for an overview of the status of major components. -LTS can be expected for anything with green or blue (stable or mature) status on -that page. For yellow (in development) subpackages, LTS *may* be provided, but -major changes may prevent backporting of complex changes, particularly if they -are connected to new features. - -Support for Alt/Az coordinates ------------------------------- - -The `~astropy.coordinates` package now supports conversion to/from AltAz -coordinates. This means `~astropy.coordinates` can now be used for planning -observations. For example:: - - >>> from astropy import units as u - >>> from astropy.time import Time - >>> from astropy.coordinates import SkyCoord, EarthLocation, AltAz - >>> greenwich = EarthLocation(lat=51.477*u.deg,lon=0*u.deg) - >>> albireo = SkyCoord('19h30m43.2805s +27d57m34.8483s') - >>> altaz = albireo.transform_to(AltAz(location=greenwich, obstime=Time('2014-6-21 0:00'))) - >>> print altaz.alt, altaz.az - 60d32m28.4576s 133d45m36.4967s - -For a more detailed outline of this new functionality, see the -:ref:`observing-example` and the `~astropy.coordinates.AltAz` documentation. - -To enable this functionality, `~astropy.coordinates` now also contains -the full IAU-sanctioned coordinate transformation stack from ICRS to AltAz. -To view the full set of coordinate frames now available, see the coordinates -:ref:`astropy-coordinates-api`. - - -New Galactocentric coordinate frame ------------------------------------ - -Added a new, customizable :class:`~astropy.coordinates.Galactocentric` -coordinate frame. The other coordinate frames (e.g., -:class:`~astropy.coordinates.ICRS`, :class:`~astropy.coordinates.Galactic`) -are all Heliocentric (or barycentric). The center of this new coordinate frame -is at the center of the Galaxy, with customizable parameters allowing the user -to specify the distance to the Galactic center (``galcen_distance``), the -ICRS position of the Galactic center (``galcen_ra``, ``galcen_dec``), the -height of the Sun above the Galactic midplane (``z_sun``), and a final roll -angle that allows for specifying the orientation of the z axis (``roll``):: - - >>> from astropy import units as u - >>> from astropy.coordinates import SkyCoord, Galactocentric - >>> c = SkyCoord(ra=152.718 * u.degree, - ... dec=-11.214 * u.degree, - ... distance=21.5 * u.kpc) - >>> c.transform_to(Galactocentric) - - >>> c.transform_to(Galactocentric(galcen_distance=8*u.kpc, z_sun=15*u.pc)) - - -.. _whatsnew_viz: - -New data visualization subpackage ---------------------------------- - -The new :ref:`Data Visualization ` package is intended -to collect functionality that can be helpful when visualizing data. At the -moment, the main functionality is image normalizing (including both scaling and -stretching) but this will be expanded in future. Included in the image -normalization functionality is the ability to compute interval limits on data, -(such as percentile limits), stretching with non-linear functions (such as -square root or arcsinh functions), and the ability to use custom stretches in -`Matplotlib `_ that are correctly reflected in the -colorbar: - -.. plot:: - :include-source: - :align: center - - import numpy as np - import matplotlib.pyplot as plt - - from astropy.visualization import SqrtStretch - from astropy.visualization.mpl_normalize import ImageNormalize - - # Generate test image - image = np.arange(65536).reshape((256, 256)) - - # Create normalizer object - norm = ImageNormalize(vmin=0., vmax=65536, stretch=SqrtStretch()) - - fig = plt.figure(figsize=(6,3)) - ax = fig.add_subplot(1,1,1) - im = ax.imshow(image, norm=norm, origin='lower', aspect='auto') - fig.colorbar(im) - -.. _whatsnew_analytical_functions: - -New analytic functions subpackage ---------------------------------- - -This subpackage provides analytic functions that are commonly used in -astronomy. These already understand `~astropy.units.Quantity`, i.e., they can -handle units of input and output parameters. For instance, to calculate the -blackbody flux for 10000K at 6000 Angstrom:: - - >>> from astropy import units as u - >>> from astropy.analytic_functions import blackbody_lambda, blackbody_nu - >>> blackbody_lambda(6000 * u.AA, 10000 * u.K) - - >>> blackbody_nu(6000 * u.AA, 10000 * u.K) - `_ ASCII file -interface (`read_csv -`_ and -`to_csv -`_). The -fast reader has parallel processing option that allows harnessing multiple -cores for input parsing to achieve even greater speed gains. - -By default, :func:`~astropy.io.ascii.read` and :func:`~astropy.io.ascii.write` -will attempt to use the fast C engine when dealing with compatible formats. -Certain features of the full read / write interface are not available in the -fast version, in which case the pure-Python version will automatically be used. - -For full details including extensive performance testing, see :ref:`fast_ascii_io`. - -Enhanced CSV format -^^^^^^^^^^^^^^^^^^^ - -One of the problems when storing a table in an ASCII format is preserving table -meta-data such as comments, keywords and column data types, units, and -descriptions. Using the newly defined `Enhanced Character Separated Values -format `_ it is -now possible to write a table to an ASCII-format file and read it back with no -loss of information. The ECSV format has been designed to be both -human-readable and compatible with most simple CSV readers. - -In the example below we show writing a table that has ``float32`` and ``bool`` -types. This illustrates the simple look of the format which has a few header -lines (starting with ``#``) in `YAML `_ format and then -the data values in CSV format. -:: - - >>> t = Table() - >>> t['x'] = Column([1.0, 2.0], unit='m', dtype='float32') - >>> t['y'] = Column([False, True], dtype='bool') - - >>> from astropy.extern.six.moves import StringIO - >>> fh = StringIO() - >>> t.write(fh, format='ascii.ecsv') # doctest: +SKIP - >>> table_string = fh.getvalue() # doctest: +SKIP - >>> print(table_string) # doctest: +SKIP - # %ECSV 0.9 - # --- - # columns: - # - {name: x, unit: m, type: float32} - # - {name: y, type: bool} - x y - 1.0 False - 2.0 True - -Without the header this table would get read back with different types -(``float64`` and ``string`` respectively) and no unit values. Instead with -the automatically-detected ECSV we get:: - - >>> Table.read(table_string, format='ascii') # doctest: +SKIP -
- x y - m - float32 bool - ------- ----- - 1.0 False - 2.0 True - -Note that using the ECSV reader requires the `PyYAML `_ -package to be installed. - -.. _whatsnew_modeling: - -New modeling features ---------------------- - -New subclasses of `~astropy.modeling.Model` are now a bit easier to define, -requiring less boilerplate code in general. Now all that is necessary to -define a new model class is an `~astropy.modeling.Model.evaluate` method that -computes the model. Optionally one can define :ref:`fittable parameters -`, a `~astropy.modeling.FittableModel.fit_deriv`, and/or -an `~astropy.modeling.Model.inverse`. The new, improved -`~astropy.modeling.custom_model` decorator reduces the boilerplate needed for -many models even more. See :ref:`modeling-new-classes` for more details. - -Array broadcasting has also been improved, enabling a broader range of -possibilities for the values of model parameters and inputs. Support has also -been improved for :ref:`modeling-model-sets` (previously referred to as -parameter sets) which can be thought of like an array of models of the same -class, each with different sets of parameters, which can be fitted -simultaneously either to the same data, or to different data sets per model. -See :ref:`modeling-instantiating` for more details. - -It is now possible to create *compound* models by combining existing models -using the standard arithmetic operators such as ``+`` and ``*``, as well as -functional composition using the ``|`` operator. This provides a powerful -and flexible new way to create more complex models without having to define -any special classes or functions. For example:: - - >>> from astropy.modeling.models import Gaussian1D - >>> gaussian1 = Gaussian1D(1, 0, 0.2) - >>> gaussian2 = Gaussian1D(2.5, 0.5, 0.1) - >>> sum_of_gaussians = gaussian1 + gaussian2 - -The resulting model works like any other model, and also works with the -fitting framework. See the -:ref:`introduction to compound models ` and full -:ref:`compound models documentation ` for more examples. - -.. _whatsnew_table: - -New Table features ------------------- - -.. |Quantity| replace:: :class:`~astropy.units.Quantity` -.. |Time| replace:: :class:`~astropy.time.Time` -.. |SkyCoord| replace:: :class:`~astropy.coordinates.SkyCoord` -.. |Table| replace:: :class:`~astropy.table.Table` -.. |Column| replace:: :class:`~astropy.table.Column` -.. |QTable| replace:: :class:`~astropy.table.QTable` - -Refactor of table infrastructure -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The underlying data container for the Astropy |Table| object has been changed -in Astropy v1.0. Previously, tables were stored internally as a Numpy structured -array object, with column access being a memory view of the corresponding Numpy -array field. Starting with this release the fundamental data container is an -ordered dictionary of individual column objects and each |Column| object is the -sole owner of its data. - -The biggest impact to users is that operations such as adding or removing -table columns is now significantly faster because there is no structured array -to rebuild each time. - -For details please see :ref:`table_implementation_change`. - -Support for 'mixin' columns -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Version v1.0 of Astropy introduces a new concept of the "Mixin -Column" in tables which allows integration of appropriate non-|Column| based -class objects within a |Table| object. These mixin column objects are not -converted in any way but are used natively. - -The available built-in mixin column classes are |Quantity|, |SkyCoord|, and -|Time|. User classes for array-like objects that support the -:ref:`mixin_protocol` can also be used in tables as mixin columns. - -.. Warning:: - - While the Astropy developers are excited about this new capability and - intend to improve it, the interface for using mixin columns is not stable at - this point and it is not recommended for use in production code. - -As an example we can create a table and add a time column:: - - >>> from astropy.table import Table - >>> from astropy.time import Time - >>> t = Table() - >>> t['index'] = [1, 2] - >>> t['time'] = Time(['2001-01-02T12:34:56', '2001-02-03T00:01:02']) - >>> print(t) - index time - ----- ----------------------- - 1 2001-01-02T12:34:56.000 - 2 2001-02-03T00:01:02.000 - -The important point here is that the ``time`` column is a bona fide |Time| object:: - - >>> t['time'] -