diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index e7bff6bfe..000000000 --- a/.coveragerc +++ /dev/null @@ -1,7 +0,0 @@ -[run] -branch = True -omit = - tqdm/tests/* - tqdm/contrib/telegram.py -[report] -show_missing = True diff --git a/.gitattributes b/.gitattributes index 1d7632345..0d1f379d9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,11 +3,7 @@ .git* export-ignore .mailmap export-ignore +.meta/ export-ignore images/ export-ignore benchmarks/ export-ignore -MANIFEST.in export-ignore -.travis.yml export-ignore -codecov.yml export-ignore asv.conf.json export-ignore -.style.yapf export-ignore -.tqdm.1.md export-ignore diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2a26028d5..d49ad0b38 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,4 +10,4 @@ demo.yml @casperdcl @martinzugnoni codecov.yml @lrq3000 setup.py @casperdcl @lrq3000 tqdm/_tqdm_notebook.py @lrq3000 @casperdcl -tqdm/tests/tests_pandas.py @casperdcl @chengs +tests/tests_pandas.py @casperdcl @chengs diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 44774f14e..5192ed5a9 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,2 +1,2 @@ github: casperdcl -custom: https://caspersci.uk.to/donate +tidelift: pypi/tqdm diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..302456a1d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: FAQs and Known Issues + url: https://github.com/tqdm/tqdm/#faq-and-known-issues + about: Frequently asked questions and known issues + - name: "StackOverflow#tqdm" + url: https://stackoverflow.com/questions/tagged/tqdm + about: "Stack Overflow questions tagged #tqdm" diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE/issue.md similarity index 67% rename from .github/ISSUE_TEMPLATE.md rename to .github/ISSUE_TEMPLATE/issue.md index 008292495..90150dea1 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE/issue.md @@ -1,3 +1,8 @@ +--- +name: Issue +about: Use this template for reporting issue(s) +--- + - [ ] I have marked all applicable categories: + [ ] exception-raising bug + [ ] visual output bug @@ -13,7 +18,7 @@ print(tqdm.__version__, sys.version, sys.platform) ``` - [source website]: https://github.com/tqdm/tqdm/ - [known issues]: https://github.com/tqdm/tqdm/#faq-and-known-issues - [issue tracker]: https://github.com/tqdm/tqdm/issues?q= - [StackOverflow#tqdm]: https://stackoverflow.com/questions/tagged/tqdm +[source website]: https://github.com/tqdm/tqdm/ +[known issues]: https://github.com/tqdm/tqdm/#faq-and-known-issues +[issue tracker]: https://github.com/tqdm/tqdm/issues?q= +[StackOverflow#tqdm]: https://stackoverflow.com/questions/tagged/tqdm diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE/pr.md similarity index 70% rename from .github/PULL_REQUEST_TEMPLATE.md rename to .github/PULL_REQUEST_TEMPLATE/pr.md index 4c7471d8b..3235dbf4f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE/pr.md @@ -1,3 +1,8 @@ +--- +name: Pull Request +about: Use this template for proposing change(s) +--- + - [ ] I have marked all applicable categories: + [ ] exception-raising fix + [ ] visual output fix @@ -17,6 +22,6 @@ Less important but also useful: print(tqdm.__version__, sys.version, sys.platform) ``` - [source website]: https://github.com/tqdm/tqdm/ - [known issues]: https://github.com/tqdm/tqdm/#faq-and-known-issues - [issue tracker]: https://github.com/tqdm/tqdm/issues?q= +[source website]: https://github.com/tqdm/tqdm/ +[known issues]: https://github.com/tqdm/tqdm/#faq-and-known-issues +[issue tracker]: https://github.com/tqdm/tqdm/issues?q= diff --git a/.github/SECURITY.md b/.github/SECURITY.md new file mode 100644 index 000000000..9955d5fb8 --- /dev/null +++ b/.github/SECURITY.md @@ -0,0 +1,14 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ----------- | ------------------ | +| >= 4.11.2 | :white_check_mark: | +| < 4.11.2 | :x: | + +## Security contact information + +To report a security vulnerability, please use the +[Tidelift security contact](https://tidelift.com/security). +Tidelift will coordinate the fix and disclosure. diff --git a/codecov.yml b/.github/codecov.yml similarity index 58% rename from codecov.yml rename to .github/codecov.yml index d09b04b0a..e1a921ab6 100644 --- a/codecov.yml +++ b/.github/codecov.yml @@ -1,8 +1,8 @@ comment: - layout: header, changes, diff + layout: diff coverage: status: patch: default: - target: '80' + threshold: 80% project: false diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 000000000..1e4432333 --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,93 @@ +name: Check +on: + push: + pull_request: + schedule: + - cron: '36 1 * * SUN' # M H d m w (Sundays at 01:36) +jobs: + check: + if: github.event_name != 'pull_request' || github.head_ref != 'devel' + name: ${{ matrix.TOXENV }} + strategy: + matrix: + TOXENV: + - setup.py + - perf + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - run: pip install -U tox + - run: tox + env: + TOXENV: ${{ matrix.TOXENV }} + asvfull: + if: (github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')) || github.event_name == 'schedule' + name: Benchmark (Full) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install + run: | + pip install -U wheel + pip install -U virtualenv asv + git checkout master && git checkout - + asv machine --machine github-actions --yes + - name: Restore previous results + uses: actions/cache@v2 + with: + path: .asv + key: asv-${{ runner.os }} + restore-keys: | + asv- + - name: Benchmark + run: | + asv run -j 8 --interleave-processes --skip-existing v3.2.0..HEAD + - name: Build pages + run: | + git config --global user.email "$GIT_AUTHOR_EMAIL" + git config --global user.name "$GIT_AUTHOR_NAME" + asv gh-pages --no-push + git push -f origin gh-pages:gh-pages + env: + GIT_AUTHOR_NAME: ${{ github.actor }} + GIT_AUTHOR_EMAIL: ${{ github.actor }}@users.noreply.github.com + testasv: + if: github.event.ref != 'refs/heads/master' && ! startsWith(github.event.ref, 'refs/tags') + name: Benchmark (Branch) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install + run: | + pip install -U wheel + pip install -U virtualenv asv + git checkout master && git checkout - + asv machine --machine github-actions --yes + - name: Restore previous results + uses: actions/cache@v2 + with: + path: .asv + key: asv-${{ runner.os }} + restore-keys: | + asv- + - name: Benchmark + run: | + asv continuous --interleave-processes --only-changed -f 1.25 master HEAD + CHANGES="$(asv compare --only-changed -f 1.25 master HEAD)" + echo "$CHANGES" + [ -z "$CHANGES" ] || exit 1 diff --git a/.github/workflows/comment-bot.yml b/.github/workflows/comment-bot.yml new file mode 100644 index 000000000..4451632e7 --- /dev/null +++ b/.github/workflows/comment-bot.yml @@ -0,0 +1,50 @@ +name: Comment Bot +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] +jobs: + tag: # /tag + if: startsWith(github.event.comment.body, '/tag ') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: React Seen + uses: actions/github-script@v2 + with: + script: | + const perm = await github.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, repo: context.repo.repo, + username: context.payload.comment.user.login}) + post = (context.eventName == "issue_comment" + ? github.reactions.createForIssueComment + : github.reactions.createForPullRequestReviewComment) + if (!["admin", "write"].includes(perm.data.permission)){ + post({ + owner: context.repo.owner, repo: context.repo.repo, + comment_id: context.payload.comment.id, content: "laugh"}) + throw "Permission denied for user " + context.payload.comment.user.login + } + post({ + owner: context.repo.owner, repo: context.repo.repo, + comment_id: context.payload.comment.id, content: "eyes"}) + - name: Tag Commit + run: | + git clone https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY} repo + git -C repo tag $(echo "$BODY" | awk '{print $2" "$3}') + git -C repo push --tags + rm -rf repo + env: + BODY: ${{ github.event.comment.body }} + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - name: React Success + uses: actions/github-script@v2 + with: + script: | + post = (context.eventName == "issue_comment" + ? github.reactions.createForIssueComment + : github.reactions.createForPullRequestReviewComment) + post({ + owner: context.repo.owner, repo: context.repo.repo, + comment_id: context.payload.comment.id, content: "rocket"}) diff --git a/.github/workflows/post-release.yml b/.github/workflows/post-release.yml new file mode 100644 index 000000000..1fff38d8d --- /dev/null +++ b/.github/workflows/post-release.yml @@ -0,0 +1,50 @@ +name: Post Release +on: + release: + types: [published] +jobs: + docs: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v2 + - name: Checkout wiki + uses: actions/checkout@v2 + with: + repository: ${{ github.repository }}.wiki + path: wiki + - name: Checkout docs + uses: actions/checkout@v2 + with: + repository: ${{ github.repository }}.github.io + path: docs + ref: src + token: ${{ secrets.GH_TOKEN }} + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install + run: | + pip install -U wheel + pip install -U -r ./docs/requirements.txt + git config --global user.email "$GIT_AUTHOR_EMAIL" + git config --global user.name "$GIT_AUTHOR_NAME" + env: + GIT_AUTHOR_NAME: ${{ github.event.sender.login }} + GIT_AUTHOR_EMAIL: ${{ github.event.sender.login }}@users.noreply.github.com + - name: Update Wiki Releases + run: | + pushd wiki + make + git commit -a -m "update release notes to ${GITHUB_REF#refs/tags/}" + git push + popd + - name: Update Docs + run: | + pushd docs + git fetch --depth=1 origin master:master + git checkout master + git push --set-upstream origin master + git checkout - + make deploy + popd diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..46ad63f05 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,219 @@ +name: Test +on: + push: + pull_request: + schedule: + - cron: '2 1 * * 6' # M H d m w (Saturdays at 1:02) +jobs: + check: + if: github.event_name != 'pull_request' || github.head_ref != 'devel' + runs-on: ubuntu-latest + name: check + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-python@v2 + - name: set PYSHA + run: echo "PYSHA=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV + - uses: actions/cache@v1 + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ env.PYSHA }}|${{ hashFiles('.pre-commit-config.yaml') }} + - name: dependencies + run: pip install -U pre-commit + - uses: reviewdog/action-setup@v1 + - if: github.event_name != 'schedule' + name: comment + run: | + if [[ $EVENT == pull_request ]]; then + REPORTER=github-pr-review + else + REPORTER=github-check + fi + pre-commit run -a todo | reviewdog -efm="%f:%l: %m" -name=TODO -tee -reporter=$REPORTER -filter-mode nofilter + pre-commit run -a flake8 | reviewdog -f=pep8 -name=flake8 -tee -reporter=$REPORTER -filter-mode nofilter + env: + REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EVENT: ${{ github.event_name }} + - run: pre-commit run -a --show-diff-on-failure + test-os: + if: github.event_name != 'pull_request' || github.head_ref != 'devel' + strategy: + matrix: + python: [2.7, 3.7] + os: [macos-latest, windows-latest] + name: py${{ matrix.python }}-${{ matrix.os }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: install + shell: bash + run: | + pip install -U tox + mkdir -p "$HOME/bin" + curl -sfL https://coverage.codacy.com/get.sh > "$HOME/bin/codacy" + chmod +x "$HOME/bin/codacy" + echo "$HOME/bin" >> $GITHUB_PATH + - run: tox -e py${PYVER/./} + shell: bash + env: + PYVER: ${{ matrix.python }} + COVERALLS_FLAG_NAME: py${{ matrix.python }}-${{ matrix.os }} + COVERALLS_PARALLEL: true + COVERALLS_SERVICE_NAME: github + # coveralls needs explicit token + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} + test: + if: github.event_name != 'pull_request' || github.head_ref != 'devel' + strategy: + matrix: + python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] + name: py${{ matrix.python }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: install + run: | + pip install -U tox + mkdir -p "$HOME/bin" + curl -sfL https://coverage.codacy.com/get.sh > "$HOME/bin/codacy" + chmod +x "$HOME/bin/codacy" + echo "$HOME/bin" >> $GITHUB_PATH + - name: tox + run: | + if [[ "$PYVER" == py* ]]; then + tox -e $PYVER # basic:pypy + elif [[ "$PYVER" == *3.9 ]]; then + tox -e py${PYVER/./} # basic + elif [[ "$PYVER" == "3.7" ]]; then + tox -e py${PYVER/./}-tf,py${PYVER/./}-tf-keras # full + else + tox -e py${PYVER/./}-tf-keras # normal + fi + env: + PYVER: ${{ matrix.python }} + COVERALLS_FLAG_NAME: py${{ matrix.python }} + COVERALLS_PARALLEL: true + COVERALLS_SERVICE_NAME: github + # coveralls needs explicit token + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} + finish: + if: github.event_name != 'pull_request' || github.head_ref != 'devel' + name: pytest cov + continue-on-error: ${{ github.event_name != 'push' }} + needs: [test, test-os] + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v2 + - name: Coveralls Finished + run: | + pip install -U coveralls + coveralls --finish || : + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Codacy Finished + run: | + curl -sfL https://coverage.codacy.com/get.sh > codacy + bash codacy final || : + env: + CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} + deploy: + if: github.event_name != 'pull_request' || github.head_ref != 'devel' + needs: [check, test, test-os] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-python@v2 + - name: Install + run: | + sudo apt-get install -yqq pandoc + pip install -r .meta/requirements-build.txt + make build .dockerignore Dockerfile snapcraft.yaml + - id: dist + uses: casperdcl/deploy-pypi@v2 + with: + password: ${{ secrets.PYPI_TOKEN }} + gpg_key: ${{ secrets.GPG_KEY }} + upload: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') }} + - id: collect_assets + name: Collect assets + run: | + if [[ $GITHUB_REF == refs/tags/v* ]]; then + echo ::set-output name=docker_tags::latest,${GITHUB_REF/refs\/tags\/v/} + echo ::set-output name=snap_channel::stable,candidate,edge + elif [[ $GITHUB_REF == refs/heads/master ]]; then + echo ::set-output name=docker_tags::master + echo ::set-output name=snap_channel::candidate,edge + elif [[ $GITHUB_REF == refs/heads/devel ]]; then + echo ::set-output name=docker_tags::devel + echo ::set-output name=snap_channel::edge + fi + git log --pretty='format:%d%n- %s%n%b---' $(git tag --sort=v:refname | tail -n2 | head -n1)..HEAD > _CHANGES.md + - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: tqdm ${{ github.ref }} stable + body_path: _CHANGES.md + draft: true + - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: dist/${{ steps.dist.outputs.whl }} + asset_name: ${{ steps.dist.outputs.whl }} + asset_content_type: application/zip + - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: dist/${{ steps.dist.outputs.whl_asc }} + asset_name: ${{ steps.dist.outputs.whl_asc }} + asset_content_type: text/plain + - uses: snapcore/action-build@v1 + id: snap_build + - if: github.event_name == 'push' && steps.collect_assets.outputs.snap_channel + uses: snapcore/action-publish@v1 + with: + store_login: ${{ secrets.SNAP_TOKEN }} + snap: ${{ steps.snap_build.outputs.snap }} + release: ${{ steps.collect_assets.outputs.snap_channel }} + - name: Docker build push + uses: elgohr/Publish-Docker-Github-Action@master + with: + name: ${{ github.repository }} + tags: ${{ steps.collect_assets.outputs.docker_tags }} + password: ${{ secrets.DOCKER_PWD }} + username: ${{ secrets.DOCKER_USR }} + no_push: ${{ steps.collect_assets.outputs.docker_tags == '' }} + - name: Docker push GitHub + uses: elgohr/Publish-Docker-Github-Action@master + with: + name: ${{ github.repository }}/tqdm + tags: ${{ steps.collect_assets.outputs.docker_tags }} + password: ${{ github.token }} + username: ${{ github.actor }} + registry: docker.pkg.github.com + no_push: ${{ steps.collect_assets.outputs.docker_tags == '' }} diff --git a/.gitignore b/.gitignore index ada6bc846..165e08ef1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,39 +1,25 @@ *.py[cod] - -# C extensions *.so +__pycache__/ +.ipynb_checkpoints/ # Packages -tqdm.egg-info -build/ -dist/ -snapcraft.yaml -tqdm_*_amd64.snap -.dockerignore +/tqdm/_dist_ver.py +/.eggs/ +/*.egg*/ +/build/ +/dist/ +/snapcraft.yaml +/tqdm_*.snap +/.dockerignore +/Dockerfile # Unit test / coverage reports -.tox/ -.coverage -__pycache__ -nosetests.xml - -# Translations -*.mo - -# Mr Developer -.mr.developer.cfg -.project -.pydevproject - -# PyCharm -.idea - -# IPython -.ipynb_checkpoints - -# asv -.asv/ -benchmarks/*.py[co] +/.tox/ +/.coverage* +/coverage.xml +/.pytest_cache/ +/.asv/ # Sumbodules /wiki/ diff --git a/.mailmap b/.mailmap index d0cd3f31c..5a8d09137 100644 --- a/.mailmap +++ b/.mailmap @@ -1,5 +1,9 @@ -Casper da Costa-Luis casperdcl +Casper da Costa-Luis +Casper da Costa-Luis +Casper da Costa-Luis +tqdm[bot] <41898282+github-actions[bot]@users.noreply.github.com> Stephen Larroque +Richard Sheridan Guangshuo Chen Guangshuo CHEN Guangshuo Chen Guangshuo Chen chengs diff --git a/.meta/.readme.rst b/.meta/.readme.rst index e1b8cbb91..d299164b9 100644 --- a/.meta/.readme.rst +++ b/.meta/.readme.rst @@ -7,7 +7,7 @@ tqdm |Build-Status| |Coverage-Status| |Branch-Coverage-Status| |Codacy-Grade| |Libraries-Rank| |PyPI-Downloads| -|DOI| |LICENCE| |OpenHub-Status| |binder-demo| |notebook-demo| |awesome-python| +|LICENCE| |OpenHub-Status| |binder-demo| |awesome-python| ``tqdm`` derives from the Arabic word *taqaddum* (تقدّم) which can mean "progress," and is an abbreviation for "I love you so much" in Spanish (*te quiero demasiado*). @@ -83,11 +83,11 @@ Latest development release on GitHub |GitHub-Status| |GitHub-Stars| |GitHub-Commits| |GitHub-Forks| |GitHub-Updated| -Pull and install in the current directory: +Pull and install pre-release ``devel`` branch: .. code:: sh - pip install -e git+https://github.com/tqdm/tqdm.git@master#egg=tqdm + pip install "git+https://github.com/tqdm/tqdm.git@devel#egg=tqdm" Latest Conda release ~~~~~~~~~~~~~~~~~~~~ @@ -111,7 +111,7 @@ There are 3 channels to choose from: snap install tqdm --candidate # master branch snap install tqdm --edge # devel branch -Note than ``snap`` binaries are purely for CLI use (not ``import``-able), and +Note that ``snap`` binaries are purely for CLI use (not ``import``-able), and automatically set up ``bash`` tab-completion. Latest Docker release @@ -139,9 +139,8 @@ Changelog The list of all changes is available either on GitHub's Releases: |GitHub-Status|, on the -`wiki `__, on the -`website `__, or on crawlers such as -`allmychanges.com `_. +`wiki `__, or on the +`website `__. Usage @@ -216,7 +215,7 @@ Perhaps the most wonderful use of ``tqdm`` is in a script or on the command line. Simply inserting ``tqdm`` (or ``python -m tqdm``) between pipes will pass through all ``stdin`` to ``stdout`` while printing progress to ``stderr``. -The example below demonstrated counting the number of lines in all Python files +The example below demonstrate counting the number of lines in all Python files in the current directory, with timing information included. .. code:: sh @@ -248,7 +247,7 @@ Backing up a large directory? .. code:: sh - tar -zcf - docs/ | tqdm --bytes --total `du -sb docs/ | cut -f1` \ + $ tar -zcf - docs/ | tqdm --bytes --total `du -sb docs/ | cut -f1` \ > backup.tgz 44%|██████████████▊ | 153M/352M [00:14<00:18, 11.0MB/s] @@ -256,8 +255,8 @@ This can be beautified further: .. code:: sh - BYTES="$(du -sb docs/ | cut -f1)" - tar -cf - docs/ \ + $ BYTES="$(du -sb docs/ | cut -f1)" + $ tar -cf - docs/ \ | tqdm --bytes --total "$BYTES" --desc Processing | gzip \ | tqdm --bytes --total "$BYTES" --desc Compressed --position 1 \ > ~/backup.tgz @@ -268,11 +267,21 @@ Or done on a file level using 7-zip: .. code:: sh - 7z a -bd -r backup.7z docs/ | grep Compressing \ + $ 7z a -bd -r backup.7z docs/ | grep Compressing \ | tqdm --total $(find docs/ -type f | wc -l) --unit files \ | grep -v Compressing 100%|██████████████████████████▉| 15327/15327 [01:00<00:00, 712.96files/s] +Pre-existing CLI programs already outputting basic progress information will +benefit from ``tqdm``'s ``--update`` and ``--update_to`` flags: + +.. code:: sh + + $ seq 3 0.1 5 | tqdm --total 5 --update_to --null + 100%|████████████████████████████████████| 5.0/5 [00:00<00:00, 9673.21it/s] + $ seq 10 | tqdm --update --null # 1 + 2 + ... + 10 = 55 iterations + 55it [00:00, 90006.52it/s] + FAQ and Known Issues -------------------- @@ -299,7 +308,7 @@ of a neat one-line progress bar. - Unicode: * Environments which report that they support unicode will have solid smooth - progressbars. The fallback is an ```ascii``-only bar. + progressbars. The fallback is an ``ascii``-only bar. * Windows consoles often only partially support unicode and thus `often require explicit ascii=True `__ (also `here `__). This is due to @@ -321,6 +330,8 @@ of a neat one-line progress bar. - `Hanging pipes in python2 `__: when using ``tqdm`` on the CLI, you may need to use Python 3.5+ for correct buffering. +- `No intermediate output in docker-compose `__: + use ``docker-compose run`` instead of ``docker-compose up`` and ``tty: true``. If you come across any other difficulties, browse and file |GitHub-Issues|. @@ -377,7 +388,7 @@ Returns def set_description(self, desc=None, refresh=True): """{DOC_tqdm.tqdm.set_description}""" - def set_postfix(self, ordered_dict=None, refresh=True, **kwargs): + def set_postfix(self, ordered_dict=None, refresh=True, **tqdm_kwargs): """{DOC_tqdm.tqdm.set_postfix}""" @classmethod @@ -391,29 +402,28 @@ Returns def display(self, msg=None, pos=None): """{DOC_tqdm.tqdm.display}""" - def trange(*args, **kwargs): - """ - A shortcut for tqdm(xrange(*args), **kwargs). - On Python3+ range is used instead of xrange. - """ - - class tqdm.gui.tqdm(tqdm.tqdm): - """Experimental GUI version""" + @classmethod + @contextmanager + def wrapattr(cls, stream, method, total=None, bytes=True, **tqdm_kwargs): + """{DOC_tqdm.tqdm.wrapattr}""" - def tqdm.gui.trange(*args, **kwargs): - """Experimental GUI version of trange""" + @classmethod + def pandas(cls, *targs, **tqdm_kwargs): + """Registers the current `tqdm` class with `pandas`.""" - class tqdm.notebook.tqdm(tqdm.tqdm): - """Experimental IPython/Jupyter Notebook widget""" + def trange(*args, **tqdm_kwargs): + """ + A shortcut for `tqdm(xrange(*args), **tqdm_kwargs)`. + On Python3+, `range` is used instead of `xrange`. + """ - def tqdm.notebook.trange(*args, **kwargs): - """Experimental IPython/Jupyter Notebook widget version of trange""" +Convenience Functions +~~~~~~~~~~~~~~~~~~~~~ - class tqdm.keras.TqdmCallback(keras.callbacks.Callback): - """`keras` callback for epoch and batch progress""" +.. code:: python def tqdm.contrib.tenumerate(iterable, start=0, total=None, - tqdm_class=tqdm.auto.tqdm, **kwargs): + tqdm_class=tqdm.auto.tqdm, **tqdm_kwargs): """Equivalent of `numpy.ndenumerate` or builtin `enumerate`.""" def tqdm.contrib.tzip(iter1, *iter2plus, **tqdm_kwargs): @@ -422,14 +432,52 @@ Returns def tqdm.contrib.tmap(function, *sequences, **tqdm_kwargs): """Equivalent of builtin `map`.""" +Submodules +~~~~~~~~~~ + +.. code:: python + + class tqdm.notebook.tqdm(tqdm.tqdm): + """IPython/Jupyter Notebook widget.""" + + class tqdm.auto.tqdm(tqdm.tqdm): + """Automatically chooses beween `tqdm.notebook` and `tqdm.tqdm`.""" + + class tqdm.asyncio.tqdm(tqdm.tqdm): + """Asynchronous version.""" + @classmethod + def as_completed(cls, fs, *, loop=None, timeout=None, total=None, + **tqdm_kwargs): + """Wrapper for `asyncio.as_completed`.""" + + class tqdm.gui.tqdm(tqdm.tqdm): + """Matplotlib GUI version.""" + + class tqdm.tk.tqdm(tqdm.tqdm): + """Tkinter GUI version.""" + + class tqdm.rich.tqdm(tqdm.tqdm): + """`rich.progress` version.""" + + class tqdm.keras.TqdmCallback(keras.callbacks.Callback): + """Keras callback for epoch and batch progress.""" + + class tqdm.dask.TqdmCallback(dask.callbacks.Callback): + """Dask callback for task progress.""" + + ``contrib`` ------------ ++++++++++++ The ``tqdm.contrib`` package also contains experimental modules: - ``tqdm.contrib.itertools``: Thin wrappers around ``itertools`` - ``tqdm.contrib.concurrent``: Thin wrappers around ``concurrent.futures`` -- ``tqdm.contrib.telegram``: Posts to `Telegram `__ bots +- ``tqdm.contrib.discord``: Posts to `Discord `__ bots +- ``tqdm.contrib.telegram``: Posts to `Telegram `__ bots +- ``tqdm.contrib.bells``: Automagically enables all optional features + + * ``auto``, ``pandas``, ``discord``, ``telegram`` Examples and Advanced Usage --------------------------- @@ -444,7 +492,7 @@ Examples and Advanced Usage on how to make a **great** progressbar; - check out the `slides from PyData London `__, or -- run the |notebook-demo| or |binder-demo|. +- run the |binder-demo|. Description and additional stats ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -546,13 +594,14 @@ available to keep nested bars on their respective lines. For manual control over positioning (e.g. for multi-processing use), you may specify ``position=n`` where ``n=0`` for the outermost bar, ``n=1`` for the next, and so on. -However, it's best to check if `tqdm` can work without manual `position` first. +However, it's best to check if ``tqdm`` can work without manual ``position`` +first. .. code:: python from time import sleep from tqdm import trange, tqdm - from multiprocessing import Pool, freeze_support + from multiprocessing import Pool, RLock, freeze_support L = list(range(9)) @@ -565,6 +614,7 @@ However, it's best to check if `tqdm` can work without manual `position` first. if __name__ == '__main__': freeze_support() # for Windows support + tqdm.set_lock(RLock()) # for managing output contention p = Pool(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),)) p.map(progresser, L) @@ -611,6 +661,7 @@ Here's an example with ``urllib``: import urllib, os from tqdm import tqdm + urllib = getattr(urllib, 'request', urllib) class TqdmUpTo(tqdm): """Provides `update_to(n)` which uses `tqdm.update(delta_n)`.""" @@ -625,7 +676,7 @@ Here's an example with ``urllib``: """ if tsize is not None: self.total = tsize - self.update(b * bsize - self.n) # will also set self.n = b * bsize + return self.update(b * bsize - self.n) # also sets self.n = b * bsize eg_link = "https://caspersci.uk.to/matryoshka.zip" with TqdmUpTo(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, @@ -672,12 +723,14 @@ down to: from tqdm import tqdm eg_link = "https://caspersci.uk.to/matryoshka.zip" + response = getattr(urllib, 'request', urllib).urlopen(eg_link) with tqdm.wrapattr(open(os.devnull, "wb"), "write", - miniters=1, desc=eg_link.split('/')[-1]) as fout: - for chunk in urllib.urlopen(eg_link): + miniters=1, desc=eg_link.split('/')[-1], + total=getattr(response, 'length', None)) as fout: + for chunk in response: fout.write(chunk) -The ``requests`` equivalent is nearly identical, albeit with a ``total``: +The ``requests`` equivalent is nearly identical: .. code:: python @@ -692,6 +745,51 @@ The ``requests`` equivalent is nearly identical, albeit with a ``total``: for chunk in response.iter_content(chunk_size=4096): fout.write(chunk) +**Custom callback** + +``tqdm`` is known for intelligently skipping unnecessary displays. To make a +custom callback take advantage of this, simply use the return value of +``update()``. This is set to ``True`` if a ``display()`` was triggered. + +.. code:: python + + from tqdm.auto import tqdm as std_tqdm + + def external_callback(*args, **kwargs): + ... + + class TqdmExt(std_tqdm): + def update(self, n=1): + displayed = super(TqdmExt, self).update(n): + if displayed: + external_callback(**self.format_dict) + return displayed + +``asyncio`` +~~~~~~~~~~~ + +Note that ``break`` isn't currently caught by asynchronous iterators. +This means that ``tqdm`` cannot clean up after itself in this case: + +.. code:: python + + from tqdm.asyncio import tqdm + + async for i in tqdm(range(9)): + if i == 2: + break + +Instead, either call ``pbar.close()`` manually or use the context manager syntax: + +.. code:: python + + from tqdm.asyncio import tqdm + + with tqdm(range(9)) as pbar: + async for i in pbar: + if i == 2: + break + Pandas Integration ~~~~~~~~~~~~~~~~~~ @@ -734,6 +832,24 @@ A ``keras`` callback is also available: model.fit(..., verbose=0, callbacks=[TqdmCallback()]) +Dask Integration +~~~~~~~~~~~~~~~~ + +A ``dask`` callback is also available: + +.. code:: python + + from tqdm.dask import TqdmCallback + + with TqdmCallback(desc="compute"): + ... + arr.compute() + + # or use callback globally + cb = TqdmCallback(desc="global") + cb.register() + arr.compute() + IPython/Jupyter Integration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -775,8 +891,34 @@ this warning. Note that notebooks will display the bar in the cell where it was created. This may be a different cell from the one where it is used. -If this is not desired, the creation of the bar must be delayed/moved to the -cell where it is desired to be displayed. +If this is not desired, either + +- delay the creation of the bar to the cell where it must be displayed, or +- create the bar with ``display=False``, and in a later cell call + ``display(bar.container)``: + +.. code:: python + + from tqdm.notebook import tqdm + pbar = tqdm(..., display=False) + +.. code:: python + + # different cell + display(pbar.container) + +The ``keras`` callback has a ``display()`` method which can be used likewise: + +.. code:: python + + from tqdm.keras import TqdmCallback + cbk = TqdmCallback(display=False) + +.. code:: python + + # different cell + cbk.display() + model.fit(..., verbose=0, callbacks=[cbk]) Another possibility is to have a single bar (near the top of the notebook) which is constantly re-used (using ``reset()`` rather than ``close()``). @@ -821,10 +963,13 @@ For further customisation, Consider overloading ``display()`` to use e.g. ``self.frontend(**self.format_dict)`` instead of ``self.sp(repr(self))``. -`tqdm/notebook.py `__ -and `tqdm/gui.py `__ -submodules are examples of inheritance which don't (yet) strictly conform to the -above recommendation. +Some submodule examples of inheritance: + +- `tqdm/notebook.py `__ +- `tqdm/gui.py `__ +- `tqdm/tk.py `__ +- `tqdm/contrib/telegram.py `__ +- `tqdm/contrib/discord.py `__ Dynamic Monitor/Meter ~~~~~~~~~~~~~~~~~~~~~ @@ -957,6 +1102,33 @@ A reusable canonical example is given below: # After the `with`, printing is restored print("Done!") +Redirecting ``logging`` +~~~~~~~~~~~~~~~~~~~~~~~ + +Similar to ``sys.stdout``/``sys.stderr`` as detailed above, console ``logging`` +may also be redirected to ``tqdm.write()``. + +Warning: if also redirecting ``sys.stdout``/``sys.stderr``, make sure to +redirect ``logging`` first if needed. + +Helper methods are available in ``tqdm.contrib.logging``. For example: + +.. code:: python + + import logging + from tqdm import trange + from tqdm.contrib.logging import logging_redirect_tqdm + + LOG = logging.getLogger(__name__) + + if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + with logging_redirect_tqdm(): + for i in trange(9): + if i == 4: + LOG.info("console logging redirected to `tqdm.write()`") + # logging restored + Monitoring thread, intervals and miniters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1004,27 +1176,28 @@ file for more information. Developers who have made significant contributions, ranked by *SLoC* (surviving lines of code, -`git fame `__ ``-wMC``), +`git fame `__ ``-wMC --excl '\.(png|gif|jpg)$'``), are: ==================== ======================================================== ==== ================================ Name ID SLoC Notes ==================== ======================================================== ==== ================================ -Casper da Costa-Luis `casperdcl `__ ~75% primary maintainer |Gift-Casper| -Stephen Larroque `lrq3000 `__ ~15% team member +Casper da Costa-Luis `casperdcl `__ ~81% primary maintainer |Gift-Casper| +Stephen Larroque `lrq3000 `__ ~10% team member Martin Zugnoni `martinzugnoni `__ ~3% +Richard Sheridan `richardsheridan `__ ~1% Guangshuo Chen `chengs `__ ~1% -Hadrien Mary `hadim `__ ~1% team member -Matthew Stevens `mjstevens777 `__ ~1% -Noam Yorav-Raphael `noamraph `__ ~1% original author -Kyle Altendorf `altendky `__ ~1% -Ivan Ivanov `obiwanus `__ ~1% -James E. King III `jeking3 `__ ~1% -Mikhail Korobov `kmike `__ ~1% team member +Kyle Altendorf `altendky `__ <1% +Matthew Stevens `mjstevens777 `__ <1% +Hadrien Mary `hadim `__ <1% team member +Ivan Ivanov `obiwanus `__ <1% +Daniel Panteleit `danielpanteleit `__ <1% +Jonas Haag `jonashaag `__ <1% +James E. King III `jeking3 `__ <1% +Noam Yorav-Raphael `noamraph `__ <1% original author +Mikhail Korobov `kmike `__ <1% team member ==================== ======================================================== ==== ================================ -|sourcerer-0| |sourcerer-1| |sourcerer-2| |sourcerer-3| |sourcerer-4| |sourcerer-5| |sourcerer-7| - Ports to Other Languages ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1037,24 +1210,24 @@ LICENCE Open Source (OSI approved): |LICENCE| -Citation information: |DOI| (publication), |DOI-code| (code) +Citation information: |DOI| |README-Hits| (Since 19 May 2016) .. |Logo| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/logo.gif -.. |Screenshot| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/tqdm.gif -.. |Video| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/video.jpg +.. |Screenshot| image:: https://raw.githubusercontent.com/tqdm/img/master/tqdm.gif +.. |Video| image:: https://raw.githubusercontent.com/tqdm/img/master/video.jpg :target: https://tqdm.github.io/video -.. |Slides| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/slides.jpg +.. |Slides| image:: https://raw.githubusercontent.com/tqdm/img/master/slides.jpg :target: https://tqdm.github.io/PyData2019/slides.html -.. |Build-Status| image:: https://img.shields.io/travis/tqdm/tqdm/master.svg?logo=travis - :target: https://travis-ci.org/tqdm/tqdm -.. |Coverage-Status| image:: https://coveralls.io/repos/tqdm/tqdm/badge.svg?branch=master +.. |Build-Status| image:: https://img.shields.io/github/workflow/status/tqdm/tqdm/Test/master?logo=GitHub + :target: https://github.com/tqdm/tqdm/actions?query=workflow%3ATest +.. |Coverage-Status| image:: https://img.shields.io/coveralls/github/tqdm/tqdm/master?logo=coveralls :target: https://coveralls.io/github/tqdm/tqdm .. |Branch-Coverage-Status| image:: https://codecov.io/gh/tqdm/tqdm/branch/master/graph/badge.svg :target: https://codecov.io/gh/tqdm/tqdm -.. |Codacy-Grade| image:: https://api.codacy.com/project/badge/Grade/3f965571598f44549c7818f29cdcf177 - :target: https://www.codacy.com/app/tqdm/tqdm/dashboard +.. |Codacy-Grade| image:: https://app.codacy.com/project/badge/Grade/3f965571598f44549c7818f29cdcf177 + :target: https://www.codacy.com/gh/tqdm/tqdm/dashboard .. |CII Best Practices| image:: https://bestpractices.coreinfrastructure.org/projects/3264/badge :target: https://bestpractices.coreinfrastructure.org/projects/3264 .. |GitHub-Status| image:: https://img.shields.io/github/tag/tqdm/tqdm.svg?maxAge=86400&logo=github&logoColor=white @@ -1074,7 +1247,7 @@ Citation information: |DOI| (publication), |DOI-code| (code) .. |GitHub-Updated| image:: https://img.shields.io/github/last-commit/tqdm/tqdm/master.svg?logo=github&logoColor=white&label=pushed :target: https://github.com/tqdm/tqdm/pulse .. |Gift-Casper| image:: https://img.shields.io/badge/dynamic/json.svg?color=ff69b4&label=gifts%20received&prefix=%C2%A3&query=%24..sum&url=https%3A%2F%2Fcaspersci.uk.to%2Fgifts.json - :target: https://caspersci.uk.to/donate + :target: https://www.cdcl.ml/sponsor .. |Versions| image:: https://img.shields.io/pypi/v/tqdm.svg :target: https://tqdm.github.io/releases .. |PyPI-Downloads| image:: https://img.shields.io/pypi/dm/tqdm.svg?label=pypi%20downloads&logo=PyPI&logoColor=white @@ -1097,32 +1270,12 @@ Citation information: |DOI| (publication), |DOI-code| (code) :target: https://github.com/vinta/awesome-python .. |LICENCE| image:: https://img.shields.io/pypi/l/tqdm.svg :target: https://raw.githubusercontent.com/tqdm/tqdm/master/LICENCE -.. |DOI| image:: https://img.shields.io/badge/DOI-10.21105/joss.01277-green.svg - :target: https://doi.org/10.21105/joss.01277 -.. |DOI-code| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.595120-blue.svg +.. |DOI| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.595120-blue.svg :target: https://doi.org/10.5281/zenodo.595120 -.. |notebook-demo| image:: https://img.shields.io/badge/launch-notebook-orange.svg?logo=jupyter - :target: https://notebooks.ai/demo/gh/tqdm/tqdm .. |binder-demo| image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/tqdm/tqdm/master?filepath=DEMO.ipynb -.. |Screenshot-Jupyter1| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/tqdm-jupyter-1.gif -.. |Screenshot-Jupyter2| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/tqdm-jupyter-2.gif -.. |Screenshot-Jupyter3| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/tqdm-jupyter-3.gif +.. |Screenshot-Jupyter1| image:: https://raw.githubusercontent.com/tqdm/img/master/jupyter-1.gif +.. |Screenshot-Jupyter2| image:: https://raw.githubusercontent.com/tqdm/img/master/jupyter-2.gif +.. |Screenshot-Jupyter3| image:: https://raw.githubusercontent.com/tqdm/img/master/jupyter-3.gif .. |README-Hits| image:: https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&style=social&r=https://github.com/tqdm/tqdm&l=https://caspersci.uk.to/images/tqdm.png&f=https://raw.githubusercontent.com/tqdm/tqdm/master/images/logo.gif :target: https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&a=plot&r=https://github.com/tqdm/tqdm&l=https://caspersci.uk.to/images/tqdm.png&f=https://raw.githubusercontent.com/tqdm/tqdm/master/images/logo.gif&style=social -.. |sourcerer-0| image:: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/0 - :target: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/0 -.. |sourcerer-1| image:: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/1 - :target: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/1 -.. |sourcerer-2| image:: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/2 - :target: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/2 -.. |sourcerer-3| image:: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/3 - :target: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/3 -.. |sourcerer-4| image:: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/4 - :target: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/4 -.. |sourcerer-5| image:: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/5 - :target: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/5 -.. |sourcerer-6| image:: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/6 - :target: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/6 -.. |sourcerer-7| image:: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/7 - :target: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/7 diff --git a/.meta/.tqdm.1.md b/.meta/.tqdm.1.md index bf0720801..f8bfe1003 100644 --- a/.meta/.tqdm.1.md +++ b/.meta/.tqdm.1.md @@ -1,6 +1,6 @@ % TQDM(1) tqdm User Manuals % tqdm developers -% 2015-2020 +% 2015-2021 # NAME diff --git a/.meta/.tqdm.gpg.enc b/.meta/.tqdm.gpg.enc deleted file mode 100644 index 828ab12c1..000000000 Binary files a/.meta/.tqdm.gpg.enc and /dev/null differ diff --git a/.meta/mkcompletion.py b/.meta/mkcompletion.py index 4c42c9370..8c56967b7 100644 --- a/.meta/mkcompletion.py +++ b/.meta/mkcompletion.py @@ -2,18 +2,18 @@ Auto-generate tqdm/completion.sh from docstrings. """ from __future__ import print_function -from io import open as io_open -from os import path + import re import sys +from io import open as io_open +from os import path sys.path.insert(0, path.dirname(path.dirname(__file__))) import tqdm # NOQA import tqdm.cli # NOQA RE_OPT = re.compile(r'(\w+) :', flags=re.M) -RE_OPT_INPUT = re.compile( - r'(\w+) : (?:str|int|float|chr|dict|tuple)', flags=re.M) +RE_OPT_INPUT = re.compile(r'(\w+) : (?:str|int|float|chr|dict|tuple)', flags=re.M) def doc2opt(doc, user_input=True): @@ -32,8 +32,7 @@ def doc2opt(doc, user_input=True): for doc in (tqdm.tqdm.__init__.__doc__, tqdm.cli.CLI_EXTRA_DOC): options.update(doc2opt(doc, user_input=False)) options_input.update(doc2opt(doc, user_input=True)) -options.difference_update( - '--' + i for i in ('name',) + tqdm.cli.UNSUPPORTED_OPTS) +options.difference_update('--' + i for i in ('name',) + tqdm.cli.UNSUPPORTED_OPTS) options_input &= options options_input -= {"--log"} # manually dealt with src_dir = path.abspath(path.dirname(__file__)) @@ -58,9 +57,7 @@ def doc2opt(doc, user_input=True): esac }} complete -F _tqdm tqdm -""".format( - opts=' '.join(sorted(options)), - opts_manual='|'.join(sorted(options_input))) +""".format(opts=' '.join(sorted(options)), opts_manual='|'.join(sorted(options_input))) if __name__ == "__main__": fncompletion = path.join(path.dirname(src_dir), 'tqdm', 'completion.sh') diff --git a/.meta/mkdocs.py b/.meta/mkdocs.py index 1fe62d5bb..4bb1a93dd 100644 --- a/.meta/mkdocs.py +++ b/.meta/mkdocs.py @@ -2,16 +2,16 @@ Auto-generate README.rst from .meta/.readme.rst and docstrings. """ from __future__ import print_function + +import sys from io import open as io_open from os import path -import sys from textwrap import dedent sys.path.insert(0, path.dirname(path.dirname(__file__))) import tqdm # NOQA import tqdm.cli # NOQA - HEAD_ARGS = """ Parameters ---------- diff --git a/.meta/.snapcraft.yml b/.meta/mksnap.py similarity index 66% rename from .meta/.snapcraft.yml rename to .meta/mksnap.py index bb1738049..1b70416f5 100644 --- a/.meta/.snapcraft.yml +++ b/.meta/mksnap.py @@ -1,4 +1,17 @@ -name: tqdm +# -*- encoding: utf-8 -*- +""" +Auto-generate snapcraft.yaml. +""" +import sys +from io import open as io_open +from os import path +from subprocess import check_output # nosec + +sys.path.insert(1, path.dirname(path.dirname(__file__))) +import tqdm # NOQA + +src_dir = path.abspath(path.dirname(__file__)) +snap_yml = r"""name: tqdm summary: A fast, extensible CLI progress bar description: | https://tqdm.github.io @@ -31,19 +44,19 @@ `tqdm` does not require any dependencies, just an environment supporting `carriage return \r` and `line feed \n` control characters. -adopt-info: tqdm grade: stable confinement: strict base: core18 -icon: {icon} +icon: logo.png +version: '{version}' license: MPL-2.0 parts: tqdm: plugin: python - python-version: python3 - source: {source} + python-packages: [disco-py] + source: . source-commit: '{commit}' - parse-info: [setup.py] + build-packages: [git] override-build: | snapcraftctl build cp $SNAPCRAFT_PART_BUILD/tqdm/completion.sh $SNAPCRAFT_PART_INSTALL/ @@ -51,3 +64,10 @@ tqdm: command: bin/tqdm completer: completion.sh +""".format(version=tqdm.__version__, commit=check_output([ + 'git', 'describe', '--always']).decode('U8').strip()) # nosec +fname = path.join(path.dirname(src_dir), 'snapcraft.yaml') + +if __name__ == "__main__": + with io_open(fname, mode='w', encoding='utf-8') as fd: + fd.write(snap_yml.decode('U8') if hasattr(snap_yml, 'decode') else snap_yml) diff --git a/.meta/requirements-build.txt b/.meta/requirements-build.txt new file mode 100644 index 000000000..5e3fc700a --- /dev/null +++ b/.meta/requirements-build.txt @@ -0,0 +1,3 @@ +py-make>=0.1.0 +twine +wheel diff --git a/.meta/requirements-test.txt b/.meta/requirements-test.txt new file mode 100644 index 000000000..05ac6d3fa --- /dev/null +++ b/.meta/requirements-test.txt @@ -0,0 +1,7 @@ +flake8 +pytest +pytest-cov +pytest-timeout +nbval +ipywidgets +# py>=37: pytest-asyncio diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..3bf198231 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,59 @@ +default_language_version: + python: python3 +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.4.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-docstring-first + - id: check-executables-have-shebangs + - id: check-toml + - id: check-merge-conflict + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: mixed-line-ending + - id: sort-simple-yaml + - id: trailing-whitespace + exclude: ^README.rst$ +- repo: local + hooks: + - id: todo + name: Check TODO + language: pygrep + entry: WIP + args: [-i] + types: [text] + exclude: ^(.pre-commit-config.yaml|.github/workflows/test.yml)$ + - id: pytest + name: pytest quick + language: python + entry: pytest + args: ['-qq', '--durations=1', '-k=not slow'] + types: [python] + pass_filenames: false + additional_dependencies: + - numpy + - pandas + - pytest-timeout + - pytest-asyncio +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.4 + hooks: + - id: flake8 + args: ['-j8'] + additional_dependencies: + - flake8-bugbear + - flake8-comprehensions + - flake8-debugger + - flake8-string-format +- repo: https://github.com/PyCQA/isort + rev: 5.7.0 + hooks: + - id: isort +- repo: https://github.com/kynan/nbstripout + rev: 0.3.9 + hooks: + - id: nbstripout + args: ['--keep-count', '--keep-output'] diff --git a/.style.yapf b/.style.yapf deleted file mode 100644 index 145fe561d..000000000 --- a/.style.yapf +++ /dev/null @@ -1,10 +0,0 @@ -[style] -allow_multiline_dictionary_keys=True -allow_multiline_lambdas=True -coalesce_brackets=True -column_limit=80 -each_dict_entry_on_separate_line=False -i18n_comment=NOQA -indent_dictionary_value=True -space_between_ending_comma_and_closing_bracket=False -split_before_named_assigns=False diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1498db381..000000000 --- a/.travis.yml +++ /dev/null @@ -1,227 +0,0 @@ -language: python -env: - global: - - PIP_CACHE_DIR="$HOME/.cache/pip" # unify pip cache location for all platforms -# use cache for big builds like pandas (to minimise build time). -# If issues, clear cache -# https://docs.travis-ci.com/user/caching/#Clearing-Caches -cache: - pip: true - directories: - - $HOME/.cache/pip -before_cache: -- rm -f $HOME/.cache/pip/log/debug.log -notifications: - email: false -# branches: # remove travis double-check on pull requests in main repo -# only: -# - master -# - /^\d\.\d+$/ -stages: -- check -- test -- name: deploy - if: repo = tqdm/tqdm AND NOT type = pull_request -- name: development - if: false -jobs: - allow_failures: - - stage: development - include: - - stage: test - name: py2.6 - python: 2.6 - env: TOXENV=py26 - dist: trusty - - name: py2.7 - python: 2.7 - env: TOXENV=py27 - - name: py3.4 - python: 3.4 - env: TOXENV=py34 - - name: py3.5 - python: 3.5 - env: TOXENV=py35 - - name: py3.6 - python: 3.6 - env: TOXENV=py36 - - name: py3.7 - python: 3.7 - env: TOXENV=py37 - - name: tf-no-keras - python: 3.7 - env: TOXENV=tf-no-keras - - name: pypy2.7 - python: pypy2.7-5.10.0 - env: TOXENV=pypy - - name: pypy3.5 - python: pypy3.5-5.10.0 - env: TOXENV=pypy3 - - stage: development - name: py2.7-win - os: windows - language: shell - env: TOXENV=py27 - before_install: &before_install_win - - | - if [[ "$TOXENV" == "py37" ]]; then - choco install python --version 3.7.4 - export PATH="/c/Python37:/c/Python37/Scripts:$PATH" - else - choco install python2 - export PATH="/c/Python27:/c/Python27/Scripts:$PATH" - fi - - python -m pip install -U pip setuptools wheel - install: &install_win - - python -m pip install tox - - python -m pip install . - script: &script_win - - python -m tox - - name: py3.7-win - os: windows - language: shell - env: TOXENV=py37 - before_install: *before_install_win - install: *install_win - script: *script_win - - name: py2.7-osx - os: osx - language: shell - env: TOXENV=py27 - - name: py3.7-osx - os: osx - osx_image: xcode11.2 # py3.7 - language: shell - env: TOXENV=py37 - - stage: check - name: style - python: 3.7 - env: TOXENV=flake8 - - name: setup - python: 3.7 - env: TOXENV=setup.py - - name: perf - python: 3.7 - env: TOXENV=perf - - stage: deploy - name: PyPI and GitHub - python: 3.7 - addons: - apt: - packages: - - pandoc - install: - script: - - pip install .[dev] - - make build - #- make submodules - #- cd wiki && make && cd .. - - openssl aes-256-cbc -K $encrypted_a6d6301302b7_key - -iv $encrypted_a6d6301302b7_iv -in .meta/.tqdm.gpg.enc -out .tqdm.gpg -d - - gpg --import .tqdm.gpg - - rm .tqdm.gpg - - git log --pretty='format:- %s%n%b---' $(git tag --sort=creatordate | tail -n2 | head -n1)..HEAD > CHANGES.md - deploy: - - provider: script - script: twine upload -s -i tqdm@caspersci.uk.to dist/tqdm-* - skip_cleanup: true - on: - tags: true - - provider: releases - api_key: $GITHUB_TOKEN - file_glob: true - file: dist/tqdm-*.whl* - skip_cleanup: true - draft: true - name: tqdm $TRAVIS_TAG stable - edge: true - tag_name: $TRAVIS_TAG - target_commitish: $TRAVIS_COMMIT - release_notes_file: CHANGES.md - on: - tags: true - - name: docker - python: 3.7 - services: - - docker - install: - script: - - echo "$DOCKER_PWD" | docker login -u $DOCKER_USR --password-stdin - - echo "$GITHUB_TOKEN" | docker login docker.pkg.github.com -u $GITHUB_USR --password-stdin - - make -B docker - - | - if [[ -n "$TRAVIS_TAG" ]]; then - docker tag tqdm/tqdm:latest tqdm/tqdm:${TRAVIS_TAG#v} - docker tag tqdm/tqdm:latest docker.pkg.github.com/tqdm/tqdm/tqdm:${TRAVIS_TAG#v} ; fi - - docker tag tqdm/tqdm:latest tqdm/tqdm:devel - - docker tag tqdm/tqdm:latest docker.pkg.github.com/tqdm/tqdm/tqdm:latest - - docker tag tqdm/tqdm:latest docker.pkg.github.com/tqdm/tqdm/tqdm:devel - deploy: - - provider: script - script: docker push tqdm/tqdm:${TRAVIS_TAG#v} - on: - tags: true - - provider: script - script: 'docker push docker.pkg.github.com/tqdm/tqdm/tqdm:${TRAVIS_TAG#v} || :' - on: - tags: true - - provider: script - script: docker push tqdm/tqdm:latest - - provider: script - script: 'docker push docker.pkg.github.com/tqdm/tqdm/tqdm:latest || :' - - provider: script - script: docker push tqdm/tqdm:devel - on: - branch: devel - - provider: script - script: 'docker push docker.pkg.github.com/tqdm/tqdm/tqdm:devel || :' - on: - branch: devel - - name: snap - python: 3.7 - addons: - snaps: - - name: snapcraft - channel: stable - confinement: classic - - name: lxd - channel: stable - env: - - SNAPCRAFT_IMAGE_INFO: '{"build_url": "$TRAVIS_BUILD_URL"}' - before_install: - - sudo /snap/bin/lxd.migrate -yes - - sudo /snap/bin/lxd waitready - - sudo /snap/bin/lxd init --auto - install: - - make snapcraft.yaml - script: - - sudo snapcraft --use-lxd - after_failure: - - sudo journalctl -u snapd - deploy: - - provider: snap - snap: tqdm*.snap - channel: stable - skip_cleanup: true - on: - tags: true - - provider: snap - snap: tqdm*.snap - channel: candidate - skip_cleanup: true - - provider: snap - snap: tqdm*.snap - channel: edge - skip_cleanup: true - on: - branch: devel -before_install: -# fix a crash with multiprocessing on Travis -# - sudo rm -rf /dev/shm -# - sudo ln -s /run/shm /dev/shm -- git fetch --tags -install: -- pip install tox -- pip install . -script: -- tox diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 319a751f3..381ad0043 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,9 +1,11 @@ # HOW TO CONTRIBUTE TO TQDM +**TL;DR: Skip to [QUICK DEV SUMMARY]** + This file describes how to - contribute changes to the project, and -- upload released to the pypi repository. +- upload released to the PyPI repository. Most of the management commands have been directly placed inside the Makefile: @@ -24,7 +26,7 @@ Contributions to the project are made using the "Fork & Pull" model. The typical steps would be: 1. create an account on [github](https://github.com) -2. fork [tqdm](https://github.com/tqdm/tqdm) +2. fork [`tqdm`](https://github.com/tqdm/tqdm) 3. make a local clone: `git clone https://github.com/your_account/tqdm.git` 4. make changes on the local copy 5. test (see below) and commit changes `git commit -a -m "my message"` @@ -54,7 +56,6 @@ However it would be helpful to bear in mind: + remember, with millions of downloads per month, `tqdm` must be extremely fast and reliable - Any other kind of change may be included in a (possibly new) submodule + submodules are likely single python files under the main [tqdm/](tqdm/) directory - * large submodules requiring a sub-folder should be included in [`MANIFEST.in`](MANIFEST.in) + submodules extending `tqdm.std.tqdm` or any other module (e.g. [`tqdm.notebook.tqdm`](tqdm/notebook.py), [`tqdm.gui.tqdm`](tqdm/gui.py)) + CLI wrapper `tqdm.cli` * if a newly added `tqdm.std.tqdm` option is not supported by the CLI, append to `tqdm.cli.UNSUPPORTED_OPTS` @@ -64,7 +65,7 @@ However it would be helpful to bear in mind: * beta: well-used; commented, perhaps still missing tests * stable: >10 users; commented, 80% coverage - `.meta/` - + A "hidden" folder containing helper utilities not strictly part of `tqdm` distribution itself + + A "hidden" folder containing helper utilities not strictly part of the `tqdm` distribution itself ## TESTING @@ -100,11 +101,11 @@ you can use `MiniConda` to install a minimal setup. You must also make sure that each distribution has an alias to call the Python interpreter: `python27` for Python 2.7's interpreter, `python32` for Python 3.2's, etc. -### Alternative unit tests with Nose +### Alternative unit tests with pytest -Alternatively, use `nose` to run the tests just for the current Python version: +Alternatively, use `pytest` to run the tests just for the current Python version: -- install `nose` and `flake8` +- install test requirements: `[python setup.py] make install_test` - run the following command: ``` @@ -119,34 +120,27 @@ This section is intended for the project's maintainers and describes how to build and upload a new release. Once again, `[python setup.py] make []` will help. Also consider `pip install`ing development utilities: -`-r requirements-dev.txt` or `tqdm[dev]`. +`[python setup.py] make install_build` at a minimum, or a more thorough `conda env create`. ## Pre-commit Hook -It's probably a good idea to add `[python setup.py] make pre-commit` to -`.git/hooks/pre-commit` for convenient local sanity-checking. +It's probably a good idea to use the `pre-commit` (`pip install pre-commit`) helper. +Run `pre-commit install` for convenient local sanity-checking. -## Semantic Versioning -The tqdm repository managers should: +## Semantic Versioning -- regularly bump the version number in the file -[_version.py](https://raw.githubusercontent.com/tqdm/tqdm/master/tqdm/_version.py) -- follow the [Semantic Versioning](https://semver.org/) convention -- take care of this (instead of users) to avoid PR conflicts -solely due to the version file bumping +The `tqdm` repository managers should: -Note: tools can be used to automate this process, such as -[bumpversion](https://github.com/peritus/bumpversion) or -[python-semanticversion](https://github.com/rbarrois/python-semanticversion/). +- follow the [Semantic Versioning](https://semver.org) convention for tagging ## Checking setup.py -To check that the `setup.py` file is compliant with PyPI requirements (e.g. -version number; reStructuredText in `README.rst`) use: +To check that the `setup.py`/`setup.cfg`/`pyproject.toml` file is compliant with PyPI +requirements (e.g. version number; reStructuredText in `README.rst`) use: ``` [python setup.py] make testsetup @@ -208,16 +202,7 @@ git merge --no-ff pr-branch-name [python setup.py] make alltests ``` -### 5 Version - -Modify `tqdm/_version.py` and amend the last (merge) commit: - -``` -git add tqdm/_version.py -git commit --amend # Add "+ bump version" in the commit message -``` - -### 6 Push to master +### 5 Push to master ``` git push origin master @@ -230,22 +215,19 @@ Formally publishing requires additional steps: testing and tagging. ### Test -- ensure that all online CI tests have passed -- check `setup.py` and `MANIFEST.in` - which define the packaging -process and info that will be uploaded to [PyPI](https://pypi.org) - -using `[python setup.py] make installdev` +Ensure that all online CI tests have passed. ### Tag -- ensure the version has been bumped, committed **and** tagged. +- ensure the version has been tagged. The tag format is `v{major}.{minor}.{patch}`, for example: `v4.4.1`. The current commit's tag is used in the version checking process. If the current commit is not tagged appropriately, the version will -display as `v{major}.{minor}.{patch}-{commit_hash}`. +display as `v{major}.{minor}.{patch}.dev{N}+g{commit_hash}`. ### Upload -Travis CI should automatically do this after pushing tags. +GitHub Actions (GHA) CI should automatically do this after pushing tags. Manual instructions are given below in case of failure. Build `tqdm` into a distributable python package: @@ -257,7 +239,7 @@ Build `tqdm` into a distributable python package: This will generate several builds in the `dist/` folder. On non-windows machines the windows `exe` installer may fail to build. This is normal. -Finally, upload everything to pypi. This can be done easily using the +Finally, upload everything to PyPI. This can be done easily using the [twine](https://github.com/pypa/twine) module: ``` @@ -321,47 +303,67 @@ following: Additionally (less maintained), there exists: - A [wiki] which is publicly editable. -- The [gh-pages project](https://tqdm.github.io/tqdm/) which is built from the +- The [gh-pages project] which is built from the [gh-pages branch](https://github.com/tqdm/tqdm/tree/gh-pages), which is - built using [asv](https://github.com/spacetelescope/asv/). -- The [gh-pages root](https://tqdm.github.io/) which is built from a separate + built using [asv](https://github.com/airspeed-velocity/asv). +- The [gh-pages root] which is built from a separate [github.io repo](https://github.com/tqdm/tqdm.github.io). +[gh-pages project]: https://tqdm.github.io/tqdm/ +[gh-pages root]: https://tqdm.github.io/ + + +## Helper Bots + +There are some helpers in +[.github/workflows](https://github.com/tqdm/tqdm/tree/master/.github/workflows) +to assist with maintenance. + +- Comment Bot + + allows maintainers to write `/tag vM.m.p commit_hash` in an issue/PR to create a tag +- Post Release + + automatically updates the [wiki] + + automatically updates the [gh-pages root] +- Benchmark + + automatically updates the [gh-pages project] + ## QUICK DEV SUMMARY -For experienced devs, once happy with local master: - -1. bump version in `tqdm/_version.py` -2. test (`[python setup.py] make alltests`) -3. `git commit [--amend] # -m "bump version"` -4. `git push` -5. wait for tests to pass - a) in case of failure, fix and go back to (2) -6. `git tag vM.m.p && git push --tags` -7. **`[AUTO:TravisCI]`** `[python setup.py] make distclean` -8. **`[AUTO:TravisCI]`** `[python setup.py] make build` -9. **`[AUTO:TravisCI]`** upload to PyPI. either: +For experienced devs, once happy with local master, follow the steps below. +Much is automated so really it's steps 1-5, then 11(a). + +1. test (`[python setup.py] make alltests` or rely on `pre-commit`) +2. `git commit [--amend] # -m "bump version"` +3. `git push` +4. wait for tests to pass + a) in case of failure, fix and go back to (1) +5. `git tag vM.m.p && git push --tags` or comment `/tag vM.m.p commit_hash` +6. **`[AUTO:GHA]`** `[python setup.py] make distclean` +7. **`[AUTO:GHA]`** `[python setup.py] make build` +8. **`[AUTO:GHA]`** upload to PyPI. either: a) `[python setup.py] make pypi`, or b) `twine upload -s -i $(git config user.signingkey) dist/tqdm-*` -10. **`[AUTO:TravisCI]`** upload to docker hub: +9. **`[AUTO:GHA]`** upload to docker hub: a) `make -B docker` b) `docker push tqdm/tqdm:latest` c) `docker push tqdm/tqdm:$(docker run -i --rm tqdm/tqdm -v)` -11. **`[AUTO:TravisCI]`** upload to snapcraft: +10. **`[AUTO:GHA]`** upload to snapcraft: a) `make snap`, and b) `snapcraft push tqdm*.snap --release stable` -12. Wait for travis to draft a new release on - a) add helpful release notes - b) **`[AUTO:TravisCI]`** attach `dist/tqdm-*` binaries +11. Wait for GHA to draft a new release on + a) replace the commit history with helpful release notes, and click publish + b) **`[AUTO:GHA]`** attach `dist/tqdm-*` binaries (usually only `*.whl*`) -13. **`[SUB]`** run `make` in the `wiki` submodule to update release notes -14. **`[SUB]`** run `make deploy` in the `docs` submodule to update website -15. **`[SUB]`** accept the automated PR in the `feedstock` submodule to update conda +12. **`[SUB][AUTO:GHA-rel]`** run `make` in the `wiki` submodule to update release notes +13. **`[SUB][AUTO:GHA-rel]`** run `make deploy` in the `docs` submodule to update website +14. **`[SUB][AUTO:GHA-rel]`** accept the automated PR in the `feedstock` submodule to update conda +15. **`[AUTO:GHA-rel]`** update the [gh-pages project] benchmarks + a) `[python setup.py] make testasvfull` + b) `asv gh-pages` Key: -- **`[AUTO:TravisCI]`**: Travis CI should automatically do this after - `git push --tags` (6) -- **`[SUB]`**: Requires one-time `make submodules` to clone - `docs`, `wiki`, and `feedstock` +- **`[AUTO:GHA]`**: GitHub Actions CI should automatically do this after `git push --tags` (5) +- **`[AUTO:GHA-rel]`**: GitHub Actions CI should automatically do this after release (11a) +- **`[SUB]`**: Requires one-time `make submodules` to clone `docs`, `wiki`, and `feedstock` diff --git a/DEMO.ipynb b/DEMO.ipynb index 67f48299f..82670daf2 100644 --- a/DEMO.ipynb +++ b/DEMO.ipynb @@ -10,11 +10,11 @@ "[![Py-Versions](https://img.shields.io/pypi/pyversions/tqdm.svg?logo=python&logoColor=white)](https://pypi.org/project/tqdm)|[![Versions](https://img.shields.io/pypi/v/tqdm.svg)](https://tqdm.github.io/releases)|[![Conda-Forge-Status](https://img.shields.io/conda/v/conda-forge/tqdm.svg?label=conda-forge&logo=conda-forge)](https://anaconda.org/conda-forge/tqdm)|[![Docker](https://img.shields.io/badge/docker-pull-blue.svg?logo=docker&logoColor=white)](https://hub.docker.com/r/tqdm/tqdm)|[![Snapcraft](https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft)](https://snapcraft.io/tqdm)\n", "-|-|-|-|-\n", "\n", - "[![Build-Status](https://img.shields.io/travis/tqdm/tqdm/master.svg?logo=travis)](https://travis-ci.org/tqdm/tqdm)|[![Coverage-Status](https://coveralls.io/repos/tqdm/tqdm/badge.svg?branch=master)](https://coveralls.io/github/tqdm/tqdm)|[![Branch-Coverage-Status](https://codecov.io/gh/tqdm/tqdm/branch/master/graph/badge.svg)](https://codecov.io/gh/tqdm/tqdm)|[![Codacy-Grade](https://api.codacy.com/project/badge/Grade/3f965571598f44549c7818f29cdcf177)](https://www.codacy.com/app/tqdm/tqdm/dashboard)|[![Libraries-Rank](https://img.shields.io/librariesio/sourcerank/pypi/tqdm.svg?logo=koding&logoColor=white)](https://libraries.io/pypi/tqdm)|[![PyPI-Downloads](https://img.shields.io/pypi/dm/tqdm.svg?label=pypi%20downloads&logo=PyPI&logoColor=white)](https://pypi.org/project/tqdm)\n", + "[![Build-Status](https://img.shields.io/github/workflow/status/tqdm/tqdm/Test/master?logo=GitHub)](https://github.com/tqdm/tqdm/actions?query=workflow%3ATest)|[![Coverage-Status](https://img.shields.io/coveralls/github/tqdm/tqdm/master?logo=coveralls)](https://coveralls.io/github/tqdm/tqdm)|[![Branch-Coverage-Status](https://codecov.io/gh/tqdm/tqdm/branch/master/graph/badge.svg)](https://codecov.io/gh/tqdm/tqdm)|[![Codacy-Grade](https://app.codacy.com/project/badge/Grade/3f965571598f44549c7818f29cdcf177)](https://www.codacy.com/gh/tqdm/tqdm/dashboard)|[![Libraries-Rank](https://img.shields.io/librariesio/sourcerank/pypi/tqdm.svg?logo=koding&logoColor=white)](https://libraries.io/pypi/tqdm)|[![PyPI-Downloads](https://img.shields.io/pypi/dm/tqdm.svg?label=pypi%20downloads&logo=PyPI&logoColor=white)](https://pypi.org/project/tqdm)\n", "-|-|-|-|-|-\n", "\n", - "[![DOI](https://img.shields.io/badge/DOI-10.21105/joss.01277-green.svg)](https://doi.org/10.21105/joss.01277)|[![LICENCE](https://img.shields.io/pypi/l/tqdm.svg)](https://raw.githubusercontent.com/tqdm/tqdm/master/LICENCE)|[![OpenHub-Status](https://www.openhub.net/p/tqdm/widgets/project_thin_badge?format=gif)](https://www.openhub.net/p/tqdm?ref=Thin+badge)|[![binder-demo](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/tqdm/tqdm/master?filepath=DEMO.ipynb)|[![notebook-demo](https://img.shields.io/badge/launch-notebook-orange.svg?logo=jupyter)](https://notebooks.ai/demo/gh/tqdm/tqdm)|[![awesome-python](https://awesome.re/mentioned-badge.svg)](https://github.com/vinta/awesome-python)\n", - "-|-|-|-|-|-\n", + "[![DOI](https://img.shields.io/badge/DOI-10.5281/zenodo.595120-blue.svg)](https://doi.org/10.5281/zenodo.595120)|[![LICENCE](https://img.shields.io/pypi/l/tqdm.svg)](https://raw.githubusercontent.com/tqdm/tqdm/master/LICENCE)|[![OpenHub-Status](https://www.openhub.net/p/tqdm/widgets/project_thin_badge?format=gif)](https://www.openhub.net/p/tqdm?ref=Thin+badge)|[![binder-demo](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/tqdm/tqdm/master?filepath=DEMO.ipynb)|[![awesome-python](https://awesome.re/mentioned-badge.svg)](https://github.com/vinta/awesome-python)\n", + "-|-|-|-|-\n", "\n", "`tqdm` derives from the Arabic word *taqaddum* (تقدّم) which can mean\n", "\"progress,\" and is an abbreviation for \"I love you so much\" in Spanish\n", @@ -74,7 +74,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "![Screenshot](https://raw.githubusercontent.com/tqdm/tqdm/master/images/tqdm.gif)|[![Video](https://raw.githubusercontent.com/tqdm/tqdm/master/images/video.jpg)](https://tqdm.github.io/video) [![Slides](https://raw.githubusercontent.com/tqdm/tqdm/master/images/slides.jpg)](https://tqdm.github.io/PyData2019/slides.html)\n", + "![Screenshot](https://raw.githubusercontent.com/tqdm/img/master/tqdm.gif)|[![Video](https://raw.githubusercontent.com/tqdm/img/master/video.jpg)](https://tqdm.github.io/video) [![Slides](https://raw.githubusercontent.com/tqdm/img/master/slides.jpg)](https://tqdm.github.io/PyData2019/slides.html)\n", "-|-\n", "\n", "It can also be executed as a module with pipes:" @@ -428,7 +428,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "\n", " Extra CLI Options\n", " -----------------\n", " delim : chr, optional\n", @@ -440,13 +439,24 @@ " bytes : bool, optional\n", " If true, will count bytes, ignore `delim`, and default\n", " `unit_scale` to True, `unit_divisor` to 1024, and `unit` to 'B'.\n", + " tee : bool, optional\n", + " If true, passes `stdin` to both `stderr` and `stdout`.\n", + " update : bool, optional\n", + " If true, will treat input as newly elapsed iterations,\n", + " i.e. numbers to pass to `update()`. Note that this is slow\n", + " (~2e5 it/s) since every input must be decoded as a number.\n", + " update_to : bool, optional\n", + " If true, will treat input as total elapsed iterations,\n", + " i.e. numbers to assign to `self.n`. Note that this is slow\n", + " (~2e5 it/s) since every input must be decoded as a number.\n", + " null : bool, optional\n", + " If true, will discard input (no stdout).\n", " manpath : str, optional\n", " Directory in which to install tqdm man pages.\n", " comppath : str, optional\n", " Directory in which to place tqdm completion.\n", " log : str, optional\n", - " CRITICAL|FATAL|ERROR|WARN(ING)|[default: 'INFO']|DEBUG|NOTSET.\n", - "\n" + " CRITICAL|FATAL|ERROR|WARN(ING)|[default: 'INFO']|DEBUG|NOTSET.\n" ] } ], @@ -533,7 +543,15 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "00:00 in total: 44%|0000. | 4/9 [00:00<00:00, 29799.67it/s]\n" + ] + } + ], "source": [ "from tqdm import tqdm\n", "from time import sleep\n", @@ -612,7 +630,7 @@ "```python\n", "from time import sleep\n", "from tqdm import trange, tqdm\n", - "from multiprocessing import Pool, freeze_support\n", + "from multiprocessing import Pool, RLock, freeze_support\n", "\n", "L = list(range(9))\n", "\n", @@ -625,6 +643,7 @@ "\n", "if __name__ == '__main__':\n", " freeze_support() # for Windows support\n", + " tqdm.set_lock(RLock()) # for managing output contention\n", " p = Pool(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),))\n", " p.map(progresser, L)\n", "```\n", @@ -689,6 +708,7 @@ "source": [ "import urllib, os\n", "from tqdm import tqdm\n", + "urllib = getattr(urllib, 'request', urllib)\n", "\n", "class TqdmUpTo(tqdm):\n", " \"\"\"Provides `update_to(n)` which uses `tqdm.update(delta_n)`.\"\"\"\n", @@ -703,7 +723,7 @@ " \"\"\"\n", " if tsize is not None:\n", " self.total = tsize\n", - " self.update(b * bsize - self.n) # will also set self.n = b * bsize\n", + " return self.update(b * bsize - self.n) # also sets self.n = b * bsize\n", "\n", "eg_link = \"https://caspersci.uk.to/matryoshka.zip\"\n", "with TqdmUpTo(unit='B', unit_scale=True, unit_divisor=1024, miniters=1,\n", @@ -759,7 +779,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "matryoshka.zip: 254kB [00:00, 334kB/s] \n" + "matryoshka.zip: 100%|██████████| 254k/254k [00:00<00:00, 602kB/s] \n" ] } ], @@ -768,9 +788,11 @@ "from tqdm import tqdm\n", "\n", "eg_link = \"https://caspersci.uk.to/matryoshka.zip\"\n", + "response = getattr(urllib, 'request', urllib).urlopen(eg_link)\n", "with tqdm.wrapattr(open(os.devnull, \"wb\"), \"write\",\n", - " miniters=1, desc=eg_link.split('/')[-1]) as fout:\n", - " for chunk in urllib.urlopen(eg_link):\n", + " miniters=1, desc=eg_link.split('/')[-1],\n", + " total=getattr(response, 'length', None)) as fout:\n", + " for chunk in response:\n", " fout.write(chunk)" ] }, @@ -778,7 +800,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The `requests` equivalent is nearly identical, albeit with a `total`:" + "The `requests` equivalent is nearly identical:" ] }, { @@ -1552,7 +1574,7 @@ "bars and colour hints (blue: normal, green: completed, red:\n", "error/interrupt, light blue: no ETA); as demonstrated below.\n", "\n", - "![Screenshot-Jupyter3](https://raw.githubusercontent.com/tqdm/tqdm/master/images/tqdm-jupyter-3.gif)\n", + "![Screenshot-Jupyter3](https://raw.githubusercontent.com/tqdm/img/master/jupyter-3.gif)\n", "\n", "The `notebook` version supports percentage or pixels for overall width\n", "(e.g.: `ncols='100%'` or `ncols='480px'`).\n", @@ -1658,11 +1680,6 @@ "specify any file-like object using the `file` argument. For example,\n", "this can be used to redirect the messages writing to a log file or class.\n", "\n", - "---\n", - "\n", - "[![sourcerer-0](https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/0)](https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/0)|[![sourcerer-1](https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/1)](https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/1)|[![sourcerer-2](https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/2)](https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/2)|[![sourcerer-3](https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/3)](https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/3)|[![sourcerer-4](https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/4)](https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/4)|[![sourcerer-5](https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/5)](https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/5)|[![sourcerer-7](https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/7)](https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/7)\n", - "-|-|-|-|-|-|-\n", - "\n", "[![README-Hits](https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&style=social&r=https://github.com/tqdm/tqdm&l=https://caspersci.uk.to/images/tqdm.png&f=https://raw.githubusercontent.com/tqdm/tqdm/master/images/logo.gif)](https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&a=plot&r=https://github.com/tqdm/tqdm&l=https://caspersci.uk.to/images/tqdm.png&f=https://raw.githubusercontent.com/tqdm/tqdm/master/images/logo.gif&style=social)|(Since 19 May 2016)\n", "-|-" ] @@ -1691,21 +1708,18 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 2", + "display_name": "Python 3", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { - "name": "ipython", - "version": 2 + "name": "ipython" }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.15" + "nbconvert_exporter": "python" } }, "nbformat": 4, diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index d7e651564..000000000 --- a/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM python:3.7-alpine -COPY setup.py tqdm/ -COPY requirements-dev.txt tqdm/ -COPY README.rst tqdm/ -COPY tqdm tqdm/tqdm -RUN pip install -U ./tqdm -ENTRYPOINT ["tqdm"] diff --git a/LICENCE b/LICENCE index 3a6c785de..35ff98640 100644 --- a/LICENCE +++ b/LICENCE @@ -7,7 +7,7 @@ Exceptions or notable authors are listed below in reverse chronological order: * files: * - MPLv2.0 2015-2020 (c) Casper da Costa-Luis + MPLv2.0 2015-2021 (c) Casper da Costa-Luis [casperdcl](https://github.com/casperdcl). * files: tqdm/_tqdm.py MIT 2016 (c) [PR #96] on behalf of Google Inc. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index a165cbc50..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,22 +0,0 @@ -# Misc -include .coveragerc -include CONTRIBUTING.md -include LICENCE -include logo.png -# include images/logo.gif -include Makefile -include tox.ini - -# Non-std submodules -recursive-include tqdm/contrib *.py - -# Test suite -recursive-include tqdm/tests *.py -include requirements-dev.txt - -# Examples/Documentation -recursive-include examples *.py -include README.rst -include tqdm/tqdm.1 -include tqdm/completion.sh -include DEMO.ipynb diff --git a/Makefile b/Makefile index 459d03039..10ee2e2ac 100644 --- a/Makefile +++ b/Makefile @@ -8,18 +8,18 @@ all flake8 test - testnose + pytest testsetup + testnb testcoverage testperf testtimer distclean coverclean - pre-commit prebuildclean clean toxclean - installdev + install_dev install build buildupload @@ -50,8 +50,8 @@ test: TOX_SKIP_ENV=perf tox --skip-missing-interpreters -p all tox -e perf -testnose: - nosetests tqdm -d -v +pytest: + pytest testsetup: @make README.rst @@ -60,16 +60,20 @@ testsetup: python setup.py check --metadata --restructuredtext --strict python setup.py make none +testnb: + pytest tests_notebook.ipynb --nbval --current-env -W=ignore --sanitize-with=setup.cfg --cov=tqdm.notebook --cov-report=term + testcoverage: @make coverclean - nosetests tqdm --with-coverage --cover-package=tqdm --cover-erase --cover-min-percentage=80 --ignore-files="tests_perf\.py" -d -v + pytest tests_notebook.ipynb --cov=tqdm --cov-report= --nbval --current-env --sanitize-with=setup.cfg -W=ignore + pytest -k "not perf" --cov=tqdm --cov-report=xml --cov-report=term --cov-append --cov-fail-under=80 testperf: # do not use coverage (which is extremely slow) - nosetests tqdm/tests/tests_perf.py -d -v + pytest -k perf testtimer: - nosetests tqdm --with-timer -d -v + pytest # another performance test, to check evolution across commits testasv: @@ -79,7 +83,7 @@ testasv: testasvfull: # Test all the commits since the beginning (full test) - asv run -j 8 v1.0.0..master + asv run --skip-existing-commits -j 8 v1.0.0..HEAD @make testasv viewasv: @@ -91,6 +95,7 @@ tqdm/tqdm.1: .meta/.tqdm.1.md tqdm/cli.py tqdm/std.py python -m tqdm --help | tail -n+5 |\ sed -r -e 's/\\/\\\\/g' \ -e 's/^ (--.*)=<(.*)> : (.*)$$/\n\\\1=*\2*\n: \3./' \ + -e 's/^ (--.*) : (.*)$$/\n\\\1\n: \2./' \ -e 's/ (-.*, )(--.*) /\n\1\\\2\n: /' |\ cat "$<" - |\ pandoc -o "$@" -s -t man @@ -101,48 +106,44 @@ tqdm/completion.sh: .meta/mkcompletion.py tqdm/std.py tqdm/cli.py README.rst: .meta/.readme.rst tqdm/std.py tqdm/cli.py @python .meta/mkdocs.py -snapcraft.yaml: .meta/.snapcraft.yml - cat "$<" | sed -e 's/{version}/'"`python -m tqdm --version`"'/g' \ - -e 's/{commit}/'"`git describe --always`"'/g' \ - -e 's/{source}/./g' -e 's/{icon}/logo.png/g' \ - -e 's/{description}/https:\/\/tqdm.github.io/g' > "$@" +snapcraft.yaml: .meta/mksnap.py + @python .meta/mksnap.py + +.dockerignore: + @+python -c "fd=open('.dockerignore', 'w'); fd.write('*\n!dist/*.whl\n')" -.dockerignore: .gitignore - cat $^ > "$@" - echo ".git" > "$@" - git clean -xdn | sed -nr 's/^Would remove (.*)$$/\1/p' >> "$@" +Dockerfile: + @+python -c 'fd=open("Dockerfile", "w"); fd.write("FROM python:3.8-alpine\nCOPY dist/*.whl .\nRUN pip install -U $$(ls ./*.whl) && rm ./*.whl\nENTRYPOINT [\"tqdm\"]\n")' distclean: @+make coverclean @+make prebuildclean @+make clean -pre-commit: - # quick sanity checks - @make testsetup - flake8 -j 8 --count --statistics tqdm/ examples/ - nosetests tqdm --ignore-files="tests_(perf|keras)\.py" -e "pandas|monitoring" -d prebuildclean: @+python -c "import shutil; shutil.rmtree('build', True)" @+python -c "import shutil; shutil.rmtree('dist', True)" @+python -c "import shutil; shutil.rmtree('tqdm.egg-info', True)" + @+python -c "import shutil; shutil.rmtree('.eggs', True)" + @+python -c "import os; os.remove('tqdm/_dist_ver.py') if os.path.exists('tqdm/_dist_ver.py') else None" coverclean: @+python -c "import os; os.remove('.coverage') if os.path.exists('.coverage') else None" + @+python -c "import os, glob; [os.remove(i) for i in glob.glob('.coverage.*')]" + @+python -c "import shutil; shutil.rmtree('tests/__pycache__', True)" + @+python -c "import shutil; shutil.rmtree('benchmarks/__pycache__', True)" @+python -c "import shutil; shutil.rmtree('tqdm/__pycache__', True)" @+python -c "import shutil; shutil.rmtree('tqdm/contrib/__pycache__', True)" - @+python -c "import shutil; shutil.rmtree('tqdm/tests/__pycache__', True)" + @+python -c "import shutil; shutil.rmtree('tqdm/examples/__pycache__', True)" clean: @+python -c "import os, glob; [os.remove(i) for i in glob.glob('*.py[co]')]" + @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tests/*.py[co]')]" + @+python -c "import os, glob; [os.remove(i) for i in glob.glob('benchmarks/*.py[co]')]" @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tqdm/*.py[co]')]" @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tqdm/contrib/*.py[co]')]" - @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tqdm/tests/*.py[co]')]" @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tqdm/examples/*.py[co]')]" toxclean: @+python -c "import shutil; shutil.rmtree('.tox', True)" -installdev: - python setup.py develop --uninstall - python setup.py develop submodules: git clone git@github.com:tqdm/tqdm.wiki wiki git clone git@github.com:tqdm/tqdm.github.io docs @@ -151,6 +152,13 @@ submodules: install: python setup.py install +install_dev: + python setup.py develop --uninstall + python setup.py develop +install_build: + python -m pip install -r .meta/requirements-dev.txt +install_test: + python -m pip install -r .meta/requirements-test.txt build: @make prebuildclean @@ -169,9 +177,9 @@ snap: @make -B snapcraft.yaml snapcraft docker: + @make build @make .dockerignore - @make coverclean - @make clean + @make Dockerfile docker build . -t tqdm/tqdm docker tag tqdm/tqdm:latest tqdm/tqdm:$(shell docker run -i --rm tqdm/tqdm -v) none: diff --git a/README.rst b/README.rst index f66df7265..c906e60cf 100644 --- a/README.rst +++ b/README.rst @@ -7,7 +7,7 @@ tqdm |Build-Status| |Coverage-Status| |Branch-Coverage-Status| |Codacy-Grade| |Libraries-Rank| |PyPI-Downloads| -|DOI| |LICENCE| |OpenHub-Status| |binder-demo| |notebook-demo| |awesome-python| +|LICENCE| |OpenHub-Status| |binder-demo| |awesome-python| ``tqdm`` derives from the Arabic word *taqaddum* (تقدّم) which can mean "progress," and is an abbreviation for "I love you so much" in Spanish (*te quiero demasiado*). @@ -83,11 +83,11 @@ Latest development release on GitHub |GitHub-Status| |GitHub-Stars| |GitHub-Commits| |GitHub-Forks| |GitHub-Updated| -Pull and install in the current directory: +Pull and install pre-release ``devel`` branch: .. code:: sh - pip install -e git+https://github.com/tqdm/tqdm.git@master#egg=tqdm + pip install "git+https://github.com/tqdm/tqdm.git@devel#egg=tqdm" Latest Conda release ~~~~~~~~~~~~~~~~~~~~ @@ -111,7 +111,7 @@ There are 3 channels to choose from: snap install tqdm --candidate # master branch snap install tqdm --edge # devel branch -Note than ``snap`` binaries are purely for CLI use (not ``import``-able), and +Note that ``snap`` binaries are purely for CLI use (not ``import``-able), and automatically set up ``bash`` tab-completion. Latest Docker release @@ -139,9 +139,8 @@ Changelog The list of all changes is available either on GitHub's Releases: |GitHub-Status|, on the -`wiki `__, on the -`website `__, or on crawlers such as -`allmychanges.com `_. +`wiki `__, or on the +`website `__. Usage @@ -216,7 +215,7 @@ Perhaps the most wonderful use of ``tqdm`` is in a script or on the command line. Simply inserting ``tqdm`` (or ``python -m tqdm``) between pipes will pass through all ``stdin`` to ``stdout`` while printing progress to ``stderr``. -The example below demonstrated counting the number of lines in all Python files +The example below demonstrate counting the number of lines in all Python files in the current directory, with timing information included. .. code:: sh @@ -248,7 +247,7 @@ Backing up a large directory? .. code:: sh - tar -zcf - docs/ | tqdm --bytes --total `du -sb docs/ | cut -f1` \ + $ tar -zcf - docs/ | tqdm --bytes --total `du -sb docs/ | cut -f1` \ > backup.tgz 44%|██████████████▊ | 153M/352M [00:14<00:18, 11.0MB/s] @@ -256,8 +255,8 @@ This can be beautified further: .. code:: sh - BYTES="$(du -sb docs/ | cut -f1)" - tar -cf - docs/ \ + $ BYTES="$(du -sb docs/ | cut -f1)" + $ tar -cf - docs/ \ | tqdm --bytes --total "$BYTES" --desc Processing | gzip \ | tqdm --bytes --total "$BYTES" --desc Compressed --position 1 \ > ~/backup.tgz @@ -268,11 +267,21 @@ Or done on a file level using 7-zip: .. code:: sh - 7z a -bd -r backup.7z docs/ | grep Compressing \ + $ 7z a -bd -r backup.7z docs/ | grep Compressing \ | tqdm --total $(find docs/ -type f | wc -l) --unit files \ | grep -v Compressing 100%|██████████████████████████▉| 15327/15327 [01:00<00:00, 712.96files/s] +Pre-existing CLI programs already outputting basic progress information will +benefit from ``tqdm``'s ``--update`` and ``--update_to`` flags: + +.. code:: sh + + $ seq 3 0.1 5 | tqdm --total 5 --update_to --null + 100%|████████████████████████████████████| 5.0/5 [00:00<00:00, 9673.21it/s] + $ seq 10 | tqdm --update --null # 1 + 2 + ... + 10 = 55 iterations + 55it [00:00, 90006.52it/s] + FAQ and Known Issues -------------------- @@ -299,7 +308,7 @@ of a neat one-line progress bar. - Unicode: * Environments which report that they support unicode will have solid smooth - progressbars. The fallback is an ```ascii``-only bar. + progressbars. The fallback is an ``ascii``-only bar. * Windows consoles often only partially support unicode and thus `often require explicit ascii=True `__ (also `here `__). This is due to @@ -321,6 +330,8 @@ of a neat one-line progress bar. - `Hanging pipes in python2 `__: when using ``tqdm`` on the CLI, you may need to use Python 3.5+ for correct buffering. +- `No intermediate output in docker-compose `__: + use ``docker-compose run`` instead of ``docker-compose up`` and ``tty: true``. If you come across any other difficulties, browse and file |GitHub-Issues|. @@ -422,7 +433,7 @@ Parameters percentage, elapsed, elapsed_s, ncols, nrows, desc, unit, rate, rate_fmt, rate_noinv, rate_noinv_fmt, rate_inv, rate_inv_fmt, postfix, unit_divisor, - remaining, remaining_s. + remaining, remaining_s, eta. Note that a trailing ": " is automatically removed after {desc} if the latter is empty. * initial : int or float, optional @@ -449,6 +460,10 @@ Parameters The screen height. If specified, hides nested bars outside this bound. If unspecified, attempts to use environment height. The fallback is 20. +* colour : str, optional + Bar colour (e.g. 'green', '#00ff00'). +* delay : float, optional + Don't display until [default: 0] seconds have elapsed. Extra CLI Options ~~~~~~~~~~~~~~~~~ @@ -462,6 +477,18 @@ Extra CLI Options * bytes : bool, optional If true, will count bytes, ignore ``delim``, and default ``unit_scale`` to True, ``unit_divisor`` to 1024, and ``unit`` to 'B'. +* tee : bool, optional + If true, passes ``stdin`` to both ``stderr`` and ``stdout``. +* update : bool, optional + If true, will treat input as newly elapsed iterations, + i.e. numbers to pass to ``update()``. Note that this is slow + (~2e5 it/s) since every input must be decoded as a number. +* update_to : bool, optional + If true, will treat input as total elapsed iterations, + i.e. numbers to assign to ``self.n``. Note that this is slow + (~2e5 it/s) since every input must be decoded as a number. +* null : bool, optional + If true, will discard input (no stdout). * manpath : str, optional Directory in which to install tqdm man pages. * comppath : str, optional @@ -497,6 +524,11 @@ Returns Increment to add to the internal counter of iterations [default: 1]. If using float, consider specifying ``{n:.3f}`` or similar in ``bar_format``, or specifying ``unit_scale``. + + Returns + ------- + out : bool or None + True if a ``display()`` was triggered. """ def close(self): @@ -544,7 +576,7 @@ Returns Forces refresh [default: True]. """ - def set_postfix(self, ordered_dict=None, refresh=True, **kwargs): + def set_postfix(self, ordered_dict=None, refresh=True, **tqdm_kwargs): """ Set/modify postfix (additional stats) with automatic formatting based on datatype. @@ -579,29 +611,38 @@ Returns (default: ``abs(self.pos)``). """ - def trange(*args, **kwargs): - """ - A shortcut for tqdm(xrange(*args), **kwargs). - On Python3+ range is used instead of xrange. - """ - - class tqdm.gui.tqdm(tqdm.tqdm): - """Experimental GUI version""" + @classmethod + @contextmanager + def wrapattr(cls, stream, method, total=None, bytes=True, **tqdm_kwargs): + """ + stream : file-like object. + method : str, "read" or "write". The result of ``read()`` and + the first argument of ``write()`` should have a ``len()``. + + >>> with tqdm.wrapattr(file_obj, "read", total=file_obj.size) as fobj: + ... while True: + ... chunk = fobj.read(chunk_size) + ... if not chunk: + ... break + """ - def tqdm.gui.trange(*args, **kwargs): - """Experimental GUI version of trange""" + @classmethod + def pandas(cls, *targs, **tqdm_kwargs): + """Registers the current `tqdm` class with `pandas`.""" - class tqdm.notebook.tqdm(tqdm.tqdm): - """Experimental IPython/Jupyter Notebook widget""" + def trange(*args, **tqdm_kwargs): + """ + A shortcut for `tqdm(xrange(*args), **tqdm_kwargs)`. + On Python3+, `range` is used instead of `xrange`. + """ - def tqdm.notebook.trange(*args, **kwargs): - """Experimental IPython/Jupyter Notebook widget version of trange""" +Convenience Functions +~~~~~~~~~~~~~~~~~~~~~ - class tqdm.keras.TqdmCallback(keras.callbacks.Callback): - """`keras` callback for epoch and batch progress""" +.. code:: python def tqdm.contrib.tenumerate(iterable, start=0, total=None, - tqdm_class=tqdm.auto.tqdm, **kwargs): + tqdm_class=tqdm.auto.tqdm, **tqdm_kwargs): """Equivalent of `numpy.ndenumerate` or builtin `enumerate`.""" def tqdm.contrib.tzip(iter1, *iter2plus, **tqdm_kwargs): @@ -610,14 +651,52 @@ Returns def tqdm.contrib.tmap(function, *sequences, **tqdm_kwargs): """Equivalent of builtin `map`.""" +Submodules +~~~~~~~~~~ + +.. code:: python + + class tqdm.notebook.tqdm(tqdm.tqdm): + """IPython/Jupyter Notebook widget.""" + + class tqdm.auto.tqdm(tqdm.tqdm): + """Automatically chooses beween `tqdm.notebook` and `tqdm.tqdm`.""" + + class tqdm.asyncio.tqdm(tqdm.tqdm): + """Asynchronous version.""" + @classmethod + def as_completed(cls, fs, *, loop=None, timeout=None, total=None, + **tqdm_kwargs): + """Wrapper for `asyncio.as_completed`.""" + + class tqdm.gui.tqdm(tqdm.tqdm): + """Matplotlib GUI version.""" + + class tqdm.tk.tqdm(tqdm.tqdm): + """Tkinter GUI version.""" + + class tqdm.rich.tqdm(tqdm.tqdm): + """`rich.progress` version.""" + + class tqdm.keras.TqdmCallback(keras.callbacks.Callback): + """Keras callback for epoch and batch progress.""" + + class tqdm.dask.TqdmCallback(dask.callbacks.Callback): + """Dask callback for task progress.""" + + ``contrib`` ------------ ++++++++++++ The ``tqdm.contrib`` package also contains experimental modules: - ``tqdm.contrib.itertools``: Thin wrappers around ``itertools`` - ``tqdm.contrib.concurrent``: Thin wrappers around ``concurrent.futures`` -- ``tqdm.contrib.telegram``: Posts to `Telegram `__ bots +- ``tqdm.contrib.discord``: Posts to `Discord `__ bots +- ``tqdm.contrib.telegram``: Posts to `Telegram `__ bots +- ``tqdm.contrib.bells``: Automagically enables all optional features + + * ``auto``, ``pandas``, ``discord``, ``telegram`` Examples and Advanced Usage --------------------------- @@ -632,7 +711,7 @@ Examples and Advanced Usage on how to make a **great** progressbar; - check out the `slides from PyData London `__, or -- run the |notebook-demo| or |binder-demo|. +- run the |binder-demo|. Description and additional stats ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -734,13 +813,14 @@ available to keep nested bars on their respective lines. For manual control over positioning (e.g. for multi-processing use), you may specify ``position=n`` where ``n=0`` for the outermost bar, ``n=1`` for the next, and so on. -However, it's best to check if `tqdm` can work without manual `position` first. +However, it's best to check if ``tqdm`` can work without manual ``position`` +first. .. code:: python from time import sleep from tqdm import trange, tqdm - from multiprocessing import Pool, freeze_support + from multiprocessing import Pool, RLock, freeze_support L = list(range(9)) @@ -753,6 +833,7 @@ However, it's best to check if `tqdm` can work without manual `position` first. if __name__ == '__main__': freeze_support() # for Windows support + tqdm.set_lock(RLock()) # for managing output contention p = Pool(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),)) p.map(progresser, L) @@ -799,6 +880,7 @@ Here's an example with ``urllib``: import urllib, os from tqdm import tqdm + urllib = getattr(urllib, 'request', urllib) class TqdmUpTo(tqdm): """Provides `update_to(n)` which uses `tqdm.update(delta_n)`.""" @@ -813,7 +895,7 @@ Here's an example with ``urllib``: """ if tsize is not None: self.total = tsize - self.update(b * bsize - self.n) # will also set self.n = b * bsize + return self.update(b * bsize - self.n) # also sets self.n = b * bsize eg_link = "https://caspersci.uk.to/matryoshka.zip" with TqdmUpTo(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, @@ -860,12 +942,14 @@ down to: from tqdm import tqdm eg_link = "https://caspersci.uk.to/matryoshka.zip" + response = getattr(urllib, 'request', urllib).urlopen(eg_link) with tqdm.wrapattr(open(os.devnull, "wb"), "write", - miniters=1, desc=eg_link.split('/')[-1]) as fout: - for chunk in urllib.urlopen(eg_link): + miniters=1, desc=eg_link.split('/')[-1], + total=getattr(response, 'length', None)) as fout: + for chunk in response: fout.write(chunk) -The ``requests`` equivalent is nearly identical, albeit with a ``total``: +The ``requests`` equivalent is nearly identical: .. code:: python @@ -880,6 +964,51 @@ The ``requests`` equivalent is nearly identical, albeit with a ``total``: for chunk in response.iter_content(chunk_size=4096): fout.write(chunk) +**Custom callback** + +``tqdm`` is known for intelligently skipping unnecessary displays. To make a +custom callback take advantage of this, simply use the return value of +``update()``. This is set to ``True`` if a ``display()`` was triggered. + +.. code:: python + + from tqdm.auto import tqdm as std_tqdm + + def external_callback(*args, **kwargs): + ... + + class TqdmExt(std_tqdm): + def update(self, n=1): + displayed = super(TqdmExt, self).update(n): + if displayed: + external_callback(**self.format_dict) + return displayed + +``asyncio`` +~~~~~~~~~~~ + +Note that ``break`` isn't currently caught by asynchronous iterators. +This means that ``tqdm`` cannot clean up after itself in this case: + +.. code:: python + + from tqdm.asyncio import tqdm + + async for i in tqdm(range(9)): + if i == 2: + break + +Instead, either call ``pbar.close()`` manually or use the context manager syntax: + +.. code:: python + + from tqdm.asyncio import tqdm + + with tqdm(range(9)) as pbar: + async for i in pbar: + if i == 2: + break + Pandas Integration ~~~~~~~~~~~~~~~~~~ @@ -922,6 +1051,24 @@ A ``keras`` callback is also available: model.fit(..., verbose=0, callbacks=[TqdmCallback()]) +Dask Integration +~~~~~~~~~~~~~~~~ + +A ``dask`` callback is also available: + +.. code:: python + + from tqdm.dask import TqdmCallback + + with TqdmCallback(desc="compute"): + ... + arr.compute() + + # or use callback globally + cb = TqdmCallback(desc="global") + cb.register() + arr.compute() + IPython/Jupyter Integration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -963,8 +1110,34 @@ this warning. Note that notebooks will display the bar in the cell where it was created. This may be a different cell from the one where it is used. -If this is not desired, the creation of the bar must be delayed/moved to the -cell where it is desired to be displayed. +If this is not desired, either + +- delay the creation of the bar to the cell where it must be displayed, or +- create the bar with ``display=False``, and in a later cell call + ``display(bar.container)``: + +.. code:: python + + from tqdm.notebook import tqdm + pbar = tqdm(..., display=False) + +.. code:: python + + # different cell + display(pbar.container) + +The ``keras`` callback has a ``display()`` method which can be used likewise: + +.. code:: python + + from tqdm.keras import TqdmCallback + cbk = TqdmCallback(display=False) + +.. code:: python + + # different cell + cbk.display() + model.fit(..., verbose=0, callbacks=[cbk]) Another possibility is to have a single bar (near the top of the notebook) which is constantly re-used (using ``reset()`` rather than ``close()``). @@ -1009,10 +1182,13 @@ For further customisation, Consider overloading ``display()`` to use e.g. ``self.frontend(**self.format_dict)`` instead of ``self.sp(repr(self))``. -`tqdm/notebook.py `__ -and `tqdm/gui.py `__ -submodules are examples of inheritance which don't (yet) strictly conform to the -above recommendation. +Some submodule examples of inheritance: + +- `tqdm/notebook.py `__ +- `tqdm/gui.py `__ +- `tqdm/tk.py `__ +- `tqdm/contrib/telegram.py `__ +- `tqdm/contrib/discord.py `__ Dynamic Monitor/Meter ~~~~~~~~~~~~~~~~~~~~~ @@ -1145,6 +1321,33 @@ A reusable canonical example is given below: # After the `with`, printing is restored print("Done!") +Redirecting ``logging`` +~~~~~~~~~~~~~~~~~~~~~~~ + +Similar to ``sys.stdout``/``sys.stderr`` as detailed above, console ``logging`` +may also be redirected to ``tqdm.write()``. + +Warning: if also redirecting ``sys.stdout``/``sys.stderr``, make sure to +redirect ``logging`` first if needed. + +Helper methods are available in ``tqdm.contrib.logging``. For example: + +.. code:: python + + import logging + from tqdm import trange + from tqdm.contrib.logging import logging_redirect_tqdm + + LOG = logging.getLogger(__name__) + + if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + with logging_redirect_tqdm(): + for i in trange(9): + if i == 4: + LOG.info("console logging redirected to `tqdm.write()`") + # logging restored + Monitoring thread, intervals and miniters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1192,27 +1395,28 @@ file for more information. Developers who have made significant contributions, ranked by *SLoC* (surviving lines of code, -`git fame `__ ``-wMC``), +`git fame `__ ``-wMC --excl '\.(png|gif|jpg)$'``), are: ==================== ======================================================== ==== ================================ Name ID SLoC Notes ==================== ======================================================== ==== ================================ -Casper da Costa-Luis `casperdcl `__ ~75% primary maintainer |Gift-Casper| -Stephen Larroque `lrq3000 `__ ~15% team member +Casper da Costa-Luis `casperdcl `__ ~81% primary maintainer |Gift-Casper| +Stephen Larroque `lrq3000 `__ ~10% team member Martin Zugnoni `martinzugnoni `__ ~3% +Richard Sheridan `richardsheridan `__ ~1% Guangshuo Chen `chengs `__ ~1% -Hadrien Mary `hadim `__ ~1% team member -Matthew Stevens `mjstevens777 `__ ~1% -Noam Yorav-Raphael `noamraph `__ ~1% original author -Kyle Altendorf `altendky `__ ~1% -Ivan Ivanov `obiwanus `__ ~1% -James E. King III `jeking3 `__ ~1% -Mikhail Korobov `kmike `__ ~1% team member +Kyle Altendorf `altendky `__ <1% +Matthew Stevens `mjstevens777 `__ <1% +Hadrien Mary `hadim `__ <1% team member +Ivan Ivanov `obiwanus `__ <1% +Daniel Panteleit `danielpanteleit `__ <1% +Jonas Haag `jonashaag `__ <1% +James E. King III `jeking3 `__ <1% +Noam Yorav-Raphael `noamraph `__ <1% original author +Mikhail Korobov `kmike `__ <1% team member ==================== ======================================================== ==== ================================ -|sourcerer-0| |sourcerer-1| |sourcerer-2| |sourcerer-3| |sourcerer-4| |sourcerer-5| |sourcerer-7| - Ports to Other Languages ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1225,24 +1429,24 @@ LICENCE Open Source (OSI approved): |LICENCE| -Citation information: |DOI| (publication), |DOI-code| (code) +Citation information: |DOI| |README-Hits| (Since 19 May 2016) .. |Logo| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/logo.gif -.. |Screenshot| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/tqdm.gif -.. |Video| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/video.jpg +.. |Screenshot| image:: https://raw.githubusercontent.com/tqdm/img/master/tqdm.gif +.. |Video| image:: https://raw.githubusercontent.com/tqdm/img/master/video.jpg :target: https://tqdm.github.io/video -.. |Slides| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/slides.jpg +.. |Slides| image:: https://raw.githubusercontent.com/tqdm/img/master/slides.jpg :target: https://tqdm.github.io/PyData2019/slides.html -.. |Build-Status| image:: https://img.shields.io/travis/tqdm/tqdm/master.svg?logo=travis - :target: https://travis-ci.org/tqdm/tqdm -.. |Coverage-Status| image:: https://coveralls.io/repos/tqdm/tqdm/badge.svg?branch=master +.. |Build-Status| image:: https://img.shields.io/github/workflow/status/tqdm/tqdm/Test/master?logo=GitHub + :target: https://github.com/tqdm/tqdm/actions?query=workflow%3ATest +.. |Coverage-Status| image:: https://img.shields.io/coveralls/github/tqdm/tqdm/master?logo=coveralls :target: https://coveralls.io/github/tqdm/tqdm .. |Branch-Coverage-Status| image:: https://codecov.io/gh/tqdm/tqdm/branch/master/graph/badge.svg :target: https://codecov.io/gh/tqdm/tqdm -.. |Codacy-Grade| image:: https://api.codacy.com/project/badge/Grade/3f965571598f44549c7818f29cdcf177 - :target: https://www.codacy.com/app/tqdm/tqdm/dashboard +.. |Codacy-Grade| image:: https://app.codacy.com/project/badge/Grade/3f965571598f44549c7818f29cdcf177 + :target: https://www.codacy.com/gh/tqdm/tqdm/dashboard .. |CII Best Practices| image:: https://bestpractices.coreinfrastructure.org/projects/3264/badge :target: https://bestpractices.coreinfrastructure.org/projects/3264 .. |GitHub-Status| image:: https://img.shields.io/github/tag/tqdm/tqdm.svg?maxAge=86400&logo=github&logoColor=white @@ -1262,7 +1466,7 @@ Citation information: |DOI| (publication), |DOI-code| (code) .. |GitHub-Updated| image:: https://img.shields.io/github/last-commit/tqdm/tqdm/master.svg?logo=github&logoColor=white&label=pushed :target: https://github.com/tqdm/tqdm/pulse .. |Gift-Casper| image:: https://img.shields.io/badge/dynamic/json.svg?color=ff69b4&label=gifts%20received&prefix=%C2%A3&query=%24..sum&url=https%3A%2F%2Fcaspersci.uk.to%2Fgifts.json - :target: https://caspersci.uk.to/donate + :target: https://www.cdcl.ml/sponsor .. |Versions| image:: https://img.shields.io/pypi/v/tqdm.svg :target: https://tqdm.github.io/releases .. |PyPI-Downloads| image:: https://img.shields.io/pypi/dm/tqdm.svg?label=pypi%20downloads&logo=PyPI&logoColor=white @@ -1285,32 +1489,12 @@ Citation information: |DOI| (publication), |DOI-code| (code) :target: https://github.com/vinta/awesome-python .. |LICENCE| image:: https://img.shields.io/pypi/l/tqdm.svg :target: https://raw.githubusercontent.com/tqdm/tqdm/master/LICENCE -.. |DOI| image:: https://img.shields.io/badge/DOI-10.21105/joss.01277-green.svg - :target: https://doi.org/10.21105/joss.01277 -.. |DOI-code| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.595120-blue.svg +.. |DOI| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.595120-blue.svg :target: https://doi.org/10.5281/zenodo.595120 -.. |notebook-demo| image:: https://img.shields.io/badge/launch-notebook-orange.svg?logo=jupyter - :target: https://notebooks.ai/demo/gh/tqdm/tqdm .. |binder-demo| image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/tqdm/tqdm/master?filepath=DEMO.ipynb -.. |Screenshot-Jupyter1| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/tqdm-jupyter-1.gif -.. |Screenshot-Jupyter2| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/tqdm-jupyter-2.gif -.. |Screenshot-Jupyter3| image:: https://raw.githubusercontent.com/tqdm/tqdm/master/images/tqdm-jupyter-3.gif +.. |Screenshot-Jupyter1| image:: https://raw.githubusercontent.com/tqdm/img/master/jupyter-1.gif +.. |Screenshot-Jupyter2| image:: https://raw.githubusercontent.com/tqdm/img/master/jupyter-2.gif +.. |Screenshot-Jupyter3| image:: https://raw.githubusercontent.com/tqdm/img/master/jupyter-3.gif .. |README-Hits| image:: https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&style=social&r=https://github.com/tqdm/tqdm&l=https://caspersci.uk.to/images/tqdm.png&f=https://raw.githubusercontent.com/tqdm/tqdm/master/images/logo.gif :target: https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&a=plot&r=https://github.com/tqdm/tqdm&l=https://caspersci.uk.to/images/tqdm.png&f=https://raw.githubusercontent.com/tqdm/tqdm/master/images/logo.gif&style=social -.. |sourcerer-0| image:: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/0 - :target: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/0 -.. |sourcerer-1| image:: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/1 - :target: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/1 -.. |sourcerer-2| image:: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/2 - :target: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/2 -.. |sourcerer-3| image:: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/3 - :target: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/3 -.. |sourcerer-4| image:: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/4 - :target: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/4 -.. |sourcerer-5| image:: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/5 - :target: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/5 -.. |sourcerer-6| image:: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/6 - :target: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/6 -.. |sourcerer-7| image:: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/images/7 - :target: https://sourcerer.io/fame/casperdcl/tqdm/tqdm/links/7 diff --git a/asv.conf.json b/asv.conf.json index b7ad42ab5..954029358 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -5,12 +5,13 @@ "repo": ".", "environment_type": "virtualenv", "show_commit_url": "https://github.com/tqdm/tqdm/commit/", - // "pythons": ["2.7", "3.3"], - // "matrix": { - // "numpy": ["1.6", "1.7"], - // "six": ["", null], // test with and without six installed - // "pip+emcee": [""], // emcee is only available for install with pip. - // }, + // "pythons": ["2.7", "3.6"], + // "conda_channels": ["conda-forge", "defaults"], + "matrix": { + "alive-progress": [""], + "progressbar2": [""], + "rich": [""], + }, // "exclude": [ // {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows // {"environment_type": "conda", "six": null}, // don't run without six on conda diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..235cb3197 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,24 @@ +# Benchmarks + +These benchmarks serve two purposes: + +1. Thorough performance tests against regression + - `tqdm` + - `tqdm(miniters=manually_optimised, smoothing=0)` + - `no-progress` (empty loop without progress wrapper) +2. Compare `tqdm`'s speed to popular alternatives + - [`rich.progress`](https://pypi.org/project/rich) + - [`progressbar2`](https://pypi.org/project/progressbar2) + - [`alive-progress`](https://pypi.org/project/alive-progress) + +Performance graphs are available at + +## Running + +These benchmarks are run automatically for all releases and pull requests. + +To run locally: + +- conda/pip install `virtualenv` and `asv` +- clone this repository +- run `asv --help` in the repository root (one directory above this file) diff --git a/benchmarks/benchmarks.py b/benchmarks/benchmarks.py index 1797fd53f..37f5df419 100644 --- a/benchmarks/benchmarks.py +++ b/benchmarks/benchmarks.py @@ -2,42 +2,93 @@ # See "Writing benchmarks" in the asv docs for more information. from __future__ import division +from functools import partial -class FractionalOverheadSuite: - """ - An example benchmark that times the performance of various kinds - of iterating over dictionaries in Python. - """ - def setup(self): + +class Comparison: + """Running time of wrapped empty loops""" + def __init__(self, length): try: from time import process_time self.time = process_time except ImportError: from time import clock self.time = clock - from tqdm import tqdm - self.tqdm = tqdm try: - self.iterable = xrange(int(6e6)) + self.iterable = xrange(int(length)) except NameError: - self.iterable = range(int(6e6)) + self.iterable = range(int(length)) + def run(self, cls): + pbar = cls(self.iterable) t0 = self.time() - [0 for _ in self.iterable] + [0 for _ in pbar] # pylint: disable=pointless-statement t1 = self.time() - self.t = t1 - t0 - - def track_tqdm(self): - with self.tqdm(self.iterable) as pbar: - t0 = self.time() - [0 for _ in pbar] - t1 = self.time() - return (t1 - t0 - self.t) / self.t - - def track_optimsed(self): - with self.tqdm(self.iterable, miniters=6e5, smoothing=0) as pbar: - # TODO: miniters=None, mininterval=0.1, smoothing=0)] - t0 = self.time() - [0 for _ in pbar] - t1 = self.time() - return (t1 - t0 - self.t) / self.t + return t1 - t0 + + def run_by_name(self, method): + return getattr(self, method.replace("-", "_"))() + + def no_progress(self): + return self.run(lambda x: x) + + def tqdm_optimised(self): + from tqdm import tqdm + return self.run(partial(tqdm, miniters=6e5, smoothing=0)) + + def tqdm(self): + from tqdm import tqdm + return self.run(tqdm) + + def alive_progress(self): + from alive_progress import alive_bar + + class wrapper: + def __init__(self, iterable): + self.iterable = iterable + + def __iter__(self): + iterable = self.iterable + with alive_bar(len(iterable)) as bar: + for i in iterable: + yield i + bar() + + return self.run(wrapper) + + # def progressbar(self): + # from progressbar.progressbar import ProgressBar + # return self.run(ProgressBar()) + + def progressbar2(self): + from progressbar import progressbar + return self.run(progressbar) + + def rich(self): + from rich.progress import track + return self.run(track) + + +# thorough test against no-progress +slow = Comparison(6e6) + + +def track_tqdm(method): + return slow.run_by_name(method) + + +track_tqdm.params = ["tqdm", "tqdm-optimised", "no-progress"] +track_tqdm.param_names = ["method"] +track_tqdm.unit = "Seconds (lower is better)" + +# quick test against alternatives +fast = Comparison(1e5) + + +def track_alternatives(library): + return fast.run_by_name(library) + + +track_alternatives.params = ["rich", "progressbar2", "alive-progress", "tqdm"] +track_alternatives.param_names = ["library"] +track_alternatives.unit = "Seconds (lower is better)" diff --git a/demo.yml b/demo.yml deleted file mode 100644 index 0b0f5f1c2..000000000 --- a/demo.yml +++ /dev/null @@ -1,2 +0,0 @@ -requirements: - - tqdm diff --git a/environment.yml b/environment.yml new file mode 100644 index 000000000..9b199cf21 --- /dev/null +++ b/environment.yml @@ -0,0 +1,45 @@ +# development environment +name: tqdm +channels: +- conda-forge +- defaults +dependencies: +# base +- python=3 +- pip +- ipykernel +- ipywidgets +- setuptools +- setuptools_scm +- toml +# test env managers +- pre-commit +- tox +- asv +# tests (native) +- pytest +- pytest-cov +- pytest-timeout +- pytest-asyncio # [py>=3.7] +- nbval +- flake8 +- flake8-bugbear +- flake8-comprehensions +- coverage +# extras +- dask # dask +- matplotlib # gui +- nbstripout # notebook editing +- numpy # pandas, keras, contrib.tenumerate +- pandas +- tensorflow # keras +- requests # contrib.telegram +- rich # rich +- argopt # `cd wiki && pymake` +- twine # `pymake pypi` +- wheel # `setup.py bdist_wheel` +- pip: + - py-make >=0.1.0 # `setup.py make/pymake` + - pydoc-markdown # `cd docs && pymake` + - flake8-debugger # flake8 + - flake8-string-format # flake8 diff --git a/examples/7zx.py b/examples/7zx.py index 5f0a5e8b9..edc5c95a1 100644 --- a/examples/7zx.py +++ b/examples/7zx.py @@ -19,17 +19,21 @@ -d, --debug-trace Print lots of debugging information (-D NOTSET) """ from __future__ import print_function -from argopt import argopt + +import io import logging -import subprocess +import os +import pty import re +import subprocess # nosec + +from argopt import argopt + from tqdm import tqdm -import pty -import os -import io + __author__ = "Casper da Costa-Luis " __licence__ = "MPLv2.0" -__version__ = "0.2.1" +__version__ = "0.2.2" __license__ = __licence__ RE_SCN = re.compile(r"([0-9]+)\s+([0-9]+)\s+(.*)$", flags=re.M) @@ -47,7 +51,7 @@ def main(): # Get compressed sizes zips = {} for fn in args.zipfiles: - info = subprocess.check_output(["7z", "l", fn]).strip() + info = subprocess.check_output(["7z", "l", fn]).strip() # nosec finfo = RE_SCN.findall(info) # size|compressed|name # builtin test: last line should be total sizes @@ -57,10 +61,9 @@ def main(): for s in range(2): # size|compressed totals totals_s = sum(map(int, (inf[s] for inf in finfo[:-1]))) if totals_s != totals[s]: - log.warn("%s: individual total %d != 7z total %d" % ( - fn, totals_s, totals[s])) - fcomp = dict((n, int(c if args.compressed else u)) - for (u, c, n) in finfo[:-1]) + log.warn("%s: individual total %d != 7z total %d", + fn, totals_s, totals[s]) + fcomp = {n: int(c if args.compressed else u) for (u, c, n) in finfo[:-1]} # log.debug(fcomp) # zips : {'zipname' : {'filename' : int(size)}} zips[fn] = fcomp @@ -69,12 +72,12 @@ def main(): cmd7zx = ["7z", "x", "-bd"] if args.yes: cmd7zx += ["-y"] - log.info("Extracting from {:d} file(s)".format(len(zips))) + log.info("Extracting from %d file(s)", len(zips)) with tqdm(total=sum(sum(fcomp.values()) for fcomp in zips.values()), unit="B", unit_scale=True) as tall: for fn, fcomp in zips.items(): md, sd = pty.openpty() - ex = subprocess.Popen( + ex = subprocess.Popen( # nosec cmd7zx + [fn], bufsize=1, stdout=md, # subprocess.PIPE, @@ -101,13 +104,12 @@ def main(): ln.startswith(i) for i in ("7-Zip ", "p7zip Version ", "Everything is Ok", "Folders: ", - "Files: ", "Size: ", - "Compressed: ")): + "Files: ", "Size: ", "Compressed: ")): if ln.startswith("Processing archive: "): if not args.silent: t.write(t.format_interval( t.start_t - tall.start_t) + ' ' + - ln.lstrip("Processing archive: ")) + ln.replace("Processing archive: ", "")) else: t.write(ln) ex.wait() diff --git a/examples/async_coroutines.py b/examples/async_coroutines.py new file mode 100644 index 000000000..40f4f249d --- /dev/null +++ b/examples/async_coroutines.py @@ -0,0 +1,38 @@ +""" +Asynchronous examples using `asyncio`, `async` and `await` on `python>=3.7`. +""" +import asyncio + +from tqdm.asyncio import tqdm, trange + + +def count(start=0, step=1): + i = start + while True: + new_start = yield i + if new_start is None: + i += step + else: + i = new_start + + +async def main(): + N = int(1e6) + async for row in tqdm(trange(N, desc="inner"), desc="outer"): + if row >= N: + break + with tqdm(count(), desc="coroutine", total=N + 2) as pbar: + async for row in pbar: + if row == N: + pbar.send(-10) + elif row < 0: + assert row == -9 + break + # should be ~1sec rather than ~50s due to async scheduling + for i in tqdm.as_completed([asyncio.sleep(0.01 * i) + for i in range(100, 0, -1)], desc="as_completed"): + await i + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/coroutine_pipe.py b/examples/coroutine_pipe.py new file mode 100644 index 000000000..deb498ad9 --- /dev/null +++ b/examples/coroutine_pipe.py @@ -0,0 +1,69 @@ +""" +Inserting `tqdm` as a "pipe" in a chain of coroutines. +Not to be confused with `asyncio.coroutine`. +""" +from functools import wraps + +from tqdm.auto import tqdm + + +def autonext(func): + @wraps(func) + def inner(*args, **kwargs): + res = func(*args, **kwargs) + next(res) + return res + return inner + + +@autonext +def tqdm_pipe(target, **tqdm_kwargs): + """ + Coroutine chain pipe `send()`ing to `target`. + + This: + >>> r = receiver() + >>> p = producer(r) + >>> next(r) + >>> next(p) + + Becomes: + >>> r = receiver() + >>> t = tqdm.pipe(r) + >>> p = producer(t) + >>> next(r) + >>> next(p) + """ + with tqdm(**tqdm_kwargs) as pbar: + while True: + obj = (yield) + target.send(obj) + pbar.update() + + +def source(target): + for i in ["foo", "bar", "baz", "pythonista", "python", "py"]: + target.send(i) + target.close() + + +@autonext +def grep(pattern, target): + while True: + line = (yield) + if pattern in line: + target.send(line) + + +@autonext +def sink(): + while True: + line = (yield) + tqdm.write(line) + + +if __name__ == "__main__": + source( + tqdm_pipe( + grep('python', + sink()))) diff --git a/examples/include_no_requirements.py b/examples/include_no_requirements.py index 3682d8845..c51a85cb9 100644 --- a/examples/include_no_requirements.py +++ b/examples/include_no_requirements.py @@ -7,3 +7,5 @@ def tqdm(*args, **kwargs): if args: return args[0] return kwargs.get('iterable', None) + +__all__ = ['tqdm'] diff --git a/examples/pandas_progress_apply.py b/examples/pandas_progress_apply.py index 1b0fc2309..4fc8f6b16 100644 --- a/examples/pandas_progress_apply.py +++ b/examples/pandas_progress_apply.py @@ -1,5 +1,6 @@ -import pandas as pd import numpy as np +import pandas as pd + from tqdm.auto import tqdm df = pd.DataFrame(np.random.randint(0, 100, (100000, 6))) diff --git a/examples/parallel_bars.py b/examples/parallel_bars.py index c2a8dcddb..498fd61df 100644 --- a/examples/parallel_bars.py +++ b/examples/parallel_bars.py @@ -1,23 +1,24 @@ from __future__ import print_function + +import sys +from concurrent.futures import ThreadPoolExecutor +from functools import partial +from multiprocessing import Pool, RLock, freeze_support +from random import random +from threading import RLock as TRLock from time import sleep + from tqdm.auto import tqdm, trange from tqdm.contrib.concurrent import process_map, thread_map -from random import random -from multiprocessing import Pool, freeze_support -from concurrent.futures import ThreadPoolExecutor -from threading import RLock -from functools import partial -import sys NUM_SUBITERS = 9 PY2 = sys.version_info[:1] <= (2,) -def progresser(n, auto_position=True, write_safe=False, blocking=True, - progress=False): - interval = random() * 0.002 / (NUM_SUBITERS - n + 2) +def progresser(n, auto_position=True, write_safe=False, blocking=True, progress=False): + interval = random() * 0.002 / (NUM_SUBITERS - n + 2) # nosec total = 5000 - text = "#{}, est. {:<04.2}s".format(n, interval * total) + text = "#{0}, est. {1:<04.2}s".format(n, interval * total) for _ in trange(total, desc=text, disable=not progress, lock_args=None if blocking else (False,), position=None if auto_position else n): @@ -47,17 +48,14 @@ def progresser(n, auto_position=True, write_safe=False, blocking=True, sleep(0.01) print("Multi-processing") + tqdm.set_lock(RLock()) p = Pool(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),)) p.map(partial(progresser, progress=True), L) - # unfortunately need ncols - # to print spaces over leftover multi-processing bars (#796) - with tqdm(leave=False) as t: - ncols = t.ncols or 80 - print(("{msg:<{ncols}}").format(msg="Multi-threading", ncols=ncols)) - - # explicitly set just threading lock for nonblocking progress - tqdm.set_lock(RLock()) - with ThreadPoolExecutor() as p: - p.map(partial(progresser, progress=True, write_safe=not PY2, - blocking=False), L) + print("Multi-threading") + tqdm.set_lock(TRLock()) + pool_args = {} + if not PY2: + pool_args.update(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),)) + with ThreadPoolExecutor(**pool_args) as p: + p.map(partial(progresser, progress=True, write_safe=not PY2, blocking=False), L) diff --git a/examples/redirect_print.py b/examples/redirect_print.py index a33e23505..0f9721ee6 100644 --- a/examples/redirect_print.py +++ b/examples/redirect_print.py @@ -11,9 +11,11 @@ A reusable canonical example is given below: """ from __future__ import print_function -from time import sleep + import contextlib import sys +from time import sleep + from tqdm import tqdm from tqdm.contrib import DummyTqdmFile diff --git a/examples/simple_examples.py b/examples/simple_examples.py index 9d54b92f1..f3401d357 100644 --- a/examples/simple_examples.py +++ b/examples/simple_examples.py @@ -47,13 +47,14 @@ pass """ +import re from time import sleep from timeit import timeit -import re # Simple demo from tqdm import trange -for i in trange(16, leave=True): + +for _ in trange(16, leave=True): sleep(0.1) # Profiling/overhead tests diff --git a/examples/tqdm_requests.py b/examples/tqdm_requests.py index ef4483502..5d03594cc 100644 --- a/examples/tqdm_requests.py +++ b/examples/tqdm_requests.py @@ -16,10 +16,10 @@ from os import devnull -from docopt import docopt import requests -from tqdm.auto import tqdm +from docopt import docopt +from tqdm.auto import tqdm opts = docopt(__doc__) diff --git a/examples/tqdm_wget.py b/examples/tqdm_wget.py index 0afdd5a0c..8663e5a39 100644 --- a/examples/tqdm_wget.py +++ b/examples/tqdm_wget.py @@ -20,10 +20,14 @@ The local file path in which to save the url [default: /dev/null]. """ +try: + from urllib import request as urllib +except ImportError: # py2 + import urllib from os import devnull from docopt import docopt -import urllib + from tqdm.auto import tqdm @@ -55,8 +59,9 @@ def update_to(b=1, bsize=1, tsize=None): """ if tsize not in (None, -1): t.total = tsize - t.update((b - last_b[0]) * bsize) + displayed = t.update((b - last_b[0]) * bsize) last_b[0] = b + return displayed return update_to @@ -81,7 +86,7 @@ def update_to(self, b=1, bsize=1, tsize=None): """ if tsize is not None: self.total = tsize - self.update(b * bsize - self.n) # will also set self.n = b * bsize + return self.update(b * bsize - self.n) # also sets self.n = b * bsize opts = docopt(__doc__) @@ -95,12 +100,14 @@ def update_to(self, b=1, bsize=1, tsize=None): # reporthook=my_hook(t), data=None) with TqdmUpTo(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, desc=eg_file) as t: # all optional kwargs - urllib.urlretrieve(eg_link, filename=eg_out, reporthook=t.update_to, - data=None) + urllib.urlretrieve( # nosec + eg_link, filename=eg_out, reporthook=t.update_to, data=None) t.total = t.n # Even simpler progress by wrapping the output file's `write()` +response = urllib.urlopen(eg_link) # nosec with tqdm.wrapattr(open(eg_out, "wb"), "write", - miniters=1, desc=eg_file) as fout: - for chunk in urllib.urlopen(eg_link): + miniters=1, desc=eg_file, + total=getattr(response, 'length', None)) as fout: + for chunk in response: fout.write(chunk) diff --git a/examples/wrapping_generators.py b/examples/wrapping_generators.py index 0ec068956..65c85bfd8 100644 --- a/examples/wrapping_generators.py +++ b/examples/wrapping_generators.py @@ -1,6 +1,7 @@ -from tqdm.contrib import tenumerate, tzip, tmap import numpy as np +from tqdm.contrib import tenumerate, tmap, tzip + for _ in tenumerate(range(int(1e6)), desc="builtin enumerate"): pass diff --git a/images/slides.jpg b/images/slides.jpg deleted file mode 100644 index 459acc44b..000000000 Binary files a/images/slides.jpg and /dev/null differ diff --git a/images/tqdm-jupyter-1.gif b/images/tqdm-jupyter-1.gif deleted file mode 100644 index 01f7681b8..000000000 Binary files a/images/tqdm-jupyter-1.gif and /dev/null differ diff --git a/images/tqdm-jupyter-2.gif b/images/tqdm-jupyter-2.gif deleted file mode 100644 index a80ea79bc..000000000 Binary files a/images/tqdm-jupyter-2.gif and /dev/null differ diff --git a/images/tqdm-jupyter-3.gif b/images/tqdm-jupyter-3.gif deleted file mode 100644 index a30059a61..000000000 Binary files a/images/tqdm-jupyter-3.gif and /dev/null differ diff --git a/images/video.jpg b/images/video.jpg deleted file mode 100644 index ef1133a5f..000000000 Binary files a/images/video.jpg and /dev/null differ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..3eb7bbcf5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +write_to = "tqdm/_dist_ver.py" +write_to_template = "__version__ = '{version}'\n" diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index fc129715e..000000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,4 +0,0 @@ -py-make>=0.1.0 # setup.py make/pymake -twine # pymake pypi -argopt # cd wiki && pymake -pydoc-markdown # cd docs && pymake diff --git a/setup.cfg b/setup.cfg index cd065ddd2..7c733eb5b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,142 @@ +[metadata] +name=tqdm +url=https://github.com/tqdm/tqdm +project_urls= + Changelog=https://tqdm.github.io/releases + Documentation=https://github.com/tqdm/tqdm#tqdm + Documentation (dev)=https://tqdm.github.io/docs/tqdm + Wiki=https://github.com/tqdm/tqdm/wiki +maintainer=tqdm developers +maintainer_email=python.tqdm@gmail.com +license=MPLv2.0, MIT Licences +license_file=LICENCE +description=Fast, Extensible Progress Meter +long_description=file: README.rst +long_description_content_type=text/x-rst +keywords=progressbar, progressmeter, progress, bar, meter, rate, eta, console, terminal, time +platforms=any +provides=tqdm +# Trove classifiers (https://pypi.org/pypi?%3Aaction=list_classifiers) +classifiers= + Development Status :: 5 - Production/Stable + Environment :: Console + Environment :: MacOS X + Environment :: Other Environment + Environment :: Win32 (MS Windows) + Environment :: X11 Applications + Framework :: IPython + Framework :: Jupyter + Intended Audience :: Developers + Intended Audience :: Education + Intended Audience :: End Users/Desktop + Intended Audience :: Other Audience + Intended Audience :: System Administrators + License :: OSI Approved :: MIT License + License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0) + Operating System :: MacOS + Operating System :: MacOS :: MacOS X + Operating System :: Microsoft + Operating System :: Microsoft :: MS-DOS + Operating System :: Microsoft :: Windows + Operating System :: POSIX + Operating System :: POSIX :: BSD + Operating System :: POSIX :: BSD :: FreeBSD + Operating System :: POSIX :: Linux + Operating System :: POSIX :: SunOS/Solaris + Operating System :: Unix + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: Implementation + Programming Language :: Python :: Implementation :: IronPython + Programming Language :: Python :: Implementation :: PyPy + Programming Language :: Unix Shell + Topic :: Desktop Environment + Topic :: Education :: Computer Aided Instruction (CAI) + Topic :: Education :: Testing + Topic :: Office/Business + Topic :: Other/Nonlisted Topic + Topic :: Software Development :: Build Tools + Topic :: Software Development :: Libraries + Topic :: Software Development :: Libraries :: Python Modules + Topic :: Software Development :: Pre-processors + Topic :: Software Development :: User Interfaces + Topic :: System :: Installation/Setup + Topic :: System :: Logging + Topic :: System :: Monitoring + Topic :: System :: Shells + Topic :: Terminals + Topic :: Utilities +[options] +setup_requires=setuptools>=42; setuptools_scm[toml]>=3.4 +python_requires=>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +tests_require=tox +include_package_data=True +packages=find: +[options.extras_require] +dev=py-make>=0.1.0; twine; wheel +telegram=requests +notebook=ipywidgets>=6 +[options.entry_points] +console_scripts= + tqdm=tqdm.cli:main +[options.packages.find] +exclude=benchmarks, tests + [bdist_wheel] -universal = 1 +universal=1 [flake8] -ignore = W503,W504,E722 -max_line_length = 80 -exclude = .asv,.tox,.ipynb_checkpoints,build,dist,.git,__pycache__ +max_line_length=88 +exclude=.asv,.eggs,.tox,.ipynb_checkpoints,build,dist,.git,__pycache__ + +[pydocstyle] +add_ignore=D400,D415 + +[yapf] +coalesce_brackets=True +column_limit=88 +each_dict_entry_on_separate_line=False +i18n_comment=NOQA +space_between_ending_comma_and_closing_bracket=False +split_before_named_assigns=False +split_before_closing_bracket=False + +[isort] +line_length=88 +multi_line_output=4 +known_first_party=tqdm,tests + +[tool:pytest] +timeout=30 +log_level=INFO +markers= + asyncio + slow +python_files=tests_*.py tests_*.ipynb +testpaths=tests +addopts=-v --tb=short -rxs -W=error --durations=0 --durations-min=0.1 +[regex1] +regex: (?<= )[\s\d.]+(it/s|s/it) +replace: ??.??it/s +[regex2] +regex: 00:0[01]<00:0[01] +replace: 00:00<00:00 + +[coverage:run] +branch=True +include=tqdm/* +omit= + tqdm/contrib/bells.py + tqdm/contrib/discord.py + tqdm/contrib/telegram.py + tqdm/contrib/utils_worker.py +relative_files=True +[coverage:report] +show_missing=True diff --git a/setup.py b/setup.py index 1aa5dd702..89dadf581 100755 --- a/setup.py +++ b/setup.py @@ -1,125 +1,16 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import os -try: - from setuptools import setup, find_packages -except ImportError: - from distutils.core import setup - - def find_packages(where='.'): - # os.walk -> list[(dirname, list[subdirs], list[files])] - return [folder.replace("/", ".").lstrip(".") - for (folder, _, fils) in os.walk(where) - if "__init__.py" in fils] import sys -from io import open as io_open +from os import path -# Get version from tqdm/_version.py -__version__ = None -src_dir = os.path.abspath(os.path.dirname(__file__)) -version_file = os.path.join(src_dir, 'tqdm', '_version.py') -with io_open(version_file, mode='r') as fd: - exec(fd.read()) +from setuptools import setup -# Executing makefile commands if specified -if sys.argv[1].lower().strip() == 'make': +src_dir = path.abspath(path.dirname(__file__)) +if sys.argv[1].lower().strip() == 'make': # exec Makefile commands import pymake - # Filename of the makefile - fpath = os.path.join(src_dir, 'Makefile') + fpath = path.join(src_dir, 'Makefile') pymake.main(['-f', fpath] + sys.argv[2:]) # Stop to avoid setup.py raising non-standard command error sys.exit(0) -extras_require = {} -requirements_dev = os.path.join(src_dir, 'requirements-dev.txt') -with io_open(requirements_dev, mode='r') as fd: - extras_require['dev'] = [i.strip().split('#', 1)[0].strip() - for i in fd.read().strip().split('\n')] - -README_rst = '' -fndoc = os.path.join(src_dir, 'README.rst') -with io_open(fndoc, mode='r', encoding='utf-8') as fd: - README_rst = fd.read() -setup( - name='tqdm', - version=__version__, - description='Fast, Extensible Progress Meter', - long_description=README_rst, - license='MPLv2.0, MIT Licences', - url='https://github.com/tqdm/tqdm', - maintainer='tqdm developers', - maintainer_email='python.tqdm@gmail.com', - platforms=['any'], - packages=['tqdm'] + ['tqdm.' + i for i in find_packages('tqdm')], - provides=['tqdm'], - extras_require=extras_require, - entry_points={'console_scripts': ['tqdm=tqdm.cli:main'], }, - package_data={'tqdm': ['CONTRIBUTING.md', 'LICENCE', 'examples/*.py', - 'tqdm.1', 'completion.sh', 'requirements-dev.txt']}, - python_requires='>=2.6, !=3.0.*, !=3.1.*', - classifiers=[ - # Trove classifiers - # (https://pypi.org/pypi?%3Aaction=list_classifiers) - 'Development Status :: 5 - Production/Stable', - 'Environment :: Console', - 'Environment :: MacOS X', - 'Environment :: Other Environment', - 'Environment :: Win32 (MS Windows)', - 'Environment :: X11 Applications', - 'Framework :: IPython', - 'Framework :: Jupyter', - 'Intended Audience :: Developers', - 'Intended Audience :: Education', - 'Intended Audience :: End Users/Desktop', - 'Intended Audience :: Other Audience', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: MIT License', - 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', - 'Operating System :: MacOS', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: Microsoft', - 'Operating System :: Microsoft :: MS-DOS', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX', - 'Operating System :: POSIX :: BSD', - 'Operating System :: POSIX :: BSD :: FreeBSD', - 'Operating System :: POSIX :: Linux', - 'Operating System :: POSIX :: SunOS/Solaris', - 'Operating System :: Unix', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: Implementation', - 'Programming Language :: Python :: Implementation :: IronPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Programming Language :: Unix Shell', - 'Topic :: Desktop Environment', - 'Topic :: Education :: Computer Aided Instruction (CAI)', - 'Topic :: Education :: Testing', - 'Topic :: Office/Business', - 'Topic :: Other/Nonlisted Topic', - 'Topic :: Software Development :: Build Tools', - 'Topic :: Software Development :: Libraries', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Software Development :: Pre-processors', - 'Topic :: Software Development :: User Interfaces', - 'Topic :: System :: Installation/Setup', - 'Topic :: System :: Logging', - 'Topic :: System :: Monitoring', - 'Topic :: System :: Shells', - 'Topic :: Terminals', - 'Topic :: Utilities' - ], - keywords='progressbar progressmeter progress bar meter' - ' rate eta console terminal time', - test_suite='nose.collector', - tests_require=['nose', 'flake8', 'coverage'], -) +setup(use_scm_version=True) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..67170442b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,41 @@ +"""Shared pytest config.""" +import sys + +from pytest import fixture + +from tqdm import tqdm + + +@fixture(autouse=True) +def pretest_posttest(): + """Fixture for all tests ensuring environment cleanup""" + try: + sys.setswitchinterval(1) + except AttributeError: + sys.setcheckinterval(100) # deprecated + + if getattr(tqdm, "_instances", False): + n = len(tqdm._instances) + if n: + tqdm._instances.clear() + raise EnvironmentError( + "{0} `tqdm` instances still in existence PRE-test".format(n)) + yield + if getattr(tqdm, "_instances", False): + n = len(tqdm._instances) + if n: + tqdm._instances.clear() + raise EnvironmentError( + "{0} `tqdm` instances still in existence POST-test".format(n)) + + +if sys.version_info[0] > 2: + @fixture + def capsysbin(capsysbinary): + """alias for capsysbinary (py3)""" + return capsysbinary +else: + @fixture + def capsysbin(capsys): + """alias for capsys (py2)""" + return capsys diff --git a/tests/py37_asyncio.py b/tests/py37_asyncio.py new file mode 100644 index 000000000..18997ca7b --- /dev/null +++ b/tests/py37_asyncio.py @@ -0,0 +1,128 @@ +import asyncio +from functools import partial +from sys import platform +from time import time + +from tqdm.asyncio import tarange, tqdm_asyncio + +from .tests_tqdm import StringIO, closing, mark + +tqdm = partial(tqdm_asyncio, miniters=0, mininterval=0) +trange = partial(tarange, miniters=0, mininterval=0) +as_completed = partial(tqdm_asyncio.as_completed, miniters=0, mininterval=0) +gather = partial(tqdm_asyncio.gather, miniters=0, mininterval=0) + + +def count(start=0, step=1): + i = start + while True: + new_start = yield i + if new_start is None: + i += step + else: + i = new_start + + +async def acount(*args, **kwargs): + for i in count(*args, **kwargs): + yield i + + +@mark.asyncio +async def test_break(): + """Test asyncio break""" + pbar = tqdm(count()) + async for _ in pbar: + break + pbar.close() + + +@mark.asyncio +async def test_generators(capsys): + """Test asyncio generators""" + with tqdm(count(), desc="counter") as pbar: + async for i in pbar: + if i >= 8: + break + _, err = capsys.readouterr() + assert '9it' in err + + with tqdm(acount(), desc="async_counter") as pbar: + async for i in pbar: + if i >= 8: + break + _, err = capsys.readouterr() + assert '9it' in err + + +@mark.asyncio +async def test_range(): + """Test asyncio range""" + with closing(StringIO()) as our_file: + async for _ in tqdm(range(9), desc="range", file=our_file): + pass + assert '9/9' in our_file.getvalue() + our_file.seek(0) + our_file.truncate() + + async for _ in trange(9, desc="trange", file=our_file): + pass + assert '9/9' in our_file.getvalue() + + +@mark.asyncio +async def test_nested(): + """Test asyncio nested""" + with closing(StringIO()) as our_file: + async for _ in tqdm(trange(9, desc="inner", file=our_file), + desc="outer", file=our_file): + pass + assert 'inner: 100%' in our_file.getvalue() + assert 'outer: 100%' in our_file.getvalue() + + +@mark.asyncio +async def test_coroutines(): + """Test asyncio coroutine.send""" + with closing(StringIO()) as our_file: + with tqdm(count(), file=our_file) as pbar: + async for i in pbar: + if i == 9: + pbar.send(-10) + elif i < 0: + assert i == -9 + break + assert '10it' in our_file.getvalue() + + +@mark.slow +@mark.asyncio +@mark.parametrize("tol", [0.2 if platform.startswith("darwin") else 0.1]) +async def test_as_completed(capsys, tol): + """Test asyncio as_completed""" + for retry in range(3): + t = time() + skew = time() - t + for i in as_completed([asyncio.sleep(0.01 * i) for i in range(30, 0, -1)]): + await i + t = time() - t - 2 * skew + try: + assert 0.3 * (1 - tol) < t < 0.3 * (1 + tol), t + _, err = capsys.readouterr() + assert '30/30' in err + except AssertionError: + if retry == 2: + raise + + +async def double(i): + return i * 2 + + +@mark.asyncio +async def test_gather(capsys): + """Test asyncio gather""" + res = await gather(list(map(double, range(30)))) + _, err = capsys.readouterr() + assert '30/30' in err + assert res == list(range(0, 30 * 2, 2)) diff --git a/tests/tests_asyncio.py b/tests/tests_asyncio.py new file mode 100644 index 000000000..6f089264c --- /dev/null +++ b/tests/tests_asyncio.py @@ -0,0 +1,11 @@ +"""Tests `tqdm.asyncio` on `python>=3.7`.""" +import sys + +if sys.version_info[:2] > (3, 6): + from .py37_asyncio import * # NOQA, pylint: disable=wildcard-import +else: + from .tests_tqdm import skip + try: + skip("async not supported", allow_module_level=True) + except TypeError: + pass diff --git a/tests/tests_concurrent.py b/tests/tests_concurrent.py new file mode 100644 index 000000000..5cd439c94 --- /dev/null +++ b/tests/tests_concurrent.py @@ -0,0 +1,49 @@ +""" +Tests for `tqdm.contrib.concurrent`. +""" +from pytest import warns + +from tqdm.contrib.concurrent import process_map, thread_map + +from .tests_tqdm import StringIO, TqdmWarning, closing, importorskip, mark, skip + + +def incr(x): + """Dummy function""" + return x + 1 + + +def test_thread_map(): + """Test contrib.concurrent.thread_map""" + with closing(StringIO()) as our_file: + a = range(9) + b = [i + 1 for i in a] + try: + assert thread_map(lambda x: x + 1, a, file=our_file) == b + except ImportError as err: + skip(str(err)) + assert thread_map(incr, a, file=our_file) == b + + +def test_process_map(): + """Test contrib.concurrent.process_map""" + with closing(StringIO()) as our_file: + a = range(9) + b = [i + 1 for i in a] + try: + assert process_map(incr, a, file=our_file) == b + except ImportError as err: + skip(str(err)) + + +@mark.parametrize("iterables,should_warn", [([], False), (['x'], False), ([()], False), + (['x', ()], False), (['x' * 1001], True), + (['x' * 100, ('x',) * 1001], True)]) +def test_chunksize_warning(iterables, should_warn): + """Test contrib.concurrent.process_map chunksize warnings""" + patch = importorskip('unittest.mock').patch + with patch('tqdm.contrib.concurrent._executor_map'): + if should_warn: + warns(TqdmWarning, process_map, incr, *iterables) + else: + process_map(incr, *iterables) diff --git a/tests/tests_contrib.py b/tests/tests_contrib.py new file mode 100644 index 000000000..3a8396251 --- /dev/null +++ b/tests/tests_contrib.py @@ -0,0 +1,71 @@ +""" +Tests for `tqdm.contrib`. +""" +import sys + +import pytest + +from tqdm import tqdm +from tqdm.contrib import tenumerate, tmap, tzip + +from .tests_tqdm import StringIO, closing, importorskip + + +def incr(x): + """Dummy function""" + return x + 1 + + +@pytest.mark.parametrize("tqdm_kwargs", [{}, {"tqdm_class": tqdm}]) +def test_enumerate(tqdm_kwargs): + """Test contrib.tenumerate""" + with closing(StringIO()) as our_file: + a = range(9) + assert list(tenumerate(a, file=our_file, **tqdm_kwargs)) == list(enumerate(a)) + assert list(tenumerate(a, 42, file=our_file, **tqdm_kwargs)) == list( + enumerate(a, 42) + ) + with closing(StringIO()) as our_file: + _ = list(tenumerate((i for i in a), file=our_file, **tqdm_kwargs)) + assert "100%" not in our_file.getvalue() + with closing(StringIO()) as our_file: + _ = list(tenumerate((i for i in a), file=our_file, total=len(a), **tqdm_kwargs)) + assert "100%" in our_file.getvalue() + + +def test_enumerate_numpy(): + """Test contrib.tenumerate(numpy.ndarray)""" + np = importorskip("numpy") + with closing(StringIO()) as our_file: + a = np.random.random((42, 7)) + assert list(tenumerate(a, file=our_file)) == list(np.ndenumerate(a)) + + +@pytest.mark.parametrize("tqdm_kwargs", [{}, {"tqdm_class": tqdm}]) +def test_zip(tqdm_kwargs): + """Test contrib.tzip""" + with closing(StringIO()) as our_file: + a = range(9) + b = [i + 1 for i in a] + if sys.version_info[:1] < (3,): + assert tzip(a, b, file=our_file, **tqdm_kwargs) == zip(a, b) + else: + gen = tzip(a, b, file=our_file, **tqdm_kwargs) + assert gen != list(zip(a, b)) + assert list(gen) == list(zip(a, b)) + + +@pytest.mark.parametrize("tqdm_kwargs", [{}, {"tqdm_class": tqdm}]) +def test_map(tqdm_kwargs): + """Test contrib.tmap""" + with closing(StringIO()) as our_file: + a = range(9) + b = [i + 1 for i in a] + if sys.version_info[:1] < (3,): + assert tmap(lambda x: x + 1, a, file=our_file, **tqdm_kwargs) == map( + incr, a + ) + else: + gen = tmap(lambda x: x + 1, a, file=our_file, **tqdm_kwargs) + assert gen != b + assert list(gen) == b diff --git a/tests/tests_contrib_logging.py b/tests/tests_contrib_logging.py new file mode 100644 index 000000000..e2affa786 --- /dev/null +++ b/tests/tests_contrib_logging.py @@ -0,0 +1,181 @@ +# pylint: disable=missing-module-docstring, missing-class-docstring +# pylint: disable=missing-function-docstring, no-self-use +from __future__ import absolute_import + +import logging +import logging.handlers +import sys +from io import StringIO + +import pytest + +from tqdm import tqdm +from tqdm.contrib.logging import _get_first_found_console_logging_formatter +from tqdm.contrib.logging import _TqdmLoggingHandler as TqdmLoggingHandler +from tqdm.contrib.logging import logging_redirect_tqdm, tqdm_logging_redirect + +from .tests_tqdm import importorskip + +LOGGER = logging.getLogger(__name__) + +TEST_LOGGING_FORMATTER = logging.Formatter() + + +class CustomTqdm(tqdm): + messages = [] + + @classmethod + def write(cls, s, **__): # pylint: disable=arguments-differ + CustomTqdm.messages.append(s) + + +class ErrorRaisingTqdm(tqdm): + exception_class = RuntimeError + + @classmethod + def write(cls, s, **__): # pylint: disable=arguments-differ + raise ErrorRaisingTqdm.exception_class('fail fast') + + +class TestTqdmLoggingHandler: + def test_should_call_tqdm_write(self): + CustomTqdm.messages = [] + logger = logging.Logger('test') + logger.handlers = [TqdmLoggingHandler(CustomTqdm)] + logger.info('test') + assert CustomTqdm.messages == ['test'] + + def test_should_call_handle_error_if_exception_was_thrown(self): + patch = importorskip('unittest.mock').patch + logger = logging.Logger('test') + ErrorRaisingTqdm.exception_class = RuntimeError + handler = TqdmLoggingHandler(ErrorRaisingTqdm) + logger.handlers = [handler] + with patch.object(handler, 'handleError') as mock: + logger.info('test') + assert mock.called + + @pytest.mark.parametrize('exception_class', [ + KeyboardInterrupt, + SystemExit + ]) + def test_should_not_swallow_certain_exceptions(self, exception_class): + logger = logging.Logger('test') + ErrorRaisingTqdm.exception_class = exception_class + handler = TqdmLoggingHandler(ErrorRaisingTqdm) + logger.handlers = [handler] + with pytest.raises(exception_class): + logger.info('test') + + +class TestGetFirstFoundConsoleLoggingFormatter: + def test_should_return_none_for_no_handlers(self): + assert _get_first_found_console_logging_formatter([]) is None + + def test_should_return_none_without_stream_handler(self): + handler = logging.handlers.MemoryHandler(capacity=1) + handler.formatter = TEST_LOGGING_FORMATTER + assert _get_first_found_console_logging_formatter([handler]) is None + + def test_should_return_none_for_stream_handler_not_stdout_or_stderr(self): + handler = logging.StreamHandler(StringIO()) + handler.formatter = TEST_LOGGING_FORMATTER + assert _get_first_found_console_logging_formatter([handler]) is None + + def test_should_return_stream_handler_formatter_if_stream_is_stdout(self): + handler = logging.StreamHandler(sys.stdout) + handler.formatter = TEST_LOGGING_FORMATTER + assert _get_first_found_console_logging_formatter( + [handler] + ) == TEST_LOGGING_FORMATTER + + def test_should_return_stream_handler_formatter_if_stream_is_stderr(self): + handler = logging.StreamHandler(sys.stderr) + handler.formatter = TEST_LOGGING_FORMATTER + assert _get_first_found_console_logging_formatter( + [handler] + ) == TEST_LOGGING_FORMATTER + + +class TestRedirectLoggingToTqdm: + def test_should_add_and_remove_tqdm_handler(self): + logger = logging.Logger('test') + with logging_redirect_tqdm(loggers=[logger]): + assert len(logger.handlers) == 1 + assert isinstance(logger.handlers[0], TqdmLoggingHandler) + assert not logger.handlers + + def test_should_remove_and_restore_console_handlers(self): + logger = logging.Logger('test') + stderr_console_handler = logging.StreamHandler(sys.stderr) + stdout_console_handler = logging.StreamHandler(sys.stderr) + logger.handlers = [stderr_console_handler, stdout_console_handler] + with logging_redirect_tqdm(loggers=[logger]): + assert len(logger.handlers) == 1 + assert isinstance(logger.handlers[0], TqdmLoggingHandler) + assert logger.handlers == [stderr_console_handler, stdout_console_handler] + + def test_should_inherit_console_logger_formatter(self): + logger = logging.Logger('test') + formatter = logging.Formatter('custom: %(message)s') + console_handler = logging.StreamHandler(sys.stderr) + console_handler.setFormatter(formatter) + logger.handlers = [console_handler] + with logging_redirect_tqdm(loggers=[logger]): + assert logger.handlers[0].formatter == formatter + + def test_should_not_remove_stream_handlers_not_fot_stdout_or_stderr(self): + logger = logging.Logger('test') + stream_handler = logging.StreamHandler(StringIO()) + logger.addHandler(stream_handler) + with logging_redirect_tqdm(loggers=[logger]): + assert len(logger.handlers) == 2 + assert logger.handlers[0] == stream_handler + assert isinstance(logger.handlers[1], TqdmLoggingHandler) + assert logger.handlers == [stream_handler] + + +class TestTqdmWithLoggingRedirect: + def test_should_add_and_remove_handler_from_root_logger_by_default(self): + original_handlers = list(logging.root.handlers) + with tqdm_logging_redirect(total=1) as pbar: + assert isinstance(logging.root.handlers[-1], TqdmLoggingHandler) + LOGGER.info('test') + pbar.update(1) + assert logging.root.handlers == original_handlers + + def test_should_add_and_remove_handler_from_custom_logger(self): + logger = logging.Logger('test') + with tqdm_logging_redirect(total=1, loggers=[logger]) as pbar: + assert len(logger.handlers) == 1 + assert isinstance(logger.handlers[0], TqdmLoggingHandler) + logger.info('test') + pbar.update(1) + assert not logger.handlers + + def test_should_not_fail_with_logger_without_console_handler(self): + logger = logging.Logger('test') + logger.handlers = [] + with tqdm_logging_redirect(total=1, loggers=[logger]): + logger.info('test') + assert not logger.handlers + + def test_should_format_message(self): + logger = logging.Logger('test') + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setFormatter(logging.Formatter( + r'prefix:%(message)s' + )) + logger.handlers = [console_handler] + CustomTqdm.messages = [] + with tqdm_logging_redirect(loggers=[logger], tqdm_class=CustomTqdm): + logger.info('test') + assert CustomTqdm.messages == ['prefix:test'] + + def test_use_root_logger_by_default_and_write_to_custom_tqdm(self): + logger = logging.root + CustomTqdm.messages = [] + with tqdm_logging_redirect(total=1, tqdm_class=CustomTqdm) as pbar: + assert isinstance(pbar, CustomTqdm) + logger.info('test') + assert CustomTqdm.messages == ['test'] diff --git a/tests/tests_dask.py b/tests/tests_dask.py new file mode 100644 index 000000000..8bf4b64f2 --- /dev/null +++ b/tests/tests_dask.py @@ -0,0 +1,20 @@ +from __future__ import division + +from time import sleep + +from .tests_tqdm import importorskip, mark + +pytestmark = mark.slow + + +def test_dask(capsys): + """Test tqdm.dask.TqdmCallback""" + ProgressBar = importorskip('tqdm.dask').TqdmCallback + dask = importorskip('dask') + + schedule = [dask.delayed(sleep)(i / 10) for i in range(5)] + with ProgressBar(desc="computing"): + dask.compute(schedule) + _, err = capsys.readouterr() + assert "computing: " in err + assert '5/5' in err diff --git a/tests/tests_gui.py b/tests/tests_gui.py new file mode 100644 index 000000000..dddd918e3 --- /dev/null +++ b/tests/tests_gui.py @@ -0,0 +1,7 @@ +"""Test `tqdm.gui`.""" +from .tests_tqdm import importorskip + + +def test_gui_import(): + """Test `tqdm.gui` import""" + importorskip('tqdm.gui') diff --git a/tqdm/tests/tests_itertools.py b/tests/tests_itertools.py similarity index 85% rename from tqdm/tests/tests_itertools.py rename to tests/tests_itertools.py index c55e07db8..8aac70da7 100644 --- a/tqdm/tests/tests_itertools.py +++ b/tests/tests_itertools.py @@ -1,10 +1,12 @@ """ Tests for `tqdm.contrib.itertools`. """ -from tqdm.contrib.itertools import product -from tests_tqdm import with_setup, pretest, posttest, StringIO, closing import itertools +from tqdm.contrib.itertools import product + +from .tests_tqdm import StringIO, closing + class NoLenIter(object): def __init__(self, iterable): @@ -15,7 +17,6 @@ def __iter__(self): yield i -@with_setup(pretest, posttest) def test_product(): """Test contrib.itertools.product""" with closing(StringIO()) as our_file: diff --git a/tests/tests_keras.py b/tests/tests_keras.py new file mode 100644 index 000000000..b26cdbb78 --- /dev/null +++ b/tests/tests_keras.py @@ -0,0 +1,82 @@ +from __future__ import division + +from .tests_tqdm import importorskip, mark + +pytestmark = mark.slow + + +@mark.filterwarnings("ignore:.*:DeprecationWarning") +def test_keras(capsys): + """Test tqdm.keras.TqdmCallback""" + TqdmCallback = importorskip('tqdm.keras').TqdmCallback + np = importorskip('numpy') + try: + import keras as K + except ImportError: + K = importorskip('tensorflow.keras') + + # 1D autoencoder + dtype = np.float32 + model = K.models.Sequential([ + K.layers.InputLayer((1, 1), dtype=dtype), K.layers.Conv1D(1, 1)]) + model.compile("adam", "mse") + x = np.random.rand(100, 1, 1).astype(dtype) + batch_size = 10 + batches = len(x) / batch_size + epochs = 5 + + # just epoch (no batch) progress + model.fit( + x, + x, + epochs=epochs, + batch_size=batch_size, + verbose=False, + callbacks=[ + TqdmCallback( + epochs, + desc="training", + data_size=len(x), + batch_size=batch_size, + verbose=0, + )], + ) + _, res = capsys.readouterr() + assert "training: " in res + assert "{epochs}/{epochs}".format(epochs=epochs) in res + assert "{batches}/{batches}".format(batches=batches) not in res + + # full (epoch and batch) progress + model.fit( + x, + x, + epochs=epochs, + batch_size=batch_size, + verbose=False, + callbacks=[ + TqdmCallback( + epochs, + desc="training", + data_size=len(x), + batch_size=batch_size, + verbose=2, + )], + ) + _, res = capsys.readouterr() + assert "training: " in res + assert "{epochs}/{epochs}".format(epochs=epochs) in res + assert "{batches}/{batches}".format(batches=batches) in res + + # auto-detect epochs and batches + model.fit( + x, + x, + epochs=epochs, + batch_size=batch_size, + verbose=False, + callbacks=[TqdmCallback(desc="training", verbose=2)], + ) + _, res = capsys.readouterr() + assert "training: " in res + assert "{epochs}/{epochs}".format(epochs=epochs) in res + assert "{batches}/{batches}".format(batches=batches) in res diff --git a/tests/tests_main.py b/tests/tests_main.py new file mode 100644 index 000000000..c02c896bb --- /dev/null +++ b/tests/tests_main.py @@ -0,0 +1,244 @@ +"""Test CLI usage.""" +import logging +import subprocess # nosec +import sys +from functools import wraps +from os import linesep + +from tqdm.cli import TqdmKeyError, TqdmTypeError, main +from tqdm.utils import IS_WIN + +from .tests_tqdm import BytesIO, _range, closing, mark, raises + + +def restore_sys(func): + """Decorates `func(capsysbin)` to save & restore `sys.(stdin|argv)`.""" + @wraps(func) + def inner(capsysbin): + """function requiring capsysbin which may alter `sys.(stdin|argv)`""" + _SYS = sys.stdin, sys.argv + try: + res = func(capsysbin) + finally: + sys.stdin, sys.argv = _SYS + return res + + return inner + + +def norm(bytestr): + """Normalise line endings.""" + return bytestr if linesep == "\n" else bytestr.replace(linesep.encode(), b"\n") + + +@mark.slow +def test_pipes(): + """Test command line pipes""" + ls_out = subprocess.check_output(['ls']) # nosec + ls = subprocess.Popen(['ls'], stdout=subprocess.PIPE) # nosec + res = subprocess.Popen( # nosec + [sys.executable, '-c', 'from tqdm.cli import main; main()'], + stdin=ls.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = res.communicate() + assert ls.poll() == 0 + + # actual test: + assert norm(ls_out) == norm(out) + assert b"it/s" in err + + +if sys.version_info[:2] >= (3, 8): + test_pipes = mark.filterwarnings("ignore:unclosed file:ResourceWarning")( + test_pipes) + + +def test_main_import(): + """Test main CLI import""" + N = 123 + _SYS = sys.stdin, sys.argv + # test direct import + sys.stdin = [str(i).encode() for i in _range(N)] + sys.argv = ['', '--desc', 'Test CLI import', + '--ascii', 'True', '--unit_scale', 'True'] + try: + import tqdm.__main__ # NOQA, pylint: disable=unused-variable + finally: + sys.stdin, sys.argv = _SYS + + +@restore_sys +def test_main_bytes(capsysbin): + """Test CLI --bytes""" + N = 123 + + # test --delim + IN_DATA = '\0'.join(map(str, _range(N))).encode() + with closing(BytesIO()) as sys.stdin: + sys.stdin.write(IN_DATA) + # sys.stdin.write(b'\xff') # TODO + sys.stdin.seek(0) + main(sys.stderr, ['--desc', 'Test CLI delim', '--ascii', 'True', + '--delim', r'\0', '--buf_size', '64']) + out, err = capsysbin.readouterr() + assert out == IN_DATA + assert str(N) + "it" in err.decode("U8") + + # test --bytes + IN_DATA = IN_DATA.replace(b'\0', b'\n') + with closing(BytesIO()) as sys.stdin: + sys.stdin.write(IN_DATA) + sys.stdin.seek(0) + main(sys.stderr, ['--ascii', '--bytes=True', '--unit_scale', 'False']) + out, err = capsysbin.readouterr() + assert out == IN_DATA + assert str(len(IN_DATA)) + "B" in err.decode("U8") + + +@mark.skipif(sys.version_info[0] == 2, reason="no caplog on py2") +def test_main_log(capsysbin, caplog): + """Test CLI --log""" + _SYS = sys.stdin, sys.argv + N = 123 + sys.stdin = [(str(i) + '\n').encode() for i in _range(N)] + IN_DATA = b''.join(sys.stdin) + try: + with caplog.at_level(logging.INFO): + main(sys.stderr, ['--log', 'INFO']) + out, err = capsysbin.readouterr() + assert norm(out) == IN_DATA and b"123/123" in err + assert not caplog.record_tuples + with caplog.at_level(logging.DEBUG): + main(sys.stderr, ['--log', 'DEBUG']) + out, err = capsysbin.readouterr() + assert norm(out) == IN_DATA and b"123/123" in err + assert caplog.record_tuples + finally: + sys.stdin, sys.argv = _SYS + + +@restore_sys +def test_main(capsysbin): + """Test misc CLI options""" + N = 123 + sys.stdin = [(str(i) + '\n').encode() for i in _range(N)] + IN_DATA = b''.join(sys.stdin) + + # test --tee + main(sys.stderr, ['--mininterval', '0', '--miniters', '1']) + out, err = capsysbin.readouterr() + assert norm(out) == IN_DATA and b"123/123" in err + assert N <= len(err.split(b"\r")) < N + 5 + + len_err = len(err) + main(sys.stderr, ['--tee', '--mininterval', '0', '--miniters', '1']) + out, err = capsysbin.readouterr() + assert norm(out) == IN_DATA and b"123/123" in err + # spaces to clear intermediate lines could increase length + assert len_err + len(norm(out)) <= len(err) + + # test --null + main(sys.stderr, ['--null']) + out, err = capsysbin.readouterr() + assert not out and b"123/123" in err + + # test integer --update + main(sys.stderr, ['--update']) + out, err = capsysbin.readouterr() + assert norm(out) == IN_DATA + assert (str(N // 2 * N) + "it").encode() in err, "expected arithmetic sum formula" + + # test integer --update_to + main(sys.stderr, ['--update-to']) + out, err = capsysbin.readouterr() + assert norm(out) == IN_DATA + assert (str(N - 1) + "it").encode() in err + assert (str(N) + "it").encode() not in err + + with closing(BytesIO()) as sys.stdin: + sys.stdin.write(IN_DATA.replace(b'\n', b'D')) + + # test integer --update --delim + sys.stdin.seek(0) + main(sys.stderr, ['--update', '--delim', 'D']) + out, err = capsysbin.readouterr() + assert out == IN_DATA.replace(b'\n', b'D') + assert (str(N // 2 * N) + "it").encode() in err, "expected arithmetic sum" + + # test integer --update_to --delim + sys.stdin.seek(0) + main(sys.stderr, ['--update-to', '--delim', 'D']) + out, err = capsysbin.readouterr() + assert out == IN_DATA.replace(b'\n', b'D') + assert (str(N - 1) + "it").encode() in err + assert (str(N) + "it").encode() not in err + + # test float --update_to + sys.stdin = [(str(i / 2.0) + '\n').encode() for i in _range(N)] + IN_DATA = b''.join(sys.stdin) + main(sys.stderr, ['--update-to']) + out, err = capsysbin.readouterr() + assert norm(out) == IN_DATA + assert (str((N - 1) / 2.0) + "it").encode() in err + assert (str(N / 2.0) + "it").encode() not in err + + +@mark.slow +@mark.skipif(IS_WIN, reason="no manpages on windows") +def test_manpath(tmp_path): + """Test CLI --manpath""" + man = tmp_path / "tqdm.1" + assert not man.exists() + with raises(SystemExit): + main(argv=['--manpath', str(tmp_path)]) + assert man.is_file() + + +@mark.slow +@mark.skipif(IS_WIN, reason="no completion on windows") +def test_comppath(tmp_path): + """Test CLI --comppath""" + man = tmp_path / "tqdm_completion.sh" + assert not man.exists() + with raises(SystemExit): + main(argv=['--comppath', str(tmp_path)]) + assert man.is_file() + + # check most important options appear + script = man.read_text() + opts = {'--help', '--desc', '--total', '--leave', '--ncols', '--ascii', + '--dynamic_ncols', '--position', '--bytes', '--nrows', '--delim', + '--manpath', '--comppath'} + assert all(args in script for args in opts) + + +@restore_sys +def test_exceptions(capsysbin): + """Test CLI Exceptions""" + N = 123 + sys.stdin = [str(i) + '\n' for i in _range(N)] + IN_DATA = ''.join(sys.stdin).encode() + + with raises(TqdmKeyError, match="bad_arg_u_ment"): + main(sys.stderr, argv=['-ascii', '-unit_scale', '--bad_arg_u_ment', 'foo']) + out, _ = capsysbin.readouterr() + assert norm(out) == IN_DATA + + with raises(TqdmTypeError, match="invalid_bool_value"): + main(sys.stderr, argv=['-ascii', '-unit_scale', 'invalid_bool_value']) + out, _ = capsysbin.readouterr() + assert norm(out) == IN_DATA + + with raises(TqdmTypeError, match="invalid_int_value"): + main(sys.stderr, argv=['-ascii', '--total', 'invalid_int_value']) + out, _ = capsysbin.readouterr() + assert norm(out) == IN_DATA + + with raises(TqdmKeyError, match="Can only have one of --"): + main(sys.stderr, argv=['--update', '--update_to']) + out, _ = capsysbin.readouterr() + assert norm(out) == IN_DATA + + # test SystemExits + for i in ('-h', '--help', '-v', '--version'): + with raises(SystemExit): + main(argv=[i]) diff --git a/tqdm/tests/tests_notebook.py b/tests/tests_notebook.py similarity index 74% rename from tqdm/tests/tests_notebook.py rename to tests/tests_notebook.py index 3af992f0c..004d7e57b 100644 --- a/tqdm/tests/tests_notebook.py +++ b/tests/tests_notebook.py @@ -1,8 +1,6 @@ from tqdm.notebook import tqdm as tqdm_notebook -from tests_tqdm import with_setup, pretest, posttest -@with_setup(pretest, posttest) def test_notebook_disabled_description(): """Test that set_description works for disabled tqdm_notebook""" with tqdm_notebook(1, disable=True) as t: diff --git a/tqdm/tests/tests_pandas.py b/tests/tests_pandas.py similarity index 75% rename from tqdm/tests/tests_pandas.py rename to tests/tests_pandas.py index 8719a7ca2..334a97cbb 100644 --- a/tqdm/tests/tests_pandas.py +++ b/tests/tests_pandas.py @@ -1,17 +1,17 @@ from tqdm import tqdm -from tests_tqdm import with_setup, pretest, posttest, SkipTest, \ - StringIO, closing + +from .tests_tqdm import StringIO, closing, importorskip, mark, skip + +pytestmark = mark.slow + +random = importorskip('numpy.random') +rand = random.rand +randint = random.randint +pd = importorskip('pandas') -@with_setup(pretest, posttest) def test_pandas_setup(): """Test tqdm.pandas()""" - try: - from numpy.random import randint - import pandas as pd - except ImportError: - raise SkipTest - with closing(StringIO()) as our_file: tqdm.pandas(file=our_file, leave=True, ascii=True, total=123) series = pd.Series(randint(0, 50, (100,))) @@ -20,15 +20,8 @@ def test_pandas_setup(): assert '100/123' in res -@with_setup(pretest, posttest) def test_pandas_rolling_expanding(): """Test pandas.(Series|DataFrame).(rolling|expanding)""" - try: - from numpy.random import randint - import pandas as pd - except ImportError: - raise SkipTest - with closing(StringIO()) as our_file: tqdm.pandas(file=our_file, leave=True, ascii=True) @@ -46,20 +39,12 @@ def test_pandas_rolling_expanding(): our_file.seek(0) if our_file.getvalue().count(exres) < 2: our_file.seek(0) - raise AssertionError( - "\nExpected:\n{0}\nIn:\n{1}\n".format( - exres + " at least twice.", our_file.read())) + raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format( + exres + " at least twice.", our_file.read())) -@with_setup(pretest, posttest) def test_pandas_series(): """Test pandas.Series.progress_apply and .progress_map""" - try: - from numpy.random import randint - import pandas as pd - except ImportError: - raise SkipTest - with closing(StringIO()) as our_file: tqdm.pandas(file=our_file, leave=True, ascii=True) @@ -77,20 +62,12 @@ def test_pandas_series(): our_file.seek(0) if our_file.getvalue().count(exres) < 2: our_file.seek(0) - raise AssertionError( - "\nExpected:\n{0}\nIn:\n{1}\n".format( - exres + " at least twice.", our_file.read())) + raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format( + exres + " at least twice.", our_file.read())) -@with_setup(pretest, posttest) def test_pandas_data_frame(): """Test pandas.DataFrame.progress_apply and .progress_applymap""" - try: - from numpy.random import randint - import pandas as pd - except ImportError: - raise SkipTest - with closing(StringIO()) as our_file: tqdm.pandas(file=our_file, leave=True, ascii=True) df = pd.DataFrame(randint(0, 50, (100, 200))) @@ -126,20 +103,12 @@ def task_func(x): our_file.seek(0) if our_file.getvalue().count(exres) < 1: our_file.seek(0) - raise AssertionError( - "\nExpected:\n{0}\nIn:\n {1}\n".format( - exres + " at least once.", our_file.read())) + raise AssertionError("\nExpected:\n{0}\nIn:\n {1}\n".format( + exres + " at least once.", our_file.read())) -@with_setup(pretest, posttest) def test_pandas_groupby_apply(): """Test pandas.DataFrame.groupby(...).progress_apply""" - try: - from numpy.random import randint, rand - import pandas as pd - except ImportError: - raise SkipTest - with closing(StringIO()) as our_file: tqdm.pandas(file=our_file, leave=False, ascii=True) @@ -149,7 +118,7 @@ def test_pandas_groupby_apply(): dfs = pd.DataFrame(randint(0, 50, (500, 3)), columns=list('abc')) dfs.groupby(['a']).progress_apply(lambda x: None) - df2 = df = pd.DataFrame(dict(a=randint(1, 8, 10000), b=rand(10000))) + df2 = df = pd.DataFrame({'a': randint(1, 8, 10000), 'b': rand(10000)}) res1 = df2.groupby("a").apply(max) res2 = df2.groupby("a").progress_apply(max) assert res1.equals(res2) @@ -187,20 +156,12 @@ def test_pandas_groupby_apply(): our_file.seek(0) if our_file.getvalue().count(exres) < 1: our_file.seek(0) - raise AssertionError( - "\nExpected:\n{0}\nIn:\n {1}\n".format( - exres + " at least once.", our_file.read())) + raise AssertionError("\nExpected:\n{0}\nIn:\n {1}\n".format( + exres + " at least once.", our_file.read())) -@with_setup(pretest, posttest) def test_pandas_leave(): """Test pandas with `leave=True`""" - try: - from numpy.random import randint - import pandas as pd - except ImportError: - raise SkipTest - with closing(StringIO()) as our_file: df = pd.DataFrame(randint(0, 100, (1000, 6))) tqdm.pandas(file=our_file, leave=True, ascii=True) @@ -211,20 +172,17 @@ def test_pandas_leave(): exres = '100%|##########| 100/100' if exres not in our_file.read(): our_file.seek(0) - raise AssertionError( - "\nExpected:\n{0}\nIn:{1}\n".format(exres, our_file.read())) + raise AssertionError("\nExpected:\n{0}\nIn:{1}\n".format( + exres, our_file.read())) -@with_setup(pretest, posttest) def test_pandas_apply_args_deprecation(): """Test warning info in `pandas.Dataframe(Series).progress_apply(func, *args)`""" try: - from numpy.random import randint from tqdm import tqdm_pandas - import pandas as pd - except ImportError: - raise SkipTest + except ImportError as err: + skip(str(err)) with closing(StringIO()) as our_file: tqdm_pandas(tqdm(file=our_file, leave=False, ascii=True, ncols=20)) @@ -232,20 +190,17 @@ def test_pandas_apply_args_deprecation(): df.progress_apply(lambda x: None, 1) # 1 shall cause a warning # Check deprecation message res = our_file.getvalue() - assert all([i in res for i in ( + assert all(i in res for i in ( "TqdmDeprecationWarning", "not supported", - "keyword arguments instead")]) + "keyword arguments instead")) -@with_setup(pretest, posttest) def test_pandas_deprecation(): """Test bar object instance as argument deprecation""" try: - from numpy.random import randint from tqdm import tqdm_pandas - import pandas as pd - except ImportError: - raise SkipTest + except ImportError as err: + skip(str(err)) with closing(StringIO()) as our_file: tqdm_pandas(tqdm(file=our_file, leave=False, ascii=True, ncols=20)) diff --git a/tqdm/tests/tests_perf.py b/tests/tests_perf.py similarity index 54% rename from tqdm/tests/tests_perf.py rename to tests/tests_perf.py index 6cb7a6ee5..ec9bd1d82 100644 --- a/tqdm/tests/tests_perf.py +++ b/tests/tests_perf.py @@ -1,17 +1,10 @@ -from __future__ import print_function, division - -from nose.plugins.skip import SkipTest - -from contextlib import contextmanager +from __future__ import division, print_function import sys +from contextlib import contextmanager +from functools import wraps from time import sleep, time -from tqdm import trange -from tqdm import tqdm - -from tests_tqdm import with_setup, pretest, posttest, StringIO, closing, _range - # Use relative/cpu timer to have reliable timings when there is a sudden load try: from time import process_time @@ -19,9 +12,11 @@ from time import clock process_time = clock +from tqdm import tqdm, trange + +from .tests_tqdm import _range, importorskip, mark, patch_lock, skip -def get_relative_time(prevtime=0): - return process_time() - prevtime +pytestmark = mark.slow def cpu_sleep(t): @@ -45,9 +40,10 @@ def checkCpuTime(sleeptime=0.2): cpu_sleep(sleeptime) t2 = process_time() - start2 - if abs(t1) < 0.0001 and (t1 < t2 / 10): + if abs(t1) < 0.0001 and t1 < t2 / 10: + checkCpuTime.passed = True return True - raise SkipTest + skip("cpu time not reliable on this machine") checkCpuTime.passed = False @@ -55,44 +51,40 @@ def checkCpuTime(sleeptime=0.2): @contextmanager def relative_timer(): + """yields a context timer function which stops ticking on exit""" start = process_time() def elapser(): return process_time() - start yield lambda: elapser() - spent = process_time() - start + spent = elapser() def elapser(): # NOQA return spent -def retry_on_except(n=3): - def wrapper(fn): - def test_inner(): +def retry_on_except(n=3, check_cpu_time=True): + """decroator for retrying `n` times before raising Exceptions""" + def wrapper(func): + """actual decorator""" + @wraps(func) + def test_inner(*args, **kwargs): + """may skip if `check_cpu_time` fails""" for i in range(1, n + 1): try: - checkCpuTime() - fn() - except SkipTest: + if check_cpu_time: + checkCpuTime() + func(*args, **kwargs) + except Exception: if i >= n: raise else: return - - test_inner.__doc__ = fn.__doc__ return test_inner - return wrapper -class MockIO(StringIO): - """Wraps StringIO to mock a file with no I/O""" - - def write(self, data): - return - - def simple_progress(iterable=None, total=None, file=sys.stdout, desc='', leave=False, miniters=1, mininterval=0.1, width=60): """Simple progress bar reproducing tqdm's major features""" @@ -132,15 +124,15 @@ def update_and_print(i=1): eta = (total - n[0]) / rate if rate > 0 else 0 eta_fmt = format_interval(eta) - # bar = "#" * int(frac * width) + # full_bar = "#" * int(frac * width) barfill = " " * int((1.0 - frac) * width) bar_length, frac_bar_length = divmod(int(frac * width * 10), 10) - bar = '#' * bar_length + full_bar = '#' * bar_length frac_bar = chr(48 + frac_bar_length) if frac_bar_length \ else ' ' file.write("\r%s %i%%|%s%s%s| %i/%i [%s<%s, %s]" % - (desc, percentage, bar, frac_bar, barfill, n[0], + (desc, percentage, full_bar, frac_bar, barfill, n[0], total, spent_fmt, eta_fmt, rate_fmt)) if n[0] == total and leave: @@ -171,82 +163,67 @@ def assert_performance(thresh, name_left, time_left, name_right, time_right): ratio=time_left / time_right, thresh=thresh)) -@with_setup(pretest, posttest) @retry_on_except() -def test_iter_overhead(): +def test_iter_basic_overhead(): """Test overhead of iteration based tqdm""" - total = int(1e6) - with closing(MockIO()) as our_file: - a = 0 - with trange(total, file=our_file) as t: - with relative_timer() as time_tqdm: - for i in t: - a += i - assert a == (total * total - total) / 2.0 - - a = 0 - with relative_timer() as time_bench: - for i in _range(total): + a = 0 + with trange(total) as t: + with relative_timer() as time_tqdm: + for i in t: a += i - our_file.write(a) + assert a == (total * total - total) / 2.0 + + a = 0 + with relative_timer() as time_bench: + for i in _range(total): + a += i + sys.stdout.write(str(a)) - assert_performance(6, 'trange', time_tqdm(), 'range', time_bench()) + assert_performance(3, 'trange', time_tqdm(), 'range', time_bench()) -@with_setup(pretest, posttest) @retry_on_except() -def test_manual_overhead(): +def test_manual_basic_overhead(): """Test overhead of manual tqdm""" - total = int(1e6) - with closing(MockIO()) as our_file: - with tqdm(total=total * 10, file=our_file, leave=True) as t: - a = 0 - with relative_timer() as time_tqdm: - for i in _range(total): - a += i - t.update(10) - + with tqdm(total=total * 10, leave=True) as t: a = 0 - with relative_timer() as time_bench: + with relative_timer() as time_tqdm: for i in _range(total): a += i - our_file.write(a) + t.update(10) - assert_performance(6, 'tqdm', time_tqdm(), 'range', time_bench()) + a = 0 + with relative_timer() as time_bench: + for i in _range(total): + a += i + sys.stdout.write(str(a)) + + assert_performance(5, 'tqdm', time_tqdm(), 'range', time_bench()) def worker(total, blocking=True): def incr_bar(x): - with closing(StringIO()) as our_file: - for _ in trange( - total, file=our_file, - lock_args=None if blocking else (False,), - miniters=1, mininterval=0, maxinterval=0): - pass + for _ in trange(total, lock_args=None if blocking else (False,), + miniters=1, mininterval=0, maxinterval=0): + pass return x + 1 return incr_bar -@with_setup(pretest, posttest) @retry_on_except() +@patch_lock(thread=True) def test_lock_args(): """Test overhead of nonblocking threads""" - try: - from concurrent.futures import ThreadPoolExecutor - from threading import RLock - except ImportError: - raise SkipTest - import sys - - total = 8 - subtotal = 1000 - - tqdm.set_lock(RLock()) - with ThreadPoolExecutor(total) as pool: + ThreadPoolExecutor = importorskip('concurrent.futures').ThreadPoolExecutor + + total = 16 + subtotal = 10000 + + with ThreadPoolExecutor() as pool: sys.stderr.write('block ... ') sys.stderr.flush() with relative_timer() as time_tqdm: @@ -258,110 +235,95 @@ def test_lock_args(): res = list(pool.map(worker(subtotal, False), range(total))) assert sum(res) == sum(range(total)) + total - assert_performance(0.2, 'noblock', time_noblock(), 'tqdm', time_tqdm()) + assert_performance(0.5, 'noblock', time_noblock(), 'tqdm', time_tqdm()) -@with_setup(pretest, posttest) -@retry_on_except() +@retry_on_except(10) def test_iter_overhead_hard(): """Test overhead of iteration based tqdm (hard)""" - total = int(1e5) - with closing(MockIO()) as our_file: - a = 0 - with trange(total, file=our_file, leave=True, miniters=1, - mininterval=0, maxinterval=0) as t: - with relative_timer() as time_tqdm: - for i in t: - a += i - assert a == (total * total - total) / 2.0 - - a = 0 - with relative_timer() as time_bench: - for i in _range(total): + a = 0 + with trange(total, leave=True, miniters=1, + mininterval=0, maxinterval=0) as t: + with relative_timer() as time_tqdm: + for i in t: a += i - our_file.write(("%i" % a) * 40) + assert a == (total * total - total) / 2.0 - assert_performance(85, 'trange', time_tqdm(), 'range', time_bench()) + a = 0 + with relative_timer() as time_bench: + for i in _range(total): + a += i + sys.stdout.write(("%i" % a) * 40) + assert_performance(130, 'trange', time_tqdm(), 'range', time_bench()) -@with_setup(pretest, posttest) -@retry_on_except() + +@retry_on_except(10) def test_manual_overhead_hard(): """Test overhead of manual tqdm (hard)""" - total = int(1e5) - with closing(MockIO()) as our_file: - t = tqdm(total=total * 10, file=our_file, leave=True, miniters=1, - mininterval=0, maxinterval=0) + with tqdm(total=total * 10, leave=True, miniters=1, + mininterval=0, maxinterval=0) as t: a = 0 with relative_timer() as time_tqdm: for i in _range(total): a += i t.update(10) - a = 0 - with relative_timer() as time_bench: - for i in _range(total): - a += i - our_file.write(("%i" % a) * 40) + a = 0 + with relative_timer() as time_bench: + for i in _range(total): + a += i + sys.stdout.write(("%i" % a) * 40) - assert_performance(85, 'tqdm', time_tqdm(), 'range', time_bench()) + assert_performance(130, 'tqdm', time_tqdm(), 'range', time_bench()) -@with_setup(pretest, posttest) -@retry_on_except() +@retry_on_except(10) def test_iter_overhead_simplebar_hard(): """Test overhead of iteration based tqdm vs simple progress bar (hard)""" - total = int(1e4) - with closing(MockIO()) as our_file: - a = 0 - with trange(total, file=our_file, leave=True, miniters=1, - mininterval=0, maxinterval=0) as t: - with relative_timer() as time_tqdm: - for i in t: - a += i - assert a == (total * total - total) / 2.0 - - a = 0 - s = simple_progress(_range(total), file=our_file, leave=True, - miniters=1, mininterval=0) - with relative_timer() as time_bench: - for i in s: + a = 0 + with trange(total, leave=True, miniters=1, + mininterval=0, maxinterval=0) as t: + with relative_timer() as time_tqdm: + for i in t: a += i + assert a == (total * total - total) / 2.0 - assert_performance( - 5, 'trange', time_tqdm(), 'simple_progress', time_bench()) + a = 0 + s = simple_progress(_range(total), leave=True, + miniters=1, mininterval=0) + with relative_timer() as time_bench: + for i in s: + a += i + assert_performance(10, 'trange', time_tqdm(), 'simple_progress', time_bench()) -@with_setup(pretest, posttest) -@retry_on_except() + +@retry_on_except(10) def test_manual_overhead_simplebar_hard(): """Test overhead of manual tqdm vs simple progress bar (hard)""" - total = int(1e4) - with closing(MockIO()) as our_file: - t = tqdm(total=total * 10, file=our_file, leave=True, miniters=1, - mininterval=0, maxinterval=0) + with tqdm(total=total * 10, leave=True, miniters=1, + mininterval=0, maxinterval=0) as t: a = 0 with relative_timer() as time_tqdm: for i in _range(total): a += i t.update(10) - simplebar_update = simple_progress( - total=total * 10, file=our_file, leave=True, miniters=1, - mininterval=0) - a = 0 - with relative_timer() as time_bench: - for i in _range(total): - a += i - simplebar_update(10) + simplebar_update = simple_progress(total=total * 10, leave=True, + miniters=1, mininterval=0) + a = 0 + with relative_timer() as time_bench: + for i in _range(total): + a += i + simplebar_update(10) - assert_performance( - 5, 'tqdm', time_tqdm(), 'simple_progress', time_bench()) + assert_performance(10, 'tqdm', time_tqdm(), 'simple_progress', time_bench()) diff --git a/tests/tests_rich.py b/tests/tests_rich.py new file mode 100644 index 000000000..c75e246d9 --- /dev/null +++ b/tests/tests_rich.py @@ -0,0 +1,10 @@ +"""Test `tqdm.rich`.""" +import sys + +from .tests_tqdm import importorskip, mark + + +@mark.skipif(sys.version_info[:3] < (3, 6, 1), reason="`rich` needs py>=3.6.1") +def test_rich_import(): + """Test `tqdm.rich` import""" + importorskip('tqdm.rich') diff --git a/tqdm/tests/tests_synchronisation.py b/tests/tests_synchronisation.py similarity index 50% rename from tqdm/tests/tests_synchronisation.py rename to tests/tests_synchronisation.py index 34f682a58..7ee55fb1d 100644 --- a/tqdm/tests/tests_synchronisation.py +++ b/tests/tests_synchronisation.py @@ -1,42 +1,95 @@ from __future__ import division -from tqdm import tqdm, trange, TMonitor -from tests_tqdm import with_setup, pretest, posttest, SkipTest, \ - StringIO, closing -from tests_tqdm import DiscreteTimer, cpu_timify -from tests_perf import retry_on_except import sys -from time import sleep +from functools import wraps from threading import Event +from time import sleep, time +from tqdm import TMonitor, tqdm, trange -class FakeSleep(object): - """Wait until the discrete timer reached the required time""" - def __init__(self, dtimer): - self.dtimer = dtimer +from .tests_perf import retry_on_except +from .tests_tqdm import StringIO, closing, importorskip, patch_lock, skip - def sleep(self, t): - end = t + self.dtimer.t - while self.dtimer.t < end: - sleep(0.0000001) # sleep a bit to interrupt (instead of pass) +class Time(object): + """Fake time class class providing an offset""" + offset = 0 + + @classmethod + def reset(cls): + """zeroes internal offset""" + cls.offset = 0 + + @classmethod + def time(cls): + """time.time() + offset""" + return time() + cls.offset + + @staticmethod + def sleep(dur): + """identical to time.sleep()""" + sleep(dur) + + @classmethod + def fake_sleep(cls, dur): + """adds `dur` to internal offset""" + cls.offset += dur + sleep(0.000001) # sleep to allow interrupt (instead of pass) -class FakeTqdm(object): - _instances = [] +def FakeEvent(): + """patched `threading.Event` where `wait()` uses `Time.fake_sleep()`""" + event = Event() # not a class in py2 so can't inherit -def make_create_fake_sleep_event(sleep): - def wait(self, timeout=None): + def wait(timeout=None): + """uses Time.fake_sleep""" if timeout is not None: - sleep(timeout) - return self.is_set() + Time.fake_sleep(timeout) + return event.is_set() + + event.wait = wait + return event + + +def patch_sleep(func): + """Temporarily makes TMonitor use Time.fake_sleep""" + @wraps(func) + def inner(*args, **kwargs): + """restores TMonitor on completion regardless of Exceptions""" + TMonitor._test["time"] = Time.time + TMonitor._test["Event"] = FakeEvent + if tqdm.monitor: + assert not tqdm.monitor.get_instances() + tqdm.monitor.exit() + del tqdm.monitor + tqdm.monitor = None + try: + return func(*args, **kwargs) + finally: + # Check that class var monitor is deleted if no instance left + tqdm.monitor_interval = 10 + if tqdm.monitor: + assert not tqdm.monitor.get_instances() + tqdm.monitor.exit() + del tqdm.monitor + tqdm.monitor = None + TMonitor._test.pop("Event") + TMonitor._test.pop("time") + + return inner + - def create_fake_sleep_event(): - event = Event() - event.wait = wait - return event +def cpu_timify(t, timer=Time): + """Force tqdm to use the specified timer instead of system-wide time""" + t._time = timer.time + t._sleep = timer.fake_sleep + t.start_t = t.last_print_t = t._time() + return timer - return create_fake_sleep_event + +class FakeTqdm(object): + _instances = set() + get_lock = tqdm.get_lock def incr(x): @@ -50,164 +103,122 @@ def incr_bar(x): return incr(x) -@with_setup(pretest, posttest) +@patch_sleep def test_monitor_thread(): """Test dummy monitoring thread""" - maxinterval = 10 - - # Setup a discrete timer - timer = DiscreteTimer() - TMonitor._time = timer.time - # And a fake sleeper - sleeper = FakeSleep(timer) - TMonitor._event = make_create_fake_sleep_event(sleeper.sleep) - - # Instanciate the monitor - monitor = TMonitor(FakeTqdm, maxinterval) + monitor = TMonitor(FakeTqdm, 10) # Test if alive, then killed assert monitor.report() monitor.exit() - timer.sleep(maxinterval * 2) # need to go out of the sleep to die assert not monitor.report() - # assert not monitor.is_alive() # not working dunno why, thread not killed + assert not monitor.is_alive() del monitor -@with_setup(pretest, posttest) +@patch_sleep def test_monitoring_and_cleanup(): """Test for stalled tqdm instance and monitor deletion""" # Note: should fix miniters for these tests, else with dynamic_miniters # it's too complicated to handle with monitoring update and maxinterval... - maxinterval = 2 - + maxinterval = tqdm.monitor_interval + assert maxinterval == 10 total = 1000 - # Setup a discrete timer - timer = DiscreteTimer() - # And a fake sleeper - sleeper = FakeSleep(timer) - # Setup TMonitor to use the timer - TMonitor._time = timer.time - TMonitor._event = make_create_fake_sleep_event(sleeper.sleep) - # Set monitor interval - tqdm.monitor_interval = maxinterval + with closing(StringIO()) as our_file: with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1, maxinterval=maxinterval) as t: - cpu_timify(t, timer) + cpu_timify(t, Time) # Do a lot of iterations in a small timeframe # (smaller than monitor interval) - timer.sleep(maxinterval / 2) # monitor won't wake up + Time.fake_sleep(maxinterval / 10) # monitor won't wake up t.update(500) # check that our fixed miniters is still there - assert t.miniters == 500 + assert t.miniters <= 500 # TODO: should really be == 500 # Then do 1 it after monitor interval, so that monitor kicks in - timer.sleep(maxinterval * 2) + Time.fake_sleep(maxinterval) t.update(1) - # Wait for the monitor to get out of sleep's loop and update tqdm.. - timeend = timer.time() + # Wait for the monitor to get out of sleep's loop and update tqdm. + timeend = Time.time() while not (t.monitor.woken >= timeend and t.miniters == 1): - timer.sleep(1) # Force monitor to wake up if it woken too soon - sleep(0.000001) # sleep to allow interrupt (instead of pass) + Time.fake_sleep(1) # Force awake up if it woken too soon assert t.miniters == 1 # check that monitor corrected miniters # Note: at this point, there may be a race condition: monitor saved - # current woken time but timer.sleep() happen just before monitor + # current woken time but Time.sleep() happen just before monitor # sleep. To fix that, either sleep here or increase time in a loop # to ensure that monitor wakes up at some point. # Try again but already at miniters = 1 so nothing will be done - timer.sleep(maxinterval * 2) + Time.fake_sleep(maxinterval) t.update(2) - timeend = timer.time() + timeend = Time.time() while t.monitor.woken < timeend: - timer.sleep(1) # Force monitor to wake up if it woken too soon - sleep(0.000001) - # Wait for the monitor to get out of sleep's loop and update tqdm.. + Time.fake_sleep(1) # Force awake if it woken too soon + # Wait for the monitor to get out of sleep's loop and update + # tqdm assert t.miniters == 1 # check that monitor corrected miniters - # Check that class var monitor is deleted if no instance left - tqdm.monitor_interval = 10 - assert tqdm.monitor is None - -@with_setup(pretest, posttest) +@patch_sleep def test_monitoring_multi(): """Test on multiple bars, one not needing miniters adjustment""" # Note: should fix miniters for these tests, else with dynamic_miniters # it's too complicated to handle with monitoring update and maxinterval... - maxinterval = 2 - + maxinterval = tqdm.monitor_interval + assert maxinterval == 10 total = 1000 - # Setup a discrete timer - timer = DiscreteTimer() - # And a fake sleeper - sleeper = FakeSleep(timer) - # Setup TMonitor to use the timer - TMonitor._time = timer.time - TMonitor._event = make_create_fake_sleep_event(sleeper.sleep) - # Set monitor interval - tqdm.monitor_interval = maxinterval + with closing(StringIO()) as our_file: with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1, maxinterval=maxinterval) as t1: # Set high maxinterval for t2 so monitor does not need to adjust it with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1, maxinterval=1E5) as t2: - cpu_timify(t1, timer) - cpu_timify(t2, timer) + cpu_timify(t1, Time) + cpu_timify(t2, Time) # Do a lot of iterations in a small timeframe - timer.sleep(maxinterval / 2) + Time.fake_sleep(maxinterval / 10) t1.update(500) t2.update(500) - assert t1.miniters == 500 + assert t1.miniters <= 500 # TODO: should really be == 500 assert t2.miniters == 500 # Then do 1 it after monitor interval, so that monitor kicks in - timer.sleep(maxinterval * 2) + Time.fake_sleep(maxinterval) t1.update(1) t2.update(1) # Wait for the monitor to get out of sleep and update tqdm - timeend = timer.time() + timeend = Time.time() while not (t1.monitor.woken >= timeend and t1.miniters == 1): - timer.sleep(1) - sleep(0.000001) + Time.fake_sleep(1) assert t1.miniters == 1 # check that monitor corrected miniters assert t2.miniters == 500 # check that t2 was not adjusted - # Check that class var monitor is deleted if no instance left - tqdm.monitor_interval = 10 - assert tqdm.monitor is None - -@with_setup(pretest, posttest) def test_imap(): """Test multiprocessing.Pool""" try: from multiprocessing import Pool - except ImportError: - raise SkipTest + except ImportError as err: + skip(str(err)) pool = Pool() res = list(tqdm(pool.imap(incr, range(100)), disable=True)) + pool.close() assert res[-1] == 100 # py2: locks won't propagate to incr_bar so may cause `AttributeError` -@retry_on_except(n=3 if sys.version_info < (3,) else 1) -@with_setup(pretest, posttest) +@retry_on_except(n=3 if sys.version_info < (3,) else 1, check_cpu_time=False) +@patch_lock(thread=True) def test_threadpool(): """Test concurrent.futures.ThreadPoolExecutor""" - try: - from concurrent.futures import ThreadPoolExecutor - from threading import RLock - except ImportError: - raise SkipTest + ThreadPoolExecutor = importorskip('concurrent.futures').ThreadPoolExecutor - tqdm.set_lock(RLock()) with ThreadPoolExecutor(8) as pool: try: res = list(tqdm(pool.map(incr_bar, range(100)), disable=True)) except AttributeError: if sys.version_info < (3,): - raise SkipTest + skip("not supported on py2") else: raise assert sum(res) == sum(range(1, 101)) diff --git a/tests/tests_tk.py b/tests/tests_tk.py new file mode 100644 index 000000000..9aa645cef --- /dev/null +++ b/tests/tests_tk.py @@ -0,0 +1,7 @@ +"""Test `tqdm.tk`.""" +from .tests_tqdm import importorskip + + +def test_tk_import(): + """Test `tqdm.tk` import""" + importorskip('tqdm.tk') diff --git a/tqdm/tests/tests_tqdm.py b/tests/tests_tqdm.py similarity index 87% rename from tqdm/tests/tests_tqdm.py rename to tests/tests_tqdm.py index 5f322e61a..8cf006533 100644 --- a/tqdm/tests/tests_tqdm.py +++ b/tests/tests_tqdm.py @@ -1,31 +1,29 @@ # -*- coding: utf-8 -*- # Advice: use repr(our_file.read()) to print the full output of tqdm # (else '\r' will replace the previous lines and you'll see only the latest. +from __future__ import print_function -import sys import csv -import re import os -from nose import with_setup -from nose.plugins.skip import SkipTest -from nose.tools import assert_raises -from nose.tools import eq_ +import re +import sys from contextlib import contextmanager +from functools import wraps from warnings import catch_warnings, simplefilter -from tqdm import tqdm -from tqdm import trange -from tqdm import TqdmDeprecationWarning -from tqdm.std import Bar +from pytest import importorskip, mark, raises, skip + +from tqdm import TqdmDeprecationWarning, TqdmWarning, tqdm, trange from tqdm.contrib import DummyTqdmFile +from tqdm.std import EMA, Bar try: from StringIO import StringIO except ImportError: from io import StringIO -from io import BytesIO from io import IOBase # to support unicode strings +from io import BytesIO class DeprecationError(Exception): @@ -61,11 +59,10 @@ def closing(arg): # List of control characters CTRLCHR = [r'\r', r'\n', r'\x1b\[A'] # Need to escape [ for regex # Regular expressions compilation -RE_rate = re.compile(r'(\d+\.\d+)it/s') +RE_rate = re.compile(r'[^\d](\d[.\d]+)it/s') RE_ctrlchr = re.compile("(%s)" % '|'.join(CTRLCHR)) # Match control chars RE_ctrlchr_excl = re.compile('|'.join(CTRLCHR)) # Match and exclude ctrl chars -RE_pos = re.compile( - r'([\r\n]+((pos\d+) bar:\s+\d+%|\s{3,6})?[^\r\n]*)') +RE_pos = re.compile(r'([\r\n]+((pos\d+) bar:\s+\d+%|\s{3,6})?[^\r\n]*)') def pos_line_diff(res_list, expected_list, raise_nonempty=True): @@ -98,7 +95,6 @@ def pos_line_diff(res_list, expected_list, raise_nonempty=True): class DiscreteTimer(object): """Virtual discrete time manager, to precisely control time for tests""" - def __init__(self): self.t = 0.0 @@ -121,33 +117,8 @@ def cpu_timify(t, timer=None): return timer -def pretest(): - # setcheckinterval is deprecated - try: - sys.setswitchinterval(1) - except AttributeError: - sys.setcheckinterval(100) - - if getattr(tqdm, "_instances", False): - n = len(tqdm._instances) - if n: - tqdm._instances.clear() - raise EnvironmentError( - "{0} `tqdm` instances still in existence PRE-test".format(n)) - - -def posttest(): - if getattr(tqdm, "_instances", False): - n = len(tqdm._instances) - if n: - tqdm._instances.clear() - raise EnvironmentError( - "{0} `tqdm` instances still in existence POST-test".format(n)) - - class UnicodeIO(IOBase): """Unicode version of StringIO""" - def __init__(self, *args, **kwargs): super(UnicodeIO, self).__init__(*args, **kwargs) self.encoding = 'U8' # io.StringIO supports unicode, but no encoding @@ -246,7 +217,7 @@ def test_format_num(): assert float(format_num(1337)) == 1337 assert format_num(int(1e6)) == '1e+6' - assert format_num(1239876) == '1''239''876' + assert format_num(1239876) == '1' '239' '876' def test_format_meter(): @@ -276,59 +247,53 @@ def test_format_meter(): "100kiB [00:13, 7.69kiB/s]" assert format_meter(100, 1000, 12, ncols=0, rate=7.33) == \ " 10% 100/1000 [00:12<02:02, 7.33it/s]" - eq_( - # ncols is small, l_bar is too large - # l_bar gets chopped - # no bar - # no r_bar + # ncols is small, l_bar is too large + # l_bar gets chopped + # no bar + # no r_bar + assert \ format_meter( 0, 1000, 13, ncols=10, - bar_format="************{bar:10}$$$$$$$$$$"), + bar_format="************{bar:10}$$$$$$$$$$") == \ "**********" # 10/12 stars since ncols is 10 - ) - eq_( - # n_cols allows for l_bar and some of bar - # l_bar displays - # bar gets chopped - # no r_bar + # n_cols allows for l_bar and some of bar + # l_bar displays + # bar gets chopped + # no r_bar + assert \ format_meter( 0, 1000, 13, ncols=20, - bar_format="************{bar:10}$$$$$$$$$$"), + bar_format="************{bar:10}$$$$$$$$$$") == \ "************ " # all 12 stars and 8/10 bar parts - ) - eq_( - # n_cols allows for l_bar, bar, and some of r_bar - # l_bar displays - # bar displays - # r_bar gets chopped + # n_cols allows for l_bar, bar, and some of r_bar + # l_bar displays + # bar displays + # r_bar gets chopped + # all 12 stars and 10 bar parts, but only 8/10 dollar signs + assert \ format_meter( 0, 1000, 13, ncols=30, - bar_format="************{bar:10}$$$$$$$$$$"), + bar_format="************{bar:10}$$$$$$$$$$") == \ "************ $$$$$$$$" - # all 12 stars and 10 bar parts, but only 8/10 dollar signs - ) - eq_( - # trim left ANSI; escape is before trim zone + # trim left ANSI; escape is before trim zone + # we only know it has ANSI codes, so we append an END code anyway + assert \ format_meter( 0, 1000, 13, ncols=10, - bar_format="*****\033[22m****\033[0m***{bar:10}$$$$$$$$$$"), + bar_format="*****\033[22m****\033[0m***{bar:10}$$$$$$$$$$") == \ "*****\033[22m****\033[0m*\033[0m" - # we only know it has ANSI codes, so we append an END code anyway - ) - eq_( - # trim left ANSI; escape is at trim zone + # trim left ANSI; escape is at trim zone + assert \ format_meter( 0, 1000, 13, ncols=10, - bar_format="*****\033[22m*****\033[0m**{bar:10}$$$$$$$$$$"), + bar_format="*****\033[22m*****\033[0m**{bar:10}$$$$$$$$$$") == \ "*****\033[22m*****\033[0m" - ) - eq_( - # trim left ANSI; escape is after trim zone + # trim left ANSI; escape is after trim zone + assert \ format_meter( 0, 1000, 13, ncols=10, - bar_format="*****\033[22m******\033[0m*{bar:10}$$$$$$$$$$"), + bar_format="*****\033[22m******\033[0m*{bar:10}$$$$$$$$$$") == \ "*****\033[22m*****\033[0m" - ) # Check that bar_format correctly adapts {bar} size to the rest assert format_meter(20, 100, 12, ncols=13, rate=8.1, bar_format=r'{l_bar}{bar}|{n_fmt}/{total_fmt}') == \ @@ -356,7 +321,7 @@ def test_format_meter(): def test_ansi_escape_codes(): """Test stripping of ANSI escape codes""" - ansi = dict(BOLD='\033[1m', RED='\033[91m', END='\033[0m') + ansi = {'BOLD': '\033[1m', 'RED': '\033[91m', 'END': '\033[0m'} desc_raw = '{BOLD}{RED}Colored{END} description' ncols = 123 @@ -387,14 +352,10 @@ def test_si_format(): assert '1.00T ' in format_meter(1, 999999999999, 1, unit_scale=True) assert '1.00P ' in format_meter(1, 999999999999999, 1, unit_scale=True) assert '1.00E ' in format_meter(1, 999999999999999999, 1, unit_scale=True) - assert '1.00Z ' in format_meter(1, 999999999999999999999, 1, - unit_scale=True) - assert '1.0Y ' in format_meter(1, 999999999999999999999999, 1, - unit_scale=True) - assert '10.0Y ' in format_meter(1, 9999999999999999999999999, 1, - unit_scale=True) - assert '100.0Y ' in format_meter(1, 99999999999999999999999999, 1, - unit_scale=True) + assert '1.00Z ' in format_meter(1, 999999999999999999999, 1, unit_scale=True) + assert '1.0Y ' in format_meter(1, 999999999999999999999999, 1, unit_scale=True) + assert '10.0Y ' in format_meter(1, 9999999999999999999999999, 1, unit_scale=True) + assert '100.0Y ' in format_meter(1, 99999999999999999999999999, 1, unit_scale=True) assert '1000.0Y ' in format_meter(1, 999999999999999999999999999, 1, unit_scale=True) @@ -408,7 +369,6 @@ def test_bar_formatspec(): assert "{0:2b}".format(Bar(0.5, 10)) == ' ' -@with_setup(pretest, posttest) def test_all_defaults(): """Test default kwargs""" with closing(UnicodeIO()) as our_file: @@ -434,7 +394,6 @@ def write(self, s): assert isinstance(s, self.expected_type) -@with_setup(pretest, posttest) def test_native_string_io_for_default_file(): """Native strings written to unspecified files""" stderr = sys.stderr @@ -449,14 +408,12 @@ def test_native_string_io_for_default_file(): sys.stderr = stderr -@with_setup(pretest, posttest) def test_unicode_string_io_for_specified_file(): """Unicode strings written to specified files""" for _ in tqdm(range(3), file=WriteTypeChecker(expected_type=type(u''))): pass -@with_setup(pretest, posttest) def test_write_bytes(): """Test write_bytes argument with and without `file`""" # specified file (and bytes) @@ -473,7 +430,6 @@ def test_write_bytes(): sys.stderr = stderr -@with_setup(pretest, posttest) def test_iterate_over_csv_rows(): """Test csv iterator""" # Create a test csv pseudo file @@ -484,14 +440,12 @@ def test_iterate_over_csv_rows(): test_csv_file.seek(0) # Test that nothing fails if we iterate over rows - reader = csv.DictReader(test_csv_file, - fieldnames=('row1', 'row2', 'row3')) + reader = csv.DictReader(test_csv_file, fieldnames=('row1', 'row2', 'row3')) with closing(StringIO()) as our_file: for _ in tqdm(reader, file=our_file): pass -@with_setup(pretest, posttest) def test_file_output(): """Test output to arbitrary file-like objects""" with closing(StringIO()) as our_file: @@ -501,7 +455,6 @@ def test_file_output(): assert '0/3' in our_file.read() -@with_setup(pretest, posttest) def test_leave_option(): """Test `leave=True` always prints info about the last iteration""" with closing(StringIO()) as our_file: @@ -517,7 +470,6 @@ def test_leave_option(): assert '| 3/3 ' not in our_file2.getvalue() -@with_setup(pretest, posttest) def test_trange(): """Test trange""" with closing(StringIO()) as our_file: @@ -531,7 +483,6 @@ def test_trange(): assert '| 3/3 ' not in our_file2.getvalue() -@with_setup(pretest, posttest) def test_min_interval(): """Test mininterval""" with closing(StringIO()) as our_file: @@ -540,7 +491,6 @@ def test_min_interval(): assert " 0%| | 0/3 [00:00<" in our_file.getvalue() -@with_setup(pretest, posttest) def test_max_interval(): """Test maxinterval""" total = 100 @@ -672,28 +622,49 @@ def test_max_interval(): t2.close() -@with_setup(pretest, posttest) +def test_delay(): + """Test delay""" + timer = DiscreteTimer() + with closing(StringIO()) as our_file: + t = tqdm(total=2, file=our_file, leave=True, delay=3) + cpu_timify(t, timer) + timer.sleep(2) + t.update(1) + assert not our_file.getvalue() + timer.sleep(2) + t.update(1) + assert our_file.getvalue() + t.close() + + def test_min_iters(): """Test miniters""" with closing(StringIO()) as our_file: - for _ in tqdm(_range(3), file=our_file, leave=True, miniters=4): - our_file.write('blank\n') - assert '\nblank\nblank\n' in our_file.getvalue() + for _ in tqdm(_range(3), file=our_file, leave=True, mininterval=0, miniters=2): + pass + + out = our_file.getvalue() + assert '| 0/3 ' in out + assert '| 1/3 ' not in out + assert '| 2/3 ' in out + assert '| 3/3 ' in out with closing(StringIO()) as our_file: - for _ in tqdm(_range(3), file=our_file, leave=True, miniters=1): - our_file.write('blank\n') - # assume automatic mininterval = 0 means intermediate output - assert '| 3/3 ' in our_file.getvalue() + for _ in tqdm(_range(3), file=our_file, leave=True, mininterval=0, miniters=1): + pass + + out = our_file.getvalue() + assert '| 0/3 ' in out + assert '| 1/3 ' in out + assert '| 2/3 ' in out + assert '| 3/3 ' in out -@with_setup(pretest, posttest) def test_dynamic_min_iters(): """Test purely dynamic miniters (and manual updates and __del__)""" with closing(StringIO()) as our_file: total = 10 - t = tqdm(total=total, file=our_file, miniters=None, mininterval=0, - smoothing=1) + t = tqdm(total=total, file=our_file, miniters=None, mininterval=0, smoothing=1) t.update() # Increase 3 iterations @@ -717,8 +688,7 @@ def test_dynamic_min_iters(): # Check with smoothing=0, miniters should be set to max update seen so far with closing(StringIO()) as our_file: total = 10 - t = tqdm(total=total, file=our_file, miniters=None, mininterval=0, - smoothing=0) + t = tqdm(total=total, file=our_file, miniters=None, mininterval=0, smoothing=0) t.update() t.update(2) @@ -756,7 +726,6 @@ def test_dynamic_min_iters(): assert not t.dynamic_miniters -@with_setup(pretest, posttest) def test_big_min_interval(): """Test large mininterval""" with closing(StringIO()) as our_file: @@ -771,37 +740,36 @@ def test_big_min_interval(): assert '50%' not in our_file.getvalue() -@with_setup(pretest, posttest) def test_smoothed_dynamic_min_iters(): """Test smoothed dynamic miniters""" timer = DiscreteTimer() with closing(StringIO()) as our_file: - with tqdm(total=100, file=our_file, miniters=None, mininterval=0, + with tqdm(total=100, file=our_file, miniters=None, mininterval=1, smoothing=0.5, maxinterval=0) as t: cpu_timify(t, timer) # Increase 10 iterations at once + timer.sleep(1) t.update(10) # The next iterations should be partially skipped for _ in _range(2): + timer.sleep(1) t.update(4) for _ in _range(20): + timer.sleep(1) t.update() - out = our_file.getvalue() assert t.dynamic_miniters + out = our_file.getvalue() assert ' 0%| | 0/100 [00:00<' in out - assert '10%' in out - assert '14%' not in out - assert '18%' in out - assert '20%' not in out + assert '20%' in out + assert '23%' not in out assert '25%' in out - assert '30%' not in out - assert '32%' in out + assert '26%' not in out + assert '28%' in out -@with_setup(pretest, posttest) def test_smoothed_dynamic_min_iters_with_min_interval(): """Test smoothed dynamic miniters with mininterval""" timer = DiscreteTimer() @@ -844,13 +812,12 @@ def test_smoothed_dynamic_min_iters_with_min_interval(): assert '14%' in out and '14%' in out2 -@with_setup(pretest, posttest) +@mark.slow def test_rlock_creation(): """Test that importing tqdm does not create multiprocessing objects.""" - import multiprocessing as mp - if sys.version_info < (3, 3): - # unittest.mock is a 3.3+ feature - raise SkipTest + mp = importorskip('multiprocessing') + if not hasattr(mp, 'get_context'): + skip("missing multiprocessing.get_context") # Use 'spawn' instead of 'fork' so that the process does not inherit any # globals that have been constructed by running other tests @@ -862,8 +829,8 @@ def test_rlock_creation(): def _rlock_creation_target(): """Check that the RLock has not been constructed.""" - from unittest.mock import patch import multiprocessing as mp + patch = importorskip('unittest.mock').patch # Patch the RLock class/method but use the original implementation with patch('multiprocessing.RLock', wraps=mp.RLock) as rlock_mock: @@ -882,7 +849,6 @@ def _rlock_creation_target(): assert rlock_mock.call_count == 1 -@with_setup(pretest, posttest) def test_disable(): """Test disable""" with closing(StringIO()) as our_file: @@ -897,7 +863,6 @@ def test_disable(): assert our_file.getvalue() == '' -@with_setup(pretest, posttest) def test_infinite_total(): """Test treatment of infinite total""" with closing(StringIO()) as our_file: @@ -905,22 +870,20 @@ def test_infinite_total(): pass -@with_setup(pretest, posttest) def test_nototal(): """Test unknown total length""" with closing(StringIO()) as our_file: - for i in tqdm((i for i in range(10)), file=our_file, unit_scale=10): + for _ in tqdm((i for i in range(10)), file=our_file, unit_scale=10): pass assert "100it" in our_file.getvalue() with closing(StringIO()) as our_file: - for i in tqdm((i for i in range(10)), file=our_file, + for _ in tqdm((i for i in range(10)), file=our_file, bar_format="{l_bar}{bar}{r_bar}"): pass assert "10/?" in our_file.getvalue() -@with_setup(pretest, posttest) def test_unit(): """Test SI unit prefix""" with closing(StringIO()) as our_file: @@ -929,7 +892,6 @@ def test_unit(): assert 'bytes/s' in our_file.getvalue() -@with_setup(pretest, posttest) def test_ascii(): """Test ascii/unicode bar""" # Test ascii autodetection @@ -958,17 +920,16 @@ def test_ascii(): assert u"20%|\u2588\u2588" in res[3] # Test custom bar - for ascii in [" .oO0", " #"]: + for bars in [" .oO0", " #"]: with closing(StringIO()) as our_file: - for _ in tqdm(_range(len(ascii) - 1), file=our_file, miniters=1, - mininterval=0, ascii=ascii, ncols=27): + for _ in tqdm(_range(len(bars) - 1), file=our_file, miniters=1, + mininterval=0, ascii=bars, ncols=27): pass res = our_file.getvalue().strip("\r").split("\r") - for bar, line in zip(ascii, res): - assert '|' + bar + '|' in line + for b, line in zip(bars, res): + assert '|' + b + '|' in line -@with_setup(pretest, posttest) def test_update(): """Test manual creation and updates""" res = None @@ -987,7 +948,6 @@ def test_update(): assert 'dynamically notify of 4 increments in total' in res -@with_setup(pretest, posttest) def test_close(): """Test manual creation and closure and n_instances""" @@ -1025,9 +985,8 @@ def test_close(): res = our_file.getvalue() assert res[-1] == '\n' if not res.startswith(exres): - raise AssertionError( - "\n<<< Expected:\n{0}\n>>> Got:\n{1}\n===".format( - exres + ', ...it/s]\n', our_file.getvalue())) + raise AssertionError("\n<<< Expected:\n{0}\n>>> Got:\n{1}\n===".format( + exres + ', ...it/s]\n', our_file.getvalue())) # Closing after the output stream has closed with closing(StringIO()) as our_file: @@ -1037,7 +996,16 @@ def test_close(): t.close() -@with_setup(pretest, posttest) +def test_ema(): + """Test exponential weighted average""" + ema = EMA(0.01) + assert round(ema(10), 2) == 10 + assert round(ema(1), 2) == 5.48 + assert round(ema(), 2) == 5.48 + assert round(ema(1), 2) == 3.97 + assert round(ema(1), 2) == 3.22 + + def test_smoothing(): """Test exponential weighted average smoothing""" timer = DiscreteTimer() @@ -1052,7 +1020,6 @@ def test_smoothing(): assert '| 3/3 ' in our_file.getvalue() # -- Test smoothing - # Compile the regex to find the rate # 1st case: no smoothing (only use average) with closing(StringIO()) as our_file2: with closing(StringIO()) as our_file: @@ -1135,11 +1102,9 @@ def test_smoothing(): assert a2 <= c2 <= b2 -@with_setup(pretest, posttest) +@mark.skipif(nt_and_no_colorama, reason="Windows without colorama") def test_deprecated_nested(): """Test nested progress bars""" - if nt_and_no_colorama: - raise SkipTest # TODO: test degradation on windows without colorama? # Artificially test nested loop printing @@ -1155,11 +1120,11 @@ def test_deprecated_nested(): raise DeprecationError("Should not allow nested kwarg") -@with_setup(pretest, posttest) def test_bar_format(): """Test custom bar formatting""" with closing(StringIO()) as our_file: - bar_format = r'{l_bar}{bar}|{n_fmt}/{total_fmt}-{n}/{total}{percentage}{rate}{rate_fmt}{elapsed}{remaining}' # NOQA + bar_format = ('{l_bar}{bar}|{n_fmt}/{total_fmt}-{n}/{total}' + '{percentage}{rate}{rate_fmt}{elapsed}{remaining}') for _ in trange(2, file=our_file, leave=True, bar_format=bar_format): pass out = our_file.getvalue() @@ -1172,7 +1137,6 @@ def test_bar_format(): assert isinstance(t.bar_format, _unicode) -@with_setup(pretest, posttest) def test_custom_format(): """Test adding additional derived format arguments""" class TqdmExtraFormat(tqdm): @@ -1185,14 +1149,23 @@ def format_dict(self): return d with closing(StringIO()) as our_file: - for i in TqdmExtraFormat( + for _ in TqdmExtraFormat( range(10), file=our_file, bar_format="{total_time}: {percentage:.0f}%|{bar}{r_bar}"): pass assert "00:00 in total" in our_file.getvalue() -@with_setup(pretest, posttest) +def test_eta(capsys): + """Test eta bar_format""" + from datetime import datetime as dt + for _ in trange(999, miniters=1, mininterval=0, leave=True, + bar_format='{l_bar}{eta:%Y-%m-%d}'): + pass + _, err = capsys.readouterr() + assert "\r100%|{eta:%Y-%m-%d}\n".format(eta=dt.now()) in err + + def test_unpause(): """Test unpause""" timer = DiscreteTimer() @@ -1215,7 +1188,18 @@ def test_unpause(): assert r_before == r_after -@with_setup(pretest, posttest) +def test_disabled_unpause(capsys): + """Test disabled unpause""" + with tqdm(total=10, disable=True) as t: + t.update() + t.unpause() + t.update() + print(t) + out, err = capsys.readouterr() + assert not err + assert out == ' 0%| | 0/10 [00:005.2f}", - postfix=[dict(name="foo"), 42]) as t: + postfix=[{'name': "foo"}, 42]) as t: for i in range(10): if i % 2: t.postfix[0]["name"] = "abcdefghij"[i] @@ -1746,21 +1733,25 @@ def std_out_err_redirect_tqdm(tqdm_file=sys.stderr): sys.stdout, sys.stderr = orig_out_err -@with_setup(pretest, posttest) def test_file_redirection(): """Test redirection of output""" with closing(StringIO()) as our_file: # Redirect stdout to tqdm.write() with std_out_err_redirect_tqdm(tqdm_file=our_file): - for _ in trange(3): + with tqdm(total=3) as pbar: print("Such fun") + pbar.update(1) + print("Such", "fun") + pbar.update(1) + print("Such ", end="") + print("fun") + pbar.update(1) res = our_file.getvalue() assert res.count("Such fun\n") == 3 assert "0/3" in res assert "3/3" in res -@with_setup(pretest, posttest) def test_external_write(): """Test external write mode""" with closing(StringIO()) as our_file: @@ -1775,7 +1766,6 @@ def test_external_write(): assert "3/3" in res -@with_setup(pretest, posttest) def test_unit_scale(): """Test numeric `unit_scale`""" with closing(StringIO()) as our_file: @@ -1786,24 +1776,42 @@ def test_unit_scale(): assert '81/81' in out -@with_setup(pretest, posttest) +def patch_lock(thread=True): + """decorator replacing tqdm's lock with vanilla threading/multiprocessing""" + try: + if thread: + from threading import RLock + else: + from multiprocessing import RLock + lock = RLock() + except (ImportError, OSError) as err: + skip(str(err)) + + def outer(func): + """actual decorator""" + @wraps(func) + def inner(*args, **kwargs): + """set & reset lock even if exceptions occur""" + default_lock = tqdm.get_lock() + try: + tqdm.set_lock(lock) + return func(*args, **kwargs) + finally: + tqdm.set_lock(default_lock) + return inner + return outer + + +@patch_lock(thread=False) def test_threading(): """Test multiprocess/thread-realted features""" - from multiprocessing import RLock - try: - mp_lock = RLock() - except OSError: - pass - else: - tqdm.set_lock(mp_lock) - # TODO: test interleaved output #445 + pass # TODO: test interleaved output #445 -@with_setup(pretest, posttest) def test_bool(): """Test boolean cast""" def internal(our_file, disable): - kwargs = dict(file=our_file, disable=disable) + kwargs = {'file': our_file, 'disable': disable} with trange(10, **kwargs) as t: assert t with trange(0, **kwargs) as t: @@ -1846,45 +1854,40 @@ def backendCheck(module): assert len(t) == 1337 -@with_setup(pretest, posttest) def test_auto(): """Test auto fallback""" - from tqdm import autonotebook, auto + from tqdm import auto, autonotebook backendCheck(autonotebook) backendCheck(auto) -@with_setup(pretest, posttest) def test_wrapattr(): """Test wrapping file-like objects""" data = "a twenty-char string" with closing(StringIO()) as our_file: with closing(StringIO()) as writer: - with tqdm.wrapattr( - writer, "write", file=our_file, bytes=True) as wrap: + with tqdm.wrapattr(writer, "write", file=our_file, bytes=True) as wrap: wrap.write(data) res = writer.getvalue() assert data == res res = our_file.getvalue() - assert ('%.1fB [' % len(data)) in res + assert '%.1fB [' % len(data) in res with closing(StringIO()) as our_file: with closing(StringIO()) as writer: - with tqdm.wrapattr( - writer, "write", file=our_file, bytes=False) as wrap: + with tqdm.wrapattr(writer, "write", file=our_file, bytes=False) as wrap: wrap.write(data) res = our_file.getvalue() - assert ('%dit [' % len(data)) in res + assert '%dit [' % len(data) in res -@with_setup(pretest, posttest) def test_float_progress(): """Test float totals""" with closing(StringIO()) as our_file: with trange(10, total=9.6, file=our_file) as t: with catch_warnings(record=True) as w: - simplefilter("always") + simplefilter("always", category=TqdmWarning) for i in t: if i < 9: assert not w @@ -1892,7 +1895,6 @@ def test_float_progress(): assert "clamping frac" in str(w[-1].message) -@with_setup(pretest, posttest) def test_screen_shape(): """Test screen shape""" # ncols @@ -1905,8 +1907,8 @@ def test_screen_shape(): # no second/third bar, leave=False with closing(StringIO()) as our_file: - kwargs = dict(file=our_file, ncols=50, nrows=2, miniters=0, - mininterval=0, leave=False) + kwargs = {'file': our_file, 'ncols': 50, 'nrows': 2, 'miniters': 0, + 'mininterval': 0, 'leave': False} with trange(10, desc="one", **kwargs) as t1: with trange(10, desc="two", **kwargs) as t2: with trange(10, desc="three", **kwargs) as t3: @@ -1926,8 +1928,8 @@ def test_screen_shape(): # all bars, leave=True with closing(StringIO()) as our_file: - kwargs = dict(file=our_file, ncols=50, nrows=2, miniters=0, - mininterval=0) + kwargs = {'file': our_file, 'ncols': 50, 'nrows': 2, + 'miniters': 0, 'mininterval': 0} with trange(10, desc="one", **kwargs) as t1: with trange(10, desc="two", **kwargs) as t2: assert "two" not in our_file.getvalue() @@ -1949,8 +1951,8 @@ def test_screen_shape(): # second bar becomes first, leave=False with closing(StringIO()) as our_file: - kwargs = dict(file=our_file, ncols=50, nrows=2, miniters=0, - mininterval=0, leave=False) + kwargs = {'file': our_file, 'ncols': 50, 'nrows': 2, 'miniters': 0, + 'mininterval': 0, 'leave': False} t1 = tqdm(total=10, desc="one", **kwargs) with tqdm(total=10, desc="two", **kwargs) as t2: t1.update() @@ -1964,3 +1966,44 @@ def test_screen_shape(): res = our_file.getvalue() assert "two" in res + + +def test_initial(): + """Test `initial`""" + with closing(StringIO()) as our_file: + for _ in tqdm(_range(9), initial=10, total=19, file=our_file, + miniters=1, mininterval=0): + pass + out = our_file.getvalue() + assert '10/19' in out + assert '19/19' in out + + +def test_colour(): + """Test `colour`""" + with closing(StringIO()) as our_file: + for _ in tqdm(_range(9), file=our_file, colour="#beefed"): + pass + out = our_file.getvalue() + assert '\x1b[38;2;%d;%d;%dm' % (0xbe, 0xef, 0xed) in out + + with catch_warnings(record=True) as w: + simplefilter("always", category=TqdmWarning) + with tqdm(total=1, file=our_file, colour="charm") as t: + assert w + t.update() + assert "Unknown colour" in str(w[-1].message) + + with closing(StringIO()) as our_file2: + for _ in tqdm(_range(9), file=our_file2, colour="blue"): + pass + out = our_file2.getvalue() + assert '\x1b[34m' in out + + +def test_closed(): + """Test writing to closed file""" + with closing(StringIO()) as our_file: + for i in trange(9, file=our_file, miniters=1, mininterval=0): + if i == 5: + our_file.close() diff --git a/tests/tests_version.py b/tests/tests_version.py new file mode 100644 index 000000000..495c797f2 --- /dev/null +++ b/tests/tests_version.py @@ -0,0 +1,14 @@ +"""Test `tqdm.__version__`.""" +import re +from ast import literal_eval + + +def test_version(): + """Test version string""" + from tqdm import __version__ + version_parts = re.split('[.-]', __version__) + if __version__ != "UNKNOWN": + assert 3 <= len(version_parts), "must have at least Major.minor.patch" + assert all( + isinstance(literal_eval(i), int) for i in version_parts[:3] + ), "Version Major.minor.patch must be 3 integers" diff --git a/tests_notebook.ipynb b/tests_notebook.ipynb new file mode 100644 index 000000000..fb8227d7d --- /dev/null +++ b/tests_notebook.ipynb @@ -0,0 +1,512 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This file is part of the [test suite](./tests) and will be moved there when [nbval#116](https://github.com/computationalmodelling/nbval/issues/116#issuecomment-793148404) is fixed.\n", + "\n", + "See [DEMO.ipynb](DEMO.ipynb) instead for notebook examples." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial\n", + "from time import sleep\n", + "\n", + "from tqdm.notebook import tqdm_notebook\n", + "from tqdm.notebook import tnrange\n", + "\n", + "# avoid displaying widgets by default (pollutes output cells)\n", + "tqdm = partial(tqdm_notebook, display=False)\n", + "trange = partial(tnrange, display=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function display in module tqdm.notebook:\n", + "\n", + "display(self, msg=None, pos=None, close=False, bar_style=None, check_delay=True)\n", + " Use `self.sp` to display `msg` in the specified `pos`.\n", + " \n", + " Consider overloading this function when inheriting to use e.g.:\n", + " `self.some_frontend(**self.format_dict)` instead of `self.sp`.\n", + " \n", + " Parameters\n", + " ----------\n", + " msg : str, optional. What to display (default: `repr(self)`).\n", + " pos : int, optional. Position to `moveto`\n", + " (default: `abs(self.pos)`).\n", + "\n" + ] + } + ], + "source": [ + "help(tqdm_notebook.display)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7c18c038bf964b55941e228503292506", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/9 [00:00=0.1.0 -commands = - python setup.py check --restructuredtext --metadata --strict - python setup.py make none +commands= + {envpython} setup.py check --restructuredtext --metadata --strict + {envpython} setup.py make none diff --git a/tqdm/__init__.py b/tqdm/__init__.py index 670d6457e..a021d16e9 100644 --- a/tqdm/__init__.py +++ b/tqdm/__init__.py @@ -1,13 +1,12 @@ -from .std import tqdm, trange -from .gui import tqdm as tqdm_gui # TODO: remove in v5.0.0 -from .gui import trange as tgrange # TODO: remove in v5.0.0 +from ._monitor import TMonitor, TqdmSynchronisationWarning from ._tqdm_pandas import tqdm_pandas from .cli import main # TODO: remove in v5.0.0 -from ._monitor import TMonitor, TqdmSynchronisationWarning -from ._version import __version__ # NOQA -from .std import TqdmTypeError, TqdmKeyError, TqdmWarning, \ - TqdmDeprecationWarning, TqdmExperimentalWarning, \ - TqdmMonitorWarning +from .gui import tqdm as tqdm_gui # TODO: remove in v5.0.0 +from .gui import trange as tgrange # TODO: remove in v5.0.0 +from .std import ( + TqdmDeprecationWarning, TqdmExperimentalWarning, TqdmKeyError, TqdmMonitorWarning, + TqdmTypeError, TqdmWarning, tqdm, trange) +from .version import __version__ __all__ = ['tqdm', 'tqdm_gui', 'trange', 'tgrange', 'tqdm_pandas', 'tqdm_notebook', 'tnrange', 'main', 'TMonitor', @@ -20,8 +19,9 @@ def tqdm_notebook(*args, **kwargs): # pragma: no cover """See tqdm.notebook.tqdm for full documentation""" - from .notebook import tqdm as _tqdm_notebook from warnings import warn + + from .notebook import tqdm as _tqdm_notebook warn("This function will be removed in tqdm==5.0.0\n" "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`", TqdmDeprecationWarning, stacklevel=2) @@ -33,8 +33,9 @@ def tnrange(*args, **kwargs): # pragma: no cover A shortcut for `tqdm.notebook.tqdm(xrange(*args), **kwargs)`. On Python3+, `range` is used instead of `xrange`. """ - from .notebook import trange as _tnrange from warnings import warn + + from .notebook import trange as _tnrange warn("Please use `tqdm.notebook.trange` instead of `tqdm.tnrange`", TqdmDeprecationWarning, stacklevel=2) return _tnrange(*args, **kwargs) diff --git a/tqdm/__main__.py b/tqdm/__main__.py index 130bc6375..4e28416e1 100644 --- a/tqdm/__main__.py +++ b/tqdm/__main__.py @@ -1,2 +1,3 @@ from .cli import main + main() diff --git a/tqdm/_main.py b/tqdm/_main.py index 07b6730b1..04fdeeff1 100644 --- a/tqdm/_main.py +++ b/tqdm/_main.py @@ -1,7 +1,9 @@ +from warnings import warn + from .cli import * # NOQA from .cli import __all__ # NOQA from .std import TqdmDeprecationWarning -from warnings import warn + warn("This function will be removed in tqdm==5.0.0\n" "Please use `tqdm.cli.*` instead of `tqdm._main.*`", TqdmDeprecationWarning, stacklevel=2) diff --git a/tqdm/_monitor.py b/tqdm/_monitor.py index e1e257069..f8443bca3 100644 --- a/tqdm/_monitor.py +++ b/tqdm/_monitor.py @@ -1,7 +1,8 @@ +import atexit from threading import Event, Thread, current_thread from time import time from warnings import warn -import atexit + __all__ = ["TMonitor", "TqdmSynchronisationWarning"] @@ -24,26 +25,16 @@ class TMonitor(Thread): sleep_interval : fload Time to sleep between monitoring checks. """ - - # internal vars for unit testing - _time = None - _event = None + _test = {} # internal vars for unit testing def __init__(self, tqdm_cls, sleep_interval): Thread.__init__(self) self.daemon = True # kill thread when main killed (KeyboardInterrupt) - self.was_killed = Event() self.woken = 0 # last time woken up, to sync with monitor self.tqdm_cls = tqdm_cls self.sleep_interval = sleep_interval - if TMonitor._time is not None: - self._time = TMonitor._time - else: - self._time = time - if TMonitor._event is not None: - self._event = TMonitor._event - else: - self._event = Event + self._time = self._test.get("time", time) + self.was_killed = self._test.get("Event", Event)() atexit.register(self.exit) self.start() @@ -90,10 +81,14 @@ def run(self): instance.miniters = 1 # Refresh now! (works only for manual tqdm) instance.refresh(nolock=True) + # Remove accidental long-lived strong reference + del instance if instances != self.get_instances(): # pragma: nocover warn("Set changed size during iteration" + " (see https://github.com/tqdm/tqdm/issues/481)", TqdmSynchronisationWarning, stacklevel=2) + # Remove accidental long-lived strong references + del instances def report(self): return not self.was_killed.is_set() diff --git a/tqdm/_tqdm.py b/tqdm/_tqdm.py index 694318ee7..7fc496277 100644 --- a/tqdm/_tqdm.py +++ b/tqdm/_tqdm.py @@ -1,7 +1,9 @@ +from warnings import warn + from .std import * # NOQA from .std import __all__ # NOQA from .std import TqdmDeprecationWarning -from warnings import warn + warn("This function will be removed in tqdm==5.0.0\n" "Please use `tqdm.std.*` instead of `tqdm._tqdm.*`", TqdmDeprecationWarning, stacklevel=2) diff --git a/tqdm/_tqdm_gui.py b/tqdm/_tqdm_gui.py index 541f104fb..f32aa894f 100644 --- a/tqdm/_tqdm_gui.py +++ b/tqdm/_tqdm_gui.py @@ -1,7 +1,9 @@ +from warnings import warn + from .gui import * # NOQA from .gui import __all__ # NOQA from .std import TqdmDeprecationWarning -from warnings import warn + warn("This function will be removed in tqdm==5.0.0\n" "Please use `tqdm.gui.*` instead of `tqdm._tqdm_gui.*`", TqdmDeprecationWarning, stacklevel=2) diff --git a/tqdm/_tqdm_notebook.py b/tqdm/_tqdm_notebook.py index dde999817..f225fbf5b 100644 --- a/tqdm/_tqdm_notebook.py +++ b/tqdm/_tqdm_notebook.py @@ -1,7 +1,9 @@ +from warnings import warn + from .notebook import * # NOQA from .notebook import __all__ # NOQA from .std import TqdmDeprecationWarning -from warnings import warn + warn("This function will be removed in tqdm==5.0.0\n" "Please use `tqdm.notebook.*` instead of `tqdm._tqdm_notebook.*`", TqdmDeprecationWarning, stacklevel=2) diff --git a/tqdm/_tqdm_pandas.py b/tqdm/_tqdm_pandas.py index 234fafffe..c4fe6efdc 100644 --- a/tqdm/_tqdm_pandas.py +++ b/tqdm/_tqdm_pandas.py @@ -4,43 +4,21 @@ __all__ = ['tqdm_pandas'] -def tqdm_pandas(tclass, *targs, **tkwargs): +def tqdm_pandas(tclass, **tqdm_kwargs): """ Registers the given `tqdm` instance with `pandas.core.groupby.DataFrameGroupBy.progress_apply`. - It will even close() the `tqdm` instance upon completion. - - Parameters - ---------- - tclass : tqdm class you want to use (eg, tqdm, tqdm_notebook, etc) - targs and tkwargs : arguments for the tqdm instance - - Examples - -------- - >>> import pandas as pd - >>> import numpy as np - >>> from tqdm import tqdm, tqdm_pandas - >>> - >>> df = pd.DataFrame(np.random.randint(0, 100, (100000, 6))) - >>> tqdm_pandas(tqdm, leave=True) # can use tqdm_gui, optional kwargs, etc - >>> # Now you can use `progress_apply` instead of `apply` - >>> df.groupby(0).progress_apply(lambda x: x**2) - - References - ---------- - https://stackoverflow.com/questions/18603270/ - progress-indicator-during-pandas-operations-python """ from tqdm import TqdmDeprecationWarning if isinstance(tclass, type) or (getattr(tclass, '__name__', '').startswith( 'tqdm_')): # delayed adapter case - TqdmDeprecationWarning("""\ -Please use `tqdm.pandas(...)` instead of `tqdm_pandas(tqdm, ...)`. -""", fp_write=getattr(tkwargs.get('file', None), 'write', sys.stderr.write)) - tclass.pandas(*targs, **tkwargs) + TqdmDeprecationWarning( + "Please use `tqdm.pandas(...)` instead of `tqdm_pandas(tqdm, ...)`.", + fp_write=getattr(tqdm_kwargs.get('file', None), 'write', sys.stderr.write)) + tclass.pandas(**tqdm_kwargs) else: - TqdmDeprecationWarning("""\ -Please use `tqdm.pandas(...)` instead of `tqdm_pandas(tqdm(...))`. -""", fp_write=getattr(tclass.fp, 'write', sys.stderr.write)) + TqdmDeprecationWarning( + "Please use `tqdm.pandas(...)` instead of `tqdm_pandas(tqdm(...))`.", + fp_write=getattr(tclass.fp, 'write', sys.stderr.write)) type(tclass).pandas(deprecated_t=tclass) diff --git a/tqdm/_utils.py b/tqdm/_utils.py index 084327f47..2228691e2 100644 --- a/tqdm/_utils.py +++ b/tqdm/_utils.py @@ -1,6 +1,12 @@ -from .utils import CUR_OS, IS_WIN, IS_NIX, RE_ANSI, _range, _unich, _unicode, colorama, WeakSet, _basestring, _OrderedDict, FormatReplace, Comparable, SimpleTextIOWrapper, _is_utf, _supports_unicode, _is_ascii, _screen_shape_wrapper, _screen_shape_windows, _screen_shape_tput, _screen_shape_linux, _environ_cols_wrapper, _term_move_up # NOQA -from .std import TqdmDeprecationWarning from warnings import warn + +from .std import TqdmDeprecationWarning +from .utils import ( # NOQA, pylint: disable=unused-import + CUR_OS, IS_NIX, IS_WIN, RE_ANSI, Comparable, FormatReplace, SimpleTextIOWrapper, + _basestring, _environ_cols_wrapper, _is_ascii, _is_utf, _range, _screen_shape_linux, + _screen_shape_tput, _screen_shape_windows, _screen_shape_wrapper, _supports_unicode, + _term_move_up, _unich, _unicode, colorama) + warn("This function will be removed in tqdm==5.0.0\n" "Please use `tqdm.utils.*` instead of `tqdm._utils.*`", TqdmDeprecationWarning, stacklevel=2) diff --git a/tqdm/_version.py b/tqdm/_version.py deleted file mode 100644 index 26606cb03..000000000 --- a/tqdm/_version.py +++ /dev/null @@ -1,59 +0,0 @@ -# Definition of the version number -import os -from io import open as io_open - -__all__ = ["__version__"] - -# major, minor, patch, -extra -version_info = 4, 46, 1 - -# Nice string for the version -__version__ = '.'.join(map(str, version_info)) - - -# auto -extra based on commit hash (if not tagged as release) -scriptdir = os.path.dirname(__file__) -gitdir = os.path.abspath(os.path.join(scriptdir, "..", ".git")) -if os.path.isdir(gitdir): # pragma: nocover - extra = None - # Open config file to check if we are in tqdm project - with io_open(os.path.join(gitdir, "config"), 'r') as fh_config: - if 'tqdm' in fh_config.read(): - # Open the HEAD file - with io_open(os.path.join(gitdir, "HEAD"), 'r') as fh_head: - extra = fh_head.readline().strip() - # in a branch => HEAD points to file containing last commit - if 'ref:' in extra: - # reference file path - ref_file = extra[5:] - branch_name = ref_file.rsplit('/', 1)[-1] - - ref_file_path = os.path.abspath(os.path.join(gitdir, ref_file)) - # check that we are in git folder - # (by stripping the git folder from the ref file path) - if os.path.relpath( - ref_file_path, gitdir).replace('\\', '/') != ref_file: - # out of git folder - extra = None - else: - # open the ref file - with io_open(ref_file_path, 'r') as fh_branch: - commit_hash = fh_branch.readline().strip() - extra = commit_hash[:8] - if branch_name != "master": - extra += '.' + branch_name - - # detached HEAD mode, already have commit hash - else: - extra = extra[:8] - - # Append commit hash (and branch) to version string if not tagged - if extra is not None: - try: - with io_open(os.path.join(gitdir, "refs", "tags", - 'v' + __version__)) as fdv: - if fdv.readline().strip()[:8] != extra[:8]: - __version__ += '-' + extra - except Exception as e: - if "No such file" not in str(e): - raise diff --git a/tqdm/asyncio.py b/tqdm/asyncio.py new file mode 100644 index 000000000..0d3ba747d --- /dev/null +++ b/tqdm/asyncio.py @@ -0,0 +1,89 @@ +""" +Asynchronous progressbar decorator for iterators. +Includes a default `range` iterator printing to `stderr`. + +Usage: +>>> from tqdm.asyncio import trange, tqdm +>>> async for i in trange(10): +... ... +""" +import asyncio + +from .std import tqdm as std_tqdm + +__author__ = {"github.com/": ["casperdcl"]} +__all__ = ['tqdm_asyncio', 'tarange', 'tqdm', 'trange'] + + +class tqdm_asyncio(std_tqdm): + """ + Asynchronous-friendly version of tqdm (Python 3.6+). + """ + def __init__(self, iterable=None, *args, **kwargs): + super(tqdm_asyncio, self).__init__(iterable, *args, **kwargs) + self.iterable_awaitable = False + if iterable is not None: + if hasattr(iterable, "__anext__"): + self.iterable_next = iterable.__anext__ + self.iterable_awaitable = True + elif hasattr(iterable, "__next__"): + self.iterable_next = iterable.__next__ + else: + self.iterable_iterator = iter(iterable) + self.iterable_next = self.iterable_iterator.__next__ + + def __aiter__(self): + return self + + async def __anext__(self): + try: + if self.iterable_awaitable: + res = await self.iterable_next() + else: + res = self.iterable_next() + self.update() + return res + except StopIteration: + self.close() + raise StopAsyncIteration + except BaseException: + self.close() + raise + + def send(self, *args, **kwargs): + return self.iterable.send(*args, **kwargs) + + @classmethod + def as_completed(cls, fs, *, loop=None, timeout=None, total=None, **tqdm_kwargs): + """ + Wrapper for `asyncio.as_completed`. + """ + if total is None: + total = len(fs) + yield from cls(asyncio.as_completed(fs, loop=loop, timeout=timeout), + total=total, **tqdm_kwargs) + + @classmethod + async def gather(cls, fs, *, loop=None, timeout=None, total=None, **tqdm_kwargs): + """ + Wrapper for `asyncio.gather`. + """ + async def wrap_awaitable(i, f): + return i, await f + + ifs = [wrap_awaitable(i, f) for i, f in enumerate(fs)] + res = [await f for f in cls.as_completed(ifs, loop=loop, timeout=timeout, + total=total, **tqdm_kwargs)] + return [i for _, i in sorted(res)] + + +def tarange(*args, **kwargs): + """ + A shortcut for `tqdm.asyncio.tqdm(range(*args), **kwargs)`. + """ + return tqdm_asyncio(range(*args), **kwargs) + + +# Aliases +tqdm = tqdm_asyncio +trange = tarange diff --git a/tqdm/auto.py b/tqdm/auto.py index 4dd171754..cffca206f 100644 --- a/tqdm/auto.py +++ b/tqdm/auto.py @@ -1,6 +1,44 @@ +""" +Enables multiple commonly used features. + +Method resolution order: + +- `tqdm.autonotebook` without import warnings +- `tqdm.asyncio` on Python3.6+ +- `tqdm.std` base class + +Usage: +>>> from tqdm.auto import trange, tqdm +>>> for i in trange(10): +... ... +""" +import sys import warnings + from .std import TqdmExperimentalWarning + with warnings.catch_warnings(): warnings.simplefilter("ignore", category=TqdmExperimentalWarning) - from .autonotebook import tqdm, trange + from .autonotebook import tqdm as notebook_tqdm + from .autonotebook import trange as notebook_trange + +if sys.version_info[:2] < (3, 6): + tqdm = notebook_tqdm + trange = notebook_trange +else: # Python3.6+ + from .asyncio import tqdm as asyncio_tqdm + from .std import tqdm as std_tqdm + + if notebook_tqdm != std_tqdm: + class tqdm(notebook_tqdm, asyncio_tqdm): # pylint: disable=inconsistent-mro + pass + else: + tqdm = asyncio_tqdm + + def trange(*args, **kwargs): + """ + A shortcut for `tqdm.auto.tqdm(range(*args), **kwargs)`. + """ + return tqdm(range(*args), **kwargs) + __all__ = ["tqdm", "trange"] diff --git a/tqdm/autonotebook.py b/tqdm/autonotebook.py index 0bcd42a13..b032061bf 100644 --- a/tqdm/autonotebook.py +++ b/tqdm/autonotebook.py @@ -1,17 +1,27 @@ +""" +Automatically choose between `tqdm.notebook` and `tqdm.std`. + +Usage: +>>> from tqdm.autonotebook import trange, tqdm +>>> for i in trange(10): +... ... +""" import os +import sys try: - from IPython import get_ipython + get_ipython = sys.modules['IPython'].get_ipython if 'IPKernelApp' not in get_ipython().config: # pragma: no cover raise ImportError("console") if 'VSCODE_PID' in os.environ: # pragma: no cover raise ImportError("vscode") -except: +except Exception: from .std import tqdm, trange else: # pragma: no cover + from warnings import warn + from .notebook import tqdm, trange from .std import TqdmExperimentalWarning - from warnings import warn warn("Using `tqdm.autonotebook.tqdm` in notebook mode." " Use `tqdm.tqdm` instead to force console mode" " (e.g. in jupyter console)", TqdmExperimentalWarning, stacklevel=2) diff --git a/tqdm/cli.py b/tqdm/cli.py index bf1cddeba..b5a16142b 100644 --- a/tqdm/cli.py +++ b/tqdm/cli.py @@ -1,13 +1,19 @@ -from .std import tqdm, TqdmTypeError, TqdmKeyError -from ._version import __version__ # NOQA -import sys -import re +""" +Module version for monitoring CLI pipes (`... | python -m tqdm | ...`). +""" import logging +import re +import sys +from ast import literal_eval as numeric + +from .std import TqdmKeyError, TqdmTypeError, tqdm +from .version import __version__ + __all__ = ["main"] +log = logging.getLogger(__name__) def cast(val, typ): - log = logging.getLogger(__name__) log.debug((val, typ)) if " or " in typ: for t in typ.split(" or "): @@ -27,40 +33,40 @@ def cast(val, typ): raise TqdmTypeError(val + ' : ' + typ) try: return eval(typ + '("' + val + '")') - except: + except Exception: if typ == 'chr': - return chr(ord(eval('"' + val + '"'))) + return chr(ord(eval('"' + val + '"'))).encode() else: raise TqdmTypeError(val + ' : ' + typ) -def posix_pipe(fin, fout, delim='\n', buf_size=256, - callback=lambda int: None # pragma: no cover - ): +def posix_pipe(fin, fout, delim=b'\\n', buf_size=256, + callback=lambda float: None, callback_len=True): """ Params ------ - fin : file with `read(buf_size : int)` method - fout : file with `write` (and optionally `flush`) methods. - callback : function(int), e.g.: `tqdm.update` + fin : binary file with `read(buf_size : int)` method + fout : binary file with `write` (and optionally `flush`) methods. + callback : function(float), e.g.: `tqdm.update` + callback_len : If (default: True) do `callback(len(buffer))`. + Otherwise, do `callback(data) for data in buffer.split(delim)`. """ fp_write = fout.write - # tmp = '' if not delim: while True: tmp = fin.read(buf_size) # flush at EOF if not tmp: - getattr(fout, 'flush', lambda: None)() # pragma: no cover + getattr(fout, 'flush', lambda: None)() return fp_write(tmp) callback(len(tmp)) # return - buf = '' + buf = b'' # n = 0 while True: tmp = fin.read(buf_size) @@ -69,8 +75,13 @@ def posix_pipe(fin, fout, delim='\n', buf_size=256, if not tmp: if buf: fp_write(buf) - callback(1 + buf.count(delim)) # n += 1 + buf.count(delim) - getattr(fout, 'flush', lambda: None)() # pragma: no cover + if callback_len: + # n += 1 + buf.count(delim) + callback(1 + buf.count(delim)) + else: + for i in buf.split(delim): + callback(i) + getattr(fout, 'flush', lambda: None)() return # n while True: @@ -81,8 +92,9 @@ def posix_pipe(fin, fout, delim='\n', buf_size=256, break else: fp_write(buf + tmp[:i + len(delim)]) - callback(1) # n += 1 - buf = '' + # n += 1 + callback(1 if callback_len else (buf + tmp[:i])) + buf = b'' tmp = tmp[i + len(delim):] @@ -109,6 +121,18 @@ def posix_pipe(fin, fout, delim='\n', buf_size=256, bytes : bool, optional If true, will count bytes, ignore `delim`, and default `unit_scale` to True, `unit_divisor` to 1024, and `unit` to 'B'. + tee : bool, optional + If true, passes `stdin` to both `stderr` and `stdout`. + update : bool, optional + If true, will treat input as newly elapsed iterations, + i.e. numbers to pass to `update()`. Note that this is slow + (~2e5 it/s) since every input must be decoded as a number. + update_to : bool, optional + If true, will treat input as total elapsed iterations, + i.e. numbers to assign to `self.n`. Note that this is slow + (~2e5 it/s) since every input must be decoded as a number. + null : bool, optional + If true, will discard input (no stdout). manpath : str, optional Directory in which to install tqdm man pages. comppath : str, optional @@ -128,7 +152,7 @@ def main(fp=sys.stderr, argv=None): if argv is None: argv = sys.argv[1:] try: - log = argv.index('--log') + log_idx = argv.index('--log') except ValueError: for i in argv: if i.startswith('--log='): @@ -137,13 +161,11 @@ def main(fp=sys.stderr, argv=None): else: logLevel = 'INFO' else: - # argv.pop(log) - # logLevel = argv.pop(log) - logLevel = argv[log + 1] - logging.basicConfig( - level=getattr(logging, logLevel), - format="%(levelname)s:%(module)s:%(lineno)d:%(message)s") - log = logging.getLogger(__name__) + # argv.pop(log_idx) + # logLevel = argv.pop(log_idx) + logLevel = argv[log_idx + 1] + logging.basicConfig(level=getattr(logging, logLevel), + format="%(levelname)s:%(module)s:%(lineno)d:%(message)s") d = tqdm.__init__.__doc__ + CLI_EXTRA_DOC @@ -158,16 +180,17 @@ def main(fp=sys.stderr, argv=None): # d = RE_OPTS.sub(r' --\1=<\1> : \2', d) split = RE_OPTS.split(d) opt_types_desc = zip(split[1::3], split[2::3], split[3::3]) - d = ''.join('\n --{0}=<{0}> : {1}{2}'.format(*otd) + d = ''.join(('\n --{0} : {2}{3}' if otd[1] == 'bool' else + '\n --{0}=<{1}> : {2}{3}').format( + otd[0].replace('_', '-'), otd[0], *otd[1:]) for otd in opt_types_desc if otd[0] not in UNSUPPORTED_OPTS) d = """Usage: tqdm [--help | options] Options: - -h, --help Print this help and exit - -v, --version Print version and exit - + -h, --help Print this help and exit. + -v, --version Print version and exit. """ + d.strip('\n') + '\n' # opts = docopt(d, version=__version__) @@ -187,28 +210,43 @@ def main(fp=sys.stderr, argv=None): tqdm_args = {'file': fp} try: for (o, v) in opts.items(): + o = o.replace('-', '_') try: tqdm_args[o] = cast(v, opt_types[o]) except KeyError as e: raise TqdmKeyError(str(e)) log.debug('args:' + str(tqdm_args)) - except: + + delim_per_char = tqdm_args.pop('bytes', False) + update = tqdm_args.pop('update', False) + update_to = tqdm_args.pop('update_to', False) + if sum((delim_per_char, update, update_to)) > 1: + raise TqdmKeyError("Can only have one of --bytes --update --update_to") + except Exception: fp.write('\nError:\nUsage:\n tqdm [--help | options]\n') for i in sys.stdin: sys.stdout.write(i) raise else: buf_size = tqdm_args.pop('buf_size', 256) - delim = tqdm_args.pop('delim', '\n') - delim_per_char = tqdm_args.pop('bytes', False) + delim = tqdm_args.pop('delim', b'\\n') + tee = tqdm_args.pop('tee', False) manpath = tqdm_args.pop('manpath', None) comppath = tqdm_args.pop('comppath', None) + if tqdm_args.pop('null', False): + class stdout(object): + @staticmethod + def write(_): + pass + else: + stdout = sys.stdout + stdout = getattr(stdout, 'buffer', stdout) stdin = getattr(sys.stdin, 'buffer', sys.stdin) - stdout = getattr(sys.stdout, 'buffer', sys.stdout) if manpath or comppath: from os import path from shutil import copyfile - from pkg_resources import resource_filename, Requirement + + from pkg_resources import Requirement, resource_filename def cp(src, dst): """copies from src path to dst""" @@ -218,10 +256,19 @@ def cp(src, dst): cp(resource_filename(Requirement.parse('tqdm'), 'tqdm/tqdm.1'), path.join(manpath, 'tqdm.1')) if comppath is not None: - cp(resource_filename(Requirement.parse('tqdm'), - 'tqdm/completion.sh'), + cp(resource_filename(Requirement.parse('tqdm'), 'tqdm/completion.sh'), path.join(comppath, 'tqdm_completion.sh')) sys.exit(0) + if tee: + stdout_write = stdout.write + fp_write = getattr(fp, 'buffer', fp).write + + class stdout(object): # pylint: disable=function-redefined + @staticmethod + def write(x): + with tqdm.external_write_mode(file=fp): + fp_write(x) + stdout_write(x) if delim_per_char: tqdm_args.setdefault('unit', 'B') tqdm_args.setdefault('unit_scale', True) @@ -229,11 +276,33 @@ def cp(src, dst): log.debug(tqdm_args) with tqdm(**tqdm_args) as t: posix_pipe(stdin, stdout, '', buf_size, t.update) - elif delim == '\n': + elif delim == b'\\n': log.debug(tqdm_args) - for i in tqdm(stdin, **tqdm_args): - stdout.write(i) + if update or update_to: + with tqdm(**tqdm_args) as t: + if update: + def callback(i): + t.update(numeric(i.decode())) + else: # update_to + def callback(i): + t.update(numeric(i.decode()) - t.n) + for i in stdin: + stdout.write(i) + callback(i) + else: + for i in tqdm(stdin, **tqdm_args): + stdout.write(i) else: log.debug(tqdm_args) with tqdm(**tqdm_args) as t: - posix_pipe(stdin, stdout, delim, buf_size, t.update) + callback_len = False + if update: + def callback(i): + t.update(numeric(i.decode())) + elif update_to: + def callback(i): + t.update(numeric(i.decode()) - t.n) + else: + callback = t.update + callback_len = True + posix_pipe(stdin, stdout, delim, buf_size, callback, callback_len) diff --git a/tqdm/completion.sh b/tqdm/completion.sh index fabd3f2d3..9f61c7f14 100755 --- a/tqdm/completion.sh +++ b/tqdm/completion.sh @@ -5,14 +5,14 @@ _tqdm(){ prv="${COMP_WORDS[COMP_CWORD - 1]}" case ${prv} in - --bar_format|--buf_size|--comppath|--delim|--desc|--initial|--lock_args|--manpath|--maxinterval|--mininterval|--miniters|--ncols|--nrows|--position|--postfix|--smoothing|--total|--unit|--unit_divisor) + --bar_format|--buf_size|--colour|--comppath|--delay|--delim|--desc|--initial|--lock_args|--manpath|--maxinterval|--mininterval|--miniters|--ncols|--nrows|--position|--postfix|--smoothing|--total|--unit|--unit_divisor) # await user input ;; "--log") COMPREPLY=($(compgen -W 'CRITICAL FATAL ERROR WARN WARNING INFO DEBUG NOTSET' -- ${cur})) ;; *) - COMPREPLY=($(compgen -W '--ascii --bar_format --buf_size --bytes --comppath --delim --desc --disable --dynamic_ncols --help --initial --leave --lock_args --log --manpath --maxinterval --mininterval --miniters --ncols --nrows --position --postfix --smoothing --total --unit --unit_divisor --unit_scale --version --write_bytes -h -v' -- ${cur})) + COMPREPLY=($(compgen -W '--ascii --bar_format --buf_size --bytes --colour --comppath --delay --delim --desc --disable --dynamic_ncols --help --initial --leave --lock_args --log --manpath --maxinterval --mininterval --miniters --ncols --nrows --null --position --postfix --smoothing --tee --total --unit --unit_divisor --unit_scale --update --update_to --version --write_bytes -h -v' -- ${cur})) ;; esac } diff --git a/tqdm/contrib/__init__.py b/tqdm/contrib/__init__.py index 01312cd05..ac1969890 100644 --- a/tqdm/contrib/__init__.py +++ b/tqdm/contrib/__init__.py @@ -3,26 +3,55 @@ Subpackages contain potentially unstable extensions. """ +import sys +from functools import wraps + from tqdm import tqdm from tqdm.auto import tqdm as tqdm_auto from tqdm.utils import ObjectWrapper -from copy import deepcopy -import functools -import sys + __author__ = {"github.com/": ["casperdcl"]} __all__ = ['tenumerate', 'tzip', 'tmap'] class DummyTqdmFile(ObjectWrapper): """Dummy file-like that will write to tqdm""" + + def __init__(self, wrapped): + super(DummyTqdmFile, self).__init__(wrapped) + self._buf = [] + def write(self, x, nolock=False): - # Avoid print() second call (useless \n) - if len(x.rstrip()) > 0: - tqdm.write(x, file=self._wrapped, nolock=nolock) + nl = b"\n" if isinstance(x, bytes) else "\n" + pre, sep, post = x.rpartition(nl) + if sep: + blank = type(nl)() + tqdm.write(blank.join(self._buf + [pre, sep]), + end=blank, file=self._wrapped, nolock=nolock) + self._buf = [post] + else: + self._buf.append(x) + def __del__(self): + if self._buf: + blank = type(self._buf[0])() + try: + tqdm.write(blank.join(self._buf), end=blank, file=self._wrapped) + except (OSError, ValueError): + pass -def tenumerate(iterable, start=0, total=None, tqdm_class=tqdm_auto, - **tqdm_kwargs): + +def builtin_iterable(func): + """Wraps `func()` output in a `list()` in py2""" + if sys.version_info[:1] < (3,): + @wraps(func) + def inner(*args, **kwargs): + return list(func(*args, **kwargs)) + return inner + return func + + +def tenumerate(iterable, start=0, total=None, tqdm_class=tqdm_auto, **tqdm_kwargs): """ Equivalent of `numpy.ndenumerate` or builtin `enumerate`. @@ -36,12 +65,13 @@ def tenumerate(iterable, start=0, total=None, tqdm_class=tqdm_auto, pass else: if isinstance(iterable, np.ndarray): - return tqdm_class(np.ndenumerate(iterable), - total=total or iterable.size, **tqdm_kwargs) - return enumerate(tqdm_class(iterable, **tqdm_kwargs), start) + return tqdm_class(np.ndenumerate(iterable), total=total or iterable.size, + **tqdm_kwargs) + return enumerate(tqdm_class(iterable, total=total, **tqdm_kwargs), start) -def _tzip(iter1, *iter2plus, **tqdm_kwargs): +@builtin_iterable +def tzip(iter1, *iter2plus, **tqdm_kwargs): """ Equivalent of builtin `zip`. @@ -49,13 +79,14 @@ def _tzip(iter1, *iter2plus, **tqdm_kwargs): ---------- tqdm_class : [default: tqdm.auto.tqdm]. """ - kwargs = deepcopy(tqdm_kwargs) + kwargs = tqdm_kwargs.copy() tqdm_class = kwargs.pop("tqdm_class", tqdm_auto) - for i in zip(tqdm_class(iter1, **tqdm_kwargs), *iter2plus): + for i in zip(tqdm_class(iter1, **kwargs), *iter2plus): yield i -def _tmap(function, *sequences, **tqdm_kwargs): +@builtin_iterable +def tmap(function, *sequences, **tqdm_kwargs): """ Equivalent of builtin `map`. @@ -63,18 +94,5 @@ def _tmap(function, *sequences, **tqdm_kwargs): ---------- tqdm_class : [default: tqdm.auto.tqdm]. """ - for i in _tzip(*sequences, **tqdm_kwargs): + for i in tzip(*sequences, **tqdm_kwargs): yield function(*i) - - -if sys.version_info[:1] < (3,): - @functools.wraps(_tzip) - def tzip(*args, **kwargs): - return list(_tzip(*args, **kwargs)) - - @functools.wraps(_tmap) - def tmap(*args, **kwargs): - return list(_tmap(*args, **kwargs)) -else: - tzip = _tzip - tmap = _tmap diff --git a/tqdm/contrib/bells.py b/tqdm/contrib/bells.py new file mode 100644 index 000000000..01e56b90e --- /dev/null +++ b/tqdm/contrib/bells.py @@ -0,0 +1,24 @@ +""" +Even more features than `tqdm.auto` (all the bells & whistles): + +- `tqdm.auto` +- `tqdm.tqdm.pandas` +- `tqdm.contrib.telegram` + + uses `${TQDM_TELEGRAM_TOKEN}` and `${TQDM_TELEGRAM_CHAT_ID}` +- `tqdm.contrib.discord` + + uses `${TQDM_DISCORD_TOKEN}` and `${TQDM_DISCORD_CHANNEL_ID}` +""" +__all__ = ['tqdm', 'trange'] +import warnings +from os import getenv + +if getenv("TQDM_TELEGRAM_TOKEN") and getenv("TQDM_TELEGRAM_CHAT_ID"): + from tqdm.contrib.telegram import tqdm, trange +elif getenv("TQDM_DISCORD_TOKEN") and getenv("TQDM_DISCORD_CHANNEL_ID"): + from tqdm.contrib.discord import tqdm, trange +else: + from tqdm.auto import tqdm, trange + +with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=FutureWarning) + tqdm.pandas() diff --git a/tqdm/contrib/concurrent.py b/tqdm/contrib/concurrent.py index 197a5f8c5..5ca683bba 100644 --- a/tqdm/contrib/concurrent.py +++ b/tqdm/contrib/concurrent.py @@ -2,9 +2,12 @@ Thin wrappers around `concurrent.futures`. """ from __future__ import absolute_import + +from contextlib import contextmanager + from tqdm import TqdmWarning from tqdm.auto import tqdm as tqdm_auto -from copy import deepcopy + try: from operator import length_hint except ImportError: @@ -23,10 +26,25 @@ def length_hint(it, default=0): def cpu_count(): return 4 import sys + __author__ = {"github.com/": ["casperdcl"]} __all__ = ['thread_map', 'process_map'] +@contextmanager +def ensure_lock(tqdm_class, lock_name=""): + """get (create if necessary) and then restore `tqdm_class`'s lock""" + old_lock = getattr(tqdm_class, '_lock', None) # don't create a new lock + lock = old_lock or tqdm_class.get_lock() # maybe create a new lock + lock = getattr(lock, lock_name, lock) # maybe subtype + tqdm_class.set_lock(lock) + yield lock + if old_lock is None: + del tqdm_class._lock + else: + tqdm_class.set_lock(old_lock) + + def _executor_map(PoolExecutor, fn, *iterables, **tqdm_kwargs): """ Implementation of `thread_map` and `process_map`. @@ -36,25 +54,26 @@ def _executor_map(PoolExecutor, fn, *iterables, **tqdm_kwargs): tqdm_class : [default: tqdm.auto.tqdm]. max_workers : [default: min(32, cpu_count() + 4)]. chunksize : [default: 1]. + lock_name : [default: "":str]. """ - kwargs = deepcopy(tqdm_kwargs) + kwargs = tqdm_kwargs.copy() if "total" not in kwargs: kwargs["total"] = len(iterables[0]) tqdm_class = kwargs.pop("tqdm_class", tqdm_auto) max_workers = kwargs.pop("max_workers", min(32, cpu_count() + 4)) chunksize = kwargs.pop("chunksize", 1) - pool_kwargs = dict(max_workers=max_workers) - sys_version = sys.version_info[:2] - if sys_version >= (3, 7): - # share lock in case workers are already using `tqdm` - pool_kwargs.update( - initializer=tqdm_class.set_lock, initargs=(tqdm_class.get_lock(),)) - map_args = {} - if not (3, 0) < sys_version < (3, 5): - map_args.update(chunksize=chunksize) - with PoolExecutor(**pool_kwargs) as ex: - return list(tqdm_class( - ex.map(fn, *iterables, **map_args), **kwargs)) + lock_name = kwargs.pop("lock_name", "") + with ensure_lock(tqdm_class, lock_name=lock_name) as lk: + pool_kwargs = {'max_workers': max_workers} + sys_version = sys.version_info[:2] + if sys_version >= (3, 7): + # share lock in case workers are already using `tqdm` + pool_kwargs.update(initializer=tqdm_class.set_lock, initargs=(lk,)) + map_args = {} + if not (3, 0) < sys_version < (3, 5): + map_args.update(chunksize=chunksize) + with PoolExecutor(**pool_kwargs) as ex: + return list(tqdm_class(ex.map(fn, *iterables, **map_args), **kwargs)) def thread_map(fn, *iterables, **tqdm_kwargs): @@ -64,9 +83,9 @@ def thread_map(fn, *iterables, **tqdm_kwargs): Parameters ---------- - tqdm_class : optional + tqdm_class : optional `tqdm` class to use for bars [default: tqdm.auto.tqdm]. - max_workers : int, optional + max_workers : int, optional Maximum number of workers to spawn; passed to `concurrent.futures.ThreadPoolExecutor.__init__`. [default: max(32, cpu_count() + 4)]. @@ -84,13 +103,15 @@ def process_map(fn, *iterables, **tqdm_kwargs): ---------- tqdm_class : optional `tqdm` class to use for bars [default: tqdm.auto.tqdm]. - max_workers : int, optional + max_workers : int, optional Maximum number of workers to spawn; passed to `concurrent.futures.ProcessPoolExecutor.__init__`. [default: min(32, cpu_count() + 4)]. - chunksize : int, optional + chunksize : int, optional Size of chunks sent to worker processes; passed to `concurrent.futures.ProcessPoolExecutor.map`. [default: 1]. + lock_name : str, optional + Member of `tqdm_class.get_lock()` to use [default: mp_lock]. """ from concurrent.futures import ProcessPoolExecutor if iterables and "chunksize" not in tqdm_kwargs: @@ -103,4 +124,7 @@ def process_map(fn, *iterables, **tqdm_kwargs): " This may seriously degrade multiprocess performance." " Set `chunksize=1` or more." % longest_iterable_len, TqdmWarning, stacklevel=2) + if "lock_name" not in tqdm_kwargs: + tqdm_kwargs = tqdm_kwargs.copy() + tqdm_kwargs["lock_name"] = "mp_lock" return _executor_map(ProcessPoolExecutor, fn, *iterables, **tqdm_kwargs) diff --git a/tqdm/contrib/discord.py b/tqdm/contrib/discord.py new file mode 100644 index 000000000..406ecf0a2 --- /dev/null +++ b/tqdm/contrib/discord.py @@ -0,0 +1,123 @@ +""" +Sends updates to a Discord bot. + +Usage: +>>> from tqdm.contrib.discord import tqdm, trange +>>> for i in tqdm(iterable, token='{token}', channel_id='{channel_id}'): +... ... + +![screenshot]( +https://raw.githubusercontent.com/tqdm/img/src/screenshot-discord.png) +""" +from __future__ import absolute_import + +import logging +from os import getenv + +try: + from disco.client import Client, ClientConfig +except ImportError: + raise ImportError("Please `pip install disco-py`") + +from tqdm.auto import tqdm as tqdm_auto +from tqdm.utils import _range + +from .utils_worker import MonoWorker + +__author__ = {"github.com/": ["casperdcl"]} +__all__ = ['DiscordIO', 'tqdm_discord', 'tdrange', 'tqdm', 'trange'] + + +class DiscordIO(MonoWorker): + """Non-blocking file-like IO using a Discord Bot.""" + def __init__(self, token, channel_id): + """Creates a new message in the given `channel_id`.""" + super(DiscordIO, self).__init__() + config = ClientConfig() + config.token = token + client = Client(config) + self.text = self.__class__.__name__ + try: + self.message = client.api.channels_messages_create(channel_id, self.text) + except Exception as e: + tqdm_auto.write(str(e)) + + def write(self, s): + """Replaces internal `message`'s text with `s`.""" + if not s: + s = "..." + s = s.replace('\r', '').strip() + if s == self.text: + return # skip duplicate message + self.text = s + try: + future = self.submit(self.message.edit, '`' + s + '`') + except Exception as e: + tqdm_auto.write(str(e)) + else: + return future + + +class tqdm_discord(tqdm_auto): + """ + Standard `tqdm.auto.tqdm` but also sends updates to a Discord Bot. + May take a few seconds to create (`__init__`). + + - create a discord bot (not public, no requirement of OAuth2 code + grant, only send message permissions) & invite it to a channel: + + - copy the bot `{token}` & `{channel_id}` and paste below + + >>> from tqdm.contrib.discord import tqdm, trange + >>> for i in tqdm(iterable, token='{token}', channel_id='{channel_id}'): + ... ... + """ + def __init__(self, *args, **kwargs): + """ + Parameters + ---------- + token : str, required. Discord token + [default: ${TQDM_DISCORD_TOKEN}]. + channel_id : int, required. Discord channel ID + [default: ${TQDM_DISCORD_CHANNEL_ID}]. + mininterval : float, optional. + Minimum of [default: 1.5] to avoid rate limit. + + See `tqdm.auto.tqdm.__init__` for other parameters. + """ + if not kwargs.get('disable'): + kwargs = kwargs.copy() + logging.getLogger("HTTPClient").setLevel(logging.WARNING) + self.dio = DiscordIO( + kwargs.pop('token', getenv("TQDM_DISCORD_TOKEN")), + kwargs.pop('channel_id', getenv("TQDM_DISCORD_CHANNEL_ID"))) + kwargs['mininterval'] = max(1.5, kwargs.get('mininterval', 1.5)) + super(tqdm_discord, self).__init__(*args, **kwargs) + + def display(self, **kwargs): + super(tqdm_discord, self).display(**kwargs) + fmt = self.format_dict + if fmt.get('bar_format', None): + fmt['bar_format'] = fmt['bar_format'].replace( + '', '{bar:10u}').replace('{bar}', '{bar:10u}') + else: + fmt['bar_format'] = '{l_bar}{bar:10u}{r_bar}' + self.dio.write(self.format_meter(**fmt)) + + def clear(self, *args, **kwargs): + super(tqdm_discord, self).clear(*args, **kwargs) + if not self.disable: + self.dio.write("") + + +def tdrange(*args, **kwargs): + """ + A shortcut for `tqdm.contrib.discord.tqdm(xrange(*args), **kwargs)`. + On Python3+, `range` is used instead of `xrange`. + """ + return tqdm_discord(_range(*args), **kwargs) + + +# Aliases +tqdm = tqdm_discord +trange = tdrange diff --git a/tqdm/contrib/itertools.py b/tqdm/contrib/itertools.py index 0f2a2a42b..c02083d07 100644 --- a/tqdm/contrib/itertools.py +++ b/tqdm/contrib/itertools.py @@ -2,9 +2,11 @@ Thin wrappers around `itertools`. """ from __future__ import absolute_import -from tqdm.auto import tqdm as tqdm_auto -from copy import deepcopy + import itertools + +from tqdm.auto import tqdm as tqdm_auto + __author__ = {"github.com/": ["casperdcl"]} __all__ = ['product'] @@ -17,7 +19,7 @@ def product(*iterables, **tqdm_kwargs): ---------- tqdm_class : [default: tqdm.auto.tqdm]. """ - kwargs = deepcopy(tqdm_kwargs) + kwargs = tqdm_kwargs.copy() tqdm_class = kwargs.pop("tqdm_class", tqdm_auto) try: lens = list(map(len, iterables)) diff --git a/tqdm/contrib/logging.py b/tqdm/contrib/logging.py new file mode 100644 index 000000000..5f70944dc --- /dev/null +++ b/tqdm/contrib/logging.py @@ -0,0 +1,126 @@ +""" +Helper functionality for interoperability with stdlib `logging`. +""" +from __future__ import absolute_import + +import logging +import sys +from contextlib import contextmanager + +try: + from typing import Iterator, List, Optional, Type # pylint: disable=unused-import +except ImportError: + pass + +from ..std import tqdm as std_tqdm + + +class _TqdmLoggingHandler(logging.StreamHandler): + def __init__( + self, + tqdm_class=std_tqdm # type: Type[std_tqdm] + ): + super(_TqdmLoggingHandler, self).__init__() + self.tqdm_class = tqdm_class + + def emit(self, record): + try: + msg = self.format(record) + self.tqdm_class.write(msg) + self.flush() + except (KeyboardInterrupt, SystemExit): + raise + except: # noqa pylint: disable=bare-except + self.handleError(record) + + +def _is_console_logging_handler(handler): + return (isinstance(handler, logging.StreamHandler) + and handler.stream in {sys.stdout, sys.stderr}) + + +def _get_first_found_console_logging_formatter(handlers): + for handler in handlers: + if _is_console_logging_handler(handler): + return handler.formatter + + +@contextmanager +def logging_redirect_tqdm( + loggers=None, # type: Optional[List[logging.Logger]], + tqdm_class=std_tqdm # type: Type[std_tqdm] +): + # type: (...) -> Iterator[None] + """ + Context manager redirecting console logging to `tqdm.write()`, leaving + other logging handlers (e.g. log files) unaffected. + + Parameters + ---------- + loggers : list, optional + Which handlers to redirect (default: [logging.root]). + tqdm_class : optional + + Example + ------- + ```python + import logging + from tqdm import trange + from tqdm.contrib.logging import logging_redirect_tqdm + + LOG = logging.getLogger(__name__) + + if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + with logging_redirect_tqdm(): + for i in trange(9): + if i == 4: + LOG.info("console logging redirected to `tqdm.write()`") + # logging restored + ``` + """ + if loggers is None: + loggers = [logging.root] + original_handlers_list = [logger.handlers for logger in loggers] + try: + for logger in loggers: + tqdm_handler = _TqdmLoggingHandler(tqdm_class) + tqdm_handler.setFormatter( + _get_first_found_console_logging_formatter(logger.handlers)) + logger.handlers = [ + handler for handler in logger.handlers + if not _is_console_logging_handler(handler)] + [tqdm_handler] + yield + finally: + for logger, original_handlers in zip(loggers, original_handlers_list): + logger.handlers = original_handlers + + +@contextmanager +def tqdm_logging_redirect( + *args, + # loggers=None, # type: Optional[List[logging.Logger]] + # tqdm=None, # type: Optional[Type[tqdm.tqdm]] + **kwargs +): + # type: (...) -> Iterator[None] + """ + Convenience shortcut for: + ```python + with tqdm_class(*args, **tqdm_kwargs) as pbar: + with logging_redirect_tqdm(loggers=loggers, tqdm_class=tqdm_class): + yield pbar + ``` + + Parameters + ---------- + tqdm_class : optional, (default: tqdm.std.tqdm). + loggers : optional, list. + **tqdm_kwargs : passed to `tqdm_class`. + """ + tqdm_kwargs = kwargs.copy() + loggers = tqdm_kwargs.pop('loggers', None) + tqdm_class = tqdm_kwargs.pop('tqdm_class', std_tqdm) + with tqdm_class(*args, **tqdm_kwargs) as pbar: + with logging_redirect_tqdm(loggers=loggers, tqdm_class=tqdm_class): + yield pbar diff --git a/tqdm/contrib/telegram.py b/tqdm/contrib/telegram.py index 5654dc99c..ab857a709 100644 --- a/tqdm/contrib/telegram.py +++ b/tqdm/contrib/telegram.py @@ -1,34 +1,45 @@ """ Sends updates to a Telegram bot. + +Usage: +>>> from tqdm.contrib.telegram import tqdm, trange +>>> for i in trange(10, token='{token}', chat_id='{chat_id}'): +... ... + +![screenshot]( +https://raw.githubusercontent.com/tqdm/img/src/screenshot-telegram.gif) """ from __future__ import absolute_import -from concurrent.futures import ThreadPoolExecutor +from os import getenv + from requests import Session from tqdm.auto import tqdm as tqdm_auto from tqdm.utils import _range + +from .utils_worker import MonoWorker + __author__ = {"github.com/": ["casperdcl"]} __all__ = ['TelegramIO', 'tqdm_telegram', 'ttgrange', 'tqdm', 'trange'] -class TelegramIO(): - """Non-blocking file-like IO to a Telegram Bot.""" +class TelegramIO(MonoWorker): + """Non-blocking file-like IO using a Telegram Bot.""" API = 'https://api.telegram.org/bot' def __init__(self, token, chat_id): """Creates a new message in the given `chat_id`.""" + super(TelegramIO, self).__init__() self.token = token self.chat_id = chat_id self.session = session = Session() self.text = self.__class__.__name__ - self.pool = ThreadPoolExecutor() - self.futures = [] try: res = session.post( self.API + '%s/sendMessage' % self.token, - data=dict(text='`' + self.text + '`', chat_id=self.chat_id, - parse_mode='MarkdownV2')) + data={'text': '`' + self.text + '`', 'chat_id': self.chat_id, + 'parse_mode': 'MarkdownV2'}) except Exception as e: tqdm_auto.write(str(e)) else: @@ -37,90 +48,70 @@ def __init__(self, token, chat_id): def write(self, s): """Replaces internal `message_id`'s text with `s`.""" if not s: - return - s = s.strip().replace('\r', '') + s = "..." + s = s.replace('\r', '').strip() if s == self.text: return # avoid duplicate message Bot error self.text = s try: - f = self.pool.submit( - self.session.post, - self.API + '%s/editMessageText' % self.token, - data=dict( - text='`' + s + '`', chat_id=self.chat_id, - message_id=self.message_id, parse_mode='MarkdownV2')) + future = self.submit( + self.session.post, self.API + '%s/editMessageText' % self.token, + data={'text': '`' + s + '`', 'chat_id': self.chat_id, + 'message_id': self.message_id, 'parse_mode': 'MarkdownV2'}) except Exception as e: tqdm_auto.write(str(e)) else: - self.futures.append(f) - return f - - def flush(self): - """Ensure the last `write` has been processed.""" - [f.cancel() for f in self.futures[-2::-1]] - try: - return self.futures[-1].result() - except IndexError: - pass - finally: - self.futures = [] - - def __del__(self): - self.flush() + return future class tqdm_telegram(tqdm_auto): """ - Standard `tqdm.auto.tqdm` but also sends updates to a Telegram bot. - May take a few seconds to create (`__init__`) and clear (`__del__`). + Standard `tqdm.auto.tqdm` but also sends updates to a Telegram Bot. + May take a few seconds to create (`__init__`). + + - create a bot + - copy its `{token}` + - add the bot to a chat and send it a message such as `/start` + - go to to find out + the `{chat_id}` + - paste the `{token}` & `{chat_id}` below >>> from tqdm.contrib.telegram import tqdm, trange - >>> for i in tqdm( - ... iterable, - ... token='1234567890:THIS1SSOMETOKEN0BTAINeDfrOmTELEGrAM', - ... chat_id='0246813579'): + >>> for i in tqdm(iterable, token='{token}', chat_id='{chat_id}'): + ... ... """ def __init__(self, *args, **kwargs): """ Parameters ---------- - token : str, required. Telegram token. - chat_id : str, required. Telegram chat ID. + token : str, required. Telegram token + [default: ${TQDM_TELEGRAM_TOKEN}]. + chat_id : str, required. Telegram chat ID + [default: ${TQDM_TELEGRAM_CHAT_ID}]. See `tqdm.auto.tqdm.__init__` for other parameters. """ - self.tgio = TelegramIO(kwargs.pop('token'), kwargs.pop('chat_id')) + if not kwargs.get('disable'): + kwargs = kwargs.copy() + self.tgio = TelegramIO( + kwargs.pop('token', getenv('TQDM_TELEGRAM_TOKEN')), + kwargs.pop('chat_id', getenv('TQDM_TELEGRAM_CHAT_ID'))) super(tqdm_telegram, self).__init__(*args, **kwargs) def display(self, **kwargs): super(tqdm_telegram, self).display(**kwargs) fmt = self.format_dict - if 'bar_format' in fmt and fmt['bar_format']: - fmt['bar_format'] = fmt['bar_format'].replace('', '{bar}') + if fmt.get('bar_format', None): + fmt['bar_format'] = fmt['bar_format'].replace( + '', '{bar:10u}').replace('{bar}', '{bar:10u}') else: - fmt['bar_format'] = '{l_bar}{bar}{r_bar}' - fmt['bar_format'] = fmt['bar_format'].replace('{bar}', '{bar:10u}') + fmt['bar_format'] = '{l_bar}{bar:10u}{r_bar}' self.tgio.write(self.format_meter(**fmt)) - def __new__(cls, *args, **kwargs): - """ - Workaround for mixed-class same-stream nested progressbars. - See [#509](https://github.com/tqdm/tqdm/issues/509) - """ - with cls.get_lock(): - try: - cls._instances = tqdm_auto._instances - except AttributeError: - pass - instance = super(tqdm_telegram, cls).__new__(cls, *args, **kwargs) - with cls.get_lock(): - try: - # `tqdm_auto` may have been changed so update - cls._instances.update(tqdm_auto._instances) - except AttributeError: - pass - tqdm_auto._instances = cls._instances - return instance + def clear(self, *args, **kwargs): + super(tqdm_telegram, self).clear(*args, **kwargs) + if not self.disable: + self.tgio.write("") def ttgrange(*args, **kwargs): diff --git a/tqdm/contrib/utils_worker.py b/tqdm/contrib/utils_worker.py new file mode 100644 index 000000000..d3fcf0b67 --- /dev/null +++ b/tqdm/contrib/utils_worker.py @@ -0,0 +1,40 @@ +""" +IO/concurrency helpers for `tqdm.contrib`. +""" +from __future__ import absolute_import + +from collections import deque +from concurrent.futures import ThreadPoolExecutor + +from tqdm.auto import tqdm as tqdm_auto + +__author__ = {"github.com/": ["casperdcl"]} +__all__ = ['MonoWorker'] + + +class MonoWorker(object): + """ + Supports one running task and one waiting task. + The waiting task is the most recent submitted (others are discarded). + """ + def __init__(self): + self.pool = ThreadPoolExecutor(max_workers=1) + self.futures = deque([], 2) + + def submit(self, func, *args, **kwargs): + """`func(*args, **kwargs)` may replace currently waiting task.""" + futures = self.futures + if len(futures) == futures.maxlen: + running = futures.popleft() + if not running.done(): + if len(futures): # clear waiting + waiting = futures.pop() + waiting.cancel() + futures.appendleft(running) # re-insert running + try: + waiting = self.pool.submit(func, *args, **kwargs) + except Exception as e: + tqdm_auto.write(str(e)) + else: + futures.append(waiting) + return waiting diff --git a/tqdm/dask.py b/tqdm/dask.py new file mode 100644 index 000000000..6fc7504c7 --- /dev/null +++ b/tqdm/dask.py @@ -0,0 +1,46 @@ +from __future__ import absolute_import + +from functools import partial + +from dask.callbacks import Callback + +from .auto import tqdm as tqdm_auto + +__author__ = {"github.com/": ["casperdcl"]} +__all__ = ['TqdmCallback'] + + +class TqdmCallback(Callback): + """Dask callback for task progress.""" + def __init__(self, start=None, pretask=None, tqdm_class=tqdm_auto, + **tqdm_kwargs): + """ + Parameters + ---------- + tqdm_class : optional + `tqdm` class to use for bars [default: `tqdm.auto.tqdm`]. + tqdm_kwargs : optional + Any other arguments used for all bars. + """ + super(TqdmCallback, self).__init__(start=start, pretask=pretask) + if tqdm_kwargs: + tqdm_class = partial(tqdm_class, **tqdm_kwargs) + self.tqdm_class = tqdm_class + + def _start_state(self, _, state): + self.pbar = self.tqdm_class(total=sum( + len(state[k]) for k in ['ready', 'waiting', 'running', 'finished'])) + + def _posttask(self, *_, **__): + self.pbar.update() + + def _finish(self, *_, **__): + self.pbar.close() + + def display(self): + """Displays in the current cell in Notebooks.""" + container = getattr(self.bar, 'container', None) + if container is None: + return + from .notebook import display + display(container) diff --git a/tqdm/gui.py b/tqdm/gui.py index 35f5c5e55..4612701d2 100644 --- a/tqdm/gui.py +++ b/tqdm/gui.py @@ -1,50 +1,47 @@ """ -GUI progressbar decorator for iterators. -Includes a default (x)range iterator printing to stderr. +Matplotlib GUI progressbar decorator for iterators. Usage: - >>> from tqdm.gui import trange[, tqdm] - >>> for i in trange(10): #same as: for i in tqdm(xrange(10)) - ... ... +>>> from tqdm.gui import trange, tqdm +>>> for i in trange(10): +... ... """ # future division is important to divide integers and get as # a result precise floating numbers (instead of truncated int) -from __future__ import division, absolute_import -# import compatibility functions and utilities -from .utils import _range -# to inherit from the tqdm class -from .std import tqdm as std_tqdm -from .std import TqdmExperimentalWarning +from __future__ import absolute_import, division + +import re from warnings import warn +# to inherit from the tqdm class +from .std import TqdmExperimentalWarning +from .std import tqdm as std_tqdm +# import compatibility functions and utilities +from .utils import _range __author__ = {"github.com/": ["casperdcl", "lrq3000"]} __all__ = ['tqdm_gui', 'tgrange', 'tqdm', 'trange'] class tqdm_gui(std_tqdm): # pragma: no cover - """ - Experimental GUI version of tqdm! - """ - + """Experimental Matplotlib GUI version of tqdm!""" # TODO: @classmethod: write() on GUI? - def __init__(self, *args, **kwargs): + from collections import deque + import matplotlib as mpl import matplotlib.pyplot as plt - from collections import deque + kwargs = kwargs.copy() kwargs['gui'] = True - + colour = kwargs.pop('colour', 'g') super(tqdm_gui, self).__init__(*args, **kwargs) - # Initialize the GUI display - if self.disable or not kwargs['gui']: + if self.disable: return - warn('GUI is experimental/alpha', TqdmExperimentalWarning, stacklevel=2) + warn("GUI is experimental/alpha", TqdmExperimentalWarning, stacklevel=2) self.mpl = mpl self.plt = plt - self.sp = None # Remember if external environment uses toolbars self.toolbar = self.mpl.rcParams['toolbar'] @@ -53,7 +50,7 @@ def __init__(self, *args, **kwargs): self.mininterval = max(self.mininterval, 0.5) self.fig, ax = plt.subplots(figsize=(9, 2.2)) # self.fig.subplots_adjust(bottom=0.2) - total = len(self) + total = self.__len__() # avoids TypeError on None #971 if total is not None: self.xdata = [] self.ydata = [] @@ -67,24 +64,22 @@ def __init__(self, *args, **kwargs): ax.set_ylim(0, 0.001) if total is not None: ax.set_xlim(0, 100) - ax.set_xlabel('percent') - self.fig.legend((self.line1, self.line2), ('cur', 'est'), + ax.set_xlabel("percent") + self.fig.legend((self.line1, self.line2), ("cur", "est"), loc='center right') # progressbar - self.hspan = plt.axhspan(0, 0.001, - xmin=0, xmax=0, color='g') + self.hspan = plt.axhspan(0, 0.001, xmin=0, xmax=0, color=colour) else: # ax.set_xlim(-60, 0) ax.set_xlim(0, 60) ax.invert_xaxis() - ax.set_xlabel('seconds') - ax.legend(('cur', 'est'), loc='lower left') + ax.set_xlabel("seconds") + ax.legend(("cur", "est"), loc='lower left') ax.grid() # ax.set_xlabel('seconds') - ax.set_ylabel((self.unit if self.unit else 'it') + '/s') + ax.set_ylabel((self.unit if self.unit else "it") + "/s") if self.unit_scale: - plt.ticklabel_format(style='sci', axis='y', - scilimits=(0, 0)) + plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0)) ax.yaxis.get_offset_text().set_x(-0.15) # Remember if external environment is interactive @@ -92,138 +87,7 @@ def __init__(self, *args, **kwargs): plt.ion() self.ax = ax - def __iter__(self): - # TODO: somehow allow the following: - # if not self.gui: - # return super(tqdm_gui, self).__iter__() - iterable = self.iterable - if self.disable: - for obj in iterable: - yield obj - return - - # ncols = self.ncols - mininterval = self.mininterval - maxinterval = self.maxinterval - miniters = self.miniters - dynamic_miniters = self.dynamic_miniters - last_print_t = self.last_print_t - last_print_n = self.last_print_n - n = self.n - # dynamic_ncols = self.dynamic_ncols - smoothing = self.smoothing - avg_time = self.avg_time - time = self._time - - for obj in iterable: - yield obj - # Update and possibly print the progressbar. - # Note: does not call self.update(1) for speed optimisation. - n += 1 - # check counter first to avoid calls to time() - if n - last_print_n >= self.miniters: - miniters = self.miniters # watch monitoring thread changes - delta_t = time() - last_print_t - if delta_t >= mininterval: - cur_t = time() - delta_it = n - last_print_n - # EMA (not just overall average) - if smoothing and delta_t and delta_it: - rate = delta_t / delta_it - avg_time = self.ema(rate, avg_time, smoothing) - self.avg_time = avg_time - - self.n = n - self.display() - - # If no `miniters` was specified, adjust automatically - # to the max iteration rate seen so far between 2 prints - if dynamic_miniters: - if maxinterval and delta_t >= maxinterval: - # Adjust miniters to time interval by rule of 3 - if mininterval: - # Set miniters to correspond to mininterval - miniters = delta_it * mininterval / delta_t - else: - # Set miniters to correspond to maxinterval - miniters = delta_it * maxinterval / delta_t - elif smoothing: - # EMA-weight miniters to converge - # towards the timeframe of mininterval - rate = delta_it - if mininterval and delta_t: - rate *= mininterval / delta_t - miniters = self.ema(rate, miniters, smoothing) - else: - # Maximum nb of iterations between 2 prints - miniters = max(miniters, delta_it) - - # Store old values for next call - self.n = self.last_print_n = last_print_n = n - self.last_print_t = last_print_t = cur_t - self.miniters = miniters - - # Closing the progress bar. - # Update some internal variables for close(). - self.last_print_n = last_print_n - self.n = n - self.miniters = miniters - self.close() - - def update(self, n=1): - # if not self.gui: - # return super(tqdm_gui, self).close() - if self.disable: - return - - if n < 0: - self.last_print_n += n # for auto-refresh logic to work - self.n += n - - # check counter first to reduce calls to time() - if self.n - self.last_print_n >= self.miniters: - delta_t = self._time() - self.last_print_t - if delta_t >= self.mininterval: - cur_t = self._time() - delta_it = self.n - self.last_print_n # >= n - # elapsed = cur_t - self.start_t - # EMA (not just overall average) - if self.smoothing and delta_t and delta_it: - rate = delta_t / delta_it - self.avg_time = self.ema( - rate, self.avg_time, self.smoothing) - - self.display() - - # If no `miniters` was specified, adjust automatically to the - # maximum iteration rate seen so far between two prints. - # e.g.: After running `tqdm.update(5)`, subsequent - # calls to `tqdm.update()` will only cause an update after - # at least 5 more iterations. - if self.dynamic_miniters: - if self.maxinterval and delta_t >= self.maxinterval: - if self.mininterval: - self.miniters = delta_it * self.mininterval \ - / delta_t - else: - self.miniters = delta_it * self.maxinterval \ - / delta_t - elif self.smoothing: - self.miniters = self.smoothing * delta_it * \ - (self.mininterval / delta_t - if self.mininterval and delta_t - else 1) + \ - (1 - self.smoothing) * self.miniters - else: - self.miniters = max(self.miniters, delta_it) - - # Store old values for next call - self.last_print_n = self.n - self.last_print_t = cur_t - def close(self): - # if not self.gui: - # return super(tqdm_gui, self).close() if self.disable: return @@ -237,10 +101,15 @@ def close(self): # Return to non-interactive mode if not self.wasion: self.plt.ioff() - if not self.leave: + if self.leave: + self.display() + else: self.plt.close(self.fig) - def display(self): + def clear(self, *_, **__): + pass + + def display(self, *_, **__): n = self.n cur_t = self._time() elapsed = cur_t - self.start_t @@ -284,8 +153,7 @@ def display(self): try: poly_lims = self.hspan.get_xy() except AttributeError: - self.hspan = self.plt.axhspan( - 0, 0.001, xmin=0, xmax=0, color='g') + self.hspan = self.plt.axhspan(0, 0.001, xmin=0, xmax=0, color='g') poly_lims = self.hspan.get_xy() poly_lims[0, 1] = ymin poly_lims[1, 1] = ymax @@ -299,12 +167,14 @@ def display(self): line1.set_data(t_ago, ydata) line2.set_data(t_ago, zdata) - ax.set_title(self.format_meter( - n, total, elapsed, 0, - self.desc, self.ascii, self.unit, self.unit_scale, - 1 / self.avg_time if self.avg_time else None, self.bar_format, - self.postfix, self.unit_divisor), - fontname="DejaVu Sans Mono", fontsize=11) + d = self.format_dict + # remove {bar} + d['bar_format'] = (d['bar_format'] or "{l_bar}{r_bar}").replace( + "{bar}", "") + msg = self.format_meter(**d) + if '' in msg: + msg = "".join(re.split(r'\|?\|?', msg, 1)) + ax.set_title(msg, fontname="DejaVu Sans Mono", fontsize=11) self.plt.pause(1e-9) diff --git a/tqdm/keras.py b/tqdm/keras.py index 7b7135496..45caf6189 100644 --- a/tqdm/keras.py +++ b/tqdm/keras.py @@ -1,6 +1,10 @@ from __future__ import absolute_import, division + +from copy import copy +from functools import partial + from .auto import tqdm as tqdm_auto -from copy import deepcopy + try: import keras except ImportError as e: @@ -13,14 +17,14 @@ class TqdmCallback(keras.callbacks.Callback): - """`keras` callback for epoch and batch progress""" + """Keras callback for epoch and batch progress.""" @staticmethod def bar2callback(bar, pop=None, delta=(lambda logs: 1)): def callback(_, logs=None): n = delta(logs) if logs: if pop: - logs = deepcopy(logs) + logs = copy(logs) [logs.pop(i, 0) for i in pop] bar.set_postfix(logs, refresh=False) bar.update(n) @@ -28,7 +32,7 @@ def callback(_, logs=None): return callback def __init__(self, epochs=None, data_size=None, batch_size=None, verbose=1, - tqdm_class=tqdm_auto): + tqdm_class=tqdm_auto, **tqdm_kwargs): """ Parameters ---------- @@ -41,9 +45,13 @@ def __init__(self, epochs=None, data_size=None, batch_size=None, verbose=1, 0: epoch, 1: batch (transient), 2: batch. [default: 1]. Will be set to `0` unless both `data_size` and `batch_size` are given. - tqdm_class : optional + tqdm_class : optional `tqdm` class to use for bars [default: `tqdm.auto.tqdm`]. + tqdm_kwargs : optional + Any other arguments used for all bars. """ + if tqdm_kwargs: + tqdm_class = partial(tqdm_class, **tqdm_kwargs) self.tqdm_class = tqdm_class self.epoch_bar = tqdm_class(total=epochs, unit='epoch') self.on_epoch_end = self.bar2callback(self.epoch_bar) @@ -53,11 +61,9 @@ def __init__(self, epochs=None, data_size=None, batch_size=None, verbose=1, self.batches = batches = None self.verbose = verbose if verbose == 1: - self.batch_bar = tqdm_class(total=batches, unit='batch', - leave=False) + self.batch_bar = tqdm_class(total=batches, unit='batch', leave=False) self.on_batch_end = self.bar2callback( - self.batch_bar, - pop=['batch', 'size'], + self.batch_bar, pop=['batch', 'size'], delta=lambda logs: logs.get('size', 1)) def on_train_begin(self, *_, **__): @@ -78,8 +84,7 @@ def on_epoch_begin(self, *_, **__): total=total, unit='batch', leave=True, unit_scale=1 / (params('batch_size', 1) or 1)) self.on_batch_end = self.bar2callback( - self.batch_bar, - pop=['batch', 'size'], + self.batch_bar, pop=['batch', 'size'], delta=lambda logs: logs.get('size', 1)) elif self.verbose == 1: self.batch_bar.unit_scale = 1 / (params('batch_size', 1) or 1) @@ -92,11 +97,25 @@ def on_train_end(self, *_, **__): self.batch_bar.close() self.epoch_bar.close() - def _implements_train_batch_hooks(self): + def display(self): + """Displays in the current cell in Notebooks.""" + container = getattr(self.epoch_bar, 'container', None) + if container is None: + return + from .notebook import display + display(container) + batch_bar = getattr(self, 'batch_bar', None) + if batch_bar is not None: + display(batch_bar.container) + + @staticmethod + def _implements_train_batch_hooks(): return True - def _implements_test_batch_hooks(self): + @staticmethod + def _implements_test_batch_hooks(): return True - def _implements_predict_batch_hooks(self): + @staticmethod + def _implements_predict_batch_hooks(): return True diff --git a/tqdm/notebook.py b/tqdm/notebook.py index 570510370..db25d8570 100644 --- a/tqdm/notebook.py +++ b/tqdm/notebook.py @@ -1,61 +1,61 @@ """ IPython/Jupyter Notebook progressbar decorator for iterators. -Includes a default (x)range iterator printing to stderr. +Includes a default `range` iterator printing to `stderr`. Usage: - >>> from tqdm.notebook import trange[, tqdm] - >>> for i in trange(10): #same as: for i in tqdm(xrange(10)) - ... ... +>>> from tqdm.notebook import trange, tqdm +>>> for i in trange(10): +... ... """ # future division is important to divide integers and get as # a result precise floating numbers (instead of truncated int) -from __future__ import division, absolute_import +from __future__ import absolute_import, division + # import compatibility functions and utilities +import re import sys -from .utils import _range + # to inherit from the tqdm class from .std import tqdm as std_tqdm - +from .utils import _range if True: # pragma: no cover # import IPython/Jupyter base widget and display utilities IPY = 0 - IPYW = 0 try: # IPython 4.x import ipywidgets IPY = 4 - try: - IPYW = int(ipywidgets.__version__.split('.')[0]) - except AttributeError: # __version__ may not exist in old versions - pass except ImportError: # IPython 3.x / 2.x IPY = 32 import warnings with warnings.catch_warnings(): warnings.filterwarnings( - 'ignore', - message=".*The `IPython.html` package has been deprecated.*") + 'ignore', message=".*The `IPython.html` package has been deprecated.*") try: - import IPython.html.widgets as ipywidgets + import IPython.html.widgets as ipywidgets # NOQA: F401 except ImportError: pass try: # IPython 4.x / 3.x if IPY == 32: + from IPython.html.widgets import HTML from IPython.html.widgets import FloatProgress as IProgress - from IPython.html.widgets import HBox, HTML + from IPython.html.widgets import HBox IPY = 3 else: + from ipywidgets import HTML from ipywidgets import FloatProgress as IProgress - from ipywidgets import HBox, HTML + from ipywidgets import HBox except ImportError: try: # IPython 2.x - from IPython.html.widgets import FloatProgressWidget as IProgress - from IPython.html.widgets import ContainerWidget as HBox from IPython.html.widgets import HTML + from IPython.html.widgets import ContainerWidget as HBox + from IPython.html.widgets import FloatProgressWidget as IProgress IPY = 2 except ImportError: IPY = 0 + IProgress = None + HBox = object try: from IPython.display import display # , clear_output @@ -68,16 +68,33 @@ except ImportError: # Py2 from cgi import escape - __author__ = {"github.com/": ["lrq3000", "casperdcl", "alexanderkuk"]} __all__ = ['tqdm_notebook', 'tnrange', 'tqdm', 'trange'] +class TqdmHBox(HBox): + """`ipywidgets.HBox` with a pretty representation""" + def _repr_json_(self, pretty=None): + if not hasattr(self, "pbar"): + return {} + d = self.pbar.format_dict + if pretty is not None: + d["ascii"] = not pretty + return d + + def __repr__(self, pretty=False): + if not hasattr(self, "pbar"): + return super(TqdmHBox, self).__repr__() + return self.pbar.format_meter(**self._repr_json_(pretty)) + + def _repr_pretty_(self, pp, *_, **__): + pp.text(self.__repr__(True)) + + class tqdm_notebook(std_tqdm): """ Experimental IPython/Jupyter Notebook widget using tqdm! """ - @staticmethod def status_printer(_, total=None, desc=None, ncols=None): """ @@ -91,28 +108,25 @@ def status_printer(_, total=None, desc=None, ncols=None): # fp = file # Prepare IPython progress bar - try: - if total: - pbar = IProgress(min=0, max=total) - else: # No total? Show info style bar with no progress tqdm status - pbar = IProgress(min=0, max=1) - pbar.value = 1 - pbar.bar_style = 'info' - except NameError: - # #187 #451 #558 + if IProgress is None: # #187 #451 #558 #872 raise ImportError( - "FloatProgress not found. Please update jupyter and ipywidgets." + "IProgress not found. Please update jupyter and ipywidgets." " See https://ipywidgets.readthedocs.io/en/stable" "/user_install.html") - + if total: + pbar = IProgress(min=0, max=total) + else: # No total? Show info style bar with no progress tqdm status + pbar = IProgress(min=0, max=1) + pbar.value = 1 + pbar.bar_style = 'info' + if ncols is None: + pbar.layout.width = "20px" + + ltext = HTML() + rtext = HTML() if desc: - pbar.description = desc - if IPYW >= 7: - pbar.style.description_width = 'initial' - # Prepare status text - ptext = HTML() - # Only way to place text to the right of the bar is to use a container - container = HBox(children=[pbar, ptext]) + ltext.value = desc + container = TqdmHBox(children=[ltext, pbar, rtext]) # Prepare layout if ncols is not None: # use default style of ipywidgets # ncols could be 100, "100px", "100%" @@ -126,13 +140,12 @@ def status_printer(_, total=None, desc=None, ncols=None): container.layout.width = ncols container.layout.display = 'inline-flex' container.layout.flex_flow = 'row wrap' - display(container) return container def display(self, msg=None, pos=None, # additional signals - close=False, bar_style=None): + close=False, bar_style=None, check_delay=True): # Note: contrary to native tqdm, msg='' does NOT clear bar # goal is to keep all infos if error happens so user knows # at which iteration the loop failed. @@ -141,38 +154,33 @@ def display(self, msg=None, pos=None, # clear_output(wait=1) if not msg and not close: - msg = self.__repr__() + d = self.format_dict + # remove {bar} + d['bar_format'] = (d['bar_format'] or "{l_bar}{r_bar}").replace( + "{bar}", "") + msg = self.format_meter(**d) - pbar, ptext = self.container.children + ltext, pbar, rtext = self.container.children pbar.value = self.n if msg: # html escape special characters (like '&') if '' in msg: - left, right = map(escape, msg.split('', 1)) + left, right = map(escape, re.split(r'\|?\|?', msg, 1)) else: left, right = '', escape(msg) - # remove inesthetical pipes - if left and left[-1] == '|': - left = left[:-1] - if right and right[0] == '|': - right = right[1:] - # Update description - pbar.description = left - if IPYW >= 7: - pbar.style.description_width = 'initial' - + ltext.value = left # never clear the bar (signal: msg='') if right: - ptext.value = right + rtext.value = right # Change bar style if bar_style: # Hack-ish way to avoid the danger bar_style being overridden by # success because the bar gets closed after the error... - if not (pbar.bar_style == 'danger' and bar_style == 'success'): + if pbar.bar_style != 'danger' or bar_style != 'success': pbar.bar_style = bar_style # Special signal to close the bar @@ -182,7 +190,30 @@ def display(self, msg=None, pos=None, except AttributeError: self.container.visible = False + if check_delay and self.delay > 0 and not self.displayed: + display(self.container) + self.displayed = True + + @property + def colour(self): + if hasattr(self, 'container'): + return self.container.children[-2].style.bar_color + + @colour.setter + def colour(self, bar_color): + if hasattr(self, 'container'): + self.container.children[-2].style.bar_color = bar_color + def __init__(self, *args, **kwargs): + """ + Supports the usual `tqdm.tqdm` parameters as well as those listed below. + + Parameters + ---------- + display : Whether to call `display(self.container)` immediately + [default: True]. + """ + kwargs = kwargs.copy() # Setup default output file_kwarg = kwargs.get('file', sys.stderr) if file_kwarg is sys.stderr or file_kwarg is None: @@ -190,13 +221,13 @@ def __init__(self, *args, **kwargs): # Initialize parent class + avoid printing by using gui=True kwargs['gui'] = True - kwargs.setdefault('bar_format', '{l_bar}{bar}{r_bar}') - kwargs['bar_format'] = kwargs['bar_format'].replace('{bar}', '') # convert disable = None to False kwargs['disable'] = bool(kwargs.get('disable', False)) + colour = kwargs.pop('colour', None) + display_here = kwargs.pop('display', True) super(tqdm_notebook, self).__init__(*args, **kwargs) if self.disable or not kwargs['gui']: - self.sp = lambda *_, **__: None + self.disp = lambda *_, **__: None return # Get bar width @@ -205,53 +236,59 @@ def __init__(self, *args, **kwargs): # Replace with IPython progress bar display (with correct total) unit_scale = 1 if self.unit_scale is True else self.unit_scale or 1 total = self.total * unit_scale if self.total else self.total - self.container = self.status_printer( - self.fp, total, self.desc, self.ncols) - self.sp = self.display + self.container = self.status_printer(self.fp, total, self.desc, self.ncols) + self.container.pbar = self + self.displayed = False + if display_here and self.delay <= 0: + display(self.container) + self.displayed = True + self.disp = self.display + self.colour = colour # Print initial bar state if not self.disable: - self.display() + self.display(check_delay=False) - def __iter__(self, *args, **kwargs): + def __iter__(self): try: - for obj in super(tqdm_notebook, self).__iter__(*args, **kwargs): + for obj in super(tqdm_notebook, self).__iter__(): # return super(tqdm...) will not catch exception yield obj # NB: except ... [ as ...] breaks IPython async KeyboardInterrupt except: # NOQA - self.sp(bar_style='danger') + self.disp(bar_style='danger') raise # NB: don't `finally: close()` # since this could be a shared bar which the user will `reset()` - def update(self, *args, **kwargs): + def update(self, n=1): try: - super(tqdm_notebook, self).update(*args, **kwargs) + return super(tqdm_notebook, self).update(n=n) # NB: except ... [ as ...] breaks IPython async KeyboardInterrupt except: # NOQA # cannot catch KeyboardInterrupt when using manual tqdm # as the interrupt will most likely happen on another statement - self.sp(bar_style='danger') + self.disp(bar_style='danger') raise # NB: don't `finally: close()` # since this could be a shared bar which the user will `reset()` - def close(self, *args, **kwargs): - super(tqdm_notebook, self).close(*args, **kwargs) + def close(self): + if self.disable: + return + super(tqdm_notebook, self).close() # Try to detect if there was an error or KeyboardInterrupt # in manual mode: if n < total, things probably got wrong if self.total and self.n < self.total: - self.sp(bar_style='danger') + self.disp(bar_style='danger', check_delay=False) else: if self.leave: - self.sp(bar_style='success') + self.disp(bar_style='success', check_delay=False) else: - self.sp(close=True) + self.disp(close=True, check_delay=False) - def moveto(self, *args, **kwargs): - # void -> avoid extraneous `\n` in IPython output cell - return + def clear(self, *_, **__): + pass def reset(self, total=None): """ @@ -263,9 +300,14 @@ def reset(self, total=None): ---------- total : int or float, optional. Total to use for the new bar. """ + if self.disable: + return super(tqdm_notebook, self).reset(total=total) + _, pbar, _ = self.container.children + pbar.bar_style = '' if total is not None: - pbar, _ = self.container.children pbar.max = total + if not self.total and self.ncols is None: # no longer unknown total + pbar.layout.width = None # reset width return super(tqdm_notebook, self).reset(total=total) diff --git a/tqdm/rich.py b/tqdm/rich.py new file mode 100644 index 000000000..0ad6545a6 --- /dev/null +++ b/tqdm/rich.py @@ -0,0 +1,153 @@ +""" +`rich.progress` decorator for iterators. + +Usage: +>>> from tqdm.rich import trange, tqdm +>>> for i in trange(10): +... ... +""" +from __future__ import absolute_import + +from warnings import warn + +from rich.progress import ( + BarColumn, Progress, ProgressColumn, Text, TimeElapsedColumn, TimeRemainingColumn, + filesize) + +from .std import TqdmExperimentalWarning +from .std import tqdm as std_tqdm +from .utils import _range + +__author__ = {"github.com/": ["casperdcl"]} +__all__ = ['tqdm_rich', 'trrange', 'tqdm', 'trange'] + + +class FractionColumn(ProgressColumn): + """Renders completed/total, e.g. '0.5/2.3 G'.""" + def __init__(self, unit_scale=False, unit_divisor=1000): + self.unit_scale = unit_scale + self.unit_divisor = unit_divisor + super().__init__() + + def render(self, task): + """Calculate common unit for completed and total.""" + completed = int(task.completed) + total = int(task.total) + if self.unit_scale: + unit, suffix = filesize.pick_unit_and_suffix( + total, + ["", "K", "M", "G", "T", "P", "E", "Z", "Y"], + self.unit_divisor, + ) + else: + unit, suffix = filesize.pick_unit_and_suffix(total, [""], 1) + precision = 0 if unit == 1 else 1 + return Text( + f"{completed/unit:,.{precision}f}/{total/unit:,.{precision}f} {suffix}", + style="progress.download") + + +class RateColumn(ProgressColumn): + """Renders human readable transfer speed.""" + def __init__(self, unit="", unit_scale=False, unit_divisor=1000): + self.unit = unit + self.unit_scale = unit_scale + self.unit_divisor = unit_divisor + super().__init__() + + def render(self, task): + """Show data transfer speed.""" + speed = task.speed + if speed is None: + return Text(f"? {self.unit}/s", style="progress.data.speed") + if self.unit_scale: + unit, suffix = filesize.pick_unit_and_suffix( + speed, + ["", "K", "M", "G", "T", "P", "E", "Z", "Y"], + self.unit_divisor, + ) + else: + unit, suffix = filesize.pick_unit_and_suffix(speed, [""], 1) + precision = 0 if unit == 1 else 1 + return Text(f"{speed/unit:,.{precision}f} {suffix}{self.unit}/s", + style="progress.data.speed") + + +class tqdm_rich(std_tqdm): # pragma: no cover + """Experimental rich.progress GUI version of tqdm!""" + # TODO: @classmethod: write()? + def __init__(self, *args, **kwargs): + """ + This class accepts the following parameters *in addition* to + the parameters accepted by `tqdm`. + + Parameters + ---------- + progress : tuple, optional + arguments for `rich.progress.Progress()`. + """ + kwargs = kwargs.copy() + kwargs['gui'] = True + # convert disable = None to False + kwargs['disable'] = bool(kwargs.get('disable', False)) + progress = kwargs.pop('progress', None) + super(tqdm_rich, self).__init__(*args, **kwargs) + + if self.disable: + return + + warn("rich is experimental/alpha", TqdmExperimentalWarning, stacklevel=2) + d = self.format_dict + if progress is None: + progress = ( + "[progress.description]{task.description}" + "[progress.percentage]{task.percentage:>4.0f}%", + BarColumn(bar_width=None), + FractionColumn( + unit_scale=d['unit_scale'], unit_divisor=d['unit_divisor']), + "[", TimeElapsedColumn(), "<", TimeRemainingColumn(), + ",", RateColumn(unit=d['unit'], unit_scale=d['unit_scale'], + unit_divisor=d['unit_divisor']), "]" + ) + self._prog = Progress(*progress, transient=not self.leave) + self._prog.__enter__() + self._task_id = self._prog.add_task(self.desc or "", **d) + + def close(self, *args, **kwargs): + if self.disable: + return + super(tqdm_rich, self).close(*args, **kwargs) + self._prog.__exit__(None, None, None) + + def clear(self, *_, **__): + pass + + def display(self, *_, **__): + if not hasattr(self, '_prog'): + return + self._prog.update(self._task_id, completed=self.n, description=self.desc) + + def reset(self, total=None): + """ + Resets to 0 iterations for repeated use. + + Parameters + ---------- + total : int or float, optional. Total to use for the new bar. + """ + if hasattr(self, '_prog'): + self._prog.reset(total=total) + super(tqdm_rich, self).reset(total=total) + + +def trrange(*args, **kwargs): + """ + A shortcut for `tqdm.rich.tqdm(xrange(*args), **kwargs)`. + On Python3+, `range` is used instead of `xrange`. + """ + return tqdm_rich(_range(*args), **kwargs) + + +# Aliases +tqdm = tqdm_rich +trange = trrange diff --git a/tqdm/std.py b/tqdm/std.py index 0cdc8e6b7..3458162f9 100644 --- a/tqdm/std.py +++ b/tqdm/std.py @@ -1,30 +1,30 @@ """ Customisable progressbar decorator for iterators. -Includes a default (x)range iterator printing to stderr. +Includes a default `range` iterator printing to `stderr`. Usage: - >>> from tqdm import trange[, tqdm] - >>> for i in trange(10): #same as: for i in tqdm(xrange(10)) - ... ... +>>> from tqdm import trange, tqdm +>>> for i in trange(10): +... ... """ from __future__ import absolute_import, division -# compatibility functions and utilities -from .utils import _supports_unicode, _screen_shape_wrapper, _range, _unich, \ - _term_move_up, _unicode, WeakSet, _basestring, _OrderedDict, \ - Comparable, _is_ascii, FormatReplace, disp_len, disp_trim, \ - SimpleTextIOWrapper, CallbackIOWrapper -from ._monitor import TMonitor -# native libraries -from contextlib import contextmanager + import sys +from collections import OrderedDict, defaultdict +from contextlib import contextmanager +from datetime import datetime, timedelta from numbers import Number from time import time -# For parallelism safety -import threading as th from warnings import warn +from weakref import WeakSet + +from ._monitor import TMonitor +from .utils import ( + CallbackIOWrapper, Comparable, DisableOnWriteError, FormatReplace, + SimpleTextIOWrapper, _basestring, _is_ascii, _range, _screen_shape_wrapper, + _supports_unicode, _term_move_up, _unich, _unicode, disp_len, disp_trim) -__author__ = {"github.com/": ["noamraph", "obiwanus", "kmike", "hadim", - "casperdcl", "lrq3000"]} +__author__ = "https://github.com/tqdm/tqdm#contributions" __all__ = ['tqdm', 'trange', 'TqdmTypeError', 'TqdmKeyError', 'TqdmWarning', 'TqdmExperimentalWarning', 'TqdmDeprecationWarning', @@ -46,8 +46,7 @@ class TqdmWarning(Warning): """ def __init__(self, msg, fp_write=None, *a, **k): if fp_write is not None: - fp_write("\n" + self.__class__.__name__ + ": " + - str(msg).rstrip() + '\n') + fp_write("\n" + self.__class__.__name__ + ": " + str(msg).rstrip() + '\n') else: super(TqdmWarning, self).__init__(msg, *a, **k) @@ -67,6 +66,15 @@ class TqdmMonitorWarning(TqdmWarning, RuntimeWarning): pass +def TRLock(*args, **kwargs): + """threading RLock""" + try: + from threading import RLock + return RLock(*args, **kwargs) + except (ImportError, OSError): # pragma: no cover + pass + + class TqdmDefaultWriteLock(object): """ Provide a default write lock for thread and multiprocessing safety. @@ -76,13 +84,22 @@ class TqdmDefaultWriteLock(object): On Windows, you need to supply the lock from the parent to the children as an argument to joblib or the parallelism lib you use. """ + # global thread lock so no setup required for multithreading. + # NB: Do not create multiprocessing lock as it sets the multiprocessing + # context, disallowing `spawn()`/`forkserver()` + th_lock = TRLock() + def __init__(self): # Create global parallelism locks to avoid racing issues with parallel # bars works only if fork available (Linux/MacOSX, but not Windows) - self.create_mp_lock() - self.create_th_lock() cls = type(self) + root_lock = cls.th_lock + if root_lock is not None: + root_lock.acquire() + cls.create_mp_lock() self.locks = [lk for lk in [cls.mp_lock, cls.th_lock] if lk is not None] + if root_lock is not None: + root_lock.release() def acquire(self, *a, **k): for lock in self.locks: @@ -103,26 +120,14 @@ def create_mp_lock(cls): if not hasattr(cls, 'mp_lock'): try: from multiprocessing import RLock - cls.mp_lock = RLock() # multiprocessing lock - except ImportError: # pragma: no cover - cls.mp_lock = None - except OSError: # pragma: no cover + cls.mp_lock = RLock() + except (ImportError, OSError): # pragma: no cover cls.mp_lock = None @classmethod def create_th_lock(cls): - if not hasattr(cls, 'th_lock'): - try: - cls.th_lock = th.RLock() # thread lock - except OSError: # pragma: no cover - cls.th_lock = None - - -# Create a thread lock before instantiation so that no setup needs to be done -# before running in a multithreaded environment. -# Do not create the multiprocessing lock because it sets the multiprocessing -# context and does not allow the user to use 'spawn' or 'forkserver' methods. -TqdmDefaultWriteLock.create_th_lock() + assert hasattr(cls, 'th_lock') + warn("create_th_lock not needed anymore", TqdmDeprecationWarning, stacklevel=2) class Bar(object): @@ -141,21 +146,50 @@ class Bar(object): ASCII = " 123456789#" UTF = u" " + u''.join(map(_unich, range(0x258F, 0x2587, -1))) BLANK = " " - - def __init__(self, frac, default_len=10, charset=UTF): - if not (0 <= frac <= 1): + COLOUR_RESET = '\x1b[0m' + COLOUR_RGB = '\x1b[38;2;%d;%d;%dm' + COLOURS = {'BLACK': '\x1b[30m', 'RED': '\x1b[31m', 'GREEN': '\x1b[32m', + 'YELLOW': '\x1b[33m', 'BLUE': '\x1b[34m', 'MAGENTA': '\x1b[35m', + 'CYAN': '\x1b[36m', 'WHITE': '\x1b[37m'} + + def __init__(self, frac, default_len=10, charset=UTF, colour=None): + if not 0 <= frac <= 1: warn("clamping frac to range [0, 1]", TqdmWarning, stacklevel=2) frac = max(0, min(1, frac)) assert default_len > 0 self.frac = frac self.default_len = default_len self.charset = charset + self.colour = colour + + @property + def colour(self): + return self._colour + + @colour.setter + def colour(self, value): + if not value: + self._colour = None + return + try: + if value.upper() in self.COLOURS: + self._colour = self.COLOURS[value.upper()] + elif value[0] == '#' and len(value) == 7: + self._colour = self.COLOUR_RGB % tuple( + int(i, 16) for i in (value[1:3], value[3:5], value[5:7])) + else: + raise KeyError + except (KeyError, AttributeError): + warn("Unknown colour (%s); valid choices: [hex (#00ff00), %s]" % ( + value, ", ".join(self.COLOURS)), + TqdmWarning, stacklevel=2) + self._colour = None def __format__(self, format_spec): if format_spec: _type = format_spec[-1].lower() try: - charset = dict(a=self.ASCII, u=self.UTF, b=self.BLANK)[_type] + charset = {'a': self.ASCII, 'u': self.UTF, 'b': self.BLANK}[_type] except KeyError: charset = self.charset else: @@ -171,17 +205,44 @@ def __format__(self, format_spec): N_BARS = self.default_len nsyms = len(charset) - 1 - bar_length, frac_bar_length = divmod( - int(self.frac * N_BARS * nsyms), nsyms) - - bar = charset[-1] * bar_length - frac_bar = charset[frac_bar_length] + bar_length, frac_bar_length = divmod(int(self.frac * N_BARS * nsyms), nsyms) - # whitespace padding - if bar_length < N_BARS: - return bar + frac_bar + \ + res = charset[-1] * bar_length + if bar_length < N_BARS: # whitespace padding + res = res + charset[frac_bar_length] + \ charset[0] * (N_BARS - bar_length - 1) - return bar + return self.colour + res + self.COLOUR_RESET if self.colour else res + + +class EMA(object): + """ + Exponential moving average: smoothing to give progressively lower + weights to older values. + + Parameters + ---------- + smoothing : float, optional + Smoothing factor in range [0, 1], [default: 0.3]. + Increase to give more weight to recent values. + Ranges from 0 (yields old value) to 1 (yields new value). + """ + def __init__(self, smoothing=0.3): + self.alpha = smoothing + self.last = 0 + self.calls = 0 + + def __call__(self, x=None): + """ + Parameters + ---------- + x : float + New value to include in EMA. + """ + beta = 1 - self.alpha + if x is not None: + self.last = self.alpha * x + beta * self.last + self.calls += 1 + return self.last / (1 - beta ** self.calls) if self.calls else self.last class tqdm(Comparable): @@ -193,6 +254,7 @@ class tqdm(Comparable): monitor_interval = 10 # set to 0 to disable the thread monitor = None + _instances = WeakSet() @staticmethod def format_sizeof(num, suffix='', divisor=1000): @@ -265,25 +327,6 @@ def format_num(n): n = str(n) return f if len(f) < len(n) else n - @staticmethod - def ema(x, mu=None, alpha=0.3): - """ - Exponential moving average: smoothing to give progressively lower - weights to older values. - - Parameters - ---------- - x : float - New value to include in EMA. - mu : float, optional - Previous EMA value. - alpha : float, optional - Smoothing factor in range [0, 1], [default: 0.3]. - Increase to give more weight to recent values. - Ranges from 0 (yields mu) to 1 (yields x). - """ - return x if mu is None else (alpha * x) + (1 - alpha) * mu - @staticmethod def status_printer(file): """ @@ -301,16 +344,16 @@ def fp_write(s): last_len = [0] def print_status(s): - len_s = len(s) + len_s = disp_len(s) fp_write('\r' + s + (' ' * max(last_len[0] - len_s, 0))) last_len[0] = len_s return print_status @staticmethod - def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, - unit='it', unit_scale=False, rate=None, bar_format=None, - postfix=None, unit_divisor=1000, **extra_kwargs): + def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, unit='it', + unit_scale=False, rate=None, bar_format=None, postfix=None, + unit_divisor=1000, initial=0, colour=None, **extra_kwargs): """ Return a string-based progress bar given some parameters @@ -355,7 +398,7 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, percentage, elapsed, elapsed_s, ncols, nrows, desc, unit, rate, rate_fmt, rate_noinv, rate_noinv_fmt, rate_inv, rate_inv_fmt, postfix, unit_divisor, - remaining, remaining_s. + remaining, remaining_s, eta. Note that a trailing ": " is automatically removed after {desc} if the latter is empty. postfix : *, optional @@ -366,6 +409,10 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, However other types are supported (#382). unit_divisor : float, optional [default: 1000], ignored unless `unit_scale` is True. + initial : int or float, optional + The initial counter value [default: 0]. + colour : str, optional + Bar colour (e.g. 'green', '#00ff00'). Returns ------- @@ -382,7 +429,7 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, total *= unit_scale n *= unit_scale if rate: - rate *= unit_scale # by default rate = 1 / self.avg_time + rate *= unit_scale # by default rate = self.avg_dn / self.avg_dt unit_scale = False elapsed_str = tqdm.format_interval(elapsed) @@ -390,15 +437,14 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, # if unspecified, attempt to use rate = average speed # (we allow manual override since predicting time is an arcane art) if rate is None and elapsed: - rate = n / elapsed + rate = (n - initial) / elapsed inv_rate = 1 / rate if rate else None format_sizeof = tqdm.format_sizeof rate_noinv_fmt = ((format_sizeof(rate) if unit_scale else - '{0:5.2f}'.format(rate)) - if rate else '?') + unit + '/s' - rate_inv_fmt = ((format_sizeof(inv_rate) if unit_scale else - '{0:5.2f}'.format(inv_rate)) - if inv_rate else '?') + 's/' + unit + '{0:5.2f}'.format(rate)) if rate else '?') + unit + '/s' + rate_inv_fmt = ( + (format_sizeof(inv_rate) if unit_scale else '{0:5.2f}'.format(inv_rate)) + if inv_rate else '?') + 's/' + unit rate_fmt = rate_inv_fmt if inv_rate and inv_rate > 1 else rate_noinv_fmt if unit_scale: @@ -416,6 +462,11 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, remaining = (total - n) / rate if rate and total else 0 remaining_str = tqdm.format_interval(remaining) if rate else '?' + try: + eta_dt = datetime.now() + timedelta(seconds=remaining) \ + if rate and total else datetime.utcfromtimestamp(0) + except OverflowError: + eta_dt = datetime.max # format the stats displayed to the left and right sides of the bar if prefix: @@ -440,9 +491,10 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, rate_noinv_fmt=rate_noinv_fmt, rate_inv=inv_rate, rate_inv_fmt=rate_inv_fmt, postfix=postfix, unit_divisor=unit_divisor, + colour=colour, # plus more useful definitions remaining=remaining_str, remaining_s=remaining, - l_bar=l_bar, r_bar=r_bar, + l_bar=l_bar, r_bar=r_bar, eta=eta_dt, **extra_kwargs) # total is known: we can predict some stats @@ -477,11 +529,10 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, return nobar # Formatting progress bar space available for bar's display - full_bar = Bar( - frac, - max(1, ncols - disp_len(nobar)) - if ncols else 10, - charset=Bar.ASCII if ascii is True else ascii or Bar.UTF) + full_bar = Bar(frac, + max(1, ncols - disp_len(nobar)) if ncols else 10, + charset=Bar.ASCII if ascii is True else ascii or Bar.UTF, + colour=colour) if not _is_ascii(full_bar.charset) and _is_ascii(bar_format): bar_format = _unicode(bar_format) res = bar_format.format(bar=full_bar, **format_dict) @@ -495,11 +546,9 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, nobar = bar_format.format(bar=full_bar, **format_dict) if not full_bar.format_called: return nobar - full_bar = Bar( - 0, - max(1, ncols - disp_len(nobar)) - if ncols else 10, - charset=Bar.BLANK) + full_bar = Bar(0, + max(1, ncols - disp_len(nobar)) if ncols else 10, + charset=Bar.BLANK, colour=colour) res = bar_format.format(bar=full_bar, **format_dict) return disp_trim(res, ncols) if ncols else res else: @@ -508,18 +557,13 @@ def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, '{0}{1} [{2}, {3}{4}]'.format( n_fmt, unit, elapsed_str, rate_fmt, postfix) - def __new__(cls, *args, **kwargs): - # Create a new instance + def __new__(cls, *_, **__): instance = object.__new__(cls) - # Construct the lock if it does not exist - with cls.get_lock(): - # Add to the list of instances - if not hasattr(cls, '_instances'): - cls._instances = WeakSet() + with cls.get_lock(): # also constructs lock if non-existent cls._instances.add(instance) - # Create the monitoring thread - if cls.monitor_interval and (cls.monitor is None or not - cls.monitor.report()): + # create monitoring thread + if cls.monitor_interval and (cls.monitor is None + or not cls.monitor.report()): try: cls.monitor = TMonitor(cls, cls.monitor_interval) except Exception as e: # pragma: nocover @@ -527,14 +571,13 @@ def __new__(cls, *args, **kwargs): " (monitor_interval = 0) due to:\n" + str(e), TqdmMonitorWarning, stacklevel=2) cls.monitor_interval = 0 - # Return the instance return instance @classmethod def _get_free_pos(cls, instance=None): """Skips specified instance.""" - positions = set(abs(inst.pos) for inst in cls._instances - if inst is not instance and hasattr(inst, "pos")) + positions = {abs(inst.pos) for inst in cls._instances + if inst is not instance and hasattr(inst, "pos")} return min(set(range(len(positions) + 1)).difference(positions)) @classmethod @@ -566,15 +609,6 @@ def _decr_instances(cls, instance): inst = min(instances, key=lambda i: i.pos) inst.clear(nolock=True) inst.pos = abs(instance.pos) - # Kill monitor if no instances are left - if not cls._instances and cls.monitor: - try: - cls.monitor.exit() - del cls.monitor - except AttributeError: # pragma: nocover - pass - else: - cls.monitor = None @classmethod def write(cls, s, file=None, end="\n", nolock=False): @@ -628,9 +662,9 @@ def get_lock(cls): return cls._lock @classmethod - def pandas(tclass, *targs, **tkwargs): + def pandas(cls, **tqdm_kwargs): """ - Registers the given `tqdm` class with + Registers the current `tqdm` class with pandas.core. ( frame.DataFrame | series.Series @@ -639,11 +673,11 @@ def pandas(tclass, *targs, **tkwargs): ).progress_apply A new instance will be create every time `progress_apply` is called, - and each instance will automatically close() upon completion. + and each instance will automatically `close()` upon completion. Parameters ---------- - targs, tkwargs : arguments for the tqdm instance + tqdm_kwargs : arguments for the tqdm instance Examples -------- @@ -659,35 +693,43 @@ def pandas(tclass, *targs, **tkwargs): References ---------- - https://stackoverflow.com/questions/18603270/ - progress-indicator-during-pandas-operations-python + """ + from warnings import catch_warnings, simplefilter + from pandas.core.frame import DataFrame from pandas.core.series import Series try: - from pandas import Panel - except ImportError: # TODO: pandas>0.25.2 + with catch_warnings(): + simplefilter("ignore", category=FutureWarning) + from pandas import Panel + except ImportError: # pandas>=1.2.0 Panel = None + Rolling, Expanding = None, None try: # pandas>=1.0.0 from pandas.core.window.rolling import _Rolling_and_Expanding except ImportError: try: # pandas>=0.18.0 from pandas.core.window import _Rolling_and_Expanding - except ImportError: # pragma: no cover - _Rolling_and_Expanding = None + except ImportError: # pandas>=1.2.0 + try: # pandas>=1.2.0 + from pandas.core.window.expanding import Expanding + from pandas.core.window.rolling import Rolling + _Rolling_and_Expanding = Rolling, Expanding + except ImportError: # pragma: no cover + _Rolling_and_Expanding = None try: # pandas>=0.25.0 - from pandas.core.groupby.generic import DataFrameGroupBy, \ - SeriesGroupBy # , NDFrameGroupBy - except ImportError: + from pandas.core.groupby.generic import SeriesGroupBy # , NDFrameGroupBy + from pandas.core.groupby.generic import DataFrameGroupBy + except ImportError: # pragma: no cover try: # pandas>=0.23.0 - from pandas.core.groupby.groupby import DataFrameGroupBy, \ - SeriesGroupBy + from pandas.core.groupby.groupby import DataFrameGroupBy, SeriesGroupBy except ImportError: - from pandas.core.groupby import DataFrameGroupBy, \ - SeriesGroupBy + from pandas.core.groupby import DataFrameGroupBy, SeriesGroupBy try: # pandas>=0.23.0 from pandas.core.groupby.groupby import GroupBy - except ImportError: + except ImportError: # pragma: no cover from pandas.core.groupby import GroupBy try: # pandas>=0.23.0 @@ -698,7 +740,8 @@ def pandas(tclass, *targs, **tkwargs): except ImportError: # pandas>=0.25.0 PanelGroupBy = None - deprecated_t = [tkwargs.pop('deprecated_t', None)] + tqdm_kwargs = tqdm_kwargs.copy() + deprecated_t = [tqdm_kwargs.pop('deprecated_t', None)] def inner_generator(df_function='apply'): def inner(df, func, *args, **kwargs): @@ -714,7 +757,7 @@ def inner(df, func, *args, **kwargs): """ # Precompute total iterations - total = tkwargs.pop("total", getattr(df, 'ngroups', None)) + total = tqdm_kwargs.pop("total", getattr(df, 'ngroups', None)) if total is None: # not grouped if df_function == 'applymap': total = df.size @@ -736,7 +779,7 @@ def inner(df, func, *args, **kwargs): t = deprecated_t[0] deprecated_t[0] = None else: - t = tclass(*targs, total=total, **tkwargs) + t = cls(total=total, **tqdm_kwargs) if len(args) > 0: # *args intentionally not supported (see #244, #299) @@ -790,17 +833,19 @@ def wrapper(*args, **kwargs): GroupBy.progress_aggregate = inner_generator('aggregate') GroupBy.progress_transform = inner_generator('transform') - if _Rolling_and_Expanding is not None: # pragma: no cover + if Rolling is not None and Expanding is not None: + Rolling.progress_apply = inner_generator() + Expanding.progress_apply = inner_generator() + elif _Rolling_and_Expanding is not None: _Rolling_and_Expanding.progress_apply = inner_generator() - def __init__(self, iterable=None, desc=None, total=None, leave=True, - file=None, ncols=None, mininterval=0.1, maxinterval=10.0, - miniters=None, ascii=None, disable=False, unit='it', - unit_scale=False, dynamic_ncols=False, smoothing=0.3, - bar_format=None, initial=0, position=None, postfix=None, - unit_divisor=1000, write_bytes=None, lock_args=None, - nrows=None, - gui=False, **kwargs): + def __init__(self, iterable=None, desc=None, total=None, leave=True, file=None, + ncols=None, mininterval=0.1, maxinterval=10.0, miniters=None, + ascii=None, disable=False, unit='it', unit_scale=False, + dynamic_ncols=False, smoothing=0.3, bar_format=None, initial=0, + position=None, postfix=None, unit_divisor=1000, write_bytes=None, + lock_args=None, nrows=None, colour=None, delay=0, gui=False, + **kwargs): """ Parameters ---------- @@ -878,7 +923,7 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, percentage, elapsed, elapsed_s, ncols, nrows, desc, unit, rate, rate_fmt, rate_noinv, rate_noinv_fmt, rate_inv, rate_inv_fmt, postfix, unit_divisor, - remaining, remaining_s. + remaining, remaining_s, eta. Note that a trailing ": " is automatically removed after {desc} if the latter is empty. initial : int or float, optional @@ -905,6 +950,10 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, The screen height. If specified, hides nested bars outside this bound. If unspecified, attempts to use environment height. The fallback is 20. + colour : str, optional + Bar colour (e.g. 'green', '#00ff00'). + delay : float, optional + Don't display until [default: 0] seconds have elapsed. gui : bool, optional WARNING: internal parameter - do not use. Use tqdm.gui.tqdm(...) instead. If set, will attempt to use @@ -926,6 +975,8 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, file = SimpleTextIOWrapper( file, encoding=getattr(file, 'encoding', None) or 'utf-8') + file = DisableOnWriteError(file, tqdm_instance=self) + if disable is None and hasattr(file, "isatty") and not file.isatty(): disable = True @@ -1018,14 +1069,19 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, self.unit = unit self.unit_scale = unit_scale self.unit_divisor = unit_divisor + self.initial = initial self.lock_args = lock_args + self.delay = delay self.gui = gui self.dynamic_ncols = dynamic_ncols self.smoothing = smoothing - self.avg_time = None - self._time = time + self._ema_dn = EMA(smoothing) + self._ema_dt = EMA(smoothing) + self._ema_miniters = EMA(smoothing) self.bar_format = bar_format self.postfix = None + self.colour = colour + self._time = time if postfix: try: self.set_postfix(refresh=False, **postfix) @@ -1047,7 +1103,8 @@ def __init__(self, iterable=None, desc=None, total=None, leave=True, if not gui: # Initialize the screen printer self.sp = self.status_printer(self.fp) - self.refresh(lock_args=self.lock_args) + if delay <= 0: + self.refresh(lock_args=self.lock_args) # Init the time counter self.last_print_t = self._time() @@ -1068,6 +1125,8 @@ def __len__(self): return self.total if self.iterable is None else \ (self.iterable.shape[0] if hasattr(self.iterable, "shape") else len(self.iterable) if hasattr(self.iterable, "__len__") + else self.iterable.__length_hint__() + if hasattr(self.iterable, "__length_hint__") else getattr(self, "total", None)) def __enter__(self): @@ -1085,7 +1144,7 @@ def __exit__(self, exc_type, exc_value, traceback): def __del__(self): self.close() - def __repr__(self): + def __str__(self): return self.format_meter(**self.format_dict) @property @@ -1109,76 +1168,28 @@ def __iter__(self): return mininterval = self.mininterval - maxinterval = self.maxinterval - miniters = self.miniters - dynamic_miniters = self.dynamic_miniters last_print_t = self.last_print_t last_print_n = self.last_print_n + min_start_t = self.start_t + self.delay n = self.n - smoothing = self.smoothing - avg_time = self.avg_time time = self._time - if not hasattr(self, 'sp'): - raise TqdmDeprecationWarning( - "Please use `tqdm.gui.tqdm(...)` instead of" - " `tqdm(..., gui=True)`\n", - fp_write=getattr(self.fp, 'write', sys.stderr.write)) - try: for obj in iterable: yield obj # Update and possibly print the progressbar. # Note: does not call self.update(1) for speed optimisation. n += 1 - # check counter first to avoid calls to time() + if n - last_print_n >= self.miniters: - miniters = self.miniters # watch monitoring thread changes - delta_t = time() - last_print_t - if delta_t >= mininterval: - cur_t = time() - delta_it = n - last_print_n - # EMA (not just overall average) - if smoothing and delta_t and delta_it: - rate = delta_t / delta_it - avg_time = self.ema(rate, avg_time, smoothing) - self.avg_time = avg_time - - self.n = n - self.refresh(lock_args=self.lock_args) - - # If no `miniters` was specified, adjust automatically - # to the max iteration rate seen so far between 2 prints - if dynamic_miniters: - if maxinterval and delta_t >= maxinterval: - # Adjust miniters to time interval by rule of 3 - if mininterval: - # Set miniters to correspond to mininterval - miniters = delta_it * mininterval / delta_t - else: - # Set miniters to correspond to maxinterval - miniters = delta_it * maxinterval / delta_t - elif smoothing: - # EMA-weight miniters to converge - # towards the timeframe of mininterval - rate = delta_it - if mininterval and delta_t: - rate *= mininterval / delta_t - miniters = self.ema(rate, miniters, smoothing) - else: - # Maximum nb of iterations between 2 prints - miniters = max(miniters, delta_it) - - # Store old values for next call - self.n = self.last_print_n = last_print_n = n - self.last_print_t = last_print_t = cur_t - self.miniters = miniters + cur_t = time() + dt = cur_t - last_print_t + if dt >= mininterval and cur_t >= min_start_t: + self.update(n - last_print_n) + last_print_n = self.last_print_n + last_print_t = self.last_print_t finally: - # Closing the progress bar. - # Update some internal variables for close(). - self.last_print_n = last_print_n self.n = n - self.miniters = miniters self.close() def update(self, n=1): @@ -1201,8 +1212,12 @@ def update(self, n=1): Increment to add to the internal counter of iterations [default: 1]. If using float, consider specifying `{n:.3f}` or similar in `bar_format`, or specifying `unit_scale`. + + Returns + ------- + out : bool or None + True if a `display()` was triggered. """ - # N.B.: see __iter__() for more comments. if self.disable: return @@ -1212,50 +1227,37 @@ def update(self, n=1): # check counter first to reduce calls to time() if self.n - self.last_print_n >= self.miniters: - delta_t = self._time() - self.last_print_t - if delta_t >= self.mininterval: + cur_t = self._time() + dt = cur_t - self.last_print_t + if dt >= self.mininterval and cur_t >= self.start_t + self.delay: cur_t = self._time() - delta_it = self.n - self.last_print_n # >= n - # elapsed = cur_t - self.start_t - # EMA (not just overall average) - if self.smoothing and delta_t and delta_it: - rate = delta_t / delta_it - self.avg_time = self.ema( - rate, self.avg_time, self.smoothing) - - if not hasattr(self, "sp"): - raise TqdmDeprecationWarning( - "Please use `tqdm.gui.tqdm(...)`" - " instead of `tqdm(..., gui=True)`\n", - fp_write=getattr(self.fp, 'write', sys.stderr.write)) - + dn = self.n - self.last_print_n # >= n + if self.smoothing and dt and dn: + # EMA (not just overall average) + self._ema_dn(dn) + self._ema_dt(dt) self.refresh(lock_args=self.lock_args) - - # If no `miniters` was specified, adjust automatically to the - # maximum iteration rate seen so far between two prints. - # e.g.: After running `tqdm.update(5)`, subsequent - # calls to `tqdm.update()` will only cause an update after - # at least 5 more iterations. if self.dynamic_miniters: - if self.maxinterval and delta_t >= self.maxinterval: - if self.mininterval: - self.miniters = delta_it * self.mininterval \ - / delta_t - else: - self.miniters = delta_it * self.maxinterval \ - / delta_t + # If no `miniters` was specified, adjust automatically to the + # maximum iteration rate seen so far between two prints. + # e.g.: After running `tqdm.update(5)`, subsequent + # calls to `tqdm.update()` will only cause an update after + # at least 5 more iterations. + if self.maxinterval and dt >= self.maxinterval: + self.miniters = dn * (self.mininterval or self.maxinterval) / dt elif self.smoothing: - self.miniters = self.smoothing * delta_it * \ - (self.mininterval / delta_t - if self.mininterval and delta_t - else 1) + \ - (1 - self.smoothing) * self.miniters + # EMA miniters update + self.miniters = self._ema_miniters( + dn * (self.mininterval / dt if self.mininterval and dt + else 1)) else: - self.miniters = max(self.miniters, delta_it) + # max iters between two prints + self.miniters = max(self.miniters, dn) # Store old values for next call self.last_print_n = self.n self.last_print_t = cur_t + return True def close(self): """Cleanup and (if leave=False) close the progressbar.""" @@ -1269,8 +1271,12 @@ def close(self): pos = abs(self.pos) self._decr_instances(self) + if self.last_print_t < self.start_t + self.delay: + # haven't ever displayed; nothing to clear + return + # GUI mode - if not hasattr(self, "sp"): + if getattr(self, 'sp', None) is None: return # annoyingly, _supports_unicode isn't good enough @@ -1289,7 +1295,7 @@ def fp_write(s): with self._lock: if leave: # stats for overall rate (no weighted average) - self.avg_time = None + self._ema_dt = lambda: None self.display(pos=0) fp_write('\n') else: @@ -1342,6 +1348,8 @@ def refresh(self, nolock=False, lock_args=None): def unpause(self): """Restart tqdm timer from last print time.""" + if self.disable: + return cur_t = self._time() self.start_t += cur_t - self.last_print_t self.last_print_t = cur_t @@ -1356,10 +1364,16 @@ def reset(self, total=None): ---------- total : int or float, optional. Total to use for the new bar. """ - self.last_print_n = self.n = 0 - self.last_print_t = self.start_t = self._time() + self.n = 0 if total is not None: self.total = total + if self.disable: + return + self.last_print_n = 0 + self.last_print_t = self.start_t = self._time() + self._ema_dn = EMA(self.smoothing) + self._ema_dt = EMA(self.smoothing) + self._ema_miniters = EMA(self.smoothing) self.refresh() def set_description(self, desc=None, refresh=True): @@ -1395,7 +1409,7 @@ def set_postfix(self, ordered_dict=None, refresh=True, **kwargs): kwargs : dict, optional """ # Sort in alphabetical order to be more deterministic - postfix = _OrderedDict([] if ordered_dict is None else ordered_dict) + postfix = OrderedDict([] if ordered_dict is None else ordered_dict) for key in sorted(kwargs.keys()): postfix[key] = kwargs[key] # Preprocess stats according to datatype @@ -1429,19 +1443,20 @@ def moveto(self, n): @property def format_dict(self): """Public API for read-only member access.""" + if self.disable and not hasattr(self, 'unit'): + return defaultdict(lambda: None, { + 'n': self.n, 'total': self.total, 'elapsed': 0, 'unit': 'it'}) if self.dynamic_ncols: self.ncols, self.nrows = self.dynamic_ncols(self.fp) - ncols, nrows = self.ncols, self.nrows - return dict( - n=self.n, total=self.total, - elapsed=self._time() - self.start_t - if hasattr(self, 'start_t') else 0, - ncols=ncols, nrows=nrows, - prefix=self.desc, ascii=self.ascii, unit=self.unit, - unit_scale=self.unit_scale, - rate=1 / self.avg_time if self.avg_time else None, - bar_format=self.bar_format, postfix=self.postfix, - unit_divisor=self.unit_divisor) + return { + 'n': self.n, 'total': self.total, + 'elapsed': self._time() - self.start_t if hasattr(self, 'start_t') else 0, + 'ncols': self.ncols, 'nrows': self.nrows, 'prefix': self.desc, + 'ascii': self.ascii, 'unit': self.unit, 'unit_scale': self.unit_scale, + 'rate': self._ema_dn() / self._ema_dt() if self._ema_dt() else None, + 'bar_format': self.bar_format, 'postfix': self.postfix, + 'unit_divisor': self.unit_divisor, 'initial': self.initial, + 'colour': self.colour} def display(self, msg=None, pos=None): """ @@ -1466,16 +1481,22 @@ def display(self, msg=None, pos=None): if msg or msg is None: # override at `nrows - 1` msg = " ... (more hidden) ..." + if not hasattr(self, "sp"): + raise TqdmDeprecationWarning( + "Please use `tqdm.gui.tqdm(...)`" + " instead of `tqdm(..., gui=True)`\n", + fp_write=getattr(self.fp, 'write', sys.stderr.write)) + if pos: self.moveto(pos) - self.sp(self.__repr__() if msg is None else msg) + self.sp(self.__str__() if msg is None else msg) if pos: self.moveto(-pos) return True @classmethod @contextmanager - def wrapattr(tclass, stream, method, total=None, bytes=True, **tkwargs): + def wrapattr(cls, stream, method, total=None, bytes=True, **tqdm_kwargs): """ stream : file-like object. method : str, "read" or "write". The result of `read()` and @@ -1487,7 +1508,7 @@ def wrapattr(tclass, stream, method, total=None, bytes=True, **tkwargs): ... if not chunk: ... break """ - with tclass(total=total, **tkwargs) as t: + with cls(total=total, **tqdm_kwargs) as t: if bytes: t.unit = "B" t.unit_scale = True diff --git a/tqdm/tests/tests_concurrent.py b/tqdm/tests/tests_concurrent.py deleted file mode 100644 index e64cb789b..000000000 --- a/tqdm/tests/tests_concurrent.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -Tests for `tqdm.contrib.concurrent`. -""" -from warnings import catch_warnings -from tqdm.contrib.concurrent import thread_map, process_map -from tests_tqdm import with_setup, pretest, posttest, SkipTest, StringIO, \ - closing - - -def incr(x): - """Dummy function""" - return x + 1 - - -@with_setup(pretest, posttest) -def test_thread_map(): - """Test contrib.concurrent.thread_map""" - with closing(StringIO()) as our_file: - a = range(9) - b = [i + 1 for i in a] - try: - assert thread_map(lambda x: x + 1, a, file=our_file) == b - except ImportError: - raise SkipTest - assert thread_map(incr, a, file=our_file) == b - - -@with_setup(pretest, posttest) -def test_process_map(): - """Test contrib.concurrent.process_map""" - with closing(StringIO()) as our_file: - a = range(9) - b = [i + 1 for i in a] - try: - assert process_map(incr, a, file=our_file) == b - except ImportError: - raise SkipTest - - -def test_chunksize_warning(): - """Test contrib.concurrent.process_map chunksize warnings""" - try: - from unittest.mock import patch - except ImportError: - raise SkipTest - - for iterables, should_warn in [ - ([], False), - (['x'], False), - ([()], False), - (['x', ()], False), - (['x' * 1001], True), - (['x' * 100, ('x',) * 1001], True), - ]: - with patch('tqdm.contrib.concurrent._executor_map'): - with catch_warnings(record=True) as w: - process_map(incr, *iterables) - assert should_warn == bool(w) diff --git a/tqdm/tests/tests_contrib.py b/tqdm/tests/tests_contrib.py deleted file mode 100644 index e79fad22e..000000000 --- a/tqdm/tests/tests_contrib.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -Tests for `tqdm.contrib`. -""" -import sys -from tqdm.contrib import tenumerate, tzip, tmap -from tests_tqdm import with_setup, pretest, posttest, SkipTest, StringIO, \ - closing - - -def incr(x): - """Dummy function""" - return x + 1 - - -@with_setup(pretest, posttest) -def test_enumerate(): - """Test contrib.tenumerate""" - with closing(StringIO()) as our_file: - a = range(9) - assert list(tenumerate(a, file=our_file)) == list(enumerate(a)) - assert list(tenumerate(a, 42, file=our_file)) == list(enumerate(a, 42)) - - -@with_setup(pretest, posttest) -def test_enumerate_numpy(): - """Test contrib.tenumerate(numpy.ndarray)""" - try: - import numpy as np - except ImportError: - raise SkipTest - with closing(StringIO()) as our_file: - a = np.random.random((42, 1337)) - assert list(tenumerate(a, file=our_file)) == list(np.ndenumerate(a)) - - -@with_setup(pretest, posttest) -def test_zip(): - """Test contrib.tzip""" - with closing(StringIO()) as our_file: - a = range(9) - b = [i + 1 for i in a] - if sys.version_info[:1] < (3,): - assert tzip(a, b, file=our_file) == zip(a, b) - else: - gen = tzip(a, b, file=our_file) - assert gen != list(zip(a, b)) - assert list(gen) == list(zip(a, b)) - - -@with_setup(pretest, posttest) -def test_map(): - """Test contrib.tmap""" - with closing(StringIO()) as our_file: - a = range(9) - b = [i + 1 for i in a] - if sys.version_info[:1] < (3,): - assert tmap(lambda x: x + 1, a, file=our_file) == map(incr, a) - else: - gen = tmap(lambda x: x + 1, a, file=our_file) - assert gen != b - assert list(gen) == b diff --git a/tqdm/tests/tests_keras.py b/tqdm/tests/tests_keras.py deleted file mode 100644 index 11684c490..000000000 --- a/tqdm/tests/tests_keras.py +++ /dev/null @@ -1,97 +0,0 @@ -from __future__ import division -from tqdm import tqdm -from tests_tqdm import with_setup, pretest, posttest, SkipTest, StringIO, \ - closing - - -@with_setup(pretest, posttest) -def test_keras(): - """Test tqdm.keras.TqdmCallback""" - try: - from tqdm.keras import TqdmCallback - import numpy as np - try: - import keras as K - except ImportError: - from tensorflow import keras as K - except ImportError: - raise SkipTest - - # 1D autoencoder - dtype = np.float32 - model = K.models.Sequential( - [K.layers.InputLayer((1, 1), dtype=dtype), K.layers.Conv1D(1, 1)] - ) - model.compile("adam", "mse") - x = np.random.rand(100, 1, 1).astype(dtype) - batch_size = 10 - batches = len(x) / batch_size - epochs = 5 - - with closing(StringIO()) as our_file: - - class Tqdm(tqdm): - """redirected I/O class""" - - def __init__(self, *a, **k): - k.setdefault("file", our_file) - super(Tqdm, self).__init__(*a, **k) - - # just epoch (no batch) progress - model.fit( - x, - x, - epochs=epochs, - batch_size=batch_size, - verbose=False, - callbacks=[ - TqdmCallback( - epochs, - data_size=len(x), - batch_size=batch_size, - verbose=0, - tqdm_class=Tqdm, - ) - ], - ) - res = our_file.getvalue() - assert "{epochs}/{epochs}".format(epochs=epochs) in res - assert "{batches}/{batches}".format(batches=batches) not in res - - # full (epoch and batch) progress - our_file.seek(0) - our_file.truncate() - model.fit( - x, - x, - epochs=epochs, - batch_size=batch_size, - verbose=False, - callbacks=[ - TqdmCallback( - epochs, - data_size=len(x), - batch_size=batch_size, - verbose=2, - tqdm_class=Tqdm, - ) - ], - ) - res = our_file.getvalue() - assert "{epochs}/{epochs}".format(epochs=epochs) in res - assert "{batches}/{batches}".format(batches=batches) in res - - # auto-detect epochs and batches - our_file.seek(0) - our_file.truncate() - model.fit( - x, - x, - epochs=epochs, - batch_size=batch_size, - verbose=False, - callbacks=[TqdmCallback(verbose=2, tqdm_class=Tqdm)], - ) - res = our_file.getvalue() - assert "{epochs}/{epochs}".format(epochs=epochs) in res - assert "{batches}/{batches}".format(batches=batches) in res diff --git a/tqdm/tests/tests_main.py b/tqdm/tests/tests_main.py deleted file mode 100644 index 75727071e..000000000 --- a/tqdm/tests/tests_main.py +++ /dev/null @@ -1,172 +0,0 @@ -import sys -import subprocess -from os import path -from shutil import rmtree -from tempfile import mkdtemp -from tqdm.cli import main, TqdmKeyError, TqdmTypeError -from tqdm.utils import IS_WIN -from io import open as io_open - -from tests_tqdm import with_setup, pretest, posttest, _range, closing, \ - UnicodeIO, StringIO, SkipTest - - -def _sh(*cmd, **kwargs): - return subprocess.Popen(cmd, stdout=subprocess.PIPE, - **kwargs).communicate()[0].decode('utf-8') - - -class Null(object): - def __call__(self, *_, **__): - return self - - def __getattr__(self, _): - return self - - -IN_DATA_LIST = map(str, _range(int(123))) -NULL = Null() - - -# WARNING: this should be the last test as it messes with sys.stdin, argv -@with_setup(pretest, posttest) -def test_main(): - """Test command line pipes""" - ls_out = _sh('ls').replace('\r\n', '\n') - ls = subprocess.Popen('ls', stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - res = _sh(sys.executable, '-c', 'from tqdm.cli import main; main()', - stdin=ls.stdout, stderr=subprocess.STDOUT) - ls.wait() - - # actual test: - - assert ls_out in res.replace('\r\n', '\n') - - # semi-fake test which gets coverage: - _SYS = sys.stdin, sys.argv - - with closing(StringIO()) as sys.stdin: - sys.argv = ['', '--desc', 'Test CLI --delim', - '--ascii', 'True', '--delim', r'\0', '--buf_size', '64'] - sys.stdin.write('\0'.join(map(str, _range(int(123))))) - # sys.stdin.write(b'\xff') # TODO - sys.stdin.seek(0) - main() - sys.stdin = IN_DATA_LIST - - sys.argv = ['', '--desc', 'Test CLI pipes', - '--ascii', 'True', '--unit_scale', 'True'] - import tqdm.__main__ # NOQA - - with closing(StringIO()) as sys.stdin: - IN_DATA = '\0'.join(IN_DATA_LIST) - sys.stdin.write(IN_DATA) - sys.stdin.seek(0) - sys.argv = ['', '--ascii', '--bytes=True', '--unit_scale', 'False'] - with closing(UnicodeIO()) as fp: - main(fp=fp) - assert str(len(IN_DATA)) in fp.getvalue() - sys.stdin = IN_DATA_LIST - - # test --log - with closing(StringIO()) as sys.stdin: - sys.stdin.write('\0'.join(map(str, _range(int(123))))) - sys.stdin.seek(0) - # with closing(UnicodeIO()) as fp: - main(argv=['--log', 'DEBUG'], fp=NULL) - # assert "DEBUG:" in sys.stdout.getvalue() - sys.stdin = IN_DATA_LIST - - # clean up - sys.stdin, sys.argv = _SYS - - -def test_manpath(): - """Test CLI --manpath""" - if IS_WIN: - raise SkipTest - tmp = mkdtemp() - man = path.join(tmp, "tqdm.1") - assert not path.exists(man) - try: - main(argv=['--manpath', tmp], fp=NULL) - except SystemExit: - pass - else: - raise SystemExit("Expected system exit") - assert path.exists(man) - rmtree(tmp, True) - - -def test_comppath(): - """Test CLI --comppath""" - if IS_WIN: - raise SkipTest - tmp = mkdtemp() - man = path.join(tmp, "tqdm_completion.sh") - assert not path.exists(man) - try: - main(argv=['--comppath', tmp], fp=NULL) - except SystemExit: - pass - else: - raise SystemExit("Expected system exit") - assert path.exists(man) - - # check most important options appear - with io_open(man, mode='r', encoding='utf-8') as fd: - script = fd.read() - opts = set([ - '--help', '--desc', '--total', '--leave', '--ncols', '--ascii', - '--dynamic_ncols', '--position', '--bytes', '--nrows', '--delim', - '--manpath', '--comppath' - ]) - assert all(args in script for args in opts) - rmtree(tmp, True) - - -def test_exceptions(): - """Test CLI Exceptions""" - _SYS = sys.stdin, sys.argv - sys.stdin = IN_DATA_LIST - - sys.argv = ['', '-ascii', '-unit_scale', '--bad_arg_u_ment', 'foo'] - try: - main(fp=NULL) - except TqdmKeyError as e: - if 'bad_arg_u_ment' not in str(e): - raise - else: - raise TqdmKeyError('bad_arg_u_ment') - - sys.argv = ['', '-ascii', '-unit_scale', 'invalid_bool_value'] - try: - main(fp=NULL) - except TqdmTypeError as e: - if 'invalid_bool_value' not in str(e): - raise - else: - raise TqdmTypeError('invalid_bool_value') - - sys.argv = ['', '-ascii', '--total', 'invalid_int_value'] - try: - main(fp=NULL) - except TqdmTypeError as e: - if 'invalid_int_value' not in str(e): - raise - else: - raise TqdmTypeError('invalid_int_value') - - # test SystemExits - for i in ('-h', '--help', '-v', '--version'): - sys.argv = ['', i] - try: - main(fp=NULL) - except SystemExit: - pass - else: - raise ValueError('expected SystemExit') - - # clean up - sys.stdin, sys.argv = _SYS diff --git a/tqdm/tests/tests_version.py b/tqdm/tests/tests_version.py deleted file mode 100644 index 226b99802..000000000 --- a/tqdm/tests/tests_version.py +++ /dev/null @@ -1,12 +0,0 @@ -import re - - -def test_version(): - """Test version string""" - from tqdm import __version__ - version_parts = re.split('[.-]', __version__) - assert 3 <= len(version_parts) # must have at least Major.minor.patch - try: - map(int, version_parts[:3]) - except ValueError: - raise TypeError('Version Major.minor.patch must be 3 integers') diff --git a/tqdm/tk.py b/tqdm/tk.py new file mode 100644 index 000000000..92adb51db --- /dev/null +++ b/tqdm/tk.py @@ -0,0 +1,207 @@ +""" +Tkinter GUI progressbar decorator for iterators. + +Usage: +>>> from tqdm.tk import trange, tqdm +>>> for i in trange(10): +... ... +""" +from __future__ import absolute_import, division + +import re +import sys +from warnings import warn + +try: + import tkinter + import tkinter.ttk as ttk +except ImportError: + import Tkinter as tkinter + import ttk as ttk + +from .std import TqdmExperimentalWarning, TqdmWarning +from .std import tqdm as std_tqdm +from .utils import _range + +__author__ = {"github.com/": ["richardsheridan", "casperdcl"]} +__all__ = ['tqdm_tk', 'ttkrange', 'tqdm', 'trange'] + + +class tqdm_tk(std_tqdm): # pragma: no cover + """ + Experimental Tkinter GUI version of tqdm! + + Note: Window interactivity suffers if `tqdm_tk` is not running within + a Tkinter mainloop and values are generated infrequently. In this case, + consider calling `tqdm_tk.refresh()` frequently in the Tk thread. + """ + + # TODO: @classmethod: write()? + + def __init__(self, *args, **kwargs): + """ + This class accepts the following parameters *in addition* to + the parameters accepted by `tqdm`. + + Parameters + ---------- + grab : bool, optional + Grab the input across all windows of the process. + tk_parent : `tkinter.Wm`, optional + Parent Tk window. + cancel_callback : Callable, optional + Create a cancel button and set `cancel_callback` to be called + when the cancel or window close button is clicked. + """ + kwargs = kwargs.copy() + kwargs['gui'] = True + # convert disable = None to False + kwargs['disable'] = bool(kwargs.get('disable', False)) + self._warn_leave = 'leave' in kwargs + grab = kwargs.pop('grab', False) + tk_parent = kwargs.pop('tk_parent', None) + self._cancel_callback = kwargs.pop('cancel_callback', None) + super(tqdm_tk, self).__init__(*args, **kwargs) + + if self.disable: + return + + if tk_parent is None: # Discover parent widget + try: + tk_parent = tkinter._default_root + except AttributeError: + raise AttributeError( + "`tk_parent` required when using `tkinter.NoDefaultRoot()`") + if tk_parent is None: # use new default root window as display + self._tk_window = tkinter.Tk() + else: # some other windows already exist + self._tk_window = tkinter.Toplevel() + else: + self._tk_window = tkinter.Toplevel(tk_parent) + + warn("GUI is experimental/alpha", TqdmExperimentalWarning, stacklevel=2) + self._tk_dispatching = self._tk_dispatching_helper() + + self._tk_window.protocol("WM_DELETE_WINDOW", self.cancel) + self._tk_window.wm_title(self.desc) + self._tk_window.wm_attributes("-topmost", 1) + self._tk_window.after(0, lambda: self._tk_window.wm_attributes("-topmost", 0)) + self._tk_n_var = tkinter.DoubleVar(self._tk_window, value=0) + self._tk_text_var = tkinter.StringVar(self._tk_window) + pbar_frame = ttk.Frame(self._tk_window, padding=5) + pbar_frame.pack() + _tk_label = ttk.Label(pbar_frame, textvariable=self._tk_text_var, + wraplength=600, anchor="center", justify="center") + _tk_label.pack() + self._tk_pbar = ttk.Progressbar( + pbar_frame, variable=self._tk_n_var, length=450) + if self.total is not None: + self._tk_pbar.configure(maximum=self.total) + else: + self._tk_pbar.configure(mode="indeterminate") + self._tk_pbar.pack() + if self._cancel_callback is not None: + _tk_button = ttk.Button(pbar_frame, text="Cancel", command=self.cancel) + _tk_button.pack() + if grab: + self._tk_window.grab_set() + + def close(self): + if self.disable: + return + + self.disable = True + + with self.get_lock(): + self._instances.remove(self) + + def _close(): + self._tk_window.after('idle', self._tk_window.destroy) + if not self._tk_dispatching: + self._tk_window.update() + + self._tk_window.protocol("WM_DELETE_WINDOW", _close) + + # if leave is set but we are self-dispatching, the left window is + # totally unresponsive unless the user manually dispatches + if not self.leave: + _close() + elif not self._tk_dispatching: + if self._warn_leave: + warn("leave flag ignored if not in tkinter mainloop", + TqdmWarning, stacklevel=2) + _close() + + def clear(self, *_, **__): + pass + + def display(self, *_, **__): + self._tk_n_var.set(self.n) + d = self.format_dict + # remove {bar} + d['bar_format'] = (d['bar_format'] or "{l_bar}{r_bar}").replace( + "{bar}", "") + msg = self.format_meter(**d) + if '' in msg: + msg = "".join(re.split(r'\|?\|?', msg, 1)) + self._tk_text_var.set(msg) + if not self._tk_dispatching: + self._tk_window.update() + + def set_description(self, desc=None, refresh=True): + self.set_description_str(desc, refresh) + + def set_description_str(self, desc=None, refresh=True): + self.desc = desc + if not self.disable: + self._tk_window.wm_title(desc) + if refresh and not self._tk_dispatching: + self._tk_window.update() + + def cancel(self): + """ + `cancel_callback()` followed by `close()` + when close/cancel buttons clicked. + """ + if self._cancel_callback is not None: + self._cancel_callback() + self.close() + + def reset(self, total=None): + """ + Resets to 0 iterations for repeated use. + + Parameters + ---------- + total : int or float, optional. Total to use for the new bar. + """ + if hasattr(self, '_tk_pbar'): + if total is None: + self._tk_pbar.configure(maximum=100, mode="indeterminate") + else: + self._tk_pbar.configure(maximum=total, mode="determinate") + super(tqdm_tk, self).reset(total=total) + + @staticmethod + def _tk_dispatching_helper(): + """determine if Tkinter mainloop is dispatching events""" + codes = {tkinter.mainloop.__code__, tkinter.Misc.mainloop.__code__} + for frame in sys._current_frames().values(): + while frame: + if frame.f_code in codes: + return True + frame = frame.f_back + return False + + +def ttkrange(*args, **kwargs): + """ + A shortcut for `tqdm.tk.tqdm(xrange(*args), **kwargs)`. + On Python3+, `range` is used instead of `xrange`. + """ + return tqdm_tk(_range(*args), **kwargs) + + +# Aliases +tqdm = tqdm_tk +trange = ttkrange diff --git a/tqdm/tqdm.1 b/tqdm/tqdm.1 index f0c692452..0533198ca 100644 --- a/tqdm/tqdm.1 +++ b/tqdm/tqdm.1 @@ -1,6 +1,6 @@ -.\" Automatically generated by Pandoc 1.19.2.1 +.\" Automatically generated by Pandoc 1.19.2 .\" -.TH "TQDM" "1" "2015\-2020" "tqdm User Manuals" "" +.TH "TQDM" "1" "2015\-2021" "tqdm User Manuals" "" .hy .SH NAME .PP @@ -36,12 +36,12 @@ counting:\ 100%|█████████|\ 432/432\ [00:00<00:00,\ 794361.83f .SH OPTIONS .TP .B \-h, \-\-help -Print this help and exit +Print this help and exit. .RS .RE .TP .B \-v, \-\-version -Print version and exit +Print version and exit. .RS .RE .TP @@ -63,7 +63,7 @@ specify an initial arbitrary large positive number, e.g. .RS .RE .TP -.B \-\-leave=\f[I]leave\f[] +.B \-\-leave bool, optional. If [default: True], keeps all traces of the progressbar upon termination of iteration. @@ -117,7 +117,7 @@ The fallback is to use ASCII characters " 123456789#". .RS .RE .TP -.B \-\-disable=\f[I]disable\f[] +.B \-\-disable bool, optional. Whether to disable the entire progressbar wrapper [default: False]. If set to None, disable on non\-TTY. @@ -131,7 +131,7 @@ it]. .RS .RE .TP -.B \-\-unit_scale=\f[I]unit_scale\f[] +.B \-\-unit\-scale=\f[I]unit_scale\f[] bool or int or float, optional. If 1 or True, the number of iterations will be reduced/scaled automatically and a metric prefix following the International System of @@ -140,7 +140,7 @@ If any other non\-zero number, will scale \f[C]total\f[] and \f[C]n\f[]. .RS .RE .TP -.B \-\-dynamic_ncols=\f[I]dynamic_ncols\f[] +.B \-\-dynamic\-ncols bool, optional. If set, constantly alters \f[C]ncols\f[] and \f[C]nrows\f[] to the environment (allowing for window resizes) [default: False]. @@ -156,7 +156,7 @@ Ranges from 0 (average speed) to 1 (current/instantaneous speed) .RS .RE .TP -.B \-\-bar_format=\f[I]bar_format\f[] +.B \-\-bar\-format=\f[I]bar_format\f[] str, optional. Specify a custom bar string formatting. May impact performance. @@ -166,7 +166,7 @@ May impact performance. vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, percentage, elapsed, elapsed_s, ncols, nrows, desc, unit, rate, rate_fmt, rate_noinv, rate_noinv_fmt, rate_inv, rate_inv_fmt, postfix, -unit_divisor, remaining, remaining_s. +unit_divisor, remaining, remaining_s, eta. Note that a trailing ": " is automatically removed after {desc} if the latter is empty. .RS @@ -196,13 +196,13 @@ Calls \f[C]set_postfix(**postfix)\f[] if possible (dict). .RS .RE .TP -.B \-\-unit_divisor=\f[I]unit_divisor\f[] +.B \-\-unit\-divisor=\f[I]unit_divisor\f[] float, optional. [default: 1000], ignored unless \f[C]unit_scale\f[] is True. .RS .RE .TP -.B \-\-write_bytes=\f[I]write_bytes\f[] +.B \-\-write\-bytes bool, optional. If (default: None) and \f[C]file\f[] is unspecified, bytes will be written in Python 2. @@ -211,7 +211,7 @@ In all other cases will default to unicode. .RS .RE .TP -.B \-\-lock_args=\f[I]lock_args\f[] +.B \-\-lock\-args=\f[I]lock_args\f[] tuple, optional. Passed to \f[C]refresh\f[] for intermediate output (initialisation, iterating, and updating). @@ -227,6 +227,19 @@ The fallback is 20. .RS .RE .TP +.B \-\-colour=\f[I]colour\f[] +str, optional. +Bar colour (e.g. +\[aq]green\[aq], \[aq]#00ff00\[aq]). +.RS +.RE +.TP +.B \-\-delay=\f[I]delay\f[] +float, optional. +Don\[aq]t display until [default: 0] seconds have elapsed. +.RS +.RE +.TP .B \-\-delim=\f[I]delim\f[] chr, optional. Delimiting character [default: \[aq]\\n\[aq]]. @@ -236,14 +249,14 @@ N.B.: on Windows systems, Python converts \[aq]\\n\[aq] to .RS .RE .TP -.B \-\-buf_size=\f[I]buf_size\f[] +.B \-\-buf\-size=\f[I]buf_size\f[] int, optional. String buffer size in bytes [default: 256] used when \f[C]delim\f[] is specified. .RS .RE .TP -.B \-\-bytes=\f[I]bytes\f[] +.B \-\-bytes bool, optional. If true, will count bytes, ignore \f[C]delim\f[], and default \f[C]unit_scale\f[] to True, \f[C]unit_divisor\f[] to 1024, and @@ -251,6 +264,37 @@ If true, will count bytes, ignore \f[C]delim\f[], and default .RS .RE .TP +.B \-\-tee +bool, optional. +If true, passes \f[C]stdin\f[] to both \f[C]stderr\f[] and +\f[C]stdout\f[]. +.RS +.RE +.TP +.B \-\-update +bool, optional. +If true, will treat input as newly elapsed iterations, i.e. +numbers to pass to \f[C]update()\f[]. +Note that this is slow (~2e5 it/s) since every input must be decoded as +a number. +.RS +.RE +.TP +.B \-\-update\-to +bool, optional. +If true, will treat input as total elapsed iterations, i.e. +numbers to assign to \f[C]self.n\f[]. +Note that this is slow (~2e5 it/s) since every input must be decoded as +a number. +.RS +.RE +.TP +.B \-\-null +bool, optional. +If true, will discard input (no stdout). +.RS +.RE +.TP .B \-\-manpath=\f[I]manpath\f[] str, optional. Directory in which to install tqdm man pages. diff --git a/tqdm/utils.py b/tqdm/utils.py index b64de297f..aae87e454 100644 --- a/tqdm/utils.py +++ b/tqdm/utils.py @@ -1,131 +1,50 @@ -from functools import wraps +""" +General helpers required for `tqdm.std`. +""" import os -from platform import system as _curos import re -import subprocess +import sys +from functools import wraps from warnings import warn -CUR_OS = _curos() -IS_WIN = CUR_OS in ['Windows', 'cli'] -IS_NIX = (not IS_WIN) and any( - CUR_OS.startswith(i) for i in - ['CYGWIN', 'MSYS', 'Linux', 'Darwin', 'SunOS', - 'FreeBSD', 'NetBSD', 'OpenBSD']) -RE_ANSI = re.compile(r"\x1b\[[;\d]*[A-Za-z]") +# py2/3 compat +try: + _range = xrange +except NameError: + _range = range +try: + _unich = unichr +except NameError: + _unich = chr -# Py2/3 compat. Empty conditional to avoid coverage -if True: # pragma: no cover - try: - _range = xrange - except NameError: - _range = range +try: + _unicode = unicode +except NameError: + _unicode = str - try: - _unich = unichr - except NameError: - _unich = chr +try: + _basestring = basestring +except NameError: + _basestring = str - try: - _unicode = unicode - except NameError: - _unicode = str +CUR_OS = sys.platform +IS_WIN = any(CUR_OS.startswith(i) for i in ['win32', 'cygwin']) +IS_NIX = any(CUR_OS.startswith(i) for i in ['aix', 'linux', 'darwin']) +RE_ANSI = re.compile(r"\x1b\[[;\d]*[A-Za-z]") - try: - if IS_WIN: - import colorama - else: - raise ImportError - except ImportError: - colorama = None +try: + if IS_WIN: + import colorama else: - try: - colorama.init(strip=False) - except TypeError: - colorama.init() - - try: - from weakref import WeakSet - except ImportError: - WeakSet = set - + raise ImportError +except ImportError: + colorama = None +else: try: - _basestring = basestring - except NameError: - _basestring = str - - try: # py>=2.7,>=3.1 - from collections import OrderedDict as _OrderedDict - except ImportError: - try: # older Python versions with backported ordereddict lib - from ordereddict import OrderedDict as _OrderedDict - except ImportError: # older Python versions without ordereddict lib - # Py2.6,3.0 compat, from PEP 372 - from collections import MutableMapping - - class _OrderedDict(dict, MutableMapping): - # Methods with direct access to underlying attributes - def __init__(self, *args, **kwds): - if len(args) > 1: - raise TypeError('expected at 1 argument, got %d', - len(args)) - if not hasattr(self, '_keys'): - self._keys = [] - self.update(*args, **kwds) - - def clear(self): - del self._keys[:] - dict.clear(self) - - def __setitem__(self, key, value): - if key not in self: - self._keys.append(key) - dict.__setitem__(self, key, value) - - def __delitem__(self, key): - dict.__delitem__(self, key) - self._keys.remove(key) - - def __iter__(self): - return iter(self._keys) - - def __reversed__(self): - return reversed(self._keys) - - def popitem(self): - if not self: - raise KeyError - key = self._keys.pop() - value = dict.pop(self, key) - return key, value - - def __reduce__(self): - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - inst_dict.pop('_keys', None) - return self.__class__, (items,), inst_dict - - # Methods with indirect access via the above methods - setdefault = MutableMapping.setdefault - update = MutableMapping.update - pop = MutableMapping.pop - keys = MutableMapping.keys - values = MutableMapping.values - items = MutableMapping.items - - def __repr__(self): - pairs = ', '.join(map('%r: %r'.__mod__, self.items())) - return '%s({%s})' % (self.__class__.__name__, pairs) - - def copy(self): - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - d = cls() - for key in iterable: - d[key] = value - return d + colorama.init(strip=False) + except TypeError: + colorama.init() class FormatReplace(object): @@ -133,7 +52,7 @@ class FormatReplace(object): >>> a = FormatReplace('something') >>> "{:5d}".format(a) 'something' - """ + """ # NOQA: P102 def __init__(self, replace=''): self.replace = replace self.format_called = 0 @@ -209,6 +128,41 @@ def __eq__(self, other): return self._wrapped == getattr(other, '_wrapped', other) +class DisableOnWriteError(ObjectWrapper): + """ + Disable the given `tqdm_instance` upon `write()` or `flush()` errors. + """ + @staticmethod + def disable_on_exception(tqdm_instance, func): + """ + Quietly set `tqdm_instance.miniters=inf` if `func` raises `errno=5`. + """ + def inner(*args, **kwargs): + try: + return func(*args, **kwargs) + except OSError as e: + if e.errno != 5: + raise + tqdm_instance.miniters = float('inf') + except ValueError as e: + if 'closed' not in str(e): + raise + tqdm_instance.miniters = float('inf') + return inner + + def __init__(self, wrapped, tqdm_instance): + super(DisableOnWriteError, self).__init__(wrapped) + if hasattr(wrapped, 'write'): + self.wrapper_setattr( + 'write', self.disable_on_exception(tqdm_instance, wrapped.write)) + if hasattr(wrapped, 'flush'): + self.wrapper_setattr( + 'flush', self.disable_on_exception(tqdm_instance, wrapped.flush)) + + def __eq__(self, other): + return self._wrapped == getattr(other, '_wrapped', other) + + class CallbackIOWrapper(ObjectWrapper): def __init__(self, callback, stream, method="read"): """ @@ -238,12 +192,12 @@ def read(*args, **kwargs): def _is_utf(encoding): try: u'\u2588\u2589'.encode(encoding) - except UnicodeEncodeError: # pragma: no cover + except UnicodeEncodeError: return False - except Exception: # pragma: no cover + except Exception: try: return encoding.lower().startswith('utf-') or ('U8' == encoding) - except: + except Exception: return False else: return True @@ -282,8 +236,8 @@ def _screen_shape_wrapper(): # pragma: no cover def _screen_shape_windows(fp): # pragma: no cover try: - from ctypes import windll, create_string_buffer import struct + from ctypes import create_string_buffer, windll from sys import stdin, stdout io_handle = -12 # assume stderr @@ -299,7 +253,7 @@ def _screen_shape_windows(fp): # pragma: no cover (_bufx, _bufy, _curx, _cury, _wattr, left, top, right, bottom, _maxx, _maxy) = struct.unpack("hhhhHhhhhhh", csbi.raw) return right - left, bottom - top # +1 - except: + except Exception: # nosec pass return None, None @@ -308,9 +262,10 @@ def _screen_shape_tput(*_): # pragma: no cover """cygwin xterm (windows)""" try: import shlex - return [int(subprocess.check_call(shlex.split('tput ' + i))) - 1 + from subprocess import check_call # nosec + return [int(check_call(shlex.split('tput ' + i))) - 1 for i in ('cols', 'lines')] - except: + except Exception: # nosec pass return None, None @@ -318,16 +273,16 @@ def _screen_shape_tput(*_): # pragma: no cover def _screen_shape_linux(fp): # pragma: no cover try: - from termios import TIOCGWINSZ - from fcntl import ioctl from array import array + from fcntl import ioctl + from termios import TIOCGWINSZ except ImportError: return None else: try: rows, cols = array('h', ioctl(fp, TIOCGWINSZ, '\0' * 8))[:2] return cols, rows - except: + except Exception: try: return [int(os.environ[i]) - 1 for i in ("COLUMNS", "LINES")] except KeyError: @@ -363,8 +318,7 @@ def _term_move_up(): # pragma: no cover _text_width = len else: def _text_width(s): - return sum( - 2 if east_asian_width(ch) in 'FW' else 1 for ch in _unicode(s)) + return sum(2 if east_asian_width(ch) in 'FW' else 1 for ch in _unicode(s)) def disp_len(data): diff --git a/tqdm/version.py b/tqdm/version.py new file mode 100644 index 000000000..11cbaea79 --- /dev/null +++ b/tqdm/version.py @@ -0,0 +1,9 @@ +"""`tqdm` version detector. Precedence: installed dist, git, 'UNKNOWN'.""" +try: + from ._dist_ver import __version__ +except ImportError: + try: + from setuptools_scm import get_version + __version__ = get_version(root='..', relative_to=__file__) + except (ImportError, LookupError): + __version__ = "UNKNOWN"