diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 958d944aa..000000000 --- a/.coveragerc +++ /dev/null @@ -1,32 +0,0 @@ -[run] -branch = True -source = . -omit = - .tox/* - /usr/* - setup.py - # Don't complain if non-runnable code isn't run - */__main__.py - pre_commit/color_windows.py - -[report] -show_missing = True -exclude_lines = - # Have to re-enable the standard pragma - \#\s*pragma: no cover - # We optionally substitute this - ${COVERAGE_IGNORE_WINDOWS} - - # Don't complain if tests don't hit defensive assertion code: - ^\s*raise AssertionError\b - ^\s*raise NotImplementedError\b - ^\s*return NotImplemented\b - ^\s*raise$ - - # Don't complain if non-runnable code isn't run: - ^if __name__ == ['"]__main__['"]:$ - -[html] -directory = coverage-html - -# vim:ft=dosini diff --git a/.gitignore b/.gitignore index a18245731..5428b0ad8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,8 @@ *.egg-info -*.iml *.py[co] -.*.sw[a-z] -.coverage -.idea -.project -.pydevproject -.tox -.venv.touch +/.coverage +/.mypy_cache +/.pytest_cache +/.tox +/dist /venv* -coverage-html -dist -.cache diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 31ea2398a..c2df486eb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,23 +1,54 @@ -- repo: https://github.com/pre-commit/pre-commit-hooks.git - sha: v0.7.0 +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - - id: autopep8-wrapper - id: check-docstring-first - id: check-json - id: check-yaml - id: debug-statements - id: name-tests-test - id: requirements-txt-fixer + - id: double-quote-string-fixer +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.7.9 + hooks: - id: flake8 -- repo: https://github.com/pre-commit/pre-commit.git - sha: v0.12.0 + additional_dependencies: [flake8-typing-imports==1.6.0] +- repo: https://github.com/pre-commit/mirrors-autopep8 + rev: v1.5 + hooks: + - id: autopep8 +- repo: https://github.com/pre-commit/pre-commit + rev: v2.1.1 hooks: - - id: validate_config - id: validate_manifest -- repo: https://github.com/asottile/reorder_python_imports.git - sha: v0.3.1 +- repo: https://github.com/asottile/pyupgrade + rev: v2.0.1 + hooks: + - id: pyupgrade + args: [--py36-plus] +- repo: https://github.com/asottile/reorder_python_imports + rev: v1.9.0 hooks: - id: reorder-python-imports - language_version: python2.7 + args: [--py3-plus] +- repo: https://github.com/asottile/add-trailing-comma + rev: v1.5.0 + hooks: + - id: add-trailing-comma + args: [--py36-plus] +- repo: https://github.com/asottile/setup-cfg-fmt + rev: v1.6.0 + hooks: + - id: setup-cfg-fmt +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.761 + hooks: + - id: mypy + exclude: ^testing/resources/ +- repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index af53043ed..ef269d133 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -1,9 +1,3 @@ -- id: validate_config - name: Validate Pre-Commit Config - description: This validator validates a pre-commit hooks config file - entry: pre-commit-validate-config - language: python - files: ^\.pre-commit-config\.yaml$ - id: validate_manifest name: Validate Pre-Commit Manifest description: This validator validates a pre-commit hooks manifest file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 5ce1af6c5..000000000 --- a/.travis.yml +++ /dev/null @@ -1,32 +0,0 @@ -language: python -dist: trusty -sudo: required -services: - - docker -matrix: - include: - - env: TOXENV=py27 - - env: TOXENV=py27 LATEST_GIT=1 - - env: TOXENV=py35 - python: 3.5 - - env: TOXENV=py36 - python: 3.6 - - env: TOXENV=pypy -install: pip install coveralls tox -script: tox -before_install: - # Our tests inspect some of *our* git history - - git fetch --unshallow - - git --version - - | - if [ "$LATEST_GIT" = "1" ]; then - ./latest-git.sh - export PATH="/tmp/git/bin:$PATH" - fi - - git --version - - './get-swift.sh && export PATH="/tmp/swift/usr/bin:$PATH"' -after_success: coveralls -cache: - directories: - - $HOME/.cache/pip - - $HOME/.pre-commit diff --git a/CHANGELOG.md b/CHANGELOG.md index d929f8942..9a6892c33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,1031 @@ -0.15.0 -====== +2.2.0 - 2020-03-12 +================== + +### Features +- Add support for the `post-checkout` hook + - #1210 issue by @domenkozar. + - #1339 PR by @andrewhare. +- Add more readable `--from-ref` / `--to-ref` aliases for `--source` / + `--origin` + - #1343 PR by @asottile. + +### Fixes +- Make sure that `--commit-msg-filename` is passed for `commit-msg` / + `prepare-commit-msg`. + - #1336 PR by @particledecay. + - #1341 PR by @particledecay. +- Fix crash when installation error is un-decodable bytes + - #1358 issue by @Guts. + - #1359 PR by @asottile. +- Fix python `healthy()` check when `python` executable goes missing. + - #1363 PR by @asottile. +- Fix crash when script executables are missing shebangs. + - #1350 issue by @chriselion. + - #1364 PR by @asottile. + +### Misc. +- pre-commit now requires python>=3.6.1 (previously 3.6.0) + - #1346 PR by @asottile. + +2.1.1 - 2020-02-24 +================== + +### Fixes +- Temporarily restore python 3.6.0 support (broken in 2.0.0) + - reported by @obestwalter. + - 081f3028 by @asottile. + +2.1.0 - 2020-02-18 +================== + +### Features +- Replace `aspy.yaml` with `sort_keys=False`. + - #1306 PR by @asottile. +- Add support for `perl`. + - #1303 PR by @scop. + +### Fixes +- Improve `.git/hooks/*` shebang creation when pythons are in `/usr/local/bin`. + - #1312 issue by @kbsezginel. + - #1319 PR by @asottile. + +### Misc. +- Add repository badge for pre-commit. + - [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) + - #1334 PR by @ddelange. + +2.0.1 - 2020-01-29 +================== + +### Fixes +- Fix `ImportError` in python 3.6.0 / 3.6.1 for `typing.NoReturn`. + - #1302 PR by @asottile. + +2.0.0 - 2020-01-28 +================== + +### Features +- Expose `PRE_COMMIT_REMOTE_NAME` and `PRE_COMMIT_REMOTE_URL` as environment + variables during `pre-push` hooks. + - #1274 issue by @dmbarreiro. + - #1288 PR by @dmbarreiro. + +### Fixes +- Fix `python -m pre_commit --version` to mention `pre-commit` instead of + `__main__.py`. + - #1273 issue by @ssbarnea. + - #1276 PR by @orcutt989. +- Don't filter `GIT_SSL_NO_VERIFY` from environment when cloning. + - #1293 PR by @schiermike. +- Allow `pre-commit init-templatedir` to succeed even if `core.hooksPath` is + set. + - #1298 issue by @damienrj. + - #1299 PR by @asottile. + +### Misc +- Fix changelog date for 1.21.0. + - #1275 PR by @flaudisio. + +### Updating +- Removed `pcre` language, use `pygrep` instead. + - #1268 PR by @asottile. +- Removed `--tags-only` argument to `pre-commit autoupdate` (it has done + nothing since 0.14.0). + - #1269 by @asottile. +- Remove python2 / python3.5 support. Note that pre-commit still supports + running hooks written in python2, but pre-commit itself requires python 3.6+. + - #1260 issue by @asottile. + - #1277 PR by @asottile. + - #1281 PR by @asottile. + - #1282 PR by @asottile. + - #1287 PR by @asottile. + - #1289 PR by @asottile. + - #1292 PR by @asottile. + +1.21.0 - 2020-01-02 +=================== + +### Features +- Add `conda` as a new `language`. + - #1204 issue by @xhochy. + - #1232 PR by @xhochy. +- Add top-level configuration `files` for file selection. + - #1220 issue by @TheButlah. + - #1248 PR by @asottile. +- Rework `--verbose` / `verbose` to be more consistent with normal runs. + - #1249 PR by @asottile. +- Add support for the `pre-merge-commit` git hook. + - #1210 PR by @asottile. + - this requires git 2.24+. +- Add `pre-commit autoupdate --freeze` which produces "frozen" revisions. + - #1068 issue by @SkypLabs. + - #1256 PR by @asottile. +- Display hook runtime duration when run with `--verbose`. + - #1144 issue by @potiuk. + - #1257 PR by @asottile. + +### Fixes +- Produce better error message when erroneously running inside of `.git`. + - #1219 issue by @Nusserdt. + - #1224 PR by @asottile. + - Note: `git` has since fixed this bug: git/git@36fd304d +- Produce better error message when hook installation fails. + - #1250 issue by @asottile. + - #1251 PR by @asottile. +- Fix cloning when `GIT_SSL_CAINFO` is necessary. + - #1253 issue by @igankevich. + - #1254 PR by @igankevich. +- Fix `pre-commit try-repo` for bare, on-disk repositories. + - #1258 issue by @webknjaz. + - #1259 PR by @asottile. +- Add some whitespace to `pre-commit autoupdate` to improve terminal autolink. + - #1261 issue by @yhoiseth. + - #1262 PR by @yhoiseth. + +### Misc. +- Minor code documentation updates. + - #1200 PR by @ryanrhee. + - #1201 PR by @ryanrhee. + +1.20.0 - 2019-10-28 +=================== + +### Features +- Allow building newer versions of `ruby`. + - #1193 issue by @choffee. + - #1195 PR by @choffee. +- Bump versions reported in `pre-commit sample-config`. + - #1197 PR by @asottile. + +### Fixes +- Fix rare race condition with multiple concurrent first-time runs. + - #1192 issue by @raholler. + - #1196 PR by @asottile. + +1.19.0 - 2019-10-26 +=================== + +### Features +- Allow `--hook-type` to be specified multiple times. + - example: `pre-commit install --hook-type pre-commit --hook-type pre-push` + - #1139 issue by @MaxymVlasov. + - #1145 PR by @asottile. +- Include more version information in crash logs. + - #1142 by @marqueewinq. +- Hook colors are now passed through on platforms which support `pty`. + - #1169 by @asottile. +- pre-commit now uses `importlib.metadata` directly when running in python 3.8 + - #1176 by @asottile. +- Normalize paths to forward slash separators on windows. + - makes it easier to match paths with `files:` regex + - avoids some quoting bugs in shell-based hooks + - #1173 issue by @steigenTI. + - #1179 PR by @asottile. + +### Fixes +- Remove some extra newlines from error messages. + - #1148 by @asottile. +- When a hook is not executable it now reports `not executable` instead of + `not found`. + - #1159 issue by @nixjdm. + - #1161 PR by @WillKoehrsen. +- Fix interleaving of stdout / stderr in hooks. + - #1168 by @asottile. +- Fix python environment `healthy()` check when current working directory + contains modules which shadow standard library names. + - issue by @vwhsu92. + - #1185 PR by @asottile. + +### Updating +- Regexes handling both backslashes and forward slashes for directory + separators now only need to handle forward slashes. + +1.18.3 - 2019-08-27 +=================== + +### Fixes +- Fix `node_modules` plugin installation on windows + - #1123 issue by @henryykt. + - #1122 PR by @henryykt. + +1.18.2 - 2019-08-15 +=================== + +### Fixes +- Make default python lookup more deterministic to avoid redundant installs + - #1117 PR by @scop. + +1.18.1 - 2019-08-11 +=================== + +### Fixes +- Fix installation of `rust` hooks with new `cargo` + - #1112 issue by @zimbatm. + - #1113 PR by @zimbatm. + +1.18.0 - 2019-08-03 +=================== + +### Features +- Use the current running executable if it matches the requested + `language_version` + - #1062 PR by @asottile. +- Print the stage when a hook is not found + - #1078 issue by @madkinsz. + - #1079 PR by @madkinsz. +- `pre-commit autoupdate` now supports non-`master` default branches + - #1089 PR by @asottile. +- Add `pre-commit init-templatedir` which makes it easier to automatically + enable `pre-commit` in cloned repositories. + - #1084 issue by @ssbarnea. + - #1090 PR by @asottile. + - #1107 PR by @asottile. +- pre-commit's color can be controlled using + `PRE_COMMIT_COLOR={auto,always,never}` + - #1073 issue by @saper. + - #1092 PR by @geieredgar. + - #1098 PR by @geieredgar. +- pre-commit's color can now be disabled using `TERM=dumb` + - #1073 issue by @saper. + - #1103 PR by @asottile. +- pre-commit now supports `docker` based hooks on windows + - #1072 by @cz-fish. + - #1093 PR by @geieredgar. + +### Fixes +- Fix shallow clone + - #1077 PR by @asottile. +- Fix autoupdate version flip flop when using shallow cloning + - #1076 issue by @mxr. + - #1088 PR by @asottile. +- Fix autoupdate when the current revision is invalid + - #1088 PR by @asottile. + +### Misc. +- Replace development instructions with `tox --devenv ...` + - #1032 issue by @yoavcaspi. + - #1067 PR by @asottile. + + +1.17.0 - 2019-06-06 +=================== + +### Features +- Produce better output on `^C` + - #1030 PR by @asottile. +- Warn on unknown keys at the top level and repo level + - #1028 PR by @yoavcaspi. + - #1048 PR by @asottile. + +### Fixes +- Fix handling of `^C` in wrapper script in python 3.x + - #1027 PR by @asottile. +- Fix `rmtree` for non-writable directories + - #1042 issue by @detailyang. + - #1043 PR by @asottile. +- Pass `--color` option to `git diff` in `--show-diff-on-failure` + - #1007 issue by @chadrik. + - #1051 PR by @mandarvaze. + +### Misc. +- Fix test when `pre-commit` is installed globally + - #1032 issue by @yoavcaspi. + - #1045 PR by @asottile. + + +1.16.1 - 2019-05-08 +=================== + +### Fixes +- Don't `UnicodeDecodeError` on unexpected non-UTF8 output in python health + check on windows. + - #1021 issue by @nicoddemus. + - #1022 PR by @asottile. + +1.16.0 - 2019-05-04 +=================== + +### Features +- Add support for `prepare-commit-msg` hook + - #1004 PR by @marcjay. + +### Fixes +- Fix repeated legacy `pre-commit install` on windows + - #1010 issue by @AbhimanyuHK. + - #1011 PR by @asottile. +- Whitespace fixup + - #1014 PR by @mxr. +- Fix CI check for working pcre support + - #1015 PR by @Myrheimb. + +### Misc. +- Switch CI from travis / appveyor to azure pipelines + - #1012 PR by @asottile. + +1.15.2 - 2019-04-16 +=================== + +### Fixes +- Fix cloning non-branch tag while in the fallback slow-clone strategy. + - #997 issue by @jpinner. + - #998 PR by @asottile. + +1.15.1 - 2019-04-01 +=================== + +### Fixes +- Fix command length calculation on posix when `SC_ARG_MAX` is not defined. + - #691 issue by @ushuz. + - #987 PR by @asottile. + +1.15.0 - 2019-03-30 +=================== + +### Features +- No longer require being in a `git` repo to run `pre-commit` `clean` / `gc` / + `sample-config`. + - #959 PR by @asottile. +- Improve command line length limit detection. + - #691 issue by @antonbabenko. + - #966 PR by @asottile. +- Use shallow cloning when possible. + - #958 PR by @DanielChabrowski. +- Add `minimum_pre_commit_version` top level key to require a new-enough + version of `pre-commit`. + - #977 PR by @asottile. +- Add helpful CI-friendly message when running + `pre-commit run --all-files --show-diff-on-failure`. + - #982 PR by @bnorquist. + +### Fixes +- Fix `try-repo` for staged untracked changes. + - #973 PR by @DanielChabrowski. +- Fix rpm build by explicitly using `#!/usr/bin/env python3` in hook template. + - #985 issue by @tim77. + - #986 PR by @tim77. +- Guard against infinite recursion when executing legacy hook script. + - #981 PR by @tristan0x. + +### Misc +- Add test for `git.no_git_env()` + - #972 PR by @javabrett. + +1.14.4 - 2019-02-18 +=================== + +### Fixes +- Don't filter `GIT_SSH_COMMAND` env variable from `git` commands + - #947 issue by @firba1. + - #948 PR by @firba1. +- Install npm packages as if they were installed from `git` + - #943 issue by @ssbarnea. + - #949 PR by @asottile. +- Don't filter `GIT_EXEC_PREFIX` env variable from `git` commands + - #664 issue by @revolter. + - #944 PR by @minrk. + +1.14.3 - 2019-02-04 +=================== + +### Fixes +- Improve performance of filename classification by 45% - 55%. + - #921 PR by @asottile. +- Fix installing `go` hooks while `GOBIN` environment variable is set. + - #924 PR by @ashanbrown. +- Fix crash while running `pre-commit migrate-config` / `pre-commit autoupdate` + with an empty configuration file. + - #929 issue by @ardakuyumcu. + - #933 PR by @jessebona. +- Require a newer virtualenv to fix metadata-based setup.cfg installs. + - #936 PR by @asottile. + +1.14.2 - 2019-01-10 +=================== + +### Fixes +- Make the hook shebang detection more timid (1.14.0 regression) + - Homebrew/homebrew-core#35825. + - #915 PR by @asottile. + +1.14.1 - 2019-01-10 +=================== + +### Fixes +- Fix python executable lookup on windows when using conda + - #913 issue by @dawelter2. + - #914 PR by @asottile. + +1.14.0 - 2019-01-08 +=================== + +### Features +- Add an `alias` configuration value to allow repeated hooks to be + differentiated + - #882 issue by @s0undt3ch. + - #886 PR by @s0undt3ch. +- Add `identity` meta hook which just prints filenames + - #865 issue by @asottile. + - #898 PR by @asottile. +- Factor out `cached-property` and improve startup performance by ~10% + - #899 PR by @asottile. +- Add a warning on unexpected keys in configuration + - #899 PR by @asottile. +- Teach `pre-commit try-repo` to clone uncommitted changes on disk. + - #589 issue by @sverhagen. + - #703 issue by @asottile. + - #904 PR by @asottile. +- Implement `pre-commit gc` which will clean up no-longer-referenced cache + repos. + - #283 issue by @jtwang. + - #906 PR by @asottile. +- Add top level config `default_language_version` to streamline overriding the + `language_version` configuration in many places + - #647 issue by @asottile. + - #908 PR by @asottile. +- Add top level config `default_stages` to streamline overriding the `stages` + configuration in many places + - #768 issue by @mattlqx. + - #909 PR by @asottile. + +### Fixes +- More intelligently pick hook shebang (`#!/usr/bin/env python3`) + - #878 issue by @fristedt. + - #893 PR by @asottile. +- Several fixes related to `--files` / `--config`: + - `pre-commit run --files x` outside of a git dir no longer stacktraces + - `pre-commit run --config ./relative` while in a sub directory of the git + repo is now able to find the configuration + - `pre-commit run --files ...` no longer runs a subprocess per file + (performance) + - #895 PR by @asottile. +- `pre-commit try-repo ./relative` while in a sub directory of the git repo is + now able to clone properly + - #903 PR by @asottile. +- Ensure `meta` repos cannot have a language other than `system` + - #905 issue by @asottile. + - #907 PR by @asottile. +- Fix committing with unstaged files that were `git add --intent-to-add` added + - #881 issue by @henniss. + - #912 PR by @asottile. + +### Misc. +- Use `--no-gpg-sign` when running tests + - #894 PR by @s0undt3ch. + + +1.13.0 - 2018-12-20 +=================== + +### Features +- Run hooks in parallel + - individual hooks may opt out of parallel execution with `require_serial: true` + - #510 issue by @chriskuehl. + - #851 PR by @chriskuehl. + +### Fixes +- Improve platform-specific `xargs` command length detection + - #691 issue by @antonbabenko. + - #839 PR by @georgeyk. +- Fix `pre-commit autoupdate` when updating to a latest tag missing a + `.pre-commit-hooks.yaml` + - #856 issue by @asottile. + - #857 PR by @runz0rd. +- Upgrade the `pre-commit-hooks` version in `pre-commit sample-config` + - #870 by @asottile. +- Improve balancing of multiprocessing by deterministic shuffling of args + - #861 issue by @Dunedan. + - #874 PR by @chriskuehl. +- `ruby` hooks work with latest `gem` by removing `--no-ri` / `--no-rdoc` and + instead using `--no-document`. + - #889 PR by @asottile. + +### Misc. +- Use `--no-gpg-sign` when running tests + - #885 PR by @s0undt3ch. + +### Updating +- If a hook requires serial execution, set `require_serial: true` to avoid the new + parallel execution. +- `ruby` hooks now require `gem>=2.0.0`. If your platform doesn't support this + by default, select a newer version using + [`language_version`](https://pre-commit.com/#overriding-language-version). + + +1.12.0 - 2018-10-23 +=================== + +### Fixes +- Install multi-hook repositories only once (performance) + - issue by @chriskuehl. + - #852 PR by @asottile. +- Improve performance by factoring out pkg_resources (performance) + - #840 issue by @RonnyPfannschmidt. + - #846 PR by @asottile. + +1.11.2 - 2018-10-10 +=================== + +### Fixes +- `check-useless-exclude` now considers `types` + - #704 issue by @asottile. + - #837 PR by @georgeyk. +- `pre-push` hook was not identifying all commits on push to new branch + - #843 issue by @prem-nuro. + - #844 PR by @asottile. + +1.11.1 - 2018-09-22 +=================== + +### Fixes +- Fix `.git` dir detection in `git<2.5` (regression introduced in + [1.10.5](#1105)) + - #831 issue by @mmacpherson. + - #832 PR by @asottile. + +1.11.0 - 2018-09-02 +=================== + +### Features +- Add new `fail` language which always fails + - light-weight way to forbid files by name. + - #812 #821 PRs by @asottile. + +### Fixes +- Fix `ResourceWarning`s for unclosed files + - #811 PR by @BoboTiG. +- Don't write ANSI colors on windows when color enabling fails + - #819 PR by @jeffreyrack. + +1.10.5 - 2018-08-06 +=================== + +### Fixes +- Work around `PATH` issue with `brew` `python` on `macos` + - Homebrew/homebrew-core#30445 issue by @asottile. + - #805 PR by @asottile. +- Support `pre-commit install` inside a worktree + - #808 issue by @s0undt3ch. + - #809 PR by @asottile. + +1.10.4 - 2018-07-22 +=================== + +### Fixes +- Replace `yaml.load` with safe alternative + - `yaml.load` can lead to arbitrary code execution, though not where it + was used + - issue by @tonybaloney. + - #779 PR by @asottile. +- Improve not found error with script paths (`./exe`) + - #782 issue by @ssbarnea. + - #785 PR by @asottile. +- Fix minor buffering issue during `--show-diff-on-failure` + - #796 PR by @asottile. +- Default `language_version: python3` for `python_venv` when running in python2 + - #794 issue by @ssbarnea. + - #797 PR by @asottile. +- `pre-commit run X` only run `X` and not hooks with `stages: [...]` + - #772 issue by @asottile. + - #803 PR by @mblayman. + +### Misc. +- Improve travis-ci build times by caching rust / swift artifacts + - #781 PR by @expobrain. +- Test against python3.7 + - #789 PR by @expobrain. + +1.10.3 - 2018-07-02 +=================== + +### Fixes +- Fix `pre-push` during a force push without a fetch + - #777 issue by @domenkozar. + - #778 PR by @asottile. + +1.10.2 - 2018-06-11 +=================== + +### Fixes +- pre-commit now invokes hooks with a consistent ordering of filenames + - issue by @mxr. + - #767 PR by @asottile. + +1.10.1 - 2018-05-28 +=================== + +### Fixes +- `python_venv` language would leak dependencies when pre-commit was installed + in a `-mvirtualenv` virtualenv + - #755 #756 issue and PR by @asottile. + +1.10.0 - 2018-05-26 +=================== + +### Features +- Add support for hooks written in `rust` + - #751 PR by @chriskuehl. + +1.9.0 - 2018-05-21 +================== + +### Features +- Add new `python_venv` language which uses the `venv` module instead of + `virtualenv` + - #631 issue by @dongyuzheng. + - #739 PR by @ojii. +- Include `LICENSE` in distribution + - #745 issue by @nicoddemus. + - #746 PR by @nicoddemus. + +### Fixes +- Normalize relative paths for `pre-commit try-repo` + - #750 PR by @asottile. + + +1.8.2 - 2018-03-17 +================== + +### Fixes +- Fix cloning relative paths (regression in 1.7.0) + - #728 issue by @jdswensen. + - #729 PR by @asottile. + + +1.8.1 - 2018-03-12 +================== + +### Fixes +- Fix integration with go 1.10 and `pkg` directory + - #725 PR by @asottile +- Restore support for `git<1.8.5` (inadvertently removed in 1.7.0) + - #723 issue by @JohnLyman. + - #724 PR by @asottile. + + +1.8.0 - 2018-03-11 +================== + +### Features +- Add a `manual` stage for cli-only interaction + - #719 issue by @hectorv. + - #720 PR by @asottile. +- Add a `--multiline` option to `pygrep` hooks + - #716 PR by @tdeo. + + +1.7.0 - 2018-03-03 +================== + +### Features +- pre-commit config validation was split to a separate `cfgv` library + - #700 PR by @asottile. +- Allow `--repo` to be specified multiple times to autoupdate + - #658 issue by @KevinHock. + - #713 PR by @asottile. +- Enable `rev` as a preferred alternative to `sha` in `.pre-commit-config.yaml` + - #106 issue by @asottile. + - #715 PR by @asottile. +- Use `--clean-src` option when invoking `nodeenv` to save ~70MB per node env + - #717 PR by @asottile. +- Refuse to install with `core.hooksPath` set + - pre-commit/pre-commit-hooks#250 issue by @revolter. + - #663 issue by @asottile. + - #718 PR by @asottile. + +### Fixes +- hooks with `additional_dependencies` now get isolated environments + - #590 issue by @coldnight. + - #711 PR by @asottile. + +### Misc. +- test against swift 4.x + - #709 by @theresama. + +### Updating + +- Run `pre-commit migrate-config` to convert `sha` to `rev` in the + `.pre-commit-config.yaml` file. + + +1.6.0 - 2018-02-04 +================== + +### Features +- Hooks now may have a `verbose` option to produce output even without failure + - #689 issue by @bagerard. + - #695 PR by @bagerard. +- Installed hook no longer requires `bash` + - #699 PR by @asottile. + +### Fixes +- legacy pre-push / commit-msg hooks are now invoked as if `git` called them + - #693 issue by @samskiter. + - #694 PR by @asottile. + - #699 PR by @asottile. + +1.5.1 - 2018-01-24 +================== + +### Fixes +- proper detection for root commit during pre-push + - #503 PR by @philipgian. + - #692 PR by @samskiter. + +1.5.0 - 2018-01-13 +================== + +### Features +- pre-commit now supports node hooks on windows. + - for now, requires python3 due to https://bugs.python.org/issue32539 + - huge thanks to @wenzowski for the tip! + - #200 issue by @asottile. + - #685 PR by @asottile. + +### Misc. +- internal reorganization of `PrefixedCommandRunner` -> `Prefix` + - #684 PR by @asottile. +- https-ify links. + - pre-commit.com is now served over https. + - #688 PR by @asottile. + + +1.4.5 - 2018-01-09 +================== + +### Fixes +- Fix `local` golang repositories with `additional_dependencies`. + - #679 #680 issue and PR by @asottile. + +### Misc. +- Replace some string literals with constants + - #678 PR by @revolter. + +1.4.4 - 2018-01-07 +================== + +### Fixes +- Invoke `git diff` without a pager during `--show-diff-on-failure`. + - #676 PR by @asottile. + +1.4.3 - 2018-01-02 +================== + +### Fixes +- `pre-commit` on windows can find pythons at non-hardcoded paths. + - #674 PR by @asottile. + +1.4.2 - 2018-01-02 +================== + +### Fixes +- `pre-commit` no longer clears `GIT_SSH` environment variable when cloning. + - #671 PR by @rp-tanium. + +1.4.1 - 2017-11-09 +================== + +### Fixes +- `pre-commit autoupdate --repo ...` no longer deletes other repos. + - #660 issue by @KevinHock. + - #661 PR by @KevinHock. + +1.4.0 - 2017-11-08 +================== + +### Features +- Lazily install repositories. + - When running `pre-commit run `, pre-commit will only install + the necessary repositories. + - #637 issue by @webknjaz. + - #639 PR by @asottile. +- Version defaulting now applies to local hooks as well. + - This extends #556 to apply to local hooks. + - #646 PR by @asottile. +- Add new `repo: meta` hooks. + - `meta` hooks expose some linters of the pre-commit configuration itself. + - `id: check-useless-excludes`: ensures that `exclude` directives actually + apply to *any* file in the repository. + - `id: check-hooks-apply`: ensures that the configured hooks apply to + at least one file in the repository. + - pre-commit/pre-commit-hooks#63 issue by @asottile. + - #405 issue by @asottile. + - #643 PR by @hackedd. + - #653 PR by @asottile. + - #654 PR by @asottile. +- Allow a specific repository to be autoupdated instead of all repositories. + - `pre-commit autoupdate --repo ...` + - #656 issue by @KevinHock. + - #657 PR by @KevinHock. + +### Fixes +- Apply selinux labelling option to docker volumes + - #642 PR by @jimmidyson. + + +1.3.0 - 2017-10-08 +================== + +### Features +- Add `pre-commit try-repo` commands + - The new `try-repo` takes a repo and will run the hooks configured in + that hook repository. + - An example invocation: + `pre-commit try-repo https://github.com/pre-commit/pre-commit-hooks` + - `pre-commit try-repo` can also take all the same arguments as + `pre-commit run`. + - It can be used to try out a repository without needing to configure it. + - It can also be used to test a hook repository while developing it. + - #589 issue by @sverhagen. + - #633 PR by @asottile. + +1.2.0 - 2017-10-03 +================== + +### Features +- Add `pygrep` language + - `pygrep` aims to be a more cross-platform alternative to `pcre` hooks. + - #630 PR by @asottile. + +### Fixes +- Use `pipes.quote` for executable path in hook template + - Fixes bash syntax error when git dir contains spaces + - #626 PR by @asottile. +- Clean up hook template + - Simplify code + - Fix `--config` not being respected in some situations + - #627 PR by @asottile. +- Use `file://` protocol for cloning under test + - Fix `file://` clone paths being treated as urls for golang + - #629 PR by @asottile. +- Add `ctypes` as an import for virtualenv healthchecks + - Fixes python3.6.2 <=> python3.6.3 virtualenv invalidation + - e70825ab by @asottile. + +1.1.2 - 2017-09-20 +================== + +### Fixes +- pre-commit can successfully install commit-msg hooks + - Due to an oversight, the commit-msg-tmpl was missing from the packaging + - #623 issue by @sobolevn. + - #624 PR by @asottile. + +1.1.1 - 2017-09-17 +================== + +### Features +- pre-commit also checks the `ssl` module for virtualenv health + - Suggestion by @merwok. + - #619 PR by @asottile. +### Fixes +- pre-commit no longer crashes with unstaged files when run for the first time + - #620 #621 issue by @Lucas-C. + - #622 PR by @asottile. + +1.1.0 - 2017-09-11 +================== + +### Features +- pre-commit configuration gains a `fail_fast` option. + - You must be using the v2 configuration format introduced in 1.0.0. + - `fail_fast` defaults to `false`. + - #240 issue by @Lucas-C. + - #616 PR by @asottile. +- pre-commit configuration gains a global `exclude` option. + - This option takes a python regular expression and can be used to exclude + files from _all_ hooks. + - You must be using the v2 configuration format introduced in 1.0.0. + - #281 issue by @asieira. + - #617 PR by @asottile. + +1.0.1 - 2017-09-07 +================== + +### Fixes +- Fix a regression in the return code of `pre-commit autoupdate` + - `pre-commit migrate-config` and `pre-commit autoupdate` return 0 when + successful. + - #614 PR by @asottile. + +1.0.0 - 2017-09-07 +================== +pre-commit will now be following [semver](https://semver.org/). Thanks to all +of the [contributors](https://github.com/pre-commit/pre-commit/graphs/contributors) +that have helped us get this far! + +### Features + +- pre-commit's cache directory has moved from `~/.pre-commit` to + `$XDG_CACHE_HOME/pre-commit` (usually `~/.cache/pre-commit`). + - `pre-commit clean` now cleans up both the old and new directory. + - If you were caching this directory in CI, you'll want to adjust the + location. + - #562 issue by @nagromc. + - #602 PR by @asottile. +- A new configuration format for `.pre-commit-config.yaml` is introduced which + will enable future development. + - The new format has a top-level map instead of a top-level list. The + new format puts the hook repositories in a `repos` key. + - Old list-based configurations will continue to be supported. + - A command `pre-commit migrate-config` has been introduced to "upgrade" + the configuration format to the new map-based configuration. + - `pre-commit autoupdate` now automatically calls `migrate-config`. + - In a later release, list-based configurations will issue a deprecation + warning. + - An example diff for upgrading a configuration: + + ```diff + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + sha: v0.9.2 + hooks: + ``` + - #414 issue by @asottile. + - #610 PR by @asottile. + +### Updating + +- Run `pre-commit migrate-config` to convert `.pre-commit-config.yaml` to the + new map format. +- Update any references from `~/.pre-commit` to `~/.cache/pre-commit`. + +0.18.3 - 2017-09-06 +=================== +- Allow --config to affect `pre-commit install` +- Tweak not found error message during `pre-push` / `commit-msg` +- Improve node support when running under cygwin. + +0.18.2 - 2017-09-05 +=================== +- Fix `--all-files`, detection of staged files, detection of manually edited + files during merge conflict, and detection of files to push for non-ascii + filenames. + +0.18.1 - 2017-09-04 +=================== +- Only mention locking when waiting for a lock. +- Fix `IOError` during locking in timeout situtation on windows under python 2. + +0.18.0 - 2017-09-02 +=================== +- Add a new `docker_image` language type. `docker_image` is intended to be a + lightweight hook type similar to `system` / `script` which allows one to use + an existing docker image that provides a hook. `docker_image` hooks can + also be used as repository `local` hooks. + +0.17.0 - 2017-08-24 +=================== +- Fix typos in help +- Allow `commit-msg` hook to be uninstalled +- Upgrade the `sample-config` +- Remove undocumented `--no-stash` and `--allow-unstaged-config` +- Remove `validate_config` hook pre-commit hook. +- Fix installation race condition when multiple `pre-commit` processes would + attempt to install the same repository. + +0.16.3 - 2017-08-10 +=================== +- autoupdate attempts to maintain config formatting. + +0.16.2 - 2017-08-06 +=================== +- Initialize submodules in hook repositories. + +0.16.1 - 2017-08-04 +=================== +- Improve node support when running under cygwin. + +0.16.0 - 2017-08-01 +=================== +- Remove backward compatibility with repositories providing metadata via + `hooks.yaml`. New repositories should provide `.pre-commit-hooks.yaml`. + Run `pre-commit autoupdate` to upgrade to the latest repositories. +- Improve golang support when running under cygwin. +- Fix crash with unstaged trailing whitespace additions while git was + configured with `apply.whitespace = error`. +- Fix crash with unstaged end-of-file crlf additions and the file's lines + ended with crlf while git was configured with `core-autocrlf = true`. + +0.15.4 - 2017-07-23 +=================== +- Add support for the `commit-msg` git hook + +0.15.3 - 2017-07-20 +=================== +- Recover from invalid python virtualenvs + + +0.15.2 - 2017-07-09 +=================== +- Work around a windows-specific virtualenv bug pypa/virtualenv#1062 + This failure mode was introduced in 0.15.1 + +0.15.1 - 2017-07-09 +=================== +- Use a more intelligent default language version for python + +0.15.0 - 2017-07-02 +=================== - Add `types` and `exclude_types` for filtering files. These options take an array of "tags" identified for each file. The tags are sourced from [identify](https://github.com/chriskuehl/identify). One can list the tags @@ -8,22 +1034,22 @@ - `always_run` + missing `files` also defaults to `files: ''` (previously it defaulted to `'^$'` (this reverses e150921c). -0.14.3 -====== +0.14.3 - 2017-06-28 +=================== - Expose `--origin` and `--source` as `PRE_COMMIT_ORIGIN` and `PRE_COMMIT_SOURCE` environment variables when running as `pre-push`. -0.14.2 -====== +0.14.2 - 2017-06-09 +=================== - Use `--no-ext-diff` when running `git diff` -0.14.1 -====== +0.14.1 - 2017-06-02 +=================== - Don't crash when `always_run` is `True` and `files` is not provided. - Set `VIRTUALENV_NO_DOWNLOAD` when making python virtualenvs. -0.14.0 -====== +0.14.0 - 2017-05-16 +=================== - Add a `pre-commit sample-config` command - Enable ansi color escapes on modern windows - `autoupdate` now defaults to `--tags-only`, use `--bleeding-edge` for the @@ -34,99 +1060,99 @@ - Add a `pass_filenames` option to allow disabling automatic filename positional arguments to hooks. -0.13.6 -====== +0.13.6 - 2017-03-27 +=================== - Fix regression in 0.13.5: allow `always_run` and `files` together despite doing nothing. -0.13.5 -====== +0.13.5 - 2017-03-26 +=================== - 0.13.4 contained incorrect files -0.13.4 -====== +0.13.4 - 2017-03-26 +=================== - Add `--show-diff-on-failure` option to `pre-commit run` - Replace `jsonschema` with better error messages -0.13.3 -====== +0.13.3 - 2017-02-23 +=================== - Add `--allow-missing-config` to install: allows `git commit` without a configuration. -0.13.2 -====== +0.13.2 - 2017-02-17 +=================== - Version the local hooks repo - Allow `minimum_pre_commit_version` for local hooks -0.13.1 -====== +0.13.1 - 2017-02-16 +=================== - Fix dummy gem for ruby local hooks -0.13.0 -====== +0.13.0 - 2017-02-16 +=================== - Autoupdate now works even when the current state is broken. - Improve pre-push fileset on new branches - Allow "language local" hooks, hooks which install dependencies using `additional_dependencies` and `language` are now allowed in `repo: local`. -0.12.2 -====== +0.12.2 - 2017-01-27 +=================== - Fix docker hooks on older (<1.12) docker -0.12.1 -====== +0.12.1 - 2017-01-25 +=================== - golang hooks now support additional_dependencies - Added a --tags-only option to pre-commit autoupdate -0.12.0 -====== +0.12.0 - 2017-01-24 +=================== - The new default file for implementing hooks in remote repositories is now .pre-commit-hooks.yaml to encourage repositories to add the metadata. As such, the previous hooks.yaml is now deprecated and generates a warning. - Fix bug with local configuration interfering with ruby hooks - Added support for hooks written in golang. -0.11.0 -====== +0.11.0 - 2017-01-20 +=================== - SwiftPM support. -0.10.1 -====== +0.10.1 - 2017-01-05 +=================== - shlex entry of docker based hooks. - Make shlex behaviour of entry more consistent. -0.10.0 -====== +0.10.0 - 2017-01-04 +=================== - Add an `install-hooks` command similar to `install --install-hooks` but without the `install` side-effects. - Adds support for docker based hooks. -0.9.4 -===== +0.9.4 - 2016-12-05 +================== - Warn when cygwin / python mismatch - Add --config for customizing configuration during run - Update rbenv + plugins to latest versions - pcre hooks now fail when grep / ggrep are not present -0.9.3 -===== +0.9.3 - 2016-11-07 +================== - Fix python hook installation when a strange setup.cfg exists -0.9.2 -===== +0.9.2 - 2016-10-25 +================== - Remove some python2.6 compatibility - UI is no longer sized to terminal width, instead 80 characters or longest necessary width. - Fix inability to create python hook environments when using venv / pyvenv on osx -0.9.1 -===== +0.9.1 - 2016-09-10 +================== - Remove some python2.6 compatibility - Fix staged-files-only with external diff tools -0.9.0 -===== +0.9.0 - 2016-08-31 +================== - Only consider forward diff in changed files - Don't run on staged deleted files that still exist - Autoupdate to tags when available @@ -134,95 +1160,95 @@ - Fix crash with staged files containing unstaged lines which have non-utf8 bytes and trailing whitespace -0.8.2 -===== +0.8.2 - 2016-05-20 +================== - Fix a crash introduced in 0.8.0 when an executable was not found -0.8.1 -===== +0.8.1 - 2016-05-17 +================== - Fix regression introduced in 0.8.0 when already using rbenv with no configured ruby hook version -0.8.0 -===== +0.8.0 - 2016-04-11 +================== - Fix --files when running in a subdir - Improve --help a bit - Switch to pyterminalsize for determining terminal size -0.7.6 -===== +0.7.6 - 2016-01-19 +================== - Work under latest virtualenv - No longer create empty directories on windows with latest virtualenv -0.7.5 -===== +0.7.5 - 2016-01-15 +================== - Consider dead symlinks as files when committing -0.7.4 -===== +0.7.4 - 2016-01-12 +================== - Produce error message instead of crashing on non-utf8 installation failure -0.7.3 -===== +0.7.3 - 2015-12-22 +================== - Fix regression introduced in 0.7.1 breaking `git commit -a` -0.7.2 -===== +0.7.2 - 2015-12-22 +================== - Add `always_run` setting for hooks to run even without file changes. -0.7.1 -===== +0.7.1 - 2015-12-19 +================== - Support running pre-commit inside submodules -0.7.0 -===== +0.7.0 - 2015-12-13 +================== - Store state about additional_dependencies for rollforward/rollback compatibility -0.6.8 -===== +0.6.8 - 2015-12-07 +================== - Build as a universal wheel - Allow '.format('-like strings in arguments - Add an option to require a minimum pre-commit version -0.6.7 -===== +0.6.7 - 2015-12-02 +================== - Print a useful message when a hook id is not present - Fix printing of non-ascii with unexpected errors - Print a message when a hook modifies files but produces no output -0.6.6 -===== +0.6.6 - 2015-11-25 +================== - Add `additional_dependencies` to hook configuration. - Fix pre-commit cloning under git 2.6 - Small improvements for windows -0.6.5 -===== +0.6.5 - 2015-11-19 +================== - Allow args for pcre hooks -0.6.4 -===== +0.6.4 - 2015-11-13 +================== - Fix regression introduced in 0.6.3 regarding hooks which make non-utf8 diffs -0.6.3 -===== +0.6.3 - 2015-11-12 +================== - Remove `expected_return_code` - Fail a hook if it makes modifications to the working directory -0.6.2 -===== +0.6.2 - 2015-10-14 +================== - Use --no-ri --no-rdoc instead of --no-document for gem to fix old gem -0.6.1 -===== +0.6.1 - 2015-10-08 +================== - Fix pre-push when pushing something that's already up to date -0.6.0 -===== +0.6.0 - 2015-10-05 +================== - Filter hooks by stage (commit, push). -0.5.5 -===== +0.5.5 - 2015-09-04 +================== - Change permissions a few files - Rename the validate entrypoints - Add --version to some entrypoints @@ -231,152 +1257,151 @@ - Suppress complaint about $TERM when no tty is attached - Support pcre hooks on osx through ggrep -0.5.4 -===== +0.5.4 - 2015-07-24 +================== - Allow hooks to produce outputs with arbitrary bytes - Fix pre-commit install when .git/hooks/pre-commit is a dead symlink - Allow an unstaged config when using --files or --all-files -0.5.3 -===== +0.5.3 - 2015-06-15 +================== - Fix autoupdate with "local" hooks - don't purge local hooks. -0.5.2 -===== +0.5.2 - 2015-06-02 +================== - Fix autoupdate with "local" hooks -0.5.1 -===== +0.5.1 - 2015-05-23 +================== - Fix bug with unknown non-ascii hook-id - Avoid crash when .git/hooks is not present in some git clients -0.5.0 -===== +0.5.0 - 2015-05-19 +================== - Add a new "local" hook type for running hooks without remote configuration. - Complain loudly when .pre-commit-config.yaml is unstaged. - Better support for multiple language versions when running hooks. - Allow exclude to be defaulted in repository configuration. -0.4.4 -===== +0.4.4 - 2015-03-29 +================== - Use sys.executable when executing virtualenv -0.4.3 -===== +0.4.3 - 2015-03-25 +================== - Use reset instead of checkout when checkout out hook repo -0.4.2 -===== +0.4.2 - 2015-02-27 +================== - Limit length of xargs arguments to workaround windows xargs bug -0.4.1 -===== +0.4.1 - 2015-02-27 +================== - Don't rename across devices when creating sqlite database -0.4.0 -===== +0.4.0 - 2015-02-27 +================== - Make ^C^C During installation not cause all subsequent runs to fail - Print while installing (instead of while cloning) - Use sqlite to manage repositories (instead of symlinks) - MVP Windows support -0.3.6 -===== +0.3.6 - 2015-02-05 +================== - `args` in venv'd languages are now property quoted. -0.3.5 -===== -- Support running during `pre-push`. See http://pre-commit.com/#advanced 'pre-commit during push'. +0.3.5 - 2015-01-15 +================== +- Support running during `pre-push`. See https://pre-commit.com/#advanced 'pre-commit during push'. -0.3.4 -===== +0.3.4 - 2015-01-13 +================== - Allow hook providers to default `args` in `hooks.yaml` -0.3.3 -===== +0.3.3 - 2015-01-06 +================== - Improve message for `CalledProcessError` -0.3.2 -===== +0.3.2 - 2014-10-07 +================== - Fix for `staged_files_only` with color.diff = always #176. -0.3.1 -===== +0.3.1 - 2014-10-03 +================== - Fix error clobbering #174. - Remove dependency on `plumbum`. - Allow pre-commit to be run from anywhere in a repository #175. -0.3.0 -===== +0.3.0 - 2014-09-18 +================== - Add `--files` option to `pre-commit run` -0.2.11 -====== +0.2.11 - 2014-09-05 +=================== - Fix terminal width detection (broken in 0.2.10) -0.2.10 -====== +0.2.10 - 2014-09-04 +=================== - Bump version of nodeenv to fix bug with ~/.npmrc - Choose `python` more intelligently when running. -0.2.9 -===== +0.2.9 - 2014-09-02 +================== - Fix bug where sys.stdout.write must take `bytes` in python 2.6 -0.2.8 -===== +0.2.8 - 2014-08-13 +================== - Allow a client to have duplicates of hooks. - Use --prebuilt instead of system for node. - Improve some fatal error messages -0.2.7 -===== +0.2.7 - 2014-07-28 +================== - Produce output when running pre-commit install --install-hooks -0.2.6 -===== +0.2.6 - 2014-07-28 +================== - Print hookid on failure - Use sys.executable for running nodeenv - Allow running as `python -m pre_commit` -0.2.5 -===== +0.2.5 - 2014-07-17 +================== - Default columns to 80 (for non-terminal execution). -0.2.4 -===== +0.2.4 - 2014-07-07 +================== - Support --install-hooks as an argument to `pre-commit install` - Install hooks before attempting to run anything - Use `python -m nodeenv` instead of `nodeenv` -0.2.3 -===== +0.2.3 - 2014-06-25 +================== - Freeze ruby building infrastructure - Fix bug that assumed diffs were utf-8 -0.2.2 -===== +0.2.2 - 2014-06-22 +================== - Fix filenames with spaces -0.2.1 -===== +0.2.1 - 2014-06-18 +================== - Use either `pre-commit` or `python -m pre_commit.main` depending on which is available - Don't use readlink -f -0.2.0 -===== +0.2.0 - 2014-06-17 +================== - Fix for merge-conflict during cherry-picking. - Add -V / --version - Add migration install mode / install -f / --overwrite - Add `pcre` "language" for perl compatible regexes - Reorganize packages. -0.1.1 -===== +0.1.1 - 2014-06-11 +================== - Fixed bug with autoupdate setting defaults on un-updated repos. - -0.1 -=== +0.1.0 - 2014-06-07 +================== - Initial Release diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1e0b2460c..2b83c8232 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,23 +2,22 @@ ## Local development -- The complete test suite depends on having at least the following installed (possibly not - a complete list) +- The complete test suite depends on having at least the following installed + (possibly not a complete list) - git (A sufficiently newer version is required to run pre-push tests) - - python - - python3.4 (Required by a test which checks different python versions) - - python3.5 (Required by a test which checks different python versions) + - python2 (Required by a test which checks different python versions) + - python3 (Required by a test which checks different python versions) - tox (or virtualenv) - ruby + gem - docker -### Setting up an environemnt +### Setting up an environment This is useful for running specific tests. The easiest way to set this up is to run: -1. `tox -e venv` -2. `. venv-pre_commit/bin/activate` +1. `tox --devenv venv` (note: requires tox>=3.13) +2. `. venv/bin/activate` This will create and put you into a virtualenv which has an editable installation of pre-commit. Hack away! Running `pre-commit` will reflect @@ -27,54 +26,121 @@ your changes immediately. ### Running a specific test Running a specific test with the environment activated is as easy as: -`py.test tests -k test_the_name_of_your_test` +`pytest tests -k test_the_name_of_your_test` ### Running all the tests -Running all the tests can be done by running `tox -e py27` (or your +Running all the tests can be done by running `tox -e py37` (or your interpreter version of choice). These often take a long time and consume significant cpu while running the slower node / ruby integration tests. Alternatively, with the environment activated you can run all of the tests using: -`py.test tests` - -To skip the slower ruby / node integration tests, you can set the environment -variable `slowtests=false`. +`pytest tests` ### Setting up the hooks With the environment activated simply run `pre-commit install`. -## Style +## Documentation + +Documentation is hosted at https://pre-commit.com + +This website is controlled through +https://github.com/pre-commit/pre-commit.github.io + +## Adding support for a new hook language + +pre-commit already supports many [programming languages](https://pre-commit.com/#supported-languages) +to write hook executables with. + +When adding support for a language, you must first decide what level of support +to implement. The current implemented languages are at varying levels: + +- 0th class - pre-commit does not require any dependencies for these languages + as they're not actually languages (current examples: fail, pygrep) +- 1st class - pre-commit will bootstrap a full interpreter requiring nothing to + be installed globally (current examples: node, ruby) +- 2nd class - pre-commit requires the user to install the language globally but + will install tools in an isolated fashion (current examples: python, go, rust, + swift, docker). +- 3rd class - pre-commit requires the user to install both the tool and the + language globally (current examples: script, system) + +"third class" is usually the easiest to implement first and is perfectly +acceptable. + +Ideally the language works on the supported platforms for pre-commit (linux, +windows, macos) but it's ok to skip one or more platforms (for example, swift +doesn't run on windows). + +When writing your new language, it's often useful to look at other examples in +the `pre_commit/languages` directory. + +It might also be useful to look at a recent pull request which added a +language, for example: + +- [rust](https://github.com/pre-commit/pre-commit/pull/751) +- [fail](https://github.com/pre-commit/pre-commit/pull/812) +- [swift](https://github.com/pre-commit/pre-commit/pull/467) + +### `language` api + +here are the apis that should be implemented for a language -This repository follows pep8 (and enforces it with flake8). There are a few -nitpicky things I also like that I'll outline below. +Note that these are also documented in [`pre_commit/languages/all.py`](https://github.com/pre-commit/pre-commit/blob/master/pre_commit/languages/all.py) -### Multi-line method invocation +#### `ENVIRONMENT_DIR` -Multiple line method invocation should look as follows +a short string which will be used for the prefix of where packages will be +installed. For example, python uses `py_env` and installs a `virtualenv` at +that location. + +this will be `None` for 0th / 3rd class languages as they don't have an install +step. + +#### `get_default_version` + +This is used to retrieve the default `language_version` for a language. If +one cannot be determined, return `'default'`. + +You generally don't need to implement this on a first pass and can just use: ```python -function_call( - argument, - argument, - argument, -) +get_default_version = helpers.basic_default_version ``` -Some notable features: -- The initial parenthesis is at the end of the line -- Parameters are indented one indentation level further than the function name -- The last parameter contains a trailing comma (This helps make `git blame` - more accurate and reduces merge conflicts when adding / removing parameters). +`python` is currently the only language which implements this api -## Documentation +#### `healthy` -Documentation is hosted at http://pre-commit.com +This is used to check whether the installed environment is considered healthy. +This function should return `True` or `False`. -This website is controlled through -https://github.com/pre-commit/pre-commit.github.io +You generally don't need to implement this on a first pass and can just use: + +```python +healthy = helpers.basic_healthy +``` + +`python` is currently the only language which implements this api, for python +it is checking whether some common dlls are still available. + +#### `install_environment` + +this is the trickiest one to implement and where all the smart parts happen. + +this api should do the following things + +- (0th / 3rd class): `install_environment = helpers.no_install` +- (1st class): install a language runtime into the hook's directory +- (2nd class): install the package at `.` into the `ENVIRONMENT_DIR` +- (2nd class, optional): install packages listed in `additional_dependencies` + into `ENVIRONMENT_DIR` (not a required feature for a first pass) + +#### `run_hook` + +This is usually the easiest to implement, most of them look the same as the +`node` hook implementation: -When adding a feature, please make a pull request to add yourself to the -contributors list and add documentation to the website if applicable. +https://github.com/pre-commit/pre-commit/blob/160238220f022035c8ef869c9a8642f622c02118/pre_commit/languages/node.py#L72-L74 diff --git a/Makefile b/Makefile deleted file mode 100644 index 16868f1e8..000000000 --- a/Makefile +++ /dev/null @@ -1,26 +0,0 @@ -REBUILD_FLAG = - -.PHONY: all -all: venv test - -.PHONY: venv -venv: .venv.touch - tox -e venv $(REBUILD_FLAG) - -.PHONY: tests test -tests: test -test: .venv.touch - tox $(REBUILD_FLAG) - - -.venv.touch: setup.py requirements-dev.txt - $(eval REBUILD_FLAG := --recreate) - touch .venv.touch - - -.PHONY: clean -clean: - find . -name '*.pyc' -delete - rm -rf .tox - rm -rf ./venv-* - rm -f .venv.touch diff --git a/README.md b/README.md index 8bbc534b1..98a6d00e0 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -[![Build Status](https://travis-ci.org/pre-commit/pre-commit.svg?branch=master)](https://travis-ci.org/pre-commit/pre-commit) -[![Coverage Status](https://coveralls.io/repos/github/pre-commit/pre-commit/badge.svg?branch=master)](https://coveralls.io/github/pre-commit/pre-commit?branch=master) -[![Build status](https://ci.appveyor.com/api/projects/status/mmcwdlfgba4esaii/branch/master?svg=true)](https://ci.appveyor.com/project/asottile/pre-commit/branch/master) +[![Build Status](https://dev.azure.com/asottile/asottile/_apis/build/status/pre-commit.pre-commit?branchName=master)](https://dev.azure.com/asottile/asottile/_build/latest?definitionId=21&branchName=master) +[![Azure DevOps coverage](https://img.shields.io/azure-devops/coverage/asottile/asottile/21/master.svg)](https://dev.azure.com/asottile/asottile/_build/latest?definitionId=21&branchName=master) +[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) ## pre-commit A framework for managing and maintaining multi-language pre-commit hooks. -For more information see: http://pre-commit.com/ +For more information see: https://pre-commit.com/ diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 013e1421a..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,26 +0,0 @@ -environment: - global: - COVERAGE_IGNORE_WINDOWS: '# pragma: windows no cover' - TOX_TESTENV_PASSENV: COVERAGE_IGNORE_WINDOWS - matrix: - - TOXENV: py27 - - TOXENV: py36 - -install: - - "SET PATH=C:\\Python36;C:\\Python36\\Scripts;%PATH%" - - pip install tox virtualenv --upgrade - - "mkdir -p C:\\Temp" - - "SET TMPDIR=C:\\Temp" - -# Not a C# project -build: false - -before_test: - # Shut up CRLF messages - - git config --global core.safecrlf false - -test_script: tox - -cache: - - '%LOCALAPPDATA%\pip\cache' - - '%USERPROFILE%\.pre-commit' diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000..9b385b4c1 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,50 @@ +trigger: + branches: + include: [master, test-me-*] + tags: + include: ['*'] + +resources: + repositories: + - repository: asottile + type: github + endpoint: github + name: asottile/azure-pipeline-templates + ref: refs/tags/v1.0.0 + +jobs: +- template: job--pre-commit.yml@asottile +- template: job--python-tox.yml@asottile + parameters: + toxenvs: [py37] + os: windows + pre_test: + - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" + displayName: Add conda to PATH + - powershell: | + Write-Host "##vso[task.prependpath]C:\Strawberry\perl\bin" + Write-Host "##vso[task.prependpath]C:\Strawberry\perl\site\bin" + Write-Host "##vso[task.prependpath]C:\Strawberry\c\bin" + displayName: Add strawberry perl to PATH +- template: job--python-tox.yml@asottile + parameters: + toxenvs: [py37] + os: linux + name_postfix: _latest_git + pre_test: + - task: UseRubyVersion@0 + - template: step--git-install.yml + - bash: | + testing/get-swift.sh + echo '##vso[task.prependpath]/tmp/swift/usr/bin' + displayName: install swift +- template: job--python-tox.yml@asottile + parameters: + toxenvs: [pypy3, py36, py37, py38] + os: linux + pre_test: + - task: UseRubyVersion@0 + - bash: | + testing/get-swift.sh + echo '##vso[task.prependpath]/tmp/swift/usr/bin' + displayName: install swift diff --git a/get-swift.sh b/get-swift.sh deleted file mode 100755 index e5cc570b3..000000000 --- a/get-swift.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -# This is a script used in travis-ci to install swift -set -ex - -. /etc/lsb-release -if [ "$DISTRIB_CODENAME" = "trusty" ]; then - SWIFT_URL='https://swift.org/builds/swift-3.0.2-release/ubuntu1404/swift-3.0.2-RELEASE/swift-3.0.2-RELEASE-ubuntu14.04.tar.gz' -else - SWIFT_URL='https://swift.org/builds/swift-3.0.2-release/ubuntu1604/swift-3.0.2-RELEASE/swift-3.0.2-RELEASE-ubuntu16.04.tar.gz' -fi - -mkdir -p /tmp/swift -pushd /tmp/swift - wget "$SWIFT_URL" -O swift.tar.gz - tar -xf swift.tar.gz --strip 1 -popd diff --git a/hooks.yaml b/hooks.yaml deleted file mode 100644 index af53043ed..000000000 --- a/hooks.yaml +++ /dev/null @@ -1,12 +0,0 @@ -- id: validate_config - name: Validate Pre-Commit Config - description: This validator validates a pre-commit hooks config file - entry: pre-commit-validate-config - language: python - files: ^\.pre-commit-config\.yaml$ -- id: validate_manifest - name: Validate Pre-Commit Manifest - description: This validator validates a pre-commit hooks manifest file - entry: pre-commit-validate-manifest - language: python - files: ^(\.pre-commit-hooks\.yaml|hooks\.yaml)$ diff --git a/latest-git.sh b/latest-git.sh deleted file mode 100755 index 75c6f62a5..000000000 --- a/latest-git.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -# This is a script used in travis-ci to have latest git -set -ex -git clone git://github.com/git/git --depth 1 /tmp/git -pushd /tmp/git -make prefix=/tmp/git -j 8 all -make prefix=/tmp/git install -popd diff --git a/pre_commit/__main__.py b/pre_commit/__main__.py index fc424d821..541406879 100644 --- a/pre_commit/__main__.py +++ b/pre_commit/__main__.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from pre_commit.main import main diff --git a/pre_commit/clientlib.py b/pre_commit/clientlib.py index 6da6db25d..56ec0dd1b 100644 --- a/pre_commit/clientlib.py +++ b/pre_commit/clientlib.py @@ -1,75 +1,80 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import argparse import functools - -from aspy.yaml import ordered_load +import logging +import shlex +import sys +from typing import Any +from typing import Dict +from typing import Optional +from typing import Sequence + +import cfgv from identify.identify import ALL_TAGS import pre_commit.constants as C -from pre_commit import schema -from pre_commit.errors import FatalError +from pre_commit.error_handler import FatalError from pre_commit.languages.all import all_languages +from pre_commit.util import parse_version +from pre_commit.util import yaml_load +logger = logging.getLogger('pre_commit') -def check_language(v): - if v not in all_languages: - raise schema.ValidationError( - 'Expected {} to be in {!r}'.format(v, all_languages), - ) +check_string_regex = cfgv.check_and(cfgv.check_string, cfgv.check_regex) -def check_type_tag(tag): +def check_type_tag(tag: str) -> None: if tag not in ALL_TAGS: - raise schema.ValidationError( - 'Type tag {!r} is not recognized. ' - 'Try upgrading identify and pre-commit?'.format(tag), + raise cfgv.ValidationError( + f'Type tag {tag!r} is not recognized. ' + f'Try upgrading identify and pre-commit?', + ) + + +def check_min_version(version: str) -> None: + if parse_version(version) > parse_version(C.VERSION): + raise cfgv.ValidationError( + f'pre-commit version {version} is required but version ' + f'{C.VERSION} is installed. ' + f'Perhaps run `pip install --upgrade pre-commit`.', ) -def _make_argparser(filenames_help): +def _make_argparser(filenames_help: str) -> argparse.ArgumentParser: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*', help=filenames_help) parser.add_argument('-V', '--version', action='version', version=C.VERSION) return parser -MANIFEST_HOOK_DICT = schema.Map( +MANIFEST_HOOK_DICT = cfgv.Map( 'Hook', 'id', - schema.Required('id', schema.check_string), - schema.Required('name', schema.check_string), - schema.Required('entry', schema.check_string), - schema.Required( - 'language', schema.check_and(schema.check_string, check_language), - ), + cfgv.Required('id', cfgv.check_string), + cfgv.Required('name', cfgv.check_string), + cfgv.Required('entry', cfgv.check_string), + cfgv.Required('language', cfgv.check_one_of(all_languages)), + cfgv.Optional('alias', cfgv.check_string, ''), - schema.Optional( - 'files', schema.check_and(schema.check_string, schema.check_regex), - '', - ), - schema.Optional( - 'exclude', - schema.check_and(schema.check_string, schema.check_regex), - '^$', - ), - schema.Optional('types', schema.check_array(check_type_tag), ['file']), - schema.Optional('exclude_types', schema.check_array(check_type_tag), []), + cfgv.Optional('files', check_string_regex, ''), + cfgv.Optional('exclude', check_string_regex, '^$'), + cfgv.Optional('types', cfgv.check_array(check_type_tag), ['file']), + cfgv.Optional('exclude_types', cfgv.check_array(check_type_tag), []), - schema.Optional( - 'additional_dependencies', schema.check_array(schema.check_string), [], + cfgv.Optional( + 'additional_dependencies', cfgv.check_array(cfgv.check_string), [], ), - schema.Optional('args', schema.check_array(schema.check_string), []), - schema.Optional('always_run', schema.check_bool, False), - schema.Optional('pass_filenames', schema.check_bool, True), - schema.Optional('description', schema.check_string, ''), - schema.Optional('language_version', schema.check_string, 'default'), - schema.Optional('log_file', schema.check_string, ''), - schema.Optional('minimum_pre_commit_version', schema.check_string, '0'), - schema.Optional('stages', schema.check_array(schema.check_string), []), + cfgv.Optional('args', cfgv.check_array(cfgv.check_string), []), + cfgv.Optional('always_run', cfgv.check_bool, False), + cfgv.Optional('pass_filenames', cfgv.check_bool, True), + cfgv.Optional('description', cfgv.check_string, ''), + cfgv.Optional('language_version', cfgv.check_string, C.DEFAULT), + cfgv.Optional('log_file', cfgv.check_string, ''), + cfgv.Optional('minimum_pre_commit_version', cfgv.check_string, '0'), + cfgv.Optional('require_serial', cfgv.check_bool, False), + cfgv.Optional('stages', cfgv.check_array(cfgv.check_one_of(C.STAGES)), []), + cfgv.Optional('verbose', cfgv.check_bool, False), ) -MANIFEST_SCHEMA = schema.Array(MANIFEST_HOOK_DICT) +MANIFEST_SCHEMA = cfgv.Array(MANIFEST_HOOK_DICT) class InvalidManifestError(FatalError): @@ -77,14 +82,14 @@ class InvalidManifestError(FatalError): load_manifest = functools.partial( - schema.load_from_filename, + cfgv.load_from_filename, schema=MANIFEST_SCHEMA, - load_strategy=ordered_load, + load_strategy=yaml_load, exc_tp=InvalidManifestError, ) -def validate_manifest_main(argv=None): +def validate_manifest_main(argv: Optional[Sequence[str]] = None) -> int: parser = _make_argparser('Manifest filenames.') args = parser.parse_args(argv) ret = 0 @@ -97,54 +102,209 @@ def validate_manifest_main(argv=None): return ret -_LOCAL_SENTINEL = 'local' -CONFIG_HOOK_DICT = schema.Map( +LOCAL = 'local' +META = 'meta' + + +class MigrateShaToRev: + key = 'rev' + + @staticmethod + def _cond(key: str) -> cfgv.Conditional: + return cfgv.Conditional( + key, cfgv.check_string, + condition_key='repo', + condition_value=cfgv.NotIn(LOCAL, META), + ensure_absent=True, + ) + + def check(self, dct: Dict[str, Any]) -> None: + if dct.get('repo') in {LOCAL, META}: + self._cond('rev').check(dct) + self._cond('sha').check(dct) + elif 'sha' in dct and 'rev' in dct: + raise cfgv.ValidationError('Cannot specify both sha and rev') + elif 'sha' in dct: + self._cond('sha').check(dct) + else: + self._cond('rev').check(dct) + + def apply_default(self, dct: Dict[str, Any]) -> None: + if 'sha' in dct: + dct['rev'] = dct.pop('sha') + + remove_default = cfgv.Required.remove_default + + +def _entry(modname: str) -> str: + """the hook `entry` is passed through `shlex.split()` by the command + runner, so to prevent issues with spaces and backslashes (on Windows) + it must be quoted here. + """ + return f'{shlex.quote(sys.executable)} -m pre_commit.meta_hooks.{modname}' + + +def warn_unknown_keys_root( + extra: Sequence[str], + orig_keys: Sequence[str], + dct: Dict[str, str], +) -> None: + logger.warning(f'Unexpected key(s) present at root: {", ".join(extra)}') + + +def warn_unknown_keys_repo( + extra: Sequence[str], + orig_keys: Sequence[str], + dct: Dict[str, str], +) -> None: + logger.warning( + f'Unexpected key(s) present on {dct["repo"]}: {", ".join(extra)}', + ) + + +_meta = ( + ( + 'check-hooks-apply', ( + ('name', 'Check hooks apply to the repository'), + ('files', C.CONFIG_FILE), + ('entry', _entry('check_hooks_apply')), + ), + ), + ( + 'check-useless-excludes', ( + ('name', 'Check for useless excludes'), + ('files', C.CONFIG_FILE), + ('entry', _entry('check_useless_excludes')), + ), + ), + ( + 'identity', ( + ('name', 'identity'), + ('verbose', True), + ('entry', _entry('identity')), + ), + ), +) + +META_HOOK_DICT = cfgv.Map( + 'Hook', 'id', + cfgv.Required('id', cfgv.check_string), + cfgv.Required('id', cfgv.check_one_of(tuple(k for k, _ in _meta))), + # language must be system + cfgv.Optional('language', cfgv.check_one_of({'system'}), 'system'), + *( + # default to the hook definition for the meta hooks + cfgv.ConditionalOptional(key, cfgv.check_any, value, 'id', hook_id) + for hook_id, values in _meta + for key, value in values + ), + *( + # default to the "manifest" parsing + cfgv.OptionalNoDefault(item.key, item.check_fn) + # these will always be defaulted above + if item.key in {'name', 'language', 'entry'} else + item + for item in MANIFEST_HOOK_DICT.items + ), +) +CONFIG_HOOK_DICT = cfgv.Map( 'Hook', 'id', - schema.Required('id', schema.check_string), + cfgv.Required('id', cfgv.check_string), # All keys in manifest hook dict are valid in a config hook dict, but # are optional. # No defaults are provided here as the config is merged on top of the # manifest. - *[ - schema.OptionalNoDefault(item.key, item.check_fn) + *( + cfgv.OptionalNoDefault(item.key, item.check_fn) for item in MANIFEST_HOOK_DICT.items if item.key != 'id' - ] + ), ) -CONFIG_REPO_DICT = schema.Map( +CONFIG_REPO_DICT = cfgv.Map( 'Repository', 'repo', - schema.Required('repo', schema.check_string), - schema.RequiredRecurse('hooks', schema.Array(CONFIG_HOOK_DICT)), + cfgv.Required('repo', cfgv.check_string), - schema.Conditional( - 'sha', schema.check_string, - condition_key='repo', condition_value=schema.Not(_LOCAL_SENTINEL), - ensure_absent=True, + cfgv.ConditionalRecurse( + 'hooks', cfgv.Array(CONFIG_HOOK_DICT), + 'repo', cfgv.NotIn(LOCAL, META), + ), + cfgv.ConditionalRecurse( + 'hooks', cfgv.Array(MANIFEST_HOOK_DICT), + 'repo', LOCAL, + ), + cfgv.ConditionalRecurse( + 'hooks', cfgv.Array(META_HOOK_DICT), + 'repo', META, ), -) -CONFIG_SCHEMA = schema.Array(CONFIG_REPO_DICT) + MigrateShaToRev(), + cfgv.WarnAdditionalKeys(('repo', 'rev', 'hooks'), warn_unknown_keys_repo), +) +DEFAULT_LANGUAGE_VERSION = cfgv.Map( + 'DefaultLanguageVersion', None, + cfgv.NoAdditionalKeys(all_languages), + *(cfgv.Optional(x, cfgv.check_string, C.DEFAULT) for x in all_languages), +) +CONFIG_SCHEMA = cfgv.Map( + 'Config', None, -def is_local_repo(repo_entry): - return repo_entry['repo'] == _LOCAL_SENTINEL + cfgv.RequiredRecurse('repos', cfgv.Array(CONFIG_REPO_DICT)), + cfgv.OptionalRecurse( + 'default_language_version', DEFAULT_LANGUAGE_VERSION, {}, + ), + cfgv.Optional( + 'default_stages', + cfgv.check_array(cfgv.check_one_of(C.STAGES)), + C.STAGES, + ), + cfgv.Optional('files', check_string_regex, ''), + cfgv.Optional('exclude', check_string_regex, '^$'), + cfgv.Optional('fail_fast', cfgv.check_bool, False), + cfgv.Optional( + 'minimum_pre_commit_version', + cfgv.check_and(cfgv.check_string, check_min_version), + '0', + ), + cfgv.WarnAdditionalKeys( + ( + 'repos', + 'default_language_version', + 'default_stages', + 'files', + 'exclude', + 'fail_fast', + 'minimum_pre_commit_version', + ), + warn_unknown_keys_root, + ), +) class InvalidConfigError(FatalError): pass +def ordered_load_normalize_legacy_config(contents: str) -> Dict[str, Any]: + data = yaml_load(contents) + if isinstance(data, list): + # TODO: Once happy, issue a deprecation warning and instructions + return {'repos': data} + else: + return data + + load_config = functools.partial( - schema.load_from_filename, + cfgv.load_from_filename, schema=CONFIG_SCHEMA, - load_strategy=ordered_load, + load_strategy=ordered_load_normalize_legacy_config, exc_tp=InvalidConfigError, ) -def validate_config_main(argv=None): +def validate_config_main(argv: Optional[Sequence[str]] = None) -> int: parser = _make_argparser('Config filenames.') args = parser.parse_args(argv) ret = 0 diff --git a/pre_commit/color.py b/pre_commit/color.py index 25fbb2568..5fa70421b 100644 --- a/pre_commit/color.py +++ b/pre_commit/color.py @@ -1,27 +1,67 @@ -from __future__ import unicode_literals - import os import sys -if os.name == 'nt': # pragma: no cover (windows) - from pre_commit.color_windows import enable_virtual_terminal_processing +if sys.platform == 'win32': # pragma: no cover (windows) + def _enable() -> None: + from ctypes import POINTER + from ctypes import windll + from ctypes import WinError + from ctypes import WINFUNCTYPE + from ctypes.wintypes import BOOL + from ctypes.wintypes import DWORD + from ctypes.wintypes import HANDLE + + STD_OUTPUT_HANDLE = -11 + ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4 + + def bool_errcheck(result, func, args): + if not result: + raise WinError() + return args + + GetStdHandle = WINFUNCTYPE(HANDLE, DWORD)( + ('GetStdHandle', windll.kernel32), ((1, 'nStdHandle'),), + ) + + GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))( + ('GetConsoleMode', windll.kernel32), + ((1, 'hConsoleHandle'), (2, 'lpMode')), + ) + GetConsoleMode.errcheck = bool_errcheck + + SetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, DWORD)( + ('SetConsoleMode', windll.kernel32), + ((1, 'hConsoleHandle'), (1, 'dwMode')), + ) + SetConsoleMode.errcheck = bool_errcheck + + # As of Windows 10, the Windows console supports (some) ANSI escape + # sequences, but it needs to be enabled using `SetConsoleMode` first. + # + # More info on the escape sequences supported: + # https://msdn.microsoft.com/en-us/library/windows/desktop/mt638032(v=vs.85).aspx + stdout = GetStdHandle(STD_OUTPUT_HANDLE) + flags = GetConsoleMode(stdout) + SetConsoleMode(stdout, flags | ENABLE_VIRTUAL_TERMINAL_PROCESSING) + try: - enable_virtual_terminal_processing() - except WindowsError: - pass + _enable() + except OSError: + terminal_supports_color = False + else: + terminal_supports_color = True +else: # pragma: win32 no cover + terminal_supports_color = True RED = '\033[41m' GREEN = '\033[42m' YELLOW = '\033[43;30m' TURQUOISE = '\033[46;30m' -NORMAL = '\033[0m' - +SUBTLE = '\033[2m' +NORMAL = '\033[m' -class InvalidColorSetting(ValueError): - pass - -def format_color(text, color, use_color_setting): +def format_color(text: str, color: str, use_color_setting: bool) -> str: """Format text with color. Args: @@ -29,25 +69,29 @@ def format_color(text, color, use_color_setting): color - The color start string use_color_setting - Whether or not to color """ - if not use_color_setting: - return text + if use_color_setting: + return f'{color}{text}{NORMAL}' else: - return '{}{}{}'.format(color, text, NORMAL) + return text COLOR_CHOICES = ('auto', 'always', 'never') -def use_color(setting): +def use_color(setting: str) -> bool: """Choose whether to use color based on the command argument. Args: setting - Either `auto`, `always`, or `never` """ if setting not in COLOR_CHOICES: - raise InvalidColorSetting(setting) + raise ValueError(setting) return ( - setting == 'always' or - (setting == 'auto' and sys.stdout.isatty()) + setting == 'always' or ( + setting == 'auto' and + sys.stdout.isatty() and + terminal_supports_color and + os.getenv('TERM') != 'dumb' + ) ) diff --git a/pre_commit/color_windows.py b/pre_commit/color_windows.py deleted file mode 100644 index d44e0b809..000000000 --- a/pre_commit/color_windows.py +++ /dev/null @@ -1,49 +0,0 @@ -from __future__ import unicode_literals - -from ctypes import POINTER -from ctypes import windll -from ctypes import WinError -from ctypes import WINFUNCTYPE -from ctypes.wintypes import BOOL -from ctypes.wintypes import DWORD -from ctypes.wintypes import HANDLE - -STD_OUTPUT_HANDLE = -11 -ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4 - - -def bool_errcheck(result, func, args): - if not result: - raise WinError() - return args - - -GetStdHandle = WINFUNCTYPE(HANDLE, DWORD)( - ("GetStdHandle", windll.kernel32), - ((1, "nStdHandle"), ) -) - -GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))( - ("GetConsoleMode", windll.kernel32), - ((1, "hConsoleHandle"), (2, "lpMode")) -) -GetConsoleMode.errcheck = bool_errcheck - -SetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, DWORD)( - ("SetConsoleMode", windll.kernel32), - ((1, "hConsoleHandle"), (1, "dwMode")) -) -SetConsoleMode.errcheck = bool_errcheck - - -def enable_virtual_terminal_processing(): - """As of Windows 10, the Windows console supports (some) ANSI escape - sequences, but it needs to be enabled using `SetConsoleMode` first. - - More info on the escape sequences supported: - https://msdn.microsoft.com/en-us/library/windows/desktop/mt638032(v=vs.85).aspx - - """ - stdout = GetStdHandle(STD_OUTPUT_HANDLE) - flags = GetConsoleMode(stdout) - SetConsoleMode(stdout, flags | ENABLE_VIRTUAL_TERMINAL_PROCESSING) diff --git a/pre_commit/commands/autoupdate.py b/pre_commit/commands/autoupdate.py index 99a5d62fd..5a9a9880c 100644 --- a/pre_commit/commands/autoupdate.py +++ b/pre_commit/commands/autoupdate.py @@ -1,109 +1,182 @@ -from __future__ import print_function -from __future__ import unicode_literals - -from collections import OrderedDict - -from aspy.yaml import ordered_dump -from aspy.yaml import ordered_load +import os.path +import re +from typing import Any +from typing import Dict +from typing import List +from typing import NamedTuple +from typing import Optional +from typing import Sequence +from typing import Tuple import pre_commit.constants as C +from pre_commit import git from pre_commit import output -from pre_commit.clientlib import CONFIG_SCHEMA -from pre_commit.clientlib import is_local_repo +from pre_commit.clientlib import InvalidManifestError from pre_commit.clientlib import load_config -from pre_commit.repository import Repository -from pre_commit.schema import remove_defaults +from pre_commit.clientlib import load_manifest +from pre_commit.clientlib import LOCAL +from pre_commit.clientlib import META +from pre_commit.commands.migrate_config import migrate_config +from pre_commit.store import Store from pre_commit.util import CalledProcessError from pre_commit.util import cmd_output -from pre_commit.util import cwd +from pre_commit.util import cmd_output_b +from pre_commit.util import tmpdir +from pre_commit.util import yaml_dump +from pre_commit.util import yaml_load -class RepositoryCannotBeUpdatedError(RuntimeError): - pass +class RevInfo(NamedTuple): + repo: str + rev: str + frozen: Optional[str] + @classmethod + def from_config(cls, config: Dict[str, Any]) -> 'RevInfo': + return cls(config['repo'], config['rev'], None) -def _update_repo(repo_config, runner, tags_only): - """Updates a repository to the tip of `master`. If the repository cannot - be updated because a hook that is configured does not exist in `master`, - this raises a RepositoryCannotBeUpdatedError - - Args: - repo_config - A config for a repository - """ - repo = Repository.create(repo_config, runner.store) - - with cwd(repo._repo_path): - cmd_output('git', 'fetch') - tag_cmd = ('git', 'describe', 'origin/master', '--tags') + def update(self, tags_only: bool, freeze: bool) -> 'RevInfo': if tags_only: - tag_cmd += ('--abbrev=0',) + tag_cmd = ('git', 'describe', 'FETCH_HEAD', '--tags', '--abbrev=0') else: - tag_cmd += ('--exact',) - try: - rev = cmd_output(*tag_cmd)[1].strip() - except CalledProcessError: - rev = cmd_output('git', 'rev-parse', 'origin/master')[1].strip() + tag_cmd = ('git', 'describe', 'FETCH_HEAD', '--tags', '--exact') + + with tmpdir() as tmp: + git.init_repo(tmp, self.repo) + cmd_output_b('git', 'fetch', 'origin', 'HEAD', '--tags', cwd=tmp) + + try: + rev = cmd_output(*tag_cmd, cwd=tmp)[1].strip() + except CalledProcessError: + cmd = ('git', 'rev-parse', 'FETCH_HEAD') + rev = cmd_output(*cmd, cwd=tmp)[1].strip() + + frozen = None + if freeze: + exact = cmd_output('git', 'rev-parse', rev, cwd=tmp)[1].strip() + if exact != rev: + rev, frozen = exact, rev + return self._replace(rev=rev, frozen=frozen) + + +class RepositoryCannotBeUpdatedError(RuntimeError): + pass - # Don't bother trying to update if our sha is the same - if rev == repo_config['sha']: - return repo_config - # Construct a new config with the head sha - new_config = OrderedDict(repo_config) - new_config['sha'] = rev - new_repo = Repository.create(new_config, runner.store) +def _check_hooks_still_exist_at_rev( + repo_config: Dict[str, Any], + info: RevInfo, + store: Store, +) -> None: + try: + path = store.clone(repo_config['repo'], info.rev) + manifest = load_manifest(os.path.join(path, C.MANIFEST_FILE)) + except InvalidManifestError as e: + raise RepositoryCannotBeUpdatedError(str(e)) # See if any of our hooks were deleted with the new commits - hooks = {hook['id'] for hook in repo.repo_config['hooks']} - hooks_missing = hooks - (hooks & set(new_repo.manifest.hooks)) + hooks = {hook['id'] for hook in repo_config['hooks']} + hooks_missing = hooks - {hook['id'] for hook in manifest} if hooks_missing: raise RepositoryCannotBeUpdatedError( - 'Cannot update because the tip of master is missing these hooks:\n' - '{}'.format(', '.join(sorted(hooks_missing))) + f'Cannot update because the tip of HEAD is missing these hooks:\n' + f'{", ".join(sorted(hooks_missing))}', ) - return new_config +REV_LINE_RE = re.compile(r'^(\s+)rev:(\s*)([^\s#]+)(.*)(\r?\n)$', re.DOTALL) + + +def _original_lines( + path: str, + rev_infos: List[Optional[RevInfo]], + retry: bool = False, +) -> Tuple[List[str], List[int]]: + """detect `rev:` lines or reformat the file""" + with open(path) as f: + original = f.read() + + lines = original.splitlines(True) + idxs = [i for i, line in enumerate(lines) if REV_LINE_RE.match(line)] + if len(idxs) == len(rev_infos): + return lines, idxs + elif retry: + raise AssertionError('could not find rev lines') + else: + with open(path, 'w') as f: + f.write(yaml_dump(yaml_load(original))) + return _original_lines(path, rev_infos, retry=True) -def autoupdate(runner, tags_only): + +def _write_new_config(path: str, rev_infos: List[Optional[RevInfo]]) -> None: + lines, idxs = _original_lines(path, rev_infos) + + for idx, rev_info in zip(idxs, rev_infos): + if rev_info is None: + continue + match = REV_LINE_RE.match(lines[idx]) + assert match is not None + new_rev_s = yaml_dump({'rev': rev_info.rev}) + new_rev = new_rev_s.split(':', 1)[1].strip() + if rev_info.frozen is not None: + comment = f' # frozen: {rev_info.frozen}' + elif match[4].strip().startswith('# frozen:'): + comment = '' + else: + comment = match[4] + lines[idx] = f'{match[1]}rev:{match[2]}{new_rev}{comment}{match[5]}' + + with open(path, 'w') as f: + f.write(''.join(lines)) + + +def autoupdate( + config_file: str, + store: Store, + tags_only: bool, + freeze: bool, + repos: Sequence[str] = (), +) -> int: """Auto-update the pre-commit config to the latest versions of repos.""" + migrate_config(config_file, quiet=True) retv = 0 - output_configs = [] + rev_infos: List[Optional[RevInfo]] = [] changed = False - input_configs = load_config( - runner.config_file_path, - load_strategy=ordered_load, - ) + config = load_config(config_file) + for repo_config in config['repos']: + if repo_config['repo'] in {LOCAL, META}: + continue - for repo_config in input_configs: - if is_local_repo(repo_config): - output_configs.append(repo_config) + info = RevInfo.from_config(repo_config) + if repos and info.repo not in repos: + rev_infos.append(None) continue - output.write('Updating {}...'.format(repo_config['repo'])) + + output.write(f'Updating {info.repo} ... ') + new_info = info.update(tags_only=tags_only, freeze=freeze) try: - new_repo_config = _update_repo(repo_config, runner, tags_only) + _check_hooks_still_exist_at_rev(repo_config, new_info, store) except RepositoryCannotBeUpdatedError as error: output.write_line(error.args[0]) - output_configs.append(repo_config) + rev_infos.append(None) retv = 1 continue - if new_repo_config['sha'] != repo_config['sha']: + if new_info.rev != info.rev: changed = True - output.write_line('updating {} -> {}.'.format( - repo_config['sha'], new_repo_config['sha'], - )) - output_configs.append(new_repo_config) + if new_info.frozen: + updated_to = f'{new_info.frozen} (frozen)' + else: + updated_to = new_info.rev + msg = f'updating {info.rev} -> {updated_to}.' + output.write_line(msg) + rev_infos.append(new_info) else: output.write_line('already up to date.') - output_configs.append(repo_config) + rev_infos.append(None) if changed: - with open(runner.config_file_path, 'w') as config_file: - config_file.write(ordered_dump( - remove_defaults(output_configs, CONFIG_SCHEMA), - **C.YAML_DUMP_KWARGS - )) + _write_new_config(config_file, rev_infos) return retv diff --git a/pre_commit/commands/clean.py b/pre_commit/commands/clean.py index 8cea6fc14..2be6c16a5 100644 --- a/pre_commit/commands/clean.py +++ b/pre_commit/commands/clean.py @@ -1,14 +1,14 @@ -from __future__ import print_function -from __future__ import unicode_literals - import os.path from pre_commit import output +from pre_commit.store import Store from pre_commit.util import rmtree -def clean(runner): - if os.path.exists(runner.store.directory): - rmtree(runner.store.directory) - output.write_line('Cleaned {}.'.format(runner.store.directory)) +def clean(store: Store) -> int: + legacy_path = os.path.expanduser('~/.pre-commit') + for directory in (store.directory, legacy_path): + if os.path.exists(directory): + rmtree(directory) + output.write_line(f'Cleaned {directory}.') return 0 diff --git a/pre_commit/commands/gc.py b/pre_commit/commands/gc.py new file mode 100644 index 000000000..7f6d31119 --- /dev/null +++ b/pre_commit/commands/gc.py @@ -0,0 +1,90 @@ +import os.path +from typing import Any +from typing import Dict +from typing import Set +from typing import Tuple + +import pre_commit.constants as C +from pre_commit import output +from pre_commit.clientlib import InvalidConfigError +from pre_commit.clientlib import InvalidManifestError +from pre_commit.clientlib import load_config +from pre_commit.clientlib import load_manifest +from pre_commit.clientlib import LOCAL +from pre_commit.clientlib import META +from pre_commit.store import Store + + +def _mark_used_repos( + store: Store, + all_repos: Dict[Tuple[str, str], str], + unused_repos: Set[Tuple[str, str]], + repo: Dict[str, Any], +) -> None: + if repo['repo'] == META: + return + elif repo['repo'] == LOCAL: + for hook in repo['hooks']: + deps = hook.get('additional_dependencies') + unused_repos.discard(( + store.db_repo_name(repo['repo'], deps), C.LOCAL_REPO_VERSION, + )) + else: + key = (repo['repo'], repo['rev']) + path = all_repos.get(key) + # can't inspect manifest if it isn't cloned + if path is None: + return + + try: + manifest = load_manifest(os.path.join(path, C.MANIFEST_FILE)) + except InvalidManifestError: + return + else: + unused_repos.discard(key) + by_id = {hook['id']: hook for hook in manifest} + + for hook in repo['hooks']: + if hook['id'] not in by_id: + continue + + deps = hook.get( + 'additional_dependencies', + by_id[hook['id']]['additional_dependencies'], + ) + unused_repos.discard(( + store.db_repo_name(repo['repo'], deps), repo['rev'], + )) + + +def _gc_repos(store: Store) -> int: + configs = store.select_all_configs() + repos = store.select_all_repos() + + # delete config paths which do not exist + dead_configs = [p for p in configs if not os.path.exists(p)] + live_configs = [p for p in configs if os.path.exists(p)] + + all_repos = {(repo, ref): path for repo, ref, path in repos} + unused_repos = set(all_repos) + for config_path in live_configs: + try: + config = load_config(config_path) + except InvalidConfigError: + dead_configs.append(config_path) + continue + else: + for repo in config['repos']: + _mark_used_repos(store, all_repos, unused_repos, repo) + + store.delete_configs(dead_configs) + for db_repo_name, ref in unused_repos: + store.delete_repo(db_repo_name, ref, all_repos[(db_repo_name, ref)]) + return len(unused_repos) + + +def gc(store: Store) -> int: + with store.exclusive_lock(): + repos_removed = _gc_repos(store) + output.write_line(f'{repos_removed} repo(s) removed.') + return 0 diff --git a/pre_commit/commands/hook_impl.py b/pre_commit/commands/hook_impl.py new file mode 100644 index 000000000..5ff4555ec --- /dev/null +++ b/pre_commit/commands/hook_impl.py @@ -0,0 +1,187 @@ +import argparse +import os.path +import subprocess +import sys +from typing import Optional +from typing import Sequence +from typing import Tuple + +from pre_commit.commands.run import run +from pre_commit.envcontext import envcontext +from pre_commit.parse_shebang import normalize_cmd +from pre_commit.store import Store + +Z40 = '0' * 40 + + +def _run_legacy( + hook_type: str, + hook_dir: str, + args: Sequence[str], +) -> Tuple[int, bytes]: + if os.environ.get('PRE_COMMIT_RUNNING_LEGACY'): + raise SystemExit( + f"bug: pre-commit's script is installed in migration mode\n" + f'run `pre-commit install -f --hook-type {hook_type}` to fix ' + f'this\n\n' + f'Please report this bug at ' + f'https://github.com/pre-commit/pre-commit/issues', + ) + + if hook_type == 'pre-push': + stdin = sys.stdin.buffer.read() + else: + stdin = b'' + + # not running in legacy mode + legacy_hook = os.path.join(hook_dir, f'{hook_type}.legacy') + if not os.access(legacy_hook, os.X_OK): + return 0, stdin + + with envcontext((('PRE_COMMIT_RUNNING_LEGACY', '1'),)): + cmd = normalize_cmd((legacy_hook, *args)) + return subprocess.run(cmd, input=stdin).returncode, stdin + + +def _validate_config( + retv: int, + config: str, + skip_on_missing_config: bool, +) -> None: + if not os.path.isfile(config): + if skip_on_missing_config or os.getenv('PRE_COMMIT_ALLOW_NO_CONFIG'): + print(f'`{config}` config file not found. Skipping `pre-commit`.') + raise SystemExit(retv) + else: + print( + f'No {config} file was found\n' + f'- To temporarily silence this, run ' + f'`PRE_COMMIT_ALLOW_NO_CONFIG=1 git ...`\n' + f'- To permanently silence this, install pre-commit with the ' + f'--allow-missing-config option\n' + f'- To uninstall pre-commit run `pre-commit uninstall`', + ) + raise SystemExit(1) + + +def _ns( + hook_type: str, + color: bool, + *, + all_files: bool = False, + from_ref: Optional[str] = None, + to_ref: Optional[str] = None, + remote_name: Optional[str] = None, + remote_url: Optional[str] = None, + commit_msg_filename: Optional[str] = None, + checkout_type: Optional[str] = None, +) -> argparse.Namespace: + return argparse.Namespace( + color=color, + hook_stage=hook_type.replace('pre-', ''), + from_ref=from_ref, + to_ref=to_ref, + remote_name=remote_name, + remote_url=remote_url, + commit_msg_filename=commit_msg_filename, + all_files=all_files, + checkout_type=checkout_type, + files=(), + hook=None, + verbose=False, + show_diff_on_failure=False, + ) + + +def _rev_exists(rev: str) -> bool: + return not subprocess.call(('git', 'rev-list', '--quiet', rev)) + + +def _pre_push_ns( + color: bool, + args: Sequence[str], + stdin: bytes, +) -> Optional[argparse.Namespace]: + remote_name = args[0] + remote_url = args[1] + + for line in stdin.decode().splitlines(): + _, local_sha, _, remote_sha = line.split() + if local_sha == Z40: + continue + elif remote_sha != Z40 and _rev_exists(remote_sha): + return _ns( + 'pre-push', color, + from_ref=remote_sha, to_ref=local_sha, + remote_name=remote_name, remote_url=remote_url, + ) + else: + # ancestors not found in remote + ancestors = subprocess.check_output(( + 'git', 'rev-list', local_sha, '--topo-order', '--reverse', + '--not', f'--remotes={remote_name}', + )).decode().strip() + if not ancestors: + continue + else: + first_ancestor = ancestors.splitlines()[0] + cmd = ('git', 'rev-list', '--max-parents=0', local_sha) + roots = set(subprocess.check_output(cmd).decode().splitlines()) + if first_ancestor in roots: + # pushing the whole tree including root commit + return _ns( + 'pre-push', color, + all_files=True, + remote_name=remote_name, remote_url=remote_url, + ) + else: + rev_cmd = ('git', 'rev-parse', f'{first_ancestor}^') + source = subprocess.check_output(rev_cmd).decode().strip() + return _ns( + 'pre-push', color, + from_ref=source, to_ref=local_sha, + remote_name=remote_name, remote_url=remote_url, + ) + + # nothing to push + return None + + +def _run_ns( + hook_type: str, + color: bool, + args: Sequence[str], + stdin: bytes, +) -> Optional[argparse.Namespace]: + if hook_type == 'pre-push': + return _pre_push_ns(color, args, stdin) + elif hook_type in {'prepare-commit-msg', 'commit-msg'}: + return _ns(hook_type, color, commit_msg_filename=args[0]) + elif hook_type in {'pre-merge-commit', 'pre-commit'}: + return _ns(hook_type, color) + elif hook_type == 'post-checkout': + return _ns( + hook_type, color, + from_ref=args[0], to_ref=args[1], checkout_type=args[2], + ) + else: + raise AssertionError(f'unexpected hook type: {hook_type}') + + +def hook_impl( + store: Store, + *, + config: str, + color: bool, + hook_type: str, + hook_dir: str, + skip_on_missing_config: bool, + args: Sequence[str], +) -> int: + retv, stdin = _run_legacy(hook_type, hook_dir, args) + _validate_config(retv, config, skip_on_missing_config) + ns = _run_ns(hook_type, color, args, stdin) + if ns is None: + return retv + else: + return retv | run(config, store, ns) diff --git a/pre_commit/commands/init_templatedir.py b/pre_commit/commands/init_templatedir.py new file mode 100644 index 000000000..f676fb192 --- /dev/null +++ b/pre_commit/commands/init_templatedir.py @@ -0,0 +1,33 @@ +import logging +import os.path +from typing import Sequence + +from pre_commit.commands.install_uninstall import install +from pre_commit.store import Store +from pre_commit.util import CalledProcessError +from pre_commit.util import cmd_output + +logger = logging.getLogger('pre_commit') + + +def init_templatedir( + config_file: str, + store: Store, + directory: str, + hook_types: Sequence[str], +) -> int: + install( + config_file, store, hook_types=hook_types, + overwrite=True, skip_on_missing_config=True, git_dir=directory, + ) + try: + _, out, _ = cmd_output('git', 'config', 'init.templateDir') + except CalledProcessError: + configured_path = None + else: + configured_path = os.path.realpath(os.path.expanduser(out.strip())) + dest = os.path.realpath(directory) + if configured_path != dest: + logger.warning('`init.templateDir` not set to the target directory') + logger.warning(f'maybe `git config --global init.templateDir {dest}`?') + return 0 diff --git a/pre_commit/commands/install_uninstall.py b/pre_commit/commands/install_uninstall.py index 6268b918f..c8b7633b6 100644 --- a/pre_commit/commands/install_uninstall.py +++ b/pre_commit/commands/install_uninstall.py @@ -1,15 +1,23 @@ -from __future__ import print_function -from __future__ import unicode_literals - -import io +import itertools +import logging import os.path +import shutil import sys +from typing import Optional +from typing import Sequence +from typing import Tuple +from pre_commit import git from pre_commit import output +from pre_commit.clientlib import load_config +from pre_commit.repository import all_hooks +from pre_commit.repository import install_hook_envs +from pre_commit.store import Store from pre_commit.util import make_executable -from pre_commit.util import mkdirp -from pre_commit.util import resource_filename +from pre_commit.util import resource_text + +logger = logging.getLogger(__name__) # This is used to identify the hook file we install PRIOR_HASHES = ( @@ -20,84 +28,148 @@ 'e358c9dae00eac5d06b38dfdb1e33a8c', ) CURRENT_HASH = '138fd403232d2ddd5efb44317e38bf03' +TEMPLATE_START = '# start templated\n' +TEMPLATE_END = '# end templated\n' +# Homebrew/homebrew-core#35825: be more timid about appropriate `PATH` +# #1312 os.defpath is too restrictive on BSD +POSIX_SEARCH_PATH = ('/usr/local/bin', '/usr/bin', '/bin') +SYS_EXE = os.path.basename(os.path.realpath(sys.executable)) + +def _hook_paths( + hook_type: str, + git_dir: Optional[str] = None, +) -> Tuple[str, str]: + git_dir = git_dir if git_dir is not None else git.get_git_dir() + pth = os.path.join(git_dir, 'hooks', hook_type) + return pth, f'{pth}.legacy' -def is_our_script(filename): - if not os.path.exists(filename): + +def is_our_script(filename: str) -> bool: + if not os.path.exists(filename): # pragma: win32 no cover (symlink) return False - contents = io.open(filename).read() + with open(filename) as f: + contents = f.read() return any(h in contents for h in (CURRENT_HASH,) + PRIOR_HASHES) -def install( - runner, overwrite=False, hooks=False, hook_type='pre-commit', - skip_on_missing_conf=False, -): - """Install the pre-commit hooks.""" - hook_path = runner.get_hook_path(hook_type) - legacy_path = hook_path + '.legacy' +def shebang() -> str: + if sys.platform == 'win32': + py = SYS_EXE + else: + exe_choices = [ + f'python{sys.version_info[0]}.{sys.version_info[1]}', + f'python{sys.version_info[0]}', + ] + # avoid searching for bare `python` as it's likely to be python 2 + if SYS_EXE != 'python': + exe_choices.append(SYS_EXE) + for path, exe in itertools.product(POSIX_SEARCH_PATH, exe_choices): + if os.access(os.path.join(path, exe), os.X_OK): + py = exe + break + else: + py = SYS_EXE + return f'#!/usr/bin/env {py}' - mkdirp(os.path.dirname(hook_path)) + +def _install_hook_script( + config_file: str, + hook_type: str, + overwrite: bool = False, + skip_on_missing_config: bool = False, + git_dir: Optional[str] = None, +) -> None: + hook_path, legacy_path = _hook_paths(hook_type, git_dir=git_dir) + + os.makedirs(os.path.dirname(hook_path), exist_ok=True) # If we have an existing hook, move it to pre-commit.legacy if os.path.lexists(hook_path) and not is_our_script(hook_path): - os.rename(hook_path, legacy_path) + shutil.move(hook_path, legacy_path) # If we specify overwrite, we simply delete the legacy file if overwrite and os.path.exists(legacy_path): os.remove(legacy_path) elif os.path.exists(legacy_path): output.write_line( - 'Running in migration mode with existing hooks at {}\n' - 'Use -f to use only pre-commit.'.format( - legacy_path, - ) + f'Running in migration mode with existing hooks at {legacy_path}\n' + f'Use -f to use only pre-commit.', ) - with io.open(hook_path, 'w') as pre_commit_file_obj: - if hook_type == 'pre-push': - with io.open(resource_filename('pre-push-tmpl')) as fp: - pre_push_contents = fp.read() - else: - pre_push_contents = '' - - skip_on_missing_conf = 'true' if skip_on_missing_conf else 'false' - contents = io.open(resource_filename('hook-tmpl')).read().format( - sys_executable=sys.executable, - hook_type=hook_type, - pre_push=pre_push_contents, - skip_on_missing_conf=skip_on_missing_conf, - ) - pre_commit_file_obj.write(contents) + args = ['hook-impl', f'--config={config_file}', f'--hook-type={hook_type}'] + if skip_on_missing_config: + args.append('--skip-on-missing-config') + params = {'INSTALL_PYTHON': sys.executable, 'ARGS': args} + + with open(hook_path, 'w') as hook_file: + contents = resource_text('hook-tmpl') + before, rest = contents.split(TEMPLATE_START) + to_template, after = rest.split(TEMPLATE_END) + + before = before.replace('#!/usr/bin/env python3', shebang()) + + hook_file.write(before + TEMPLATE_START) + for line in to_template.splitlines(): + var = line.split()[0] + hook_file.write(f'{var} = {params[var]!r}\n') + hook_file.write(TEMPLATE_END + after) make_executable(hook_path) - output.write_line('pre-commit installed at {}'.format(hook_path)) + output.write_line(f'pre-commit installed at {hook_path}') + + +def install( + config_file: str, + store: Store, + hook_types: Sequence[str], + overwrite: bool = False, + hooks: bool = False, + skip_on_missing_config: bool = False, + git_dir: Optional[str] = None, +) -> int: + if git_dir is None and git.has_core_hookpaths_set(): + logger.error( + 'Cowardly refusing to install hooks with `core.hooksPath` set.\n' + 'hint: `git config --unset-all core.hooksPath`', + ) + return 1 + + for hook_type in hook_types: + _install_hook_script( + config_file, hook_type, + overwrite=overwrite, + skip_on_missing_config=skip_on_missing_config, + git_dir=git_dir, + ) - # If they requested we install all of the hooks, do so. if hooks: - install_hooks(runner) + install_hooks(config_file, store) return 0 -def install_hooks(runner): - for repository in runner.repositories: - repository.require_installed() +def install_hooks(config_file: str, store: Store) -> int: + install_hook_envs(all_hooks(load_config(config_file), store), store) + return 0 -def uninstall(runner, hook_type='pre-commit'): - """Uninstall the pre-commit hooks.""" - hook_path = runner.get_hook_path(hook_type) - legacy_path = hook_path + '.legacy' +def _uninstall_hook_script(hook_type: str) -> None: + hook_path, legacy_path = _hook_paths(hook_type) + # If our file doesn't exist or it isn't ours, gtfo. if not os.path.exists(hook_path) or not is_our_script(hook_path): - return 0 + return os.remove(hook_path) - output.write_line('{} uninstalled'.format(hook_type)) + output.write_line(f'{hook_type} uninstalled') if os.path.exists(legacy_path): os.rename(legacy_path, hook_path) - output.write_line('Restored previous hooks to {}'.format(hook_path)) + output.write_line(f'Restored previous hooks to {hook_path}') + +def uninstall(hook_types: Sequence[str]) -> int: + for hook_type in hook_types: + _uninstall_hook_script(hook_type) return 0 diff --git a/pre_commit/commands/migrate_config.py b/pre_commit/commands/migrate_config.py new file mode 100644 index 000000000..d83b8e9cf --- /dev/null +++ b/pre_commit/commands/migrate_config.py @@ -0,0 +1,59 @@ +import re + +import yaml + +from pre_commit.util import yaml_load + + +def _indent(s: str) -> str: + lines = s.splitlines(True) + return ''.join(' ' * 4 + line if line.strip() else line for line in lines) + + +def _is_header_line(line: str) -> bool: + return line.startswith(('#', '---')) or not line.strip() + + +def _migrate_map(contents: str) -> str: + # Find the first non-header line + lines = contents.splitlines(True) + i = 0 + # Only loop on non empty configuration file + while i < len(lines) and _is_header_line(lines[i]): + i += 1 + + header = ''.join(lines[:i]) + rest = ''.join(lines[i:]) + + if isinstance(yaml_load(contents), list): + # If they are using the "default" flow style of yaml, this operation + # will yield a valid configuration + try: + trial_contents = f'{header}repos:\n{rest}' + yaml_load(trial_contents) + contents = trial_contents + except yaml.YAMLError: + contents = f'{header}repos:\n{_indent(rest)}' + + return contents + + +def _migrate_sha_to_rev(contents: str) -> str: + return re.sub(r'(\n\s+)sha:', r'\1rev:', contents) + + +def migrate_config(config_file: str, quiet: bool = False) -> int: + with open(config_file) as f: + orig_contents = contents = f.read() + + contents = _migrate_map(contents) + contents = _migrate_sha_to_rev(contents) + + if contents != orig_contents: + with open(config_file, 'w') as f: + f.write(contents) + + print('Configuration has been migrated.') + elif not quiet: + print('Configuration is already migrated.') + return 0 diff --git a/pre_commit/commands/run.py b/pre_commit/commands/run.py index 99d3a189e..2f745782e 100644 --- a/pre_commit/commands/run.py +++ b/pre_commit/commands/run.py @@ -1,171 +1,212 @@ -from __future__ import print_function -from __future__ import unicode_literals - +import argparse +import contextlib +import functools import logging import os +import re import subprocess -import sys +import time +from typing import Any +from typing import Collection +from typing import Dict +from typing import List +from typing import Sequence +from typing import Set +from typing import Tuple from identify.identify import tags_from_path from pre_commit import color from pre_commit import git from pre_commit import output -from pre_commit.output import get_hook_message +from pre_commit.clientlib import load_config +from pre_commit.hook import Hook +from pre_commit.languages.all import languages +from pre_commit.repository import all_hooks +from pre_commit.repository import install_hook_envs from pre_commit.staged_files_only import staged_files_only -from pre_commit.util import cmd_output -from pre_commit.util import memoize_by_cwd -from pre_commit.util import noop_context +from pre_commit.store import Store +from pre_commit.util import cmd_output_b +from pre_commit.util import EnvironT logger = logging.getLogger('pre_commit') -tags_from_path = memoize_by_cwd(tags_from_path) - - -def _get_skips(environ): +def _start_msg(*, start: str, cols: int, end_len: int) -> str: + dots = '.' * (cols - len(start) - end_len - 1) + return f'{start}{dots}' + + +def _full_msg( + *, + start: str, + cols: int, + end_msg: str, + end_color: str, + use_color: bool, + postfix: str = '', +) -> str: + dots = '.' * (cols - len(start) - len(postfix) - len(end_msg) - 1) + end = color.format_color(end_msg, end_color, use_color) + return f'{start}{dots}{postfix}{end}\n' + + +def filter_by_include_exclude( + names: Collection[str], + include: str, + exclude: str, +) -> List[str]: + include_re, exclude_re = re.compile(include), re.compile(exclude) + return [ + filename for filename in names + if include_re.search(filename) + if not exclude_re.search(filename) + ] + + +class Classifier: + def __init__(self, filenames: Sequence[str]) -> None: + # on windows we normalize all filenames to use forward slashes + # this makes it easier to filter using the `files:` regex + # this also makes improperly quoted shell-based hooks work better + # see #1173 + if os.altsep == '/' and os.sep == '\\': + filenames = [f.replace(os.sep, os.altsep) for f in filenames] + self.filenames = [f for f in filenames if os.path.lexists(f)] + + @functools.lru_cache(maxsize=None) + def _types_for_file(self, filename: str) -> Set[str]: + return tags_from_path(filename) + + def by_types( + self, + names: Sequence[str], + types: Collection[str], + exclude_types: Collection[str], + ) -> List[str]: + types, exclude_types = frozenset(types), frozenset(exclude_types) + ret = [] + for filename in names: + tags = self._types_for_file(filename) + if tags >= types and not tags & exclude_types: + ret.append(filename) + return ret + + def filenames_for_hook(self, hook: Hook) -> Tuple[str, ...]: + names = self.filenames + names = filter_by_include_exclude(names, hook.files, hook.exclude) + names = self.by_types(names, hook.types, hook.exclude_types) + return tuple(names) + + +def _get_skips(environ: EnvironT) -> Set[str]: skips = environ.get('SKIP', '') return {skip.strip() for skip in skips.split(',') if skip.strip()} -def _hook_msg_start(hook, verbose): - return '{}{}'.format( - '[{}] '.format(hook['id']) if verbose else '', - hook['name'], - ) - - -def get_changed_files(new, old): - return cmd_output( - 'git', 'diff', '--no-ext-diff', '--name-only', - '{}...{}'.format(old, new), - )[1].splitlines() - - -def filter_filenames_by_types(filenames, types, exclude_types): - types, exclude_types = frozenset(types), frozenset(exclude_types) - ret = [] - for filename in filenames: - tags = tags_from_path(filename) - if tags >= types and not tags & exclude_types: - ret.append(filename) - return tuple(ret) - - -def get_filenames(args, include_expr, exclude_expr): - if args.origin and args.source: - getter = git.get_files_matching( - lambda: get_changed_files(args.origin, args.source), - ) - elif args.files: - getter = git.get_files_matching(lambda: args.files) - elif args.all_files: - getter = git.get_all_files_matching - elif git.is_in_merge_conflict(): - getter = git.get_conflicted_files_matching - else: - getter = git.get_staged_files_matching - return getter(include_expr, exclude_expr) - - SKIPPED = 'Skipped' NO_FILES = '(no files to check)' -def _run_single_hook(hook, repo, args, skips, cols): - filenames = get_filenames(args, hook['files'], hook['exclude']) - filenames = filter_filenames_by_types( - filenames, hook['types'], hook['exclude_types'], - ) - if hook['id'] in skips: - output.write(get_hook_message( - _hook_msg_start(hook, args.verbose), - end_msg=SKIPPED, - end_color=color.YELLOW, - use_color=args.color, - cols=cols, - )) - return 0 - elif not filenames and not hook['always_run']: - output.write(get_hook_message( - _hook_msg_start(hook, args.verbose), - postfix=NO_FILES, - end_msg=SKIPPED, - end_color=color.TURQUOISE, - use_color=args.color, - cols=cols, - )) - return 0 - - # Print the hook and the dots first in case the hook takes hella long to - # run. - output.write(get_hook_message( - _hook_msg_start(hook, args.verbose), end_len=6, cols=cols, - )) - sys.stdout.flush() - - diff_before = cmd_output( - 'git', 'diff', '--no-ext-diff', retcode=None, encoding=None, - ) - retcode, stdout, stderr = repo.run_hook( - hook, - tuple(filenames) if hook['pass_filenames'] else (), - ) - diff_after = cmd_output( - 'git', 'diff', '--no-ext-diff', retcode=None, encoding=None, - ) - - file_modifications = diff_before != diff_after +def _subtle_line(s: str, use_color: bool) -> None: + output.write_line(color.format_color(s, color.SUBTLE, use_color)) + + +def _run_single_hook( + classifier: Classifier, + hook: Hook, + skips: Set[str], + cols: int, + verbose: bool, + use_color: bool, +) -> bool: + filenames = classifier.filenames_for_hook(hook) + + if hook.id in skips or hook.alias in skips: + output.write( + _full_msg( + start=hook.name, + end_msg=SKIPPED, + end_color=color.YELLOW, + use_color=use_color, + cols=cols, + ), + ) + duration = None + retcode = 0 + files_modified = False + out = b'' + elif not filenames and not hook.always_run: + output.write( + _full_msg( + start=hook.name, + postfix=NO_FILES, + end_msg=SKIPPED, + end_color=color.TURQUOISE, + use_color=use_color, + cols=cols, + ), + ) + duration = None + retcode = 0 + files_modified = False + out = b'' + else: + # print hook and dots first in case the hook takes a while to run + output.write(_start_msg(start=hook.name, end_len=6, cols=cols)) + + diff_cmd = ('git', 'diff', '--no-ext-diff') + diff_before = cmd_output_b(*diff_cmd, retcode=None) + if not hook.pass_filenames: + filenames = () + time_before = time.time() + language = languages[hook.language] + retcode, out = language.run_hook(hook, filenames, use_color) + duration = round(time.time() - time_before, 2) or 0 + diff_after = cmd_output_b(*diff_cmd, retcode=None) + + # if the hook makes changes, fail the commit + files_modified = diff_before != diff_after + + if retcode or files_modified: + print_color = color.RED + status = 'Failed' + else: + print_color = color.GREEN + status = 'Passed' - # If the hook makes changes, fail the commit - if file_modifications: - retcode = 1 + output.write_line(color.format_color(status, print_color, use_color)) - if retcode: - retcode = 1 - print_color = color.RED - pass_fail = 'Failed' - else: - retcode = 0 - print_color = color.GREEN - pass_fail = 'Passed' + if verbose or hook.verbose or retcode or files_modified: + _subtle_line(f'- hook id: {hook.id}', use_color) - output.write_line(color.format_color(pass_fail, print_color, args.color)) + if (verbose or hook.verbose) and duration is not None: + _subtle_line(f'- duration: {duration}s', use_color) - if (stdout or stderr or file_modifications) and (retcode or args.verbose): - output.write_line('hookid: {}\n'.format(hook['id'])) + if retcode: + _subtle_line(f'- exit code: {retcode}', use_color) # Print a message if failing due to file modifications - if file_modifications: - output.write('Files were modified by this hook.') - - if stdout or stderr: - output.write_line(' Additional output:') + if files_modified: + _subtle_line('- files were modified by this hook', use_color) + if out.strip(): + output.write_line() + output.write_line_b(out.strip(), logfile_name=hook.log_file) output.write_line() - for out in (stdout, stderr): - assert type(out) is bytes, type(out) - if out.strip(): - output.write_line(out.strip(), logfile_name=hook['log_file']) - output.write_line() - - return retcode + return files_modified or bool(retcode) -def _compute_cols(hooks, verbose): +def _compute_cols(hooks: Sequence[Hook]) -> int: """Compute the number of columns to display hook messages. The widest that will be displayed is in the no files skipped case: Hook name...(no files to check) Skipped - - or in the verbose case - - Hook name [hookid]...(no files to check) Skipped """ if hooks: - name_len = max(len(_hook_msg_start(hook, verbose)) for hook in hooks) + name_len = max(len(hook.name) for hook in hooks) else: name_len = 0 @@ -173,98 +214,147 @@ def _compute_cols(hooks, verbose): return max(cols, 80) -def _run_hooks(repo_hooks, args, environ): +def _all_filenames(args: argparse.Namespace) -> Collection[str]: + if args.hook_stage == 'post-checkout': # no files for post-checkout + return () + elif args.hook_stage in {'prepare-commit-msg', 'commit-msg'}: + return (args.commit_msg_filename,) + elif args.from_ref and args.to_ref: + return git.get_changed_files(args.from_ref, args.to_ref) + elif args.files: + return args.files + elif args.all_files: + return git.get_all_files() + elif git.is_in_merge_conflict(): + return git.get_conflicted_files() + else: + return git.get_staged_files() + + +def _run_hooks( + config: Dict[str, Any], + hooks: Sequence[Hook], + args: argparse.Namespace, + environ: EnvironT, +) -> int: """Actually run the hooks.""" skips = _get_skips(environ) - cols = _compute_cols([hook for _, hook in repo_hooks], args.verbose) + cols = _compute_cols(hooks) + filenames = filter_by_include_exclude( + _all_filenames(args), config['files'], config['exclude'], + ) + classifier = Classifier(filenames) retval = 0 - for repo, hook in repo_hooks: - retval |= _run_single_hook(hook, repo, args, skips, cols) - if ( - retval and - args.show_diff_on_failure and - subprocess.call(('git', 'diff', '--quiet', '--no-ext-diff')) != 0 - ): - print('All changes made by hooks:') - subprocess.call(('git', 'diff', '--no-ext-diff')) - return retval - + for hook in hooks: + retval |= _run_single_hook( + classifier, hook, skips, cols, + verbose=args.verbose, use_color=args.color, + ) + if retval and config['fail_fast']: + break + if retval and args.show_diff_on_failure and git.has_diff(): + if args.all_files: + output.write_line( + 'pre-commit hook(s) made changes.\n' + 'If you are seeing this message in CI, ' + 'reproduce locally with: `pre-commit run --all-files`.\n' + 'To run `pre-commit` as part of git workflow, use ' + '`pre-commit install`.', + ) + output.write_line('All changes made by hooks:') + # args.color is a boolean. + # See user_color function in color.py + git_color_opt = 'always' if args.color else 'never' + subprocess.call(( + 'git', '--no-pager', 'diff', '--no-ext-diff', + f'--color={git_color_opt}', + )) -def get_repo_hooks(runner): - for repo in runner.repositories: - for _, hook in repo.hooks: - yield (repo, hook) + return retval -def _has_unmerged_paths(runner): - _, stdout, _ = runner.cmd_runner.run(['git', 'ls-files', '--unmerged']) +def _has_unmerged_paths() -> bool: + _, stdout, _ = cmd_output_b('git', 'ls-files', '--unmerged') return bool(stdout.strip()) -def _has_unstaged_config(runner): - retcode, _, _ = runner.cmd_runner.run( - ( - 'git', 'diff', '--no-ext-diff', '--exit-code', - runner.config_file_path, - ), +def _has_unstaged_config(config_file: str) -> bool: + retcode, _, _ = cmd_output_b( + 'git', 'diff', '--no-ext-diff', '--exit-code', config_file, retcode=None, ) # be explicit, other git errors don't mean it has an unstaged config. return retcode == 1 -def run(runner, args, environ=os.environ): - no_stash = args.no_stash or args.all_files or bool(args.files) +def run( + config_file: str, + store: Store, + args: argparse.Namespace, + environ: EnvironT = os.environ, +) -> int: + stash = not args.all_files and not args.files # Check if we have unresolved merge conflict files and fail fast. - if _has_unmerged_paths(runner): + if _has_unmerged_paths(): logger.error('Unmerged files. Resolve before committing.') return 1 - if bool(args.source) != bool(args.origin): - logger.error('Specify both --origin and --source.') + if bool(args.from_ref) != bool(args.to_ref): + logger.error('Specify both --from-ref and --to-ref.') return 1 - if _has_unstaged_config(runner) and not no_stash: - if args.allow_unstaged_config: - logger.warn( - 'You have an unstaged config file and have specified the ' - '--allow-unstaged-config option.\n' - 'Note that your config will be stashed before the config is ' - 'parsed unless --no-stash is specified.', - ) - else: - logger.error( - 'Your .pre-commit-config.yaml is unstaged.\n' - '`git add .pre-commit-config.yaml` to fix this.\n' - 'Run pre-commit with --allow-unstaged-config to silence this.' + if stash and _has_unstaged_config(config_file): + logger.error( + f'Your pre-commit configuration is unstaged.\n' + f'`git add {config_file}` to fix this.', + ) + return 1 + if ( + args.hook_stage in {'prepare-commit-msg', 'commit-msg'} and + not args.commit_msg_filename + ): + logger.error( + f'`--commit-msg-filename` is required for ' + f'`--hook-stage {args.hook_stage}`', + ) + return 1 + + # Expose from-ref / to-ref as environment variables for hooks to consume + if args.from_ref and args.to_ref: + # legacy names + environ['PRE_COMMIT_ORIGIN'] = args.from_ref + environ['PRE_COMMIT_SOURCE'] = args.to_ref + # new names + environ['PRE_COMMIT_FROM_REF'] = args.from_ref + environ['PRE_COMMIT_TO_REF'] = args.to_ref + + if args.remote_name and args.remote_url: + environ['PRE_COMMIT_REMOTE_NAME'] = args.remote_name + environ['PRE_COMMIT_REMOTE_URL'] = args.remote_url + + if args.checkout_type: + environ['PRE_COMMIT_CHECKOUT_TYPE'] = args.checkout_type + + with contextlib.ExitStack() as exit_stack: + if stash: + exit_stack.enter_context(staged_files_only(store.directory)) + + config = load_config(config_file) + hooks = [ + hook + for hook in all_hooks(config, store) + if not args.hook or hook.id == args.hook or hook.alias == args.hook + if args.hook_stage in hook.stages + ] + + if args.hook and not hooks: + output.write_line( + f'No hook with id `{args.hook}` in stage `{args.hook_stage}`', ) return 1 - # Expose origin / source as environment variables for hooks to consume - if args.origin and args.source: - environ['PRE_COMMIT_ORIGIN'] = args.origin - environ['PRE_COMMIT_SOURCE'] = args.source + install_hook_envs(hooks, store) - if no_stash: - ctx = noop_context() - else: - ctx = staged_files_only(runner.cmd_runner) - - with ctx: - repo_hooks = list(get_repo_hooks(runner)) - - if args.hook: - repo_hooks = [ - (repo, hook) for repo, hook in repo_hooks - if hook['id'] == args.hook - ] - if not repo_hooks: - output.write_line('No hook with id `{}`'.format(args.hook)) - return 1 - - # Filter hooks for stages - repo_hooks = [ - (repo, hook) for repo, hook in repo_hooks - if not hook['stages'] or args.hook_stage in hook['stages'] - ] + return _run_hooks(config, hooks, args, environ) - return _run_hooks(repo_hooks, args, environ) + # https://github.com/python/mypy/issues/7726 + raise AssertionError('unreachable') diff --git a/pre_commit/commands/sample_config.py b/pre_commit/commands/sample_config.py index f38d655f0..d435faa8c 100644 --- a/pre_commit/commands/sample_config.py +++ b/pre_commit/commands/sample_config.py @@ -1,17 +1,13 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - - # TODO: maybe `git ls-remote git://github.com/pre-commit/pre-commit-hooks` to # determine the latest revision? This adds ~200ms from my tests (and is # significantly faster than https:// or http://). For now, periodically # manually updating the revision is fine. SAMPLE_CONFIG = '''\ -# See http://pre-commit.com for more information -# See http://pre-commit.com/hooks.html for more hooks +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: - repo: https://github.com/pre-commit/pre-commit-hooks - sha: v0.7.1 + rev: v2.4.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -20,6 +16,6 @@ ''' -def sample_config(): +def sample_config() -> int: print(SAMPLE_CONFIG, end='') return 0 diff --git a/pre_commit/commands/try_repo.py b/pre_commit/commands/try_repo.py new file mode 100644 index 000000000..4aee209c6 --- /dev/null +++ b/pre_commit/commands/try_repo.py @@ -0,0 +1,77 @@ +import argparse +import logging +import os.path +from typing import Optional +from typing import Tuple + +import pre_commit.constants as C +from pre_commit import git +from pre_commit import output +from pre_commit.clientlib import load_manifest +from pre_commit.commands.run import run +from pre_commit.store import Store +from pre_commit.util import cmd_output_b +from pre_commit.util import tmpdir +from pre_commit.util import yaml_dump +from pre_commit.xargs import xargs + +logger = logging.getLogger(__name__) + + +def _repo_ref(tmpdir: str, repo: str, ref: Optional[str]) -> Tuple[str, str]: + # if `ref` is explicitly passed, use it + if ref is not None: + return repo, ref + + ref = git.head_rev(repo) + # if it exists on disk, we'll try and clone it with the local changes + if os.path.exists(repo) and git.has_diff('HEAD', repo=repo): + logger.warning('Creating temporary repo with uncommitted changes...') + + shadow = os.path.join(tmpdir, 'shadow-repo') + cmd_output_b('git', 'clone', repo, shadow) + cmd_output_b('git', 'checkout', ref, '-b', '_pc_tmp', cwd=shadow) + + idx = git.git_path('index', repo=shadow) + objs = git.git_path('objects', repo=shadow) + env = dict(os.environ, GIT_INDEX_FILE=idx, GIT_OBJECT_DIRECTORY=objs) + + staged_files = git.get_staged_files(cwd=repo) + if staged_files: + xargs(('git', 'add', '--'), staged_files, cwd=repo, env=env) + + cmd_output_b('git', 'add', '-u', cwd=repo, env=env) + git.commit(repo=shadow) + + return shadow, git.head_rev(shadow) + else: + return repo, ref + + +def try_repo(args: argparse.Namespace) -> int: + with tmpdir() as tempdir: + repo, ref = _repo_ref(tempdir, args.repo, args.ref) + + store = Store(tempdir) + if args.hook: + hooks = [{'id': args.hook}] + else: + repo_path = store.clone(repo, ref) + manifest = load_manifest(os.path.join(repo_path, C.MANIFEST_FILE)) + manifest = sorted(manifest, key=lambda hook: hook['id']) + hooks = [{'id': hook['id']} for hook in manifest] + + config = {'repos': [{'repo': repo, 'rev': ref, 'hooks': hooks}]} + config_s = yaml_dump(config) + + config_filename = os.path.join(tempdir, C.CONFIG_FILE) + with open(config_filename, 'w') as cfg: + cfg.write(config_s) + + output.write_line('=' * 79) + output.write_line('Using config:') + output.write_line('=' * 79) + output.write(config_s) + output.write_line('=' * 79) + + return run(config_filename, store, args) diff --git a/pre_commit/constants.py b/pre_commit/constants.py index 3f81c8020..e2b8e3aca 100644 --- a/pre_commit/constants.py +++ b/pre_commit/constants.py @@ -1,24 +1,24 @@ -from __future__ import unicode_literals +import sys -import pkg_resources +if sys.version_info < (3, 8): # pragma: no cover ( str: return ''.join( - env.get(part.name, part.default) - if isinstance(part, Var) - else part + env.get(part.name, part.default) if isinstance(part, Var) else part for part in parts ) @contextlib.contextmanager -def envcontext(patch, _env=None): +def envcontext( + patch: PatchesT, + _env: Optional[EnvironT] = None, +) -> Generator[None, None, None]: """In this context, `os.environ` is modified according to `patch`. `patch` is an iterable of 2-tuples (key, value): diff --git a/pre_commit/error_handler.py b/pre_commit/error_handler.py index a661cc4fc..b2321ae0d 100644 --- a/pre_commit/error_handler.py +++ b/pre_commit/error_handler.py @@ -1,56 +1,64 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import contextlib +import functools import os.path +import sys import traceback +from typing import Generator -import six - -from pre_commit import five +import pre_commit.constants as C from pre_commit import output -from pre_commit.errors import FatalError from pre_commit.store import Store +from pre_commit.util import force_bytes -# For testing purposes -class PreCommitSystemExit(SystemExit): +class FatalError(RuntimeError): pass -def _to_bytes(exc): - try: - return bytes(exc) - except Exception: - return six.text_type(exc).encode('UTF-8') - - -def _log_and_exit(msg, exc, formatted): - error_msg = b''.join(( - five.to_bytes(msg), b': ', - five.to_bytes(type(exc).__name__), b': ', - _to_bytes(exc), b'\n', - )) - output.write(error_msg) - output.write_line('Check the log at ~/.pre-commit/pre-commit.log') - store = Store() - store.require_created() - with open(os.path.join(store.directory, 'pre-commit.log'), 'wb') as log: - output.write(error_msg, stream=log) - output.write_line(formatted, stream=log) - raise PreCommitSystemExit(1) +def _log_and_exit(msg: str, exc: BaseException, formatted: str) -> None: + error_msg = f'{msg}: {type(exc).__name__}: '.encode() + force_bytes(exc) + output.write_line_b(error_msg) + log_path = os.path.join(Store().directory, 'pre-commit.log') + output.write_line(f'Check the log at {log_path}') + + with open(log_path, 'wb') as log: + _log_line = functools.partial(output.write_line, stream=log) + _log_line_b = functools.partial(output.write_line_b, stream=log) + + _log_line('### version information') + _log_line() + _log_line('```') + _log_line(f'pre-commit version: {C.VERSION}') + _log_line('sys.version:') + for line in sys.version.splitlines(): + _log_line(f' {line}') + _log_line(f'sys.executable: {sys.executable}') + _log_line(f'os.name: {os.name}') + _log_line(f'sys.platform: {sys.platform}') + _log_line('```') + _log_line() + + _log_line('### error information') + _log_line() + _log_line('```') + _log_line_b(error_msg) + _log_line('```') + _log_line() + _log_line('```') + _log_line(formatted) + _log_line('```') + raise SystemExit(1) @contextlib.contextmanager -def error_handler(): +def error_handler() -> Generator[None, None, None]: try: yield - except FatalError as e: - _log_and_exit('An error has occurred', e, traceback.format_exc()) - except Exception as e: - _log_and_exit( - 'An unexpected error has occurred', - e, - traceback.format_exc(), - ) + except (Exception, KeyboardInterrupt) as e: + if isinstance(e, FatalError): + msg = 'An error has occurred' + elif isinstance(e, KeyboardInterrupt): + msg = 'Interrupted (^C)' + else: + msg = 'An unexpected error has occurred' + _log_and_exit(msg, e, traceback.format_exc()) diff --git a/pre_commit/errors.py b/pre_commit/errors.py deleted file mode 100644 index 4dedbfc2d..000000000 --- a/pre_commit/errors.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - - -class FatalError(RuntimeError): - pass diff --git a/pre_commit/file_lock.py b/pre_commit/file_lock.py new file mode 100644 index 000000000..ff0dc5e64 --- /dev/null +++ b/pre_commit/file_lock.py @@ -0,0 +1,76 @@ +import contextlib +import errno +import os +from typing import Callable +from typing import Generator + + +if os.name == 'nt': # pragma: no cover (windows) + import msvcrt + + # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/locking + + # on windows we lock "regions" of files, we don't care about the actual + # byte region so we'll just pick *some* number here. + _region = 0xffff + + @contextlib.contextmanager + def _locked( + fileno: int, + blocked_cb: Callable[[], None], + ) -> Generator[None, None, None]: + try: + # TODO: https://github.com/python/typeshed/pull/3607 + msvcrt.locking(fileno, msvcrt.LK_NBLCK, _region) # type: ignore + except OSError: + blocked_cb() + while True: + try: + # TODO: https://github.com/python/typeshed/pull/3607 + msvcrt.locking(fileno, msvcrt.LK_LOCK, _region) # type: ignore # noqa: E501 + except OSError as e: + # Locking violation. Returned when the _LK_LOCK or _LK_RLCK + # flag is specified and the file cannot be locked after 10 + # attempts. + if e.errno != errno.EDEADLOCK: + raise + else: + break + + try: + yield + finally: + # From cursory testing, it seems to get unlocked when the file is + # closed so this may not be necessary. + # The documentation however states: + # "Regions should be locked only briefly and should be unlocked + # before closing a file or exiting the program." + # TODO: https://github.com/python/typeshed/pull/3607 + msvcrt.locking(fileno, msvcrt.LK_UNLCK, _region) # type: ignore +else: # pragma: win32 no cover + import fcntl + + @contextlib.contextmanager + def _locked( + fileno: int, + blocked_cb: Callable[[], None], + ) -> Generator[None, None, None]: + try: + fcntl.flock(fileno, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: # pragma: no cover (tests are single-threaded) + blocked_cb() + fcntl.flock(fileno, fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(fileno, fcntl.LOCK_UN) + + +@contextlib.contextmanager +def lock( + path: str, + blocked_cb: Callable[[], None], +) -> Generator[None, None, None]: + with open(path, 'a+') as f: + with _locked(f.fileno(), blocked_cb): + yield diff --git a/pre_commit/five.py b/pre_commit/five.py deleted file mode 100644 index de017267f..000000000 --- a/pre_commit/five.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import unicode_literals - -import six - - -def to_text(s): - return s if isinstance(s, six.text_type) else s.decode('UTF-8') - - -def to_bytes(s): - return s if isinstance(s, bytes) else s.encode('UTF-8') - - -n = to_bytes if six.PY2 else to_text diff --git a/pre_commit/git.py b/pre_commit/git.py index 754514aa8..7e757f247 100644 --- a/pre_commit/git.py +++ b/pre_commit/git.py @@ -1,43 +1,67 @@ -from __future__ import unicode_literals - -import functools import logging import os.path -import re import sys +from typing import Dict +from typing import List +from typing import Optional +from typing import Set -from pre_commit.errors import FatalError -from pre_commit.util import CalledProcessError from pre_commit.util import cmd_output -from pre_commit.util import memoize_by_cwd - - -logger = logging.getLogger('pre_commit') +from pre_commit.util import cmd_output_b +from pre_commit.util import EnvironT + + +logger = logging.getLogger(__name__) + + +def zsplit(s: str) -> List[str]: + s = s.strip('\0') + if s: + return s.split('\0') + else: + return [] + + +def no_git_env(_env: Optional[EnvironT] = None) -> Dict[str, str]: + # Too many bugs dealing with environment variables and GIT: + # https://github.com/pre-commit/pre-commit/issues/300 + # In git 2.6.3 (maybe others), git exports GIT_WORK_TREE while running + # pre-commit hooks + # In git 1.9.1 (maybe others), git exports GIT_DIR and GIT_INDEX_FILE + # while running pre-commit hooks in submodules. + # GIT_DIR: Causes git clone to clone wrong thing + # GIT_INDEX_FILE: Causes 'error invalid object ...' during commit + _env = _env if _env is not None else os.environ + return { + k: v for k, v in _env.items() + if not k.startswith('GIT_') or + k in { + 'GIT_EXEC_PATH', 'GIT_SSH', 'GIT_SSH_COMMAND', 'GIT_SSL_CAINFO', + 'GIT_SSL_NO_VERIFY', + } + } -def get_root(): - try: - return cmd_output('git', 'rev-parse', '--show-toplevel')[1].strip() - except CalledProcessError: - raise FatalError( - 'git failed. Is it installed, and are you in a Git repository ' - 'directory?' - ) +def get_root() -> str: + return cmd_output('git', 'rev-parse', '--show-toplevel')[1].strip() -def get_git_dir(git_root): - return os.path.normpath(os.path.join( - git_root, - cmd_output('git', 'rev-parse', '--git-dir', cwd=git_root)[1].strip(), - )) +def get_git_dir(git_root: str = '.') -> str: + opts = ('--git-common-dir', '--git-dir') + _, out, _ = cmd_output('git', 'rev-parse', *opts, cwd=git_root) + for line, opt in zip(out.splitlines(), opts): + if line != opt: # pragma: no branch (git < 2.5) + return os.path.normpath(os.path.join(git_root, line)) + else: + raise AssertionError('unreachable: no git dir') -def get_remote_url(git_root): - ret = cmd_output('git', 'config', 'remote.origin.url', cwd=git_root)[1] - return ret.strip() +def get_remote_url(git_root: str) -> str: + _, out, _ = cmd_output('git', 'config', 'remote.origin.url', cwd=git_root) + return out.strip() -def is_in_merge_conflict(): +def is_in_merge_conflict() -> bool: git_dir = get_git_dir('.') return ( os.path.exists(os.path.join(git_dir, 'MERGE_MSG')) and @@ -45,73 +69,114 @@ def is_in_merge_conflict(): ) -def parse_merge_msg_for_conflicts(merge_msg): +def parse_merge_msg_for_conflicts(merge_msg: bytes) -> List[str]: # Conflicted files start with tabs return [ - line.lstrip(b'#').strip().decode('UTF-8') + line.lstrip(b'#').strip().decode() for line in merge_msg.splitlines() # '#\t' for git 2.4.1 if line.startswith((b'\t', b'#\t')) ] -@memoize_by_cwd -def get_conflicted_files(): +def get_conflicted_files() -> Set[str]: logger.info('Checking merge-conflict files only.') # Need to get the conflicted files from the MERGE_MSG because they could # have resolved the conflict by choosing one side or the other - merge_msg = open(os.path.join(get_git_dir('.'), 'MERGE_MSG'), 'rb').read() + with open(os.path.join(get_git_dir('.'), 'MERGE_MSG'), 'rb') as f: + merge_msg = f.read() merge_conflict_filenames = parse_merge_msg_for_conflicts(merge_msg) # This will get the rest of the changes made after the merge. # If they resolved the merge conflict by choosing a mesh of both sides # this will also include the conflicted files tree_hash = cmd_output('git', 'write-tree')[1].strip() - merge_diff_filenames = cmd_output( - 'git', 'diff', '--no-ext-diff', - '-m', tree_hash, 'HEAD', 'MERGE_HEAD', '--name-only', - )[1].splitlines() + merge_diff_filenames = zsplit( + cmd_output( + 'git', 'diff', '--name-only', '--no-ext-diff', '-z', + '-m', tree_hash, 'HEAD', 'MERGE_HEAD', + )[1], + ) return set(merge_conflict_filenames) | set(merge_diff_filenames) -@memoize_by_cwd -def get_staged_files(): - return cmd_output( - 'git', 'diff', '--staged', '--name-only', '--no-ext-diff', - # Everything except for D - '--diff-filter=ACMRTUXB' - )[1].splitlines() - - -@memoize_by_cwd -def get_all_files(): - return cmd_output('git', 'ls-files')[1].splitlines() - - -def get_files_matching(all_file_list_strategy): - @functools.wraps(all_file_list_strategy) - @memoize_by_cwd - def wrapper(include_expr, exclude_expr): - include_regex = re.compile(include_expr) - exclude_regex = re.compile(exclude_expr) - return { - filename - for filename in all_file_list_strategy() - if ( - include_regex.search(filename) and - not exclude_regex.search(filename) and - os.path.lexists(filename) - ) - } - return wrapper +def get_staged_files(cwd: Optional[str] = None) -> List[str]: + return zsplit( + cmd_output( + 'git', 'diff', '--staged', '--name-only', '--no-ext-diff', '-z', + # Everything except for D + '--diff-filter=ACMRTUXB', + cwd=cwd, + )[1], + ) + + +def intent_to_add_files() -> List[str]: + _, stdout, _ = cmd_output('git', 'status', '--porcelain', '-z') + parts = list(reversed(zsplit(stdout))) + intent_to_add = [] + while parts: + line = parts.pop() + status, filename = line[:3], line[3:] + if status[0] in {'C', 'R'}: # renames / moves have an additional arg + parts.pop() + if status[1] == 'A': + intent_to_add.append(filename) + return intent_to_add + + +def get_all_files() -> List[str]: + return zsplit(cmd_output('git', 'ls-files', '-z')[1]) + + +def get_changed_files(old: str, new: str) -> List[str]: + return zsplit( + cmd_output( + 'git', 'diff', '--name-only', '--no-ext-diff', '-z', + f'{old}...{new}', + )[1], + ) + + +def head_rev(remote: str) -> str: + _, out, _ = cmd_output('git', 'ls-remote', '--exit-code', remote, 'HEAD') + return out.split()[0] + + +def has_diff(*args: str, repo: str = '.') -> bool: + cmd = ('git', 'diff', '--quiet', '--no-ext-diff', *args) + return cmd_output_b(*cmd, cwd=repo, retcode=None)[0] == 1 + + +def has_core_hookpaths_set() -> bool: + _, out, _ = cmd_output_b('git', 'config', 'core.hooksPath', retcode=None) + return bool(out.strip()) + + +def init_repo(path: str, remote: str) -> None: + if os.path.isdir(remote): + remote = os.path.abspath(remote) + + env = no_git_env() + cmd_output_b('git', 'init', path, env=env) + cmd_output_b('git', 'remote', 'add', 'origin', remote, cwd=path, env=env) + + +def commit(repo: str = '.') -> None: + env = no_git_env() + name, email = 'pre-commit', 'asottile+pre-commit@umich.edu' + env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = name + env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = email + cmd = ('git', 'commit', '--no-edit', '--no-gpg-sign', '-n', '-minit') + cmd_output_b(*cmd, cwd=repo, env=env) -get_staged_files_matching = get_files_matching(get_staged_files) -get_all_files_matching = get_files_matching(get_all_files) -get_conflicted_files_matching = get_files_matching(get_conflicted_files) +def git_path(name: str, repo: str = '.') -> str: + _, out, _ = cmd_output('git', 'rev-parse', '--git-path', name, cwd=repo) + return os.path.join(repo, out.strip()) -def check_for_cygwin_mismatch(): +def check_for_cygwin_mismatch() -> None: """See https://github.com/pre-commit/pre-commit/issues/354""" if sys.platform in ('cygwin', 'win32'): # pragma: no cover (windows) is_cygwin_python = sys.platform == 'cygwin' @@ -121,14 +186,11 @@ def check_for_cygwin_mismatch(): if is_cygwin_python ^ is_cygwin_git: exe_type = {True: '(cygwin)', False: '(windows)'} logger.warn( - 'pre-commit has detected a mix of cygwin python / git\n' - 'This combination is not supported, it is likely you will ' - 'receive an error later in the program.\n' - 'Make sure to use cygwin git+python while using cygwin\n' - 'These can be installed through the cygwin installer.\n' - ' - python {}\n' - ' - git {}\n'.format( - exe_type[is_cygwin_python], - exe_type[is_cygwin_git], - ) + f'pre-commit has detected a mix of cygwin python / git\n' + f'This combination is not supported, it is likely you will ' + f'receive an error later in the program.\n' + f'Make sure to use cygwin git+python while using cygwin\n' + f'These can be installed through the cygwin installer.\n' + f' - python {exe_type[is_cygwin_python]}\n' + f' - git {exe_type[is_cygwin_git]}\n', ) diff --git a/pre_commit/hook.py b/pre_commit/hook.py new file mode 100644 index 000000000..b65ac42b0 --- /dev/null +++ b/pre_commit/hook.py @@ -0,0 +1,63 @@ +import logging +import shlex +from typing import Any +from typing import Dict +from typing import NamedTuple +from typing import Sequence +from typing import Tuple + +from pre_commit.prefix import Prefix + +logger = logging.getLogger('pre_commit') + + +class Hook(NamedTuple): + src: str + prefix: Prefix + id: str + name: str + entry: str + language: str + alias: str + files: str + exclude: str + types: Sequence[str] + exclude_types: Sequence[str] + additional_dependencies: Sequence[str] + args: Sequence[str] + always_run: bool + pass_filenames: bool + description: str + language_version: str + log_file: str + minimum_pre_commit_version: str + require_serial: bool + stages: Sequence[str] + verbose: bool + + @property + def cmd(self) -> Tuple[str, ...]: + return (*shlex.split(self.entry), *self.args) + + @property + def install_key(self) -> Tuple[Prefix, str, str, Tuple[str, ...]]: + return ( + self.prefix, + self.language, + self.language_version, + tuple(self.additional_dependencies), + ) + + @classmethod + def create(cls, src: str, prefix: Prefix, dct: Dict[str, Any]) -> 'Hook': + # TODO: have cfgv do this (?) + extra_keys = set(dct) - _KEYS + if extra_keys: + logger.warning( + f'Unexpected key(s) present on {src} => {dct["id"]}: ' + f'{", ".join(sorted(extra_keys))}', + ) + return cls(src=src, prefix=prefix, **{k: dct[k] for k in _KEYS}) + + +_KEYS = frozenset(set(Hook._fields) - {'src', 'prefix'}) diff --git a/pre_commit/languages/all.py b/pre_commit/languages/all.py index f441ddd27..8f4ffa8c5 100644 --- a/pre_commit/languages/all.py +++ b/pre_commit/languages/all.py @@ -1,58 +1,60 @@ -from __future__ import unicode_literals +from typing import Callable +from typing import NamedTuple +from typing import Optional +from typing import Sequence +from typing import Tuple +from pre_commit.hook import Hook +from pre_commit.languages import conda from pre_commit.languages import docker +from pre_commit.languages import docker_image +from pre_commit.languages import fail from pre_commit.languages import golang from pre_commit.languages import node -from pre_commit.languages import pcre +from pre_commit.languages import perl +from pre_commit.languages import pygrep from pre_commit.languages import python +from pre_commit.languages import python_venv from pre_commit.languages import ruby +from pre_commit.languages import rust from pre_commit.languages import script from pre_commit.languages import swift from pre_commit.languages import system +from pre_commit.prefix import Prefix -# A language implements the following constant and two functions in its module: -# -# # Use None for no environment -# ENVIRONMENT_DIR = 'foo_env' -# -# def install_environment( -# repo_cmd_runner, -# version='default', -# additional_dependencies=(), -# ): -# """Installs a repository in the given repository. Note that the current -# working directory will already be inside the repository. -# -# Args: -# repo_cmd_runner - `PrefixedCommandRunner` bound to the repository. -# version - A version specified in the hook configuration or -# 'default'. -# """ -# -# def run_hook(repo_cmd_runner, hook, file_args): -# """Runs a hook and returns the returncode and output of running that -# hook. -# -# Args: -# repo_cmd_runner - `PrefixedCommandRunner` bound to the repository. -# hook - Hook dictionary -# file_args - The files to be run -# -# Returns: -# (returncode, stdout, stderr) -# """ -languages = { - 'docker': docker, - 'golang': golang, - 'node': node, - 'pcre': pcre, - 'python': python, - 'ruby': ruby, - 'script': script, - 'swift': swift, - 'system': system, -} +class Language(NamedTuple): + name: str + # Use `None` for no installation / environment + ENVIRONMENT_DIR: Optional[str] + # return a value to replace `'default` for `language_version` + get_default_version: Callable[[], str] + # return whether the environment is healthy (or should be rebuilt) + healthy: Callable[[Prefix, str], bool] + # install a repository for the given language and language_version + install_environment: Callable[[Prefix, str, Sequence[str]], None] + # execute a hook and return the exit code and output + run_hook: 'Callable[[Hook, Sequence[str], bool], Tuple[int, bytes]]' -all_languages = languages.keys() +# TODO: back to modules + Protocol: https://github.com/python/mypy/issues/5018 +languages = { + # BEGIN GENERATED (testing/gen-languages-all) + 'conda': Language(name='conda', ENVIRONMENT_DIR=conda.ENVIRONMENT_DIR, get_default_version=conda.get_default_version, healthy=conda.healthy, install_environment=conda.install_environment, run_hook=conda.run_hook), # noqa: E501 + 'docker': Language(name='docker', ENVIRONMENT_DIR=docker.ENVIRONMENT_DIR, get_default_version=docker.get_default_version, healthy=docker.healthy, install_environment=docker.install_environment, run_hook=docker.run_hook), # noqa: E501 + 'docker_image': Language(name='docker_image', ENVIRONMENT_DIR=docker_image.ENVIRONMENT_DIR, get_default_version=docker_image.get_default_version, healthy=docker_image.healthy, install_environment=docker_image.install_environment, run_hook=docker_image.run_hook), # noqa: E501 + 'fail': Language(name='fail', ENVIRONMENT_DIR=fail.ENVIRONMENT_DIR, get_default_version=fail.get_default_version, healthy=fail.healthy, install_environment=fail.install_environment, run_hook=fail.run_hook), # noqa: E501 + 'golang': Language(name='golang', ENVIRONMENT_DIR=golang.ENVIRONMENT_DIR, get_default_version=golang.get_default_version, healthy=golang.healthy, install_environment=golang.install_environment, run_hook=golang.run_hook), # noqa: E501 + 'node': Language(name='node', ENVIRONMENT_DIR=node.ENVIRONMENT_DIR, get_default_version=node.get_default_version, healthy=node.healthy, install_environment=node.install_environment, run_hook=node.run_hook), # noqa: E501 + 'perl': Language(name='perl', ENVIRONMENT_DIR=perl.ENVIRONMENT_DIR, get_default_version=perl.get_default_version, healthy=perl.healthy, install_environment=perl.install_environment, run_hook=perl.run_hook), # noqa: E501 + 'pygrep': Language(name='pygrep', ENVIRONMENT_DIR=pygrep.ENVIRONMENT_DIR, get_default_version=pygrep.get_default_version, healthy=pygrep.healthy, install_environment=pygrep.install_environment, run_hook=pygrep.run_hook), # noqa: E501 + 'python': Language(name='python', ENVIRONMENT_DIR=python.ENVIRONMENT_DIR, get_default_version=python.get_default_version, healthy=python.healthy, install_environment=python.install_environment, run_hook=python.run_hook), # noqa: E501 + 'python_venv': Language(name='python_venv', ENVIRONMENT_DIR=python_venv.ENVIRONMENT_DIR, get_default_version=python_venv.get_default_version, healthy=python_venv.healthy, install_environment=python_venv.install_environment, run_hook=python_venv.run_hook), # noqa: E501 + 'ruby': Language(name='ruby', ENVIRONMENT_DIR=ruby.ENVIRONMENT_DIR, get_default_version=ruby.get_default_version, healthy=ruby.healthy, install_environment=ruby.install_environment, run_hook=ruby.run_hook), # noqa: E501 + 'rust': Language(name='rust', ENVIRONMENT_DIR=rust.ENVIRONMENT_DIR, get_default_version=rust.get_default_version, healthy=rust.healthy, install_environment=rust.install_environment, run_hook=rust.run_hook), # noqa: E501 + 'script': Language(name='script', ENVIRONMENT_DIR=script.ENVIRONMENT_DIR, get_default_version=script.get_default_version, healthy=script.healthy, install_environment=script.install_environment, run_hook=script.run_hook), # noqa: E501 + 'swift': Language(name='swift', ENVIRONMENT_DIR=swift.ENVIRONMENT_DIR, get_default_version=swift.get_default_version, healthy=swift.healthy, install_environment=swift.install_environment, run_hook=swift.run_hook), # noqa: E501 + 'system': Language(name='system', ENVIRONMENT_DIR=system.ENVIRONMENT_DIR, get_default_version=system.get_default_version, healthy=system.healthy, install_environment=system.install_environment, run_hook=system.run_hook), # noqa: E501 + # END GENERATED +} +all_languages = sorted(languages) diff --git a/pre_commit/languages/conda.py b/pre_commit/languages/conda.py new file mode 100644 index 000000000..071757a1f --- /dev/null +++ b/pre_commit/languages/conda.py @@ -0,0 +1,84 @@ +import contextlib +import os +from typing import Generator +from typing import Sequence +from typing import Tuple + +from pre_commit.envcontext import envcontext +from pre_commit.envcontext import PatchesT +from pre_commit.envcontext import SubstitutionT +from pre_commit.envcontext import UNSET +from pre_commit.envcontext import Var +from pre_commit.hook import Hook +from pre_commit.languages import helpers +from pre_commit.prefix import Prefix +from pre_commit.util import clean_path_on_failure +from pre_commit.util import cmd_output_b + +ENVIRONMENT_DIR = 'conda' +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy + + +def get_env_patch(env: str) -> PatchesT: + # On non-windows systems executable live in $CONDA_PREFIX/bin, on Windows + # they can be in $CONDA_PREFIX/bin, $CONDA_PREFIX/Library/bin, + # $CONDA_PREFIX/Scripts and $CONDA_PREFIX. Whereas the latter only + # seems to be used for python.exe. + path: SubstitutionT = (os.path.join(env, 'bin'), os.pathsep, Var('PATH')) + if os.name == 'nt': # pragma: no cover (platform specific) + path = (env, os.pathsep, *path) + path = (os.path.join(env, 'Scripts'), os.pathsep, *path) + path = (os.path.join(env, 'Library', 'bin'), os.pathsep, *path) + + return ( + ('PYTHONHOME', UNSET), + ('VIRTUAL_ENV', UNSET), + ('CONDA_PREFIX', env), + ('PATH', path), + ) + + +@contextlib.contextmanager +def in_env( + prefix: Prefix, + language_version: str, +) -> Generator[None, None, None]: + directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version) + envdir = prefix.path(directory) + with envcontext(get_env_patch(envdir)): + yield + + +def install_environment( + prefix: Prefix, + version: str, + additional_dependencies: Sequence[str], +) -> None: + helpers.assert_version_default('conda', version) + directory = helpers.environment_dir(ENVIRONMENT_DIR, version) + + env_dir = prefix.path(directory) + with clean_path_on_failure(env_dir): + cmd_output_b( + 'conda', 'env', 'create', '-p', env_dir, '--file', + 'environment.yml', cwd=prefix.prefix_dir, + ) + if additional_dependencies: + cmd_output_b( + 'conda', 'install', '-p', env_dir, *additional_dependencies, + cwd=prefix.prefix_dir, + ) + + +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: + # TODO: Some rare commands need to be run using `conda run` but mostly we + # can run them withot which is much quicker and produces a better + # output. + # cmd = ('conda', 'run', '-p', env_dir) + hook.cmd + with in_env(hook.prefix, hook.language_version): + return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/languages/docker.py b/pre_commit/languages/docker.py index 7d3f8d041..f4495847d 100644 --- a/pre_commit/languages/docker.py +++ b/pre_commit/languages/docker.py @@ -1,99 +1,114 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import hashlib import os +from typing import Sequence +from typing import Tuple -from pre_commit import five +import pre_commit.constants as C +from pre_commit.hook import Hook from pre_commit.languages import helpers +from pre_commit.prefix import Prefix from pre_commit.util import CalledProcessError from pre_commit.util import clean_path_on_failure -from pre_commit.util import cmd_output -from pre_commit.xargs import xargs - +from pre_commit.util import cmd_output_b ENVIRONMENT_DIR = 'docker' PRE_COMMIT_LABEL = 'PRE_COMMIT' +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy -def md5(s): # pragma: windows no cover - return hashlib.md5(five.to_bytes(s)).hexdigest() +def md5(s: str) -> str: # pragma: win32 no cover + return hashlib.md5(s.encode()).hexdigest() -def docker_tag(repo_cmd_runner): # pragma: windows no cover - return 'pre-commit-{}'.format( - md5(os.path.basename(repo_cmd_runner.path())) - ).lower() +def docker_tag(prefix: Prefix) -> str: # pragma: win32 no cover + md5sum = md5(os.path.basename(prefix.prefix_dir)).lower() + return f'pre-commit-{md5sum}' -def docker_is_running(): # pragma: windows no cover +def docker_is_running() -> bool: # pragma: win32 no cover try: - return cmd_output('docker', 'ps')[0] == 0 + cmd_output_b('docker', 'ps') except CalledProcessError: return False + else: + return True -def assert_docker_available(): # pragma: windows no cover +def assert_docker_available() -> None: # pragma: win32 no cover assert docker_is_running(), ( 'Docker is either not running or not configured in this environment' ) -def build_docker_image(repo_cmd_runner, **kwargs): # pragma: windows no cover - pull = kwargs.pop('pull') - assert not kwargs, kwargs - cmd = ( +def build_docker_image( + prefix: Prefix, + *, + pull: bool, +) -> None: # pragma: win32 no cover + cmd: Tuple[str, ...] = ( 'docker', 'build', - '--tag', docker_tag(repo_cmd_runner), + '--tag', docker_tag(prefix), '--label', PRE_COMMIT_LABEL, ) if pull: cmd += ('--pull',) # This must come last for old versions of docker. See #477 cmd += ('.',) - helpers.run_setup_cmd(repo_cmd_runner, cmd) + helpers.run_setup_cmd(prefix, cmd) def install_environment( - repo_cmd_runner, - version='default', - additional_dependencies=(), -): # pragma: windows no cover - assert repo_cmd_runner.exists('Dockerfile'), ( - 'No Dockerfile was found in the hook repository' - ) + prefix: Prefix, version: str, additional_dependencies: Sequence[str], +) -> None: # pragma: win32 no cover helpers.assert_version_default('docker', version) helpers.assert_no_additional_deps('docker', additional_dependencies) assert_docker_available() - directory = repo_cmd_runner.path( - helpers.environment_dir(ENVIRONMENT_DIR, 'default'), + directory = prefix.path( + helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), ) # Docker doesn't really have relevant disk environment, but pre-commit - # still needs to cleanup it's state files on failure + # still needs to cleanup its state files on failure with clean_path_on_failure(directory): - build_docker_image(repo_cmd_runner, pull=True) + build_docker_image(prefix, pull=True) os.mkdir(directory) -def run_hook(repo_cmd_runner, hook, file_args): # pragma: windows no cover - assert_docker_available() - # Rebuild the docker image in case it has gone missing, as many people do - # automated cleanup of docker images. - build_docker_image(repo_cmd_runner, pull=False) +def get_docker_user() -> str: # pragma: win32 no cover + try: + return f'{os.getuid()}:{os.getgid()}' + except AttributeError: + return '1000:1000' - hook_cmd = helpers.to_cmd(hook) - entry_executable, cmd_rest = hook_cmd[0], hook_cmd[1:] - cmd = ( +def docker_cmd() -> Tuple[str, ...]: # pragma: win32 no cover + return ( 'docker', 'run', '--rm', - '-u', '{}:{}'.format(os.getuid(), os.getgid()), - '-v', '{}:/src:rw'.format(os.getcwd()), + '-u', get_docker_user(), + # https://docs.docker.com/engine/reference/commandline/run/#mount-volumes-from-container-volumes-from + # The `Z` option tells Docker to label the content with a private + # unshared label. Only the current container can use a private volume. + '-v', f'{os.getcwd()}:/src:rw,Z', '--workdir', '/src', - '--entrypoint', entry_executable, - docker_tag(repo_cmd_runner) - ) + cmd_rest + ) + + +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: # pragma: win32 no cover + assert_docker_available() + # Rebuild the docker image in case it has gone missing, as many people do + # automated cleanup of docker images. + build_docker_image(hook.prefix, pull=False) + + hook_cmd = hook.cmd + entry_exe, cmd_rest = hook.cmd[0], hook_cmd[1:] - return xargs(cmd, file_args) + entry_tag = ('--entrypoint', entry_exe, docker_tag(hook.prefix)) + cmd = docker_cmd() + entry_tag + cmd_rest + return helpers.run_xargs(hook, cmd, file_args, color=color) diff --git a/pre_commit/languages/docker_image.py b/pre_commit/languages/docker_image.py new file mode 100644 index 000000000..0c51df628 --- /dev/null +++ b/pre_commit/languages/docker_image.py @@ -0,0 +1,22 @@ +from typing import Sequence +from typing import Tuple + +from pre_commit.hook import Hook +from pre_commit.languages import helpers +from pre_commit.languages.docker import assert_docker_available +from pre_commit.languages.docker import docker_cmd + +ENVIRONMENT_DIR = None +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy +install_environment = helpers.no_install + + +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: # pragma: win32 no cover + assert_docker_available() + cmd = docker_cmd() + hook.cmd + return helpers.run_xargs(hook, cmd, file_args, color=color) diff --git a/pre_commit/languages/fail.py b/pre_commit/languages/fail.py new file mode 100644 index 000000000..d2b02d23e --- /dev/null +++ b/pre_commit/languages/fail.py @@ -0,0 +1,20 @@ +from typing import Sequence +from typing import Tuple + +from pre_commit.hook import Hook +from pre_commit.languages import helpers + +ENVIRONMENT_DIR = None +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy +install_environment = helpers.no_install + + +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: + out = f'{hook.entry}\n\n'.encode() + out += b'\n'.join(f.encode() for f in file_args) + b'\n' + return 1, out diff --git a/pre_commit/languages/golang.py b/pre_commit/languages/golang.py index c0bfbcbcb..91ade1e99 100644 --- a/pre_commit/languages/golang.py +++ b/pre_commit/languages/golang.py @@ -1,41 +1,51 @@ -from __future__ import unicode_literals - import contextlib import os.path +import sys +from typing import Generator +from typing import Sequence +from typing import Tuple +import pre_commit.constants as C from pre_commit import git from pre_commit.envcontext import envcontext +from pre_commit.envcontext import PatchesT from pre_commit.envcontext import Var +from pre_commit.hook import Hook from pre_commit.languages import helpers +from pre_commit.prefix import Prefix from pre_commit.util import clean_path_on_failure from pre_commit.util import cmd_output +from pre_commit.util import cmd_output_b from pre_commit.util import rmtree -from pre_commit.xargs import xargs - ENVIRONMENT_DIR = 'golangenv' +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy -def get_env_patch(venv): +def get_env_patch(venv: str) -> PatchesT: return ( ('PATH', (os.path.join(venv, 'bin'), os.pathsep, Var('PATH'))), ) @contextlib.contextmanager -def in_env(repo_cmd_runner): - envdir = repo_cmd_runner.path( - helpers.environment_dir(ENVIRONMENT_DIR, 'default'), +def in_env(prefix: Prefix) -> Generator[None, None, None]: + envdir = prefix.path( + helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), ) with envcontext(get_env_patch(envdir)): yield -def guess_go_dir(remote_url): +def guess_go_dir(remote_url: str) -> str: if remote_url.endswith('.git'): remote_url = remote_url[:-1 * len('.git')] + looks_like_url = ( + not remote_url.startswith('file://') and + ('//' in remote_url or '@' in remote_url) + ) remote_url = remote_url.replace(':', '/') - looks_like_url = '//' in remote_url or '@' in remote_url if looks_like_url: _, _, remote_url = remote_url.rpartition('//') _, _, remote_url = remote_url.rpartition('@') @@ -45,33 +55,43 @@ def guess_go_dir(remote_url): def install_environment( - repo_cmd_runner, - version='default', - additional_dependencies=(), -): + prefix: Prefix, + version: str, + additional_dependencies: Sequence[str], +) -> None: helpers.assert_version_default('golang', version) - directory = repo_cmd_runner.path( - helpers.environment_dir(ENVIRONMENT_DIR, 'default'), + directory = prefix.path( + helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), ) with clean_path_on_failure(directory): - remote = git.get_remote_url(repo_cmd_runner.path()) + remote = git.get_remote_url(prefix.prefix_dir) repo_src_dir = os.path.join(directory, 'src', guess_go_dir(remote)) # Clone into the goenv we'll create - helpers.run_setup_cmd( - repo_cmd_runner, ('git', 'clone', '.', repo_src_dir), - ) - - env = dict(os.environ, GOPATH=directory) - cmd_output('go', 'get', './...', cwd=repo_src_dir, env=env) + helpers.run_setup_cmd(prefix, ('git', 'clone', '.', repo_src_dir)) + + if sys.platform == 'cygwin': # pragma: no cover + _, gopath, _ = cmd_output('cygpath', '-w', directory) + gopath = gopath.strip() + else: + gopath = directory + env = dict(os.environ, GOPATH=gopath) + env.pop('GOBIN', None) + cmd_output_b('go', 'get', './...', cwd=repo_src_dir, env=env) for dependency in additional_dependencies: - cmd_output('go', 'get', dependency, cwd=repo_src_dir, env=env) + cmd_output_b('go', 'get', dependency, cwd=repo_src_dir, env=env) # Same some disk space, we don't need these after installation - rmtree(repo_cmd_runner.path(directory, 'src')) - rmtree(repo_cmd_runner.path(directory, 'pkg')) - - -def run_hook(repo_cmd_runner, hook, file_args): - with in_env(repo_cmd_runner): - return xargs(helpers.to_cmd(hook), file_args) + rmtree(prefix.path(directory, 'src')) + pkgdir = prefix.path(directory, 'pkg') + if os.path.exists(pkgdir): # pragma: no cover (go<1.10) + rmtree(pkgdir) + + +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: + with in_env(hook.prefix): + return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/languages/helpers.py b/pre_commit/languages/helpers.py index a6c93de18..b5c95e522 100644 --- a/pre_commit/languages/helpers.py +++ b/pre_commit/languages/helpers.py @@ -1,35 +1,109 @@ -from __future__ import unicode_literals +import multiprocessing +import os +import random +from typing import Any +from typing import List +from typing import Optional +from typing import overload +from typing import Sequence +from typing import Tuple +from typing import TYPE_CHECKING -import shlex +import pre_commit.constants as C +from pre_commit.hook import Hook +from pre_commit.prefix import Prefix +from pre_commit.util import cmd_output_b +from pre_commit.xargs import xargs -from pre_commit.util import cmd_output +if TYPE_CHECKING: + from typing import NoReturn +FIXED_RANDOM_SEED = 1542676186 -def run_setup_cmd(runner, cmd): - cmd_output(*cmd, cwd=runner.prefix_dir, encoding=None) +def run_setup_cmd(prefix: Prefix, cmd: Tuple[str, ...]) -> None: + cmd_output_b(*cmd, cwd=prefix.prefix_dir) -def environment_dir(ENVIRONMENT_DIR, language_version): - if ENVIRONMENT_DIR is None: - return None - else: - return '{}-{}'.format(ENVIRONMENT_DIR, language_version) + +@overload +def environment_dir(d: None, language_version: str) -> None: ... +@overload +def environment_dir(d: str, language_version: str) -> str: ... -def to_cmd(hook): - return tuple(shlex.split(hook['entry'])) + tuple(hook['args']) +def environment_dir(d: Optional[str], language_version: str) -> Optional[str]: + if d is None: + return None + else: + return f'{d}-{language_version}' -def assert_version_default(binary, version): - if version != 'default': +def assert_version_default(binary: str, version: str) -> None: + if version != C.DEFAULT: raise AssertionError( - 'For now, pre-commit requires system-installed {}'.format(binary), + f'For now, pre-commit requires system-installed {binary}', ) -def assert_no_additional_deps(lang, additional_deps): +def assert_no_additional_deps( + lang: str, + additional_deps: Sequence[str], +) -> None: if additional_deps: raise AssertionError( - 'For now, pre-commit does not support ' - 'additional_dependencies for {}'.format(lang), + f'For now, pre-commit does not support ' + f'additional_dependencies for {lang}', ) + + +def basic_get_default_version() -> str: + return C.DEFAULT + + +def basic_healthy(prefix: Prefix, language_version: str) -> bool: + return True + + +def no_install( + prefix: Prefix, + version: str, + additional_dependencies: Sequence[str], +) -> 'NoReturn': + raise AssertionError('This type is not installable') + + +def target_concurrency(hook: Hook) -> int: + if hook.require_serial or 'PRE_COMMIT_NO_CONCURRENCY' in os.environ: + return 1 + else: + # Travis appears to have a bunch of CPUs, but we can't use them all. + if 'TRAVIS' in os.environ: + return 2 + else: + try: + return multiprocessing.cpu_count() + except NotImplementedError: + return 1 + + +def _shuffled(seq: Sequence[str]) -> List[str]: + """Deterministically shuffle""" + fixed_random = random.Random() + fixed_random.seed(FIXED_RANDOM_SEED, version=1) + + seq = list(seq) + random.shuffle(seq, random=fixed_random.random) + return seq + + +def run_xargs( + hook: Hook, + cmd: Tuple[str, ...], + file_args: Sequence[str], + **kwargs: Any, +) -> Tuple[int, bytes]: + # Shuffle the files so that they more evenly fill out the xargs partitions, + # but do it deterministically in case a hook cares about ordering. + file_args = _shuffled(file_args) + kwargs['target_concurrency'] = target_concurrency(hook) + return xargs(cmd, file_args, **kwargs) diff --git a/pre_commit/languages/node.py b/pre_commit/languages/node.py index ef557a16b..79ff807a5 100644 --- a/pre_commit/languages/node.py +++ b/pre_commit/languages/node.py @@ -1,66 +1,93 @@ -from __future__ import unicode_literals - import contextlib import os import sys +from typing import Generator +from typing import Sequence +from typing import Tuple +import pre_commit.constants as C from pre_commit.envcontext import envcontext +from pre_commit.envcontext import PatchesT from pre_commit.envcontext import Var +from pre_commit.hook import Hook from pre_commit.languages import helpers +from pre_commit.languages.python import bin_dir +from pre_commit.prefix import Prefix from pre_commit.util import clean_path_on_failure -from pre_commit.xargs import xargs - +from pre_commit.util import cmd_output +from pre_commit.util import cmd_output_b ENVIRONMENT_DIR = 'node_env' +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy -def get_env_patch(venv): # pragma: windows no cover +def _envdir(prefix: Prefix, version: str) -> str: + directory = helpers.environment_dir(ENVIRONMENT_DIR, version) + return prefix.path(directory) + + +def get_env_patch(venv: str) -> PatchesT: + if sys.platform == 'cygwin': # pragma: no cover + _, win_venv, _ = cmd_output('cygpath', '-w', venv) + install_prefix = fr'{win_venv.strip()}\bin' + lib_dir = 'lib' + elif sys.platform == 'win32': # pragma: no cover + install_prefix = bin_dir(venv) + lib_dir = 'Scripts' + else: # pragma: win32 no cover + install_prefix = venv + lib_dir = 'lib' return ( ('NODE_VIRTUAL_ENV', venv), - ('NPM_CONFIG_PREFIX', venv), - ('npm_config_prefix', venv), - ('NODE_PATH', os.path.join(venv, 'lib', 'node_modules')), - ('PATH', (os.path.join(venv, 'bin'), os.pathsep, Var('PATH'))), + ('NPM_CONFIG_PREFIX', install_prefix), + ('npm_config_prefix', install_prefix), + ('NODE_PATH', os.path.join(venv, lib_dir, 'node_modules')), + ('PATH', (bin_dir(venv), os.pathsep, Var('PATH'))), ) @contextlib.contextmanager -def in_env(repo_cmd_runner, language_version): # pragma: windows no cover - envdir = repo_cmd_runner.path( - helpers.environment_dir(ENVIRONMENT_DIR, language_version), - ) - with envcontext(get_env_patch(envdir)): +def in_env( + prefix: Prefix, + language_version: str, +) -> Generator[None, None, None]: + with envcontext(get_env_patch(_envdir(prefix, language_version))): yield def install_environment( - repo_cmd_runner, - version='default', - additional_dependencies=(), -): # pragma: windows no cover + prefix: Prefix, version: str, additional_dependencies: Sequence[str], +) -> None: additional_dependencies = tuple(additional_dependencies) - assert repo_cmd_runner.exists('package.json') - directory = helpers.environment_dir(ENVIRONMENT_DIR, version) + assert prefix.exists('package.json') + envdir = _envdir(prefix, version) - env_dir = repo_cmd_runner.path(directory) - with clean_path_on_failure(env_dir): + # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx?f=255&MSPPError=-2147217396#maxpath + if sys.platform == 'win32': # pragma: no cover + envdir = f'\\\\?\\{os.path.normpath(envdir)}' + with clean_path_on_failure(envdir): cmd = [ - sys.executable, '-m', 'nodeenv', '--prebuilt', - '{{prefix}}{}'.format(directory), + sys.executable, '-mnodeenv', '--prebuilt', '--clean-src', envdir, ] - - if version != 'default': + if version != C.DEFAULT: cmd.extend(['-n', version]) + cmd_output_b(*cmd) - repo_cmd_runner.run(cmd) - - with in_env(repo_cmd_runner, version): + with in_env(prefix, version): + # https://npm.community/t/npm-install-g-git-vs-git-clone-cd-npm-install-g/5449 + # install as if we installed from git + helpers.run_setup_cmd(prefix, ('npm', 'install')) helpers.run_setup_cmd( - repo_cmd_runner, - ('npm', 'install', '-g', '.') + additional_dependencies, + prefix, + ('npm', 'install', '-g', '.', *additional_dependencies), ) -def run_hook(repo_cmd_runner, hook, file_args): # pragma: windows no cover - with in_env(repo_cmd_runner, hook['language_version']): - return xargs(helpers.to_cmd(hook), file_args) +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: + with in_env(hook.prefix, hook.language_version): + return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/languages/pcre.py b/pre_commit/languages/pcre.py deleted file mode 100644 index 314ea090d..000000000 --- a/pre_commit/languages/pcre.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import unicode_literals - -import sys - -from pre_commit.xargs import xargs - - -ENVIRONMENT_DIR = None -GREP = 'ggrep' if sys.platform == 'darwin' else 'grep' - - -def install_environment( - repo_cmd_runner, - version='default', - additional_dependencies=(), -): - """Installation for pcre type is a noop.""" - raise AssertionError('Cannot install pcre repo.') - - -def run_hook(repo_cmd_runner, hook, file_args): - # For PCRE the entry is the regular expression to match - cmd = (GREP, '-H', '-n', '-P') + tuple(hook['args']) + (hook['entry'],) - - # Grep usually returns 0 for matches, and nonzero for non-matches so we - # negate it here. - return xargs(cmd, file_args, negate=True) diff --git a/pre_commit/languages/perl.py b/pre_commit/languages/perl.py new file mode 100644 index 000000000..bbf550494 --- /dev/null +++ b/pre_commit/languages/perl.py @@ -0,0 +1,67 @@ +import contextlib +import os +import shlex +from typing import Generator +from typing import Sequence +from typing import Tuple + +from pre_commit.envcontext import envcontext +from pre_commit.envcontext import PatchesT +from pre_commit.envcontext import Var +from pre_commit.hook import Hook +from pre_commit.languages import helpers +from pre_commit.prefix import Prefix +from pre_commit.util import clean_path_on_failure + +ENVIRONMENT_DIR = 'perl_env' +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy + + +def _envdir(prefix: Prefix, version: str) -> str: + directory = helpers.environment_dir(ENVIRONMENT_DIR, version) + return prefix.path(directory) + + +def get_env_patch(venv: str) -> PatchesT: + return ( + ('PATH', (os.path.join(venv, 'bin'), os.pathsep, Var('PATH'))), + ('PERL5LIB', os.path.join(venv, 'lib', 'perl5')), + ('PERL_MB_OPT', f'--install_base {shlex.quote(venv)}'), + ( + 'PERL_MM_OPT', ( + f'INSTALL_BASE={shlex.quote(venv)} ' + f'INSTALLSITEMAN1DIR=none INSTALLSITEMAN3DIR=none' + ), + ), + ) + + +@contextlib.contextmanager +def in_env( + prefix: Prefix, + language_version: str, +) -> Generator[None, None, None]: + with envcontext(get_env_patch(_envdir(prefix, language_version))): + yield + + +def install_environment( + prefix: Prefix, version: str, additional_dependencies: Sequence[str], +) -> None: + helpers.assert_version_default('perl', version) + + with clean_path_on_failure(_envdir(prefix, version)): + with in_env(prefix, version): + helpers.run_setup_cmd( + prefix, ('cpan', '-T', '.', *additional_dependencies), + ) + + +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: + with in_env(hook.prefix, hook.language_version): + return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/languages/pygrep.py b/pre_commit/languages/pygrep.py new file mode 100644 index 000000000..40adba0f7 --- /dev/null +++ b/pre_commit/languages/pygrep.py @@ -0,0 +1,87 @@ +import argparse +import re +import sys +from typing import Optional +from typing import Pattern +from typing import Sequence +from typing import Tuple + +from pre_commit import output +from pre_commit.hook import Hook +from pre_commit.languages import helpers +from pre_commit.xargs import xargs + +ENVIRONMENT_DIR = None +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy +install_environment = helpers.no_install + + +def _process_filename_by_line(pattern: Pattern[bytes], filename: str) -> int: + retv = 0 + with open(filename, 'rb') as f: + for line_no, line in enumerate(f, start=1): + if pattern.search(line): + retv = 1 + output.write(f'{filename}:{line_no}:') + output.write_line_b(line.rstrip(b'\r\n')) + return retv + + +def _process_filename_at_once(pattern: Pattern[bytes], filename: str) -> int: + retv = 0 + with open(filename, 'rb') as f: + contents = f.read() + match = pattern.search(contents) + if match: + retv = 1 + line_no = contents[:match.start()].count(b'\n') + output.write(f'{filename}:{line_no + 1}:') + + matched_lines = match[0].split(b'\n') + matched_lines[0] = contents.split(b'\n')[line_no] + + output.write_line_b(b'\n'.join(matched_lines)) + return retv + + +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: + exe = (sys.executable, '-m', __name__) + tuple(hook.args) + (hook.entry,) + return xargs(exe, file_args, color=color) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser( + description=( + 'grep-like finder using python regexes. Unlike grep, this tool ' + 'returns nonzero when it finds a match and zero otherwise. The ' + 'idea here being that matches are "problems".' + ), + ) + parser.add_argument('-i', '--ignore-case', action='store_true') + parser.add_argument('--multiline', action='store_true') + parser.add_argument('pattern', help='python regex pattern.') + parser.add_argument('filenames', nargs='*') + args = parser.parse_args(argv) + + flags = re.IGNORECASE if args.ignore_case else 0 + if args.multiline: + flags |= re.MULTILINE | re.DOTALL + + pattern = re.compile(args.pattern.encode(), flags) + + retv = 0 + for filename in args.filenames: + if args.multiline: + retv |= _process_filename_at_once(pattern, filename) + else: + retv |= _process_filename_by_line(pattern, filename) + return retv + + +if __name__ == '__main__': + exit(main()) diff --git a/pre_commit/languages/python.py b/pre_commit/languages/python.py index 634abe587..5073a8bce 100644 --- a/pre_commit/languages/python.py +++ b/pre_commit/languages/python.py @@ -1,28 +1,38 @@ -from __future__ import unicode_literals - import contextlib -import distutils.spawn +import functools import os import sys - +from typing import Callable +from typing import ContextManager +from typing import Generator +from typing import Optional +from typing import Sequence +from typing import Tuple + +import pre_commit.constants as C from pre_commit.envcontext import envcontext +from pre_commit.envcontext import PatchesT from pre_commit.envcontext import UNSET from pre_commit.envcontext import Var +from pre_commit.hook import Hook from pre_commit.languages import helpers +from pre_commit.parse_shebang import find_executable +from pre_commit.prefix import Prefix +from pre_commit.util import CalledProcessError from pre_commit.util import clean_path_on_failure -from pre_commit.xargs import xargs - +from pre_commit.util import cmd_output +from pre_commit.util import cmd_output_b ENVIRONMENT_DIR = 'py_env' -def bin_dir(venv): +def bin_dir(venv: str) -> str: """On windows there's a different directory for the virtualenv""" bin_part = 'Scripts' if os.name == 'nt' else 'bin' return os.path.join(venv, bin_part) -def get_env_patch(venv): +def get_env_patch(venv: str) -> PatchesT: return ( ('PYTHONHOME', UNSET), ('VIRTUAL_ENV', venv), @@ -30,57 +40,171 @@ def get_env_patch(venv): ) -@contextlib.contextmanager -def in_env(repo_cmd_runner, language_version): - envdir = repo_cmd_runner.path( - helpers.environment_dir(ENVIRONMENT_DIR, language_version), - ) - with envcontext(get_env_patch(envdir)): - yield - +def _find_by_py_launcher( + version: str, +) -> Optional[str]: # pragma: no cover (windows only) + if version.startswith('python'): + num = version[len('python'):] + try: + cmd = ('py', f'-{num}', '-c', 'import sys; print(sys.executable)') + return cmd_output(*cmd)[1].strip() + except CalledProcessError: + pass + return None + + +def _find_by_sys_executable() -> Optional[str]: + def _norm(path: str) -> Optional[str]: + _, exe = os.path.split(path.lower()) + exe, _, _ = exe.partition('.exe') + if exe not in {'python', 'pythonw'} and find_executable(exe): + return exe + return None + + # On linux, I see these common sys.executables: + # + # system `python`: /usr/bin/python -> python2.7 + # system `python2`: /usr/bin/python2 -> python2.7 + # virtualenv v: v/bin/python (will not return from this loop) + # virtualenv v -ppython2: v/bin/python -> python2 + # virtualenv v -ppython2.7: v/bin/python -> python2.7 + # virtualenv v -ppypy: v/bin/python -> v/bin/pypy + for path in (sys.executable, os.path.realpath(sys.executable)): + exe = _norm(path) + if exe: + return exe + return None + + +@functools.lru_cache(maxsize=1) +def get_default_version() -> str: # pragma: no cover (platform dependent) + # First attempt from `sys.executable` (or the realpath) + exe = _find_by_sys_executable() + if exe: + return exe + + # Next try the `pythonX.X` executable + exe = f'python{sys.version_info[0]}.{sys.version_info[1]}' + if find_executable(exe): + return exe + + if _find_by_py_launcher(exe): + return exe + + # Give a best-effort try for windows + default_folder_name = exe.replace('.', '') + if os.path.exists(fr'C:\{default_folder_name}\python.exe'): + return exe + + # We tried! + return C.DEFAULT + + +def _sys_executable_matches(version: str) -> bool: + if version == 'python': + return True + elif not version.startswith('python'): + return False + + try: + info = tuple(int(p) for p in version[len('python'):].split('.')) + except ValueError: + return False + + return sys.version_info[:len(info)] == info + + +def norm_version(version: str) -> str: + # first see if our current executable is appropriate + if _sys_executable_matches(version): + return sys.executable -def norm_version(version): if os.name == 'nt': # pragma: no cover (windows) + version_exec = _find_by_py_launcher(version) + if version_exec: + return version_exec + # Try looking up by name - if distutils.spawn.find_executable(version): - return version + version_exec = find_executable(version) + if version_exec and version_exec != version: + return version_exec # If it is in the form pythonx.x search in the default # place on windows if version.startswith('python'): - return r'C:\{}\python.exe'.format(version.replace('.', '')) + default_folder_name = version.replace('.', '') + return fr'C:\{default_folder_name}\python.exe' - # Otherwise assume it is a path + # Otherwise assume it is a path return os.path.expanduser(version) -def install_environment( - repo_cmd_runner, - version='default', - additional_dependencies=(), -): - additional_dependencies = tuple(additional_dependencies) - directory = helpers.environment_dir(ENVIRONMENT_DIR, version) - - # Install a virtualenv - with clean_path_on_failure(repo_cmd_runner.path(directory)): - venv_cmd = [ - sys.executable, '-m', 'virtualenv', - '{{prefix}}{}'.format(directory) - ] - if version != 'default': - venv_cmd.extend(['-p', norm_version(version)]) - else: - venv_cmd.extend(['-p', os.path.realpath(sys.executable)]) - venv_env = dict(os.environ, VIRTUALENV_NO_DOWNLOAD='1') - repo_cmd_runner.run(venv_cmd, cwd='/', env=venv_env) - with in_env(repo_cmd_runner, version): - helpers.run_setup_cmd( - repo_cmd_runner, - ('pip', 'install', '.') + additional_dependencies, +def py_interface( + _dir: str, + _make_venv: Callable[[str, str], None], +) -> Tuple[ + Callable[[Prefix, str], ContextManager[None]], + Callable[[Prefix, str], bool], + Callable[[Hook, Sequence[str], bool], Tuple[int, bytes]], + Callable[[Prefix, str, Sequence[str]], None], +]: + @contextlib.contextmanager + def in_env( + prefix: Prefix, + language_version: str, + ) -> Generator[None, None, None]: + envdir = prefix.path(helpers.environment_dir(_dir, language_version)) + with envcontext(get_env_patch(envdir)): + yield + + def healthy(prefix: Prefix, language_version: str) -> bool: + envdir = helpers.environment_dir(_dir, language_version) + exe_name = 'python.exe' if sys.platform == 'win32' else 'python' + py_exe = prefix.path(bin_dir(envdir), exe_name) + with in_env(prefix, language_version): + retcode, _, _ = cmd_output_b( + py_exe, '-c', 'import ctypes, datetime, io, os, ssl, weakref', + cwd='/', + retcode=None, ) - - -def run_hook(repo_cmd_runner, hook, file_args): - with in_env(repo_cmd_runner, hook['language_version']): - return xargs(helpers.to_cmd(hook), file_args) + return retcode == 0 + + def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, + ) -> Tuple[int, bytes]: + with in_env(hook.prefix, hook.language_version): + return helpers.run_xargs(hook, hook.cmd, file_args, color=color) + + def install_environment( + prefix: Prefix, + version: str, + additional_dependencies: Sequence[str], + ) -> None: + additional_dependencies = tuple(additional_dependencies) + directory = helpers.environment_dir(_dir, version) + + env_dir = prefix.path(directory) + with clean_path_on_failure(env_dir): + if version != C.DEFAULT: + python = norm_version(version) + else: + python = os.path.realpath(sys.executable) + _make_venv(env_dir, python) + with in_env(prefix, version): + helpers.run_setup_cmd( + prefix, ('pip', 'install', '.') + additional_dependencies, + ) + + return in_env, healthy, run_hook, install_environment + + +def make_venv(envdir: str, python: str) -> None: + env = dict(os.environ, VIRTUALENV_NO_DOWNLOAD='1') + cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python) + cmd_output_b(*cmd, env=env, cwd='/') + + +_interface = py_interface(ENVIRONMENT_DIR, make_venv) +in_env, healthy, run_hook, install_environment = _interface diff --git a/pre_commit/languages/python_venv.py b/pre_commit/languages/python_venv.py new file mode 100644 index 000000000..5404c8be5 --- /dev/null +++ b/pre_commit/languages/python_venv.py @@ -0,0 +1,46 @@ +import os.path + +from pre_commit.languages import python +from pre_commit.util import CalledProcessError +from pre_commit.util import cmd_output +from pre_commit.util import cmd_output_b + +ENVIRONMENT_DIR = 'py_venv' +get_default_version = python.get_default_version + + +def orig_py_exe(exe: str) -> str: # pragma: no cover (platform specific) + """A -mvenv virtualenv made from a -mvirtualenv virtualenv installs + packages to the incorrect location. Attempt to find the _original_ exe + and invoke `-mvenv` from there. + + See: + - https://github.com/pre-commit/pre-commit/issues/755 + - https://github.com/pypa/virtualenv/issues/1095 + - https://bugs.python.org/issue30811 + """ + try: + prefix_script = 'import sys; print(sys.real_prefix)' + _, prefix, _ = cmd_output(exe, '-c', prefix_script) + prefix = prefix.strip() + except CalledProcessError: + # not created from -mvirtualenv + return exe + + if os.name == 'nt': + expected = os.path.join(prefix, 'python.exe') + else: + expected = os.path.join(prefix, 'bin', os.path.basename(exe)) + + if os.path.exists(expected): + return expected + else: + return exe + + +def make_venv(envdir: str, python: str) -> None: + cmd_output_b(orig_py_exe(python), '-mvenv', envdir, cwd='/') + + +_interface = python.py_interface(ENVIRONMENT_DIR, make_venv) +in_env, healthy, run_hook, install_environment = _interface diff --git a/pre_commit/languages/ruby.py b/pre_commit/languages/ruby.py index d3896d904..61241f855 100644 --- a/pre_commit/languages/ruby.py +++ b/pre_commit/languages/ruby.py @@ -1,133 +1,126 @@ -from __future__ import unicode_literals - import contextlib -import io import os.path import shutil import tarfile +from typing import Generator +from typing import Sequence +from typing import Tuple +import pre_commit.constants as C from pre_commit.envcontext import envcontext +from pre_commit.envcontext import PatchesT from pre_commit.envcontext import Var +from pre_commit.hook import Hook from pre_commit.languages import helpers +from pre_commit.prefix import Prefix from pre_commit.util import CalledProcessError from pre_commit.util import clean_path_on_failure -from pre_commit.util import resource_filename -from pre_commit.xargs import xargs - +from pre_commit.util import resource_bytesio ENVIRONMENT_DIR = 'rbenv' +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy -def get_env_patch(venv, language_version): # pragma: windows no cover - patches = ( +def get_env_patch( + venv: str, + language_version: str, +) -> PatchesT: # pragma: win32 no cover + patches: PatchesT = ( ('GEM_HOME', os.path.join(venv, 'gems')), ('RBENV_ROOT', venv), ('BUNDLE_IGNORE_CONFIG', '1'), - ('PATH', ( - os.path.join(venv, 'gems', 'bin'), os.pathsep, - os.path.join(venv, 'shims'), os.pathsep, - os.path.join(venv, 'bin'), os.pathsep, Var('PATH'), - )), + ( + 'PATH', ( + os.path.join(venv, 'gems', 'bin'), os.pathsep, + os.path.join(venv, 'shims'), os.pathsep, + os.path.join(venv, 'bin'), os.pathsep, Var('PATH'), + ), + ), ) - if language_version != 'default': + if language_version != C.DEFAULT: patches += (('RBENV_VERSION', language_version),) return patches -@contextlib.contextmanager -def in_env(repo_cmd_runner, language_version): # pragma: windows no cover - envdir = repo_cmd_runner.path( +@contextlib.contextmanager # pragma: win32 no cover +def in_env( + prefix: Prefix, + language_version: str, +) -> Generator[None, None, None]: + envdir = prefix.path( helpers.environment_dir(ENVIRONMENT_DIR, language_version), ) with envcontext(get_env_patch(envdir, language_version)): yield +def _extract_resource(filename: str, dest: str) -> None: + with resource_bytesio(filename) as bio: + with tarfile.open(fileobj=bio) as tf: + tf.extractall(dest) + + def _install_rbenv( - repo_cmd_runner, version='default', -): # pragma: windows no cover + prefix: Prefix, + version: str = C.DEFAULT, +) -> None: # pragma: win32 no cover directory = helpers.environment_dir(ENVIRONMENT_DIR, version) - with tarfile.open(resource_filename('rbenv.tar.gz')) as tf: - tf.extractall(repo_cmd_runner.path('.')) - shutil.move( - repo_cmd_runner.path('rbenv'), repo_cmd_runner.path(directory), - ) + _extract_resource('rbenv.tar.gz', prefix.path('.')) + shutil.move(prefix.path('rbenv'), prefix.path(directory)) # Only install ruby-build if the version is specified - if version != 'default': - # ruby-download - with tarfile.open(resource_filename('ruby-download.tar.gz')) as tf: - tf.extractall(repo_cmd_runner.path(directory, 'plugins')) - - # ruby-build - with tarfile.open(resource_filename('ruby-build.tar.gz')) as tf: - tf.extractall(repo_cmd_runner.path(directory, 'plugins')) - - activate_path = repo_cmd_runner.path(directory, 'bin', 'activate') - with io.open(activate_path, 'w') as activate_file: - # This is similar to how you would install rbenv to your home directory - # However we do a couple things to make the executables exposed and - # configure it to work in our directory. - # We also modify the PS1 variable for manual debugging sake. - activate_file.write( - '#!/usr/bin/env bash\n' - "export RBENV_ROOT='{directory}'\n" - 'export PATH="$RBENV_ROOT/bin:$PATH"\n' - 'eval "$(rbenv init -)"\n' - 'export PS1="(rbenv)$PS1"\n' - # This lets us install gems in an isolated and repeatable - # directory - "export GEM_HOME='{directory}/gems'\n" - 'export PATH="$GEM_HOME/bin:$PATH"\n' - '\n'.format(directory=repo_cmd_runner.path(directory)) - ) - - # If we aren't using the system ruby, add a version here - if version != 'default': - activate_file.write('export RBENV_VERSION="{}"\n'.format(version)) - - -def _install_ruby(runner, version): # pragma: windows no cover + if version != C.DEFAULT: + plugins_dir = prefix.path(directory, 'plugins') + _extract_resource('ruby-download.tar.gz', plugins_dir) + _extract_resource('ruby-build.tar.gz', plugins_dir) + + +def _install_ruby( + prefix: Prefix, + version: str, +) -> None: # pragma: win32 no cover try: - helpers.run_setup_cmd(runner, ('rbenv', 'download', version)) + helpers.run_setup_cmd(prefix, ('rbenv', 'download', version)) except CalledProcessError: # pragma: no cover (usually find with download) # Failed to download from mirror for some reason, build it instead - helpers.run_setup_cmd(runner, ('rbenv', 'install', version)) + helpers.run_setup_cmd(prefix, ('rbenv', 'install', version)) def install_environment( - repo_cmd_runner, - version='default', - additional_dependencies=(), -): # pragma: windows no cover + prefix: Prefix, version: str, additional_dependencies: Sequence[str], +) -> None: # pragma: win32 no cover additional_dependencies = tuple(additional_dependencies) directory = helpers.environment_dir(ENVIRONMENT_DIR, version) - with clean_path_on_failure(repo_cmd_runner.path(directory)): + with clean_path_on_failure(prefix.path(directory)): # TODO: this currently will fail if there's no version specified and # there's no system ruby installed. Is this ok? - _install_rbenv(repo_cmd_runner, version=version) - with in_env(repo_cmd_runner, version): + _install_rbenv(prefix, version=version) + with in_env(prefix, version): # Need to call this before installing so rbenv's directories are # set up - helpers.run_setup_cmd(repo_cmd_runner, ('rbenv', 'init', '-')) - if version != 'default': - _install_ruby(repo_cmd_runner, version) + helpers.run_setup_cmd(prefix, ('rbenv', 'init', '-')) + if version != C.DEFAULT: + _install_ruby(prefix, version) # Need to call this after installing to set up the shims - helpers.run_setup_cmd(repo_cmd_runner, ('rbenv', 'rehash')) + helpers.run_setup_cmd(prefix, ('rbenv', 'rehash')) helpers.run_setup_cmd( - repo_cmd_runner, - ('gem', 'build') + repo_cmd_runner.star('.gemspec'), + prefix, ('gem', 'build', *prefix.star('.gemspec')), ) helpers.run_setup_cmd( - repo_cmd_runner, + prefix, ( - ('gem', 'install', '--no-ri', '--no-rdoc') + - repo_cmd_runner.star('.gem') + additional_dependencies + 'gem', 'install', '--no-document', + *prefix.star('.gem'), *additional_dependencies, ), ) -def run_hook(repo_cmd_runner, hook, file_args): # pragma: windows no cover - with in_env(repo_cmd_runner, hook['language_version']): - return xargs(helpers.to_cmd(hook), file_args) +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: # pragma: win32 no cover + with in_env(hook.prefix, hook.language_version): + return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/languages/rust.py b/pre_commit/languages/rust.py new file mode 100644 index 000000000..7ea3f5406 --- /dev/null +++ b/pre_commit/languages/rust.py @@ -0,0 +1,106 @@ +import contextlib +import os.path +from typing import Generator +from typing import Sequence +from typing import Set +from typing import Tuple + +import toml + +import pre_commit.constants as C +from pre_commit.envcontext import envcontext +from pre_commit.envcontext import PatchesT +from pre_commit.envcontext import Var +from pre_commit.hook import Hook +from pre_commit.languages import helpers +from pre_commit.prefix import Prefix +from pre_commit.util import clean_path_on_failure +from pre_commit.util import cmd_output_b + +ENVIRONMENT_DIR = 'rustenv' +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy + + +def get_env_patch(target_dir: str) -> PatchesT: + return ( + ('PATH', (os.path.join(target_dir, 'bin'), os.pathsep, Var('PATH'))), + ) + + +@contextlib.contextmanager +def in_env(prefix: Prefix) -> Generator[None, None, None]: + target_dir = prefix.path( + helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), + ) + with envcontext(get_env_patch(target_dir)): + yield + + +def _add_dependencies( + cargo_toml_path: str, + additional_dependencies: Set[str], +) -> None: + with open(cargo_toml_path, 'r+') as f: + cargo_toml = toml.load(f) + cargo_toml.setdefault('dependencies', {}) + for dep in additional_dependencies: + name, _, spec = dep.partition(':') + cargo_toml['dependencies'][name] = spec or '*' + f.seek(0) + toml.dump(cargo_toml, f) + f.truncate() + + +def install_environment( + prefix: Prefix, + version: str, + additional_dependencies: Sequence[str], +) -> None: + helpers.assert_version_default('rust', version) + directory = prefix.path( + helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), + ) + + # There are two cases where we might want to specify more dependencies: + # as dependencies for the library being built, and as binary packages + # to be `cargo install`'d. + # + # Unlike e.g. Python, if we just `cargo install` a library, it won't be + # used for compilation. And if we add a crate providing a binary to the + # `Cargo.toml`, the binary won't be built. + # + # Because of this, we allow specifying "cli" dependencies by prefixing + # with 'cli:'. + cli_deps = { + dep for dep in additional_dependencies if dep.startswith('cli:') + } + lib_deps = set(additional_dependencies) - cli_deps + + if len(lib_deps) > 0: + _add_dependencies(prefix.path('Cargo.toml'), lib_deps) + + with clean_path_on_failure(directory): + packages_to_install: Set[Tuple[str, ...]] = {('--path', '.')} + for cli_dep in cli_deps: + cli_dep = cli_dep[len('cli:'):] + package, _, version = cli_dep.partition(':') + if version != '': + packages_to_install.add((package, '--version', version)) + else: + packages_to_install.add((package,)) + + for args in packages_to_install: + cmd_output_b( + 'cargo', 'install', '--bins', '--root', directory, *args, + cwd=prefix.prefix_dir, + ) + + +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: + with in_env(hook.prefix): + return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/languages/script.py b/pre_commit/languages/script.py index 762ae7634..a5e1365c0 100644 --- a/pre_commit/languages/script.py +++ b/pre_commit/languages/script.py @@ -1,22 +1,19 @@ -from __future__ import unicode_literals +from typing import Sequence +from typing import Tuple +from pre_commit.hook import Hook from pre_commit.languages import helpers -from pre_commit.xargs import xargs - ENVIRONMENT_DIR = None +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy +install_environment = helpers.no_install -def install_environment( - repo_cmd_runner, - version='default', - additional_dependencies=(), -): - """Installation for script type is a noop.""" - raise AssertionError('Cannot install script repo.') - - -def run_hook(repo_cmd_runner, hook, file_args): - cmd = helpers.to_cmd(hook) - cmd = (repo_cmd_runner.prefix_dir + cmd[0],) + cmd[1:] - return xargs(cmd, file_args) +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: + cmd = (hook.prefix.path(hook.cmd[0]), *hook.cmd[1:]) + return helpers.run_xargs(hook, cmd, file_args, color=color) diff --git a/pre_commit/languages/swift.py b/pre_commit/languages/swift.py index 4d171c5be..66aadc8b2 100644 --- a/pre_commit/languages/swift.py +++ b/pre_commit/languages/swift.py @@ -1,55 +1,64 @@ -from __future__ import unicode_literals - import contextlib import os +from typing import Generator +from typing import Sequence +from typing import Tuple +import pre_commit.constants as C from pre_commit.envcontext import envcontext +from pre_commit.envcontext import PatchesT from pre_commit.envcontext import Var +from pre_commit.hook import Hook from pre_commit.languages import helpers +from pre_commit.prefix import Prefix from pre_commit.util import clean_path_on_failure -from pre_commit.xargs import xargs +from pre_commit.util import cmd_output_b ENVIRONMENT_DIR = 'swift_env' +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy BUILD_DIR = '.build' BUILD_CONFIG = 'release' -def get_env_patch(venv): # pragma: windows no cover +def get_env_patch(venv: str) -> PatchesT: # pragma: win32 no cover bin_path = os.path.join(venv, BUILD_DIR, BUILD_CONFIG) return (('PATH', (bin_path, os.pathsep, Var('PATH'))),) -@contextlib.contextmanager -def in_env(repo_cmd_runner): # pragma: windows no cover - envdir = repo_cmd_runner.path( - helpers.environment_dir(ENVIRONMENT_DIR, 'default'), +@contextlib.contextmanager # pragma: win32 no cover +def in_env(prefix: Prefix) -> Generator[None, None, None]: + envdir = prefix.path( + helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), ) with envcontext(get_env_patch(envdir)): yield def install_environment( - repo_cmd_runner, - version='default', - additional_dependencies=(), -): # pragma: windows no cover + prefix: Prefix, version: str, additional_dependencies: Sequence[str], +) -> None: # pragma: win32 no cover helpers.assert_version_default('swift', version) helpers.assert_no_additional_deps('swift', additional_dependencies) - directory = repo_cmd_runner.path( - helpers.environment_dir(ENVIRONMENT_DIR, 'default'), + directory = prefix.path( + helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), ) # Build the swift package with clean_path_on_failure(directory): os.mkdir(directory) - repo_cmd_runner.run(( + cmd_output_b( 'swift', 'build', - '-C', '{prefix}', + '-C', prefix.prefix_dir, '-c', BUILD_CONFIG, '--build-path', os.path.join(directory, BUILD_DIR), - )) + ) -def run_hook(repo_cmd_runner, hook, file_args): # pragma: windows no cover - with in_env(repo_cmd_runner): - return xargs(helpers.to_cmd(hook), file_args) +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: # pragma: win32 no cover + with in_env(hook.prefix): + return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/languages/system.py b/pre_commit/languages/system.py index c9e1c5dca..139f45d13 100644 --- a/pre_commit/languages/system.py +++ b/pre_commit/languages/system.py @@ -1,20 +1,19 @@ -from __future__ import unicode_literals +from typing import Sequence +from typing import Tuple +from pre_commit.hook import Hook from pre_commit.languages import helpers -from pre_commit.xargs import xargs ENVIRONMENT_DIR = None +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy +install_environment = helpers.no_install -def install_environment( - repo_cmd_runner, - version='default', - additional_dependencies=(), -): - """Installation for system type is a noop.""" - raise AssertionError('Cannot install system repo.') - - -def run_hook(repo_cmd_runner, hook, file_args): - return xargs(helpers.to_cmd(hook), file_args) +def run_hook( + hook: Hook, + file_args: Sequence[str], + color: bool, +) -> Tuple[int, bytes]: + return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/logging_handler.py b/pre_commit/logging_handler.py index 78c2827af..ba05295da 100644 --- a/pre_commit/logging_handler.py +++ b/pre_commit/logging_handler.py @@ -1,11 +1,10 @@ -from __future__ import unicode_literals - +import contextlib import logging +from typing import Generator from pre_commit import color from pre_commit import output - logger = logging.getLogger('pre_commit') LOG_LEVEL_COLORS = { @@ -17,23 +16,25 @@ class LoggingHandler(logging.Handler): - def __init__(self, use_color): - super(LoggingHandler, self).__init__() + def __init__(self, use_color: bool) -> None: + super().__init__() self.use_color = use_color - def emit(self, record): - output.write_line( - '{}{}'.format( - color.format_color( - '[{}]'.format(record.levelname), - LOG_LEVEL_COLORS[record.levelname], - self.use_color, - ) + ' ', - record.getMessage(), - ) + def emit(self, record: logging.LogRecord) -> None: + level_msg = color.format_color( + f'[{record.levelname}]', + LOG_LEVEL_COLORS[record.levelname], + self.use_color, ) + output.write_line(f'{level_msg} {record.getMessage()}') -def add_logging_handler(*args, **kwargs): - logger.addHandler(LoggingHandler(*args, **kwargs)) +@contextlib.contextmanager +def logging_handler(use_color: bool) -> Generator[None, None, None]: + handler = LoggingHandler(use_color) + logger.addHandler(handler) logger.setLevel(logging.INFO) + try: + yield + finally: + logger.removeHandler(handler) diff --git a/pre_commit/main.py b/pre_commit/main.py index baaf84b69..790b34773 100644 --- a/pre_commit/main.py +++ b/pre_commit/main.py @@ -1,24 +1,32 @@ -from __future__ import unicode_literals - import argparse import logging import os import sys +from typing import Any +from typing import Optional +from typing import Sequence +from typing import Union import pre_commit.constants as C from pre_commit import color -from pre_commit import five from pre_commit import git from pre_commit.commands.autoupdate import autoupdate from pre_commit.commands.clean import clean +from pre_commit.commands.gc import gc +from pre_commit.commands.hook_impl import hook_impl +from pre_commit.commands.init_templatedir import init_templatedir from pre_commit.commands.install_uninstall import install from pre_commit.commands.install_uninstall import install_hooks from pre_commit.commands.install_uninstall import uninstall +from pre_commit.commands.migrate_config import migrate_config from pre_commit.commands.run import run from pre_commit.commands.sample_config import sample_config +from pre_commit.commands.try_repo import try_repo from pre_commit.error_handler import error_handler -from pre_commit.logging_handler import add_logging_handler -from pre_commit.runner import Runner +from pre_commit.error_handler import FatalError +from pre_commit.logging_handler import logging_handler +from pre_commit.store import Store +from pre_commit.util import CalledProcessError logger = logging.getLogger('pre_commit') @@ -30,35 +38,215 @@ os.environ.pop('__PYVENV_LAUNCHER__', None) -def _add_color_option(parser): +COMMANDS_NO_GIT = {'clean', 'gc', 'init-templatedir', 'sample-config'} + + +def _add_color_option(parser: argparse.ArgumentParser) -> None: parser.add_argument( - '--color', default='auto', type=color.use_color, + '--color', default=os.environ.get('PRE_COMMIT_COLOR', 'auto'), + type=color.use_color, metavar='{' + ','.join(color.COLOR_CHOICES) + '}', help='Whether to use color in output. Defaults to `%(default)s`.', ) -def _add_config_option(parser): +def _add_config_option(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + '-c', '--config', default=C.CONFIG_FILE, + help='Path to alternate config file', + ) + + +class AppendReplaceDefault(argparse.Action): + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.appended = False + + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + values: Union[str, Sequence[str], None], + option_string: Optional[str] = None, + ) -> None: + if not self.appended: + setattr(namespace, self.dest, []) + self.appended = True + getattr(namespace, self.dest).append(values) + + +def _add_hook_type_option(parser: argparse.ArgumentParser) -> None: parser.add_argument( - '-c', '--config', default='.pre-commit-config.yaml', - help='Path to alternate config file' + '-t', '--hook-type', choices=( + 'pre-commit', 'pre-merge-commit', 'pre-push', + 'prepare-commit-msg', 'commit-msg', 'post-checkout', + ), + action=AppendReplaceDefault, + default=['pre-commit'], + dest='hook_types', ) -def main(argv=None): +def _add_run_options(parser: argparse.ArgumentParser) -> None: + parser.add_argument('hook', nargs='?', help='A single hook-id to run') + parser.add_argument('--verbose', '-v', action='store_true', default=False) + mutex_group = parser.add_mutually_exclusive_group(required=False) + mutex_group.add_argument( + '--all-files', '-a', action='store_true', default=False, + help='Run on all the files in the repo.', + ) + mutex_group.add_argument( + '--files', nargs='*', default=[], + help='Specific filenames to run hooks on.', + ) + parser.add_argument( + '--show-diff-on-failure', action='store_true', + help='When hooks fail, run `git diff` directly afterward.', + ) + parser.add_argument( + '--hook-stage', choices=C.STAGES, default='commit', + help='The stage during which the hook is fired. One of %(choices)s', + ) + parser.add_argument( + '--from-ref', '--source', '-s', + help=( + '(for usage with `--from-ref`) -- this option represents the ' + 'original ref in a `from_ref...to_ref` diff expression. ' + 'For `pre-push` hooks, this represents the branch you are pushing ' + 'to. ' + 'For `post-checkout` hooks, this represents the branch that was ' + 'previously checked out.' + ), + ) + parser.add_argument( + '--to-ref', '--origin', '-o', + help=( + '(for usage with `--to-ref`) -- this option represents the ' + 'destination ref in a `from_ref...to_ref` diff expression. ' + 'For `pre-push` hooks, this represents the branch being pushed. ' + 'For `post-checkout` hooks, this represents the branch that is ' + 'now checked out.' + ), + ) + parser.add_argument( + '--commit-msg-filename', + help='Filename to check when running during `commit-msg`', + ) + parser.add_argument( + '--remote-name', help='Remote name used by `git push`.', + ) + parser.add_argument('--remote-url', help='Remote url used by `git push`.') + parser.add_argument( + '--checkout-type', + help=( + 'Indicates whether the checkout was a branch checkout ' + '(changing branches, flag=1) or a file checkout (retrieving a ' + 'file from the index, flag=0).' + ), + ) + + +def _adjust_args_and_chdir(args: argparse.Namespace) -> None: + # `--config` was specified relative to the non-root working directory + if os.path.exists(args.config): + args.config = os.path.abspath(args.config) + if args.command in {'run', 'try-repo'}: + args.files = [os.path.abspath(filename) for filename in args.files] + if args.command == 'try-repo' and os.path.exists(args.repo): + args.repo = os.path.abspath(args.repo) + + try: + toplevel = git.get_root() + except CalledProcessError: + raise FatalError( + 'git failed. Is it installed, and are you in a Git repository ' + 'directory?', + ) + else: + if toplevel == '': # pragma: no cover (old git) + raise FatalError( + 'git toplevel unexpectedly empty! make sure you are not ' + 'inside the `.git` directory of your repository.', + ) + else: + os.chdir(toplevel) + + args.config = os.path.relpath(args.config) + if args.command in {'run', 'try-repo'}: + args.files = [os.path.relpath(filename) for filename in args.files] + if args.command == 'try-repo' and os.path.exists(args.repo): + args.repo = os.path.relpath(args.repo) + + +def main(argv: Optional[Sequence[str]] = None) -> int: argv = argv if argv is not None else sys.argv[1:] - argv = [five.to_text(arg) for arg in argv] - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(prog='pre-commit') - # http://stackoverflow.com/a/8521644/812183 + # https://stackoverflow.com/a/8521644/812183 parser.add_argument( '-V', '--version', action='version', - version='%(prog)s {}'.format(C.VERSION), + version=f'%(prog)s {C.VERSION}', ) subparsers = parser.add_subparsers(dest='command') + autoupdate_parser = subparsers.add_parser( + 'autoupdate', + help="Auto-update pre-commit config to the latest repos' versions.", + ) + _add_color_option(autoupdate_parser) + _add_config_option(autoupdate_parser) + autoupdate_parser.add_argument( + '--bleeding-edge', action='store_true', + help=( + 'Update to the bleeding edge of `master` instead of the latest ' + 'tagged version (the default behavior).' + ), + ) + autoupdate_parser.add_argument( + '--freeze', action='store_true', + help='Store "frozen" hashes in `rev` instead of tag names', + ) + autoupdate_parser.add_argument( + '--repo', dest='repos', action='append', metavar='REPO', + help='Only update this repository -- may be specified multiple times.', + ) + + clean_parser = subparsers.add_parser( + 'clean', help='Clean out pre-commit files.', + ) + _add_color_option(clean_parser) + _add_config_option(clean_parser) + + hook_impl_parser = subparsers.add_parser('hook-impl') + _add_color_option(hook_impl_parser) + _add_config_option(hook_impl_parser) + hook_impl_parser.add_argument('--hook-type') + hook_impl_parser.add_argument('--hook-dir') + hook_impl_parser.add_argument( + '--skip-on-missing-config', action='store_true', + ) + hook_impl_parser.add_argument(dest='rest', nargs=argparse.REMAINDER) + + gc_parser = subparsers.add_parser('gc', help='Clean unused cached repos.') + _add_color_option(gc_parser) + _add_config_option(gc_parser) + + init_templatedir_parser = subparsers.add_parser( + 'init-templatedir', + help=( + 'Install hook script in a directory intended for use with ' + '`git config init.templateDir`.' + ), + ) + _add_color_option(init_templatedir_parser) + _add_config_option(init_templatedir_parser) + init_templatedir_parser.add_argument( + 'directory', help='The directory in which to write the hook script.', + ) + _add_hook_type_option(init_templatedir_parser) + install_parser = subparsers.add_parser( 'install', help='Install the pre-commit script.', ) @@ -75,14 +263,11 @@ def main(argv=None): 'in the config file.' ), ) - install_parser.add_argument( - '-t', '--hook-type', choices=('pre-commit', 'pre-push'), - default='pre-commit', - ) + _add_hook_type_option(install_parser) install_parser.add_argument( '--allow-missing-config', action='store_true', default=False, help=( - 'Whether to allow a missing `pre-config` configuration file ' + 'Whether to allow a missing `pre-commit` configuration file ' 'or exit with a failure code.' ), ) @@ -98,141 +283,126 @@ def main(argv=None): _add_color_option(install_hooks_parser) _add_config_option(install_hooks_parser) - uninstall_parser = subparsers.add_parser( - 'uninstall', help='Uninstall the pre-commit script.', - ) - _add_color_option(uninstall_parser) - _add_config_option(uninstall_parser) - uninstall_parser.add_argument( - '-t', '--hook-type', choices=('pre-commit', 'pre-push'), - default='pre-commit', - ) - - clean_parser = subparsers.add_parser( - 'clean', help='Clean out pre-commit files.', - ) - _add_color_option(clean_parser) - _add_config_option(clean_parser) - autoupdate_parser = subparsers.add_parser( - 'autoupdate', - help="Auto-update pre-commit config to the latest repos' versions.", - ) - _add_color_option(autoupdate_parser) - _add_config_option(autoupdate_parser) - autoupdate_parser.add_argument( - '--tags-only', action='store_true', help='LEGACY: for compatibility', - ) - autoupdate_parser.add_argument( - '--bleeding-edge', action='store_true', - help=( - 'Update to the bleeding edge of `master` instead of the latest ' - 'tagged version (the default behavior).' - ), + migrate_config_parser = subparsers.add_parser( + 'migrate-config', + help='Migrate list configuration to new map configuration.', ) + _add_color_option(migrate_config_parser) + _add_config_option(migrate_config_parser) run_parser = subparsers.add_parser('run', help='Run hooks.') _add_color_option(run_parser) _add_config_option(run_parser) - run_parser.add_argument('hook', nargs='?', help='A single hook-id to run') - run_parser.add_argument( - '--no-stash', default=False, action='store_true', - help='Use this option to prevent auto stashing of unstaged files.', - ) - run_parser.add_argument( - '--verbose', '-v', action='store_true', default=False, + _add_run_options(run_parser) + + sample_config_parser = subparsers.add_parser( + 'sample-config', help=f'Produce a sample {C.CONFIG_FILE} file', ) - run_parser.add_argument( - '--origin', '-o', - help="The origin branch's commit_id when using `git push`.", + _add_color_option(sample_config_parser) + _add_config_option(sample_config_parser) + + try_repo_parser = subparsers.add_parser( + 'try-repo', + help='Try the hooks in a repository, useful for developing new hooks.', ) - run_parser.add_argument( - '--source', '-s', - help="The remote branch's commit_id when using `git push`.", + _add_color_option(try_repo_parser) + _add_config_option(try_repo_parser) + try_repo_parser.add_argument( + 'repo', help='Repository to source hooks from.', ) - run_parser.add_argument( - '--allow-unstaged-config', default=False, action='store_true', + try_repo_parser.add_argument( + '--ref', '--rev', help=( - 'Allow an unstaged config to be present. Note that this will ' - 'be stashed before parsing unless --no-stash is specified.' + 'Manually select a rev to run against, otherwise the `HEAD` ' + 'revision will be used.' ), ) - run_parser.add_argument( - '--hook-stage', choices=('commit', 'push'), default='commit', - help='The stage during which the hook is fired e.g. commit or push.', - ) - run_parser.add_argument( - '--show-diff-on-failure', action='store_true', - help='When hooks fail, run `git diff` directly afterward.', - ) - run_mutex_group = run_parser.add_mutually_exclusive_group(required=False) - run_mutex_group.add_argument( - '--all-files', '-a', action='store_true', default=False, - help='Run on all the files in the repo. Implies --no-stash.', - ) - run_mutex_group.add_argument( - '--files', nargs='*', default=[], - help='Specific filenames to run hooks on.', - ) + _add_run_options(try_repo_parser) - sample_config_parser = subparsers.add_parser( - 'sample-config', help='Produce a sample {} file'.format(C.CONFIG_FILE), + uninstall_parser = subparsers.add_parser( + 'uninstall', help='Uninstall the pre-commit script.', ) - _add_color_option(sample_config_parser) - _add_config_option(sample_config_parser) + _add_color_option(uninstall_parser) + _add_config_option(uninstall_parser) + _add_hook_type_option(uninstall_parser) help = subparsers.add_parser( 'help', help='Show help for a specific command.', ) help.add_argument('help_cmd', nargs='?', help='Command to show help for.') - # Argparse doesn't really provide a way to use a `default` subparser + # argparse doesn't really provide a way to use a `default` subparser if len(argv) == 0: argv = ['run'] args = parser.parse_args(argv) - if args.command == 'run': - args.files = [ - os.path.relpath(os.path.abspath(filename), git.get_root()) - for filename in args.files - ] - - if args.command == 'help': - if args.help_cmd: - parser.parse_args([args.help_cmd, '--help']) - else: - parser.parse_args(['--help']) - with error_handler(): - add_logging_handler(args.color) - runner = Runner.create(args.config) + if args.command == 'help' and args.help_cmd: + parser.parse_args([args.help_cmd, '--help']) + elif args.command == 'help': + parser.parse_args(['--help']) + + with error_handler(), logging_handler(args.color): + if args.command not in COMMANDS_NO_GIT: + _adjust_args_and_chdir(args) + git.check_for_cygwin_mismatch() - if args.command == 'install': - return install( - runner, overwrite=args.overwrite, hooks=args.install_hooks, + store = Store() + store.mark_config_used(args.config) + + if args.command == 'autoupdate': + return autoupdate( + args.config, store, + tags_only=not args.bleeding_edge, + freeze=args.freeze, + repos=args.repos, + ) + elif args.command == 'clean': + return clean(store) + elif args.command == 'gc': + return gc(store) + elif args.command == 'hook-impl': + return hook_impl( + store, + config=args.config, + color=args.color, hook_type=args.hook_type, - skip_on_missing_conf=args.allow_missing_config, + hook_dir=args.hook_dir, + skip_on_missing_config=args.skip_on_missing_config, + args=args.rest[1:], + ) + elif args.command == 'install': + return install( + args.config, store, + hook_types=args.hook_types, + overwrite=args.overwrite, + hooks=args.install_hooks, + skip_on_missing_config=args.allow_missing_config, + ) + elif args.command == 'init-templatedir': + return init_templatedir( + args.config, store, args.directory, + hook_types=args.hook_types, ) elif args.command == 'install-hooks': - return install_hooks(runner) - elif args.command == 'uninstall': - return uninstall(runner, hook_type=args.hook_type) - elif args.command == 'clean': - return clean(runner) - elif args.command == 'autoupdate': - if args.tags_only: - logger.warning('--tags-only is the default') - return autoupdate(runner, tags_only=not args.bleeding_edge) + return install_hooks(args.config, store) + elif args.command == 'migrate-config': + return migrate_config(args.config) elif args.command == 'run': - return run(runner, args) + return run(args.config, store, args) elif args.command == 'sample-config': return sample_config() + elif args.command == 'try-repo': + return try_repo(args) + elif args.command == 'uninstall': + return uninstall(hook_types=args.hook_types) else: raise NotImplementedError( - 'Command {} not implemented.'.format(args.command) + f'Command {args.command} not implemented.', ) raise AssertionError( - 'Command {} failed to exit with a returncode'.format(args.command) + f'Command {args.command} failed to exit with a returncode', ) diff --git a/pre_commit/make_archives.py b/pre_commit/make_archives.py index 4baaaa18f..c31bcd714 100644 --- a/pre_commit/make_archives.py +++ b/pre_commit/make_archives.py @@ -1,13 +1,11 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - +import argparse import os.path import tarfile +from typing import Optional +from typing import Sequence from pre_commit import output -from pre_commit.util import cmd_output -from pre_commit.util import cwd +from pre_commit.util import cmd_output_b from pre_commit.util import rmtree from pre_commit.util import tmpdir @@ -17,8 +15,8 @@ REPOS = ( - ('rbenv', 'git://github.com/rbenv/rbenv', 'e60ad4a'), - ('ruby-build', 'git://github.com/rbenv/ruby-build', '9bc9971'), + ('rbenv', 'git://github.com/rbenv/rbenv', 'a3fa9b7'), + ('ruby-build', 'git://github.com/rbenv/ruby-build', '1a902f3'), ( 'ruby-download', 'git://github.com/garnieretienne/rvm-download', @@ -27,12 +25,7 @@ ) -RESOURCES_DIR = os.path.abspath( - os.path.join(os.path.dirname(__file__), 'resources') -) - - -def make_archive(name, repo, ref, destdir): +def make_archive(name: str, repo: str, ref: str, destdir: str) -> str: """Makes an archive of a repository in the given destdir. :param text name: Name to give the archive. For instance foo. The file @@ -41,12 +34,11 @@ def make_archive(name, repo, ref, destdir): :param text ref: Tag/SHA/branch to check out. :param text destdir: Directory to place archives in. """ - output_path = os.path.join(destdir, name + '.tar.gz') + output_path = os.path.join(destdir, f'{name}.tar.gz') with tmpdir() as tempdir: # Clone the repository to the temporary directory - cmd_output('git', 'clone', repo, tempdir) - with cwd(tempdir): - cmd_output('git', 'checkout', ref) + cmd_output_b('git', 'clone', repo, tempdir) + cmd_output_b('git', 'checkout', ref, cwd=tempdir) # We don't want the '.git' directory # It adds a bunch of size to the archive and we don't use it at @@ -59,12 +51,14 @@ def make_archive(name, repo, ref, destdir): return output_path -def main(): +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument('--dest', default='pre_commit/resources') + args = parser.parse_args(argv) for archive_name, repo, ref in REPOS: - output.write_line('Making {}.tar.gz for {}@{}'.format( - archive_name, repo, ref, - )) - make_archive(archive_name, repo, ref, RESOURCES_DIR) + output.write_line(f'Making {archive_name}.tar.gz for {repo}@{ref}') + make_archive(archive_name, repo, ref, args.dest) + return 0 if __name__ == '__main__': diff --git a/pre_commit/manifest.py b/pre_commit/manifest.py deleted file mode 100644 index 888ad6dda..000000000 --- a/pre_commit/manifest.py +++ /dev/null @@ -1,41 +0,0 @@ -from __future__ import unicode_literals - -import logging -import os.path - -from cached_property import cached_property - -import pre_commit.constants as C -from pre_commit.clientlib import load_manifest - - -logger = logging.getLogger('pre_commit') - - -class Manifest(object): - def __init__(self, repo_path, repo_url): - self.repo_path = repo_path - self.repo_url = repo_url - - @cached_property - def manifest_contents(self): - default_path = os.path.join(self.repo_path, C.MANIFEST_FILE) - legacy_path = os.path.join(self.repo_path, C.MANIFEST_FILE_LEGACY) - if os.path.exists(legacy_path) and not os.path.exists(default_path): - logger.warning( - '{} uses legacy {} to provide hooks.\n' - 'In newer versions, this file is called {}\n' - 'This will work in this version of pre-commit but will be ' - 'removed at a later time.\n' - 'If `pre-commit autoupdate` does not silence this warning ' - 'consider making an issue / pull request.'.format( - self.repo_url, C.MANIFEST_FILE_LEGACY, C.MANIFEST_FILE, - ) - ) - return load_manifest(legacy_path) - else: - return load_manifest(default_path) - - @cached_property - def hooks(self): - return {hook['id']: hook for hook in self.manifest_contents} diff --git a/testing/resources/arbitrary_bytes_repo/python3_hook/__init__.py b/pre_commit/meta_hooks/__init__.py similarity index 100% rename from testing/resources/arbitrary_bytes_repo/python3_hook/__init__.py rename to pre_commit/meta_hooks/__init__.py diff --git a/pre_commit/meta_hooks/check_hooks_apply.py b/pre_commit/meta_hooks/check_hooks_apply.py new file mode 100644 index 000000000..d0244a944 --- /dev/null +++ b/pre_commit/meta_hooks/check_hooks_apply.py @@ -0,0 +1,39 @@ +import argparse +from typing import Optional +from typing import Sequence + +import pre_commit.constants as C +from pre_commit import git +from pre_commit.clientlib import load_config +from pre_commit.commands.run import Classifier +from pre_commit.repository import all_hooks +from pre_commit.store import Store + + +def check_all_hooks_match_files(config_file: str) -> int: + classifier = Classifier(git.get_all_files()) + retv = 0 + + for hook in all_hooks(load_config(config_file), Store()): + if hook.always_run or hook.language == 'fail': + continue + elif not classifier.filenames_for_hook(hook): + print(f'{hook.id} does not apply to this repository') + retv = 1 + + return retv + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument('filenames', nargs='*', default=[C.CONFIG_FILE]) + args = parser.parse_args(argv) + + retv = 0 + for filename in args.filenames: + retv |= check_all_hooks_match_files(filename) + return retv + + +if __name__ == '__main__': + exit(main()) diff --git a/pre_commit/meta_hooks/check_useless_excludes.py b/pre_commit/meta_hooks/check_useless_excludes.py new file mode 100644 index 000000000..30b8d8101 --- /dev/null +++ b/pre_commit/meta_hooks/check_useless_excludes.py @@ -0,0 +1,72 @@ +import argparse +import re +from typing import Optional +from typing import Sequence + +from cfgv import apply_defaults + +import pre_commit.constants as C +from pre_commit import git +from pre_commit.clientlib import load_config +from pre_commit.clientlib import MANIFEST_HOOK_DICT +from pre_commit.commands.run import Classifier + + +def exclude_matches_any( + filenames: Sequence[str], + include: str, + exclude: str, +) -> bool: + if exclude == '^$': + return True + include_re, exclude_re = re.compile(include), re.compile(exclude) + for filename in filenames: + if include_re.search(filename) and exclude_re.search(filename): + return True + return False + + +def check_useless_excludes(config_file: str) -> int: + config = load_config(config_file) + classifier = Classifier(git.get_all_files()) + retv = 0 + + exclude = config['exclude'] + if not exclude_matches_any(classifier.filenames, '', exclude): + print( + f'The global exclude pattern {exclude!r} does not match any files', + ) + retv = 1 + + for repo in config['repos']: + for hook in repo['hooks']: + # Not actually a manifest dict, but this more accurately reflects + # the defaults applied during runtime + hook = apply_defaults(hook, MANIFEST_HOOK_DICT) + names = classifier.filenames + types, exclude_types = hook['types'], hook['exclude_types'] + names = classifier.by_types(names, types, exclude_types) + include, exclude = hook['files'], hook['exclude'] + if not exclude_matches_any(names, include, exclude): + print( + f'The exclude pattern {exclude!r} for {hook["id"]} does ' + f'not match any files', + ) + retv = 1 + + return retv + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument('filenames', nargs='*', default=[C.CONFIG_FILE]) + args = parser.parse_args(argv) + + retv = 0 + for filename in args.filenames: + retv |= check_useless_excludes(filename) + return retv + + +if __name__ == '__main__': + exit(main()) diff --git a/pre_commit/meta_hooks/identity.py b/pre_commit/meta_hooks/identity.py new file mode 100644 index 000000000..730d0ec00 --- /dev/null +++ b/pre_commit/meta_hooks/identity.py @@ -0,0 +1,16 @@ +import sys +from typing import Optional +from typing import Sequence + +from pre_commit import output + + +def main(argv: Optional[Sequence[str]] = None) -> int: + argv = argv if argv is not None else sys.argv[1:] + for arg in argv: + output.write_line(arg) + return 0 + + +if __name__ == '__main__': + exit(main()) diff --git a/pre_commit/output.py b/pre_commit/output.py index 365960906..24f9d8465 100644 --- a/pre_commit/output.py +++ b/pre_commit/output.py @@ -1,88 +1,32 @@ -from __future__ import unicode_literals - +import contextlib import sys +from typing import Any +from typing import IO +from typing import Optional -from pre_commit import color -from pre_commit import five -from pre_commit.util import noop_context - - -def get_hook_message( - start, - postfix='', - end_msg=None, - end_len=0, - end_color=None, - use_color=None, - cols=80, -): - """Prints a message for running a hook. - - This currently supports three approaches: - - # Print `start` followed by dots, leaving 6 characters at the end - >>> print_hook_message('start', end_len=6) - start............................................................... - - # Print `start` followed by dots with the end message colored if coloring - # is specified and a newline afterwards - >>> print_hook_message( - 'start', - end_msg='end', - end_color=color.RED, - use_color=True, - ) - start...................................................................end - - # Print `start` followed by dots, followed by the `postfix` message - # uncolored, followed by the `end_msg` colored if specified and a newline - # afterwards - >>> print_hook_message( - 'start', - postfix='postfix ', - end_msg='end', - end_color=color.RED, - use_color=True, - ) - start...........................................................postfix end - """ - if bool(end_msg) == bool(end_len): - raise ValueError('Expected one of (`end_msg`, `end_len`)') - if end_msg is not None and (end_color is None or use_color is None): - raise ValueError( - '`end_color` and `use_color` are required with `end_msg`' - ) - if end_len: - return start + '.' * (cols - len(start) - end_len - 1) - else: - return '{}{}{}{}\n'.format( - start, - '.' * (cols - len(start) - len(postfix) - len(end_msg) - 1), - postfix, - color.format_color(end_msg, end_color, use_color), - ) - - -stdout_byte_stream = getattr(sys.stdout, 'buffer', sys.stdout) - - -def write(s, stream=stdout_byte_stream): - stream.write(five.to_bytes(s)) +def write(s: str, stream: IO[bytes] = sys.stdout.buffer) -> None: + stream.write(s.encode()) stream.flush() -def write_line(s=None, stream=stdout_byte_stream, logfile_name=None): - output_streams = [stream] - if logfile_name: - ctx = open(logfile_name, 'ab') - output_streams.append(ctx) - else: - ctx = noop_context() +def write_line_b( + s: Optional[bytes] = None, + stream: IO[bytes] = sys.stdout.buffer, + logfile_name: Optional[str] = None, +) -> None: + with contextlib.ExitStack() as exit_stack: + output_streams = [stream] + if logfile_name: + stream = exit_stack.enter_context(open(logfile_name, 'ab')) + output_streams.append(stream) - with ctx: for output_stream in output_streams: if s is not None: - output_stream.write(five.to_bytes(s)) + output_stream.write(s) output_stream.write(b'\n') output_stream.flush() + + +def write_line(s: Optional[str] = None, **kwargs: Any) -> None: + write_line_b(s.encode() if s is not None else s, **kwargs) diff --git a/pre_commit/parse_shebang.py b/pre_commit/parse_shebang.py index 4419cbfc5..d344a1dab 100644 --- a/pre_commit/parse_shebang.py +++ b/pre_commit/parse_shebang.py @@ -1,24 +1,30 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import os.path +from typing import Mapping +from typing import Optional +from typing import Tuple +from typing import TYPE_CHECKING from identify.identify import parse_shebang_from_file +if TYPE_CHECKING: + from typing import NoReturn + class ExecutableNotFoundError(OSError): - def to_output(self): - return (1, self.args[0].encode('UTF-8'), b'') + def to_output(self) -> Tuple[int, bytes, None]: + return (1, self.args[0].encode(), None) -def parse_filename(filename): +def parse_filename(filename: str) -> Tuple[str, ...]: if not os.path.exists(filename): return () else: return parse_shebang_from_file(filename) -def find_executable(exe, _environ=None): +def find_executable( + exe: str, _environ: Optional[Mapping[str, str]] = None, +) -> Optional[str]: exe = os.path.normpath(exe) if os.sep in exe: return exe @@ -26,10 +32,8 @@ def find_executable(exe, _environ=None): environ = _environ if _environ is not None else os.environ if 'PATHEXT' in environ: - possible_exe_names = tuple( - exe + ext.lower() for ext in environ['PATHEXT'].split(os.pathsep) - ) + (exe,) - + exts = environ['PATHEXT'].split(os.pathsep) + possible_exe_names = tuple(f'{exe}{ext}' for ext in exts) + (exe,) else: possible_exe_names = (exe,) @@ -42,21 +46,28 @@ def find_executable(exe, _environ=None): return None -def normexe(orig_exe): - if os.sep not in orig_exe: - exe = find_executable(orig_exe) +def normexe(orig: str) -> str: + def _error(msg: str) -> 'NoReturn': + raise ExecutableNotFoundError(f'Executable `{orig}` {msg}') + + if os.sep not in orig and (not os.altsep or os.altsep not in orig): + exe = find_executable(orig) if exe is None: - raise ExecutableNotFoundError( - 'Executable `{}` not found'.format(orig_exe), - ) + _error('not found') return exe + elif os.path.isdir(orig): + _error('is a directory') + elif not os.path.isfile(orig): + _error('not found') + elif not os.access(orig, os.X_OK): # pragma: win32 no cover + _error('is not executable') else: - return orig_exe + return orig -def normalize_cmd(cmd): +def normalize_cmd(cmd: Tuple[str, ...]) -> Tuple[str, ...]: """Fixes for the following issues on windows - - http://bugs.python.org/issue8557 + - https://bugs.python.org/issue8557 - windows does not parse shebangs This function also makes deep-path shebangs work just fine diff --git a/pre_commit/prefix.py b/pre_commit/prefix.py new file mode 100644 index 000000000..0e3ebbd89 --- /dev/null +++ b/pre_commit/prefix.py @@ -0,0 +1,17 @@ +import os.path +from typing import NamedTuple +from typing import Tuple + + +class Prefix(NamedTuple): + prefix_dir: str + + def path(self, *parts: str) -> str: + return os.path.normpath(os.path.join(self.prefix_dir, *parts)) + + def exists(self, *parts: str) -> bool: + return os.path.exists(self.path(*parts)) + + def star(self, end: str) -> Tuple[str, ...]: + paths = os.listdir(self.prefix_dir) + return tuple(path for path in paths if path.endswith(end)) diff --git a/pre_commit/prefixed_command_runner.py b/pre_commit/prefixed_command_runner.py deleted file mode 100644 index 6ae850997..000000000 --- a/pre_commit/prefixed_command_runner.py +++ /dev/null @@ -1,50 +0,0 @@ -from __future__ import unicode_literals - -import os.path -import subprocess - -from pre_commit.util import cmd_output - - -class PrefixedCommandRunner(object): - """A PrefixedCommandRunner allows you to run subprocess commands with - comand substitution. - - For instance: - PrefixedCommandRunner('/tmp/foo').run(['{prefix}foo.sh', 'bar', 'baz']) - - will run ['/tmp/foo/foo.sh', 'bar', 'baz'] - """ - - def __init__( - self, - prefix_dir, - popen=subprocess.Popen, - makedirs=os.makedirs - ): - self.prefix_dir = prefix_dir.rstrip(os.sep) + os.sep - self.__popen = popen - self.__makedirs = makedirs - - def _create_path_if_not_exists(self): - if not os.path.exists(self.prefix_dir): - self.__makedirs(self.prefix_dir) - - def run(self, cmd, **kwargs): - self._create_path_if_not_exists() - replaced_cmd = [ - part.replace('{prefix}', self.prefix_dir) for part in cmd - ] - return cmd_output(*replaced_cmd, __popen=self.__popen, **kwargs) - - def path(self, *parts): - path = os.path.join(self.prefix_dir, *parts) - return os.path.normpath(path) - - def exists(self, *parts): - return os.path.exists(self.path(*parts)) - - def star(self, end): - return tuple( - path for path in os.listdir(self.prefix_dir) if path.endswith(end) - ) diff --git a/pre_commit/repository.py b/pre_commit/repository.py index 2c1eedb3d..77734ee64 100644 --- a/pre_commit/repository.py +++ b/pre_commit/repository.py @@ -1,243 +1,208 @@ -from __future__ import unicode_literals - -import io import json import logging import os -import shutil -from collections import defaultdict - -import pkg_resources -from cached_property import cached_property +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Sequence +from typing import Set +from typing import Tuple import pre_commit.constants as C -from pre_commit import five -from pre_commit import git -from pre_commit.clientlib import is_local_repo -from pre_commit.clientlib import MANIFEST_HOOK_DICT +from pre_commit.clientlib import load_manifest +from pre_commit.clientlib import LOCAL +from pre_commit.clientlib import META +from pre_commit.hook import Hook from pre_commit.languages.all import languages from pre_commit.languages.helpers import environment_dir -from pre_commit.manifest import Manifest -from pre_commit.prefixed_command_runner import PrefixedCommandRunner -from pre_commit.schema import apply_defaults -from pre_commit.schema import validate +from pre_commit.prefix import Prefix +from pre_commit.store import Store +from pre_commit.util import parse_version +from pre_commit.util import rmtree logger = logging.getLogger('pre_commit') -def _state(additional_deps): +def _state(additional_deps: Sequence[str]) -> object: return {'additional_dependencies': sorted(additional_deps)} -def _state_filename(cmd_runner, venv): - return cmd_runner.path( - venv, '.install_state_v' + C.INSTALLED_STATE_VERSION, - ) +def _state_filename(prefix: Prefix, venv: str) -> str: + return prefix.path(venv, f'.install_state_v{C.INSTALLED_STATE_VERSION}') -def _read_installed_state(cmd_runner, venv): - filename = _state_filename(cmd_runner, venv) +def _read_state(prefix: Prefix, venv: str) -> Optional[object]: + filename = _state_filename(prefix, venv) if not os.path.exists(filename): return None else: - return json.loads(io.open(filename).read()) + with open(filename) as f: + return json.load(f) -def _write_installed_state(cmd_runner, venv, state): - state_filename = _state_filename(cmd_runner, venv) - staging = state_filename + 'staging' - with io.open(staging, 'w') as state_file: - state_file.write(five.to_text(json.dumps(state))) +def _write_state(prefix: Prefix, venv: str, state: object) -> None: + state_filename = _state_filename(prefix, venv) + staging = f'{state_filename}staging' + with open(staging, 'w') as state_file: + state_file.write(json.dumps(state)) # Move the file into place atomically to indicate we've installed os.rename(staging, state_filename) -def _installed(cmd_runner, language_name, language_version, additional_deps): - language = languages[language_name] - venv = environment_dir(language.ENVIRONMENT_DIR, language_version) +def _hook_installed(hook: Hook) -> bool: + lang = languages[hook.language] + venv = environment_dir(lang.ENVIRONMENT_DIR, hook.language_version) return ( - venv is None or - _read_installed_state(cmd_runner, venv) == _state(additional_deps) + venv is None or ( + ( + _read_state(hook.prefix, venv) == + _state(hook.additional_dependencies) + ) and + lang.healthy(hook.prefix, hook.language_version) + ) ) -def _install_all(venvs, repo_url): - """Tuple of (cmd_runner, language, version, deps)""" - need_installed = tuple( - (cmd_runner, language_name, version, deps) - for cmd_runner, language_name, version, deps in venvs - if not _installed(cmd_runner, language_name, version, deps) - ) +def _hook_install(hook: Hook) -> None: + logger.info(f'Installing environment for {hook.src}.') + logger.info('Once installed this environment will be reused.') + logger.info('This may take a few minutes...') - if need_installed: - logger.info( - 'Installing environment for {}.'.format(repo_url) - ) - logger.info('Once installed this environment will be reused.') - logger.info('This may take a few minutes...') + lang = languages[hook.language] + assert lang.ENVIRONMENT_DIR is not None + venv = environment_dir(lang.ENVIRONMENT_DIR, hook.language_version) - for cmd_runner, language_name, version, deps in need_installed: - language = languages[language_name] - venv = environment_dir(language.ENVIRONMENT_DIR, version) + # There's potentially incomplete cleanup from previous runs + # Clean it up! + if hook.prefix.exists(venv): + rmtree(hook.prefix.path(venv)) - # There's potentially incomplete cleanup from previous runs - # Clean it up! - if cmd_runner.exists(venv): - shutil.rmtree(cmd_runner.path(venv)) + lang.install_environment( + hook.prefix, hook.language_version, hook.additional_dependencies, + ) + # Write our state to indicate we're installed + _write_state(hook.prefix, venv, _state(hook.additional_dependencies)) - language.install_environment(cmd_runner, version, deps) - # Write our state to indicate we're installed - state = _state(deps) - _write_installed_state(cmd_runner, venv, state) +def _hook( + *hook_dicts: Dict[str, Any], + root_config: Dict[str, Any], +) -> Dict[str, Any]: + ret, rest = dict(hook_dicts[0]), hook_dicts[1:] + for dct in rest: + ret.update(dct) -def _validate_minimum_version(hook): - hook_version = pkg_resources.parse_version( - hook['minimum_pre_commit_version'], - ) - if hook_version > C.VERSION_PARSED: + version = ret['minimum_pre_commit_version'] + if parse_version(version) > parse_version(C.VERSION): logger.error( - 'The hook `{}` requires pre-commit version {} but ' - 'version {} is installed. ' - 'Perhaps run `pip install --upgrade pre-commit`.'.format( - hook['id'], hook_version, C.VERSION_PARSED, - ) + f'The hook `{ret["id"]}` requires pre-commit version {version} ' + f'but version {C.VERSION} is installed. ' + f'Perhaps run `pip install --upgrade pre-commit`.', ) exit(1) - return hook + lang = ret['language'] + if ret['language_version'] == C.DEFAULT: + ret['language_version'] = root_config['default_language_version'][lang] + if ret['language_version'] == C.DEFAULT: + ret['language_version'] = languages[lang].get_default_version() -class Repository(object): - def __init__(self, repo_config, store): - self.repo_config = repo_config - self.store = store - self.__installed = False + if not ret['stages']: + ret['stages'] = root_config['default_stages'] - @classmethod - def create(cls, config, store): - if is_local_repo(config): - return LocalRepository(config, store) - else: - return cls(config, store) + return ret - @cached_property - def _repo_path(self): - return self.store.clone( - self.repo_config['repo'], self.repo_config['sha'], - ) - @cached_property - def _cmd_runner(self): - return PrefixedCommandRunner(self._repo_path) - - def _cmd_runner_from_deps(self, language_name, deps): - return self._cmd_runner - - @cached_property - def manifest(self): - return Manifest(self._repo_path, self.repo_config['repo']) - - @cached_property - def hooks(self): - for hook in self.repo_config['hooks']: - if hook['id'] not in self.manifest.hooks: - logger.error( - '`{}` is not present in repository {}. ' - 'Typo? Perhaps it is introduced in a newer version? ' - 'Often `pre-commit autoupdate` fixes this.'.format( - hook['id'], self.repo_config['repo'], - ) - ) - exit(1) - - _validate_minimum_version(self.manifest.hooks[hook['id']]) - - return tuple( - (hook['id'], dict(self.manifest.hooks[hook['id']], **hook)) - for hook in self.repo_config['hooks'] - ) - - @cached_property - def _venvs(self): - deps_dict = defaultdict(_UniqueList) - for _, hook in self.hooks: - deps_dict[(hook['language'], hook['language_version'])].update( - hook['additional_dependencies'], - ) - ret = [] - for (language, version), deps in deps_dict.items(): - ret.append((self._cmd_runner, language, version, deps)) - return tuple(ret) - - def require_installed(self): - if not self.__installed: - _install_all(self._venvs, self.repo_config['repo']) - self.__installed = True - - def run_hook(self, hook, file_args): - """Run a hook. - - :param dict hook: - :param tuple file_args: all the files to run the hook on - """ - self.require_installed() - language_name = hook['language'] - deps = hook['additional_dependencies'] - cmd_runner = self._cmd_runner_from_deps(language_name, deps) - return languages[language_name].run_hook(cmd_runner, hook, file_args) - - -class LocalRepository(Repository): - def _cmd_runner_from_deps(self, language_name, deps): - """local repositories have a cmd runner per hook""" +def _non_cloned_repository_hooks( + repo_config: Dict[str, Any], + store: Store, + root_config: Dict[str, Any], +) -> Tuple[Hook, ...]: + def _prefix(language_name: str, deps: Sequence[str]) -> Prefix: language = languages[language_name] - # pcre / script / system do not have environments so they work out - # of the current directory + # pygrep / script / system / docker_image do not have + # environments so they work out of the current directory if language.ENVIRONMENT_DIR is None: - return PrefixedCommandRunner(git.get_root()) + return Prefix(os.getcwd()) else: - return PrefixedCommandRunner(self.store.make_local(deps)) + return Prefix(store.make_local(deps)) - @cached_property - def manifest(self): - raise NotImplementedError + return tuple( + Hook.create( + repo_config['repo'], + _prefix(hook['language'], hook['additional_dependencies']), + _hook(hook, root_config=root_config), + ) + for hook in repo_config['hooks'] + ) - @cached_property - def hooks(self): - return tuple( - ( - hook['id'], - _validate_minimum_version( - apply_defaults( - validate(hook, MANIFEST_HOOK_DICT), - MANIFEST_HOOK_DICT, - ), - ), + +def _cloned_repository_hooks( + repo_config: Dict[str, Any], + store: Store, + root_config: Dict[str, Any], +) -> Tuple[Hook, ...]: + repo, rev = repo_config['repo'], repo_config['rev'] + manifest_path = os.path.join(store.clone(repo, rev), C.MANIFEST_FILE) + by_id = {hook['id']: hook for hook in load_manifest(manifest_path)} + + for hook in repo_config['hooks']: + if hook['id'] not in by_id: + logger.error( + f'`{hook["id"]}` is not present in repository {repo}. ' + f'Typo? Perhaps it is introduced in a newer version? ' + f'Often `pre-commit autoupdate` fixes this.', ) - for hook in self.repo_config['hooks'] + exit(1) + + hook_dcts = [ + _hook(by_id[hook['id']], hook, root_config=root_config) + for hook in repo_config['hooks'] + ] + return tuple( + Hook.create( + repo_config['repo'], + Prefix(store.clone(repo, rev, hook['additional_dependencies'])), + hook, ) + for hook in hook_dcts + ) + + +def _repository_hooks( + repo_config: Dict[str, Any], + store: Store, + root_config: Dict[str, Any], +) -> Tuple[Hook, ...]: + if repo_config['repo'] in {LOCAL, META}: + return _non_cloned_repository_hooks(repo_config, store, root_config) + else: + return _cloned_repository_hooks(repo_config, store, root_config) + - @cached_property - def _venvs(self): +def install_hook_envs(hooks: Sequence[Hook], store: Store) -> None: + def _need_installed() -> List[Hook]: + seen: Set[Tuple[Prefix, str, str, Tuple[str, ...]]] = set() ret = [] - for _, hook in self.hooks: - language = hook['language'] - version = hook['language_version'] - deps = hook['additional_dependencies'] - ret.append(( - self._cmd_runner_from_deps(language, deps), - language, version, deps, - )) - return tuple(ret) - - -class _UniqueList(list): - def __init__(self): - self._set = set() - - def update(self, obj): - for item in obj: - if item not in self._set: - self._set.add(item) - self.append(item) + for hook in hooks: + if hook.install_key not in seen and not _hook_installed(hook): + ret.append(hook) + seen.add(hook.install_key) + return ret + + if not _need_installed(): + return + with store.exclusive_lock(): + # Another process may have already completed this work + for hook in _need_installed(): + _hook_install(hook) + + +def all_hooks(root_config: Dict[str, Any], store: Store) -> Tuple[Hook, ...]: + return tuple( + hook + for repo in root_config['repos'] + for hook in _repository_hooks(repo, store, root_config) + ) diff --git a/testing/resources/python3_hooks_repo/python3_hook/__init__.py b/pre_commit/resources/__init__.py similarity index 100% rename from testing/resources/python3_hooks_repo/python3_hook/__init__.py rename to pre_commit/resources/__init__.py diff --git a/pre_commit/resources/empty_template/.npmignore b/pre_commit/resources/empty_template_.npmignore similarity index 100% rename from pre_commit/resources/empty_template/.npmignore rename to pre_commit/resources/empty_template_.npmignore diff --git a/pre_commit/resources/empty_template_Cargo.toml b/pre_commit/resources/empty_template_Cargo.toml new file mode 100644 index 000000000..3dfeffafb --- /dev/null +++ b/pre_commit/resources/empty_template_Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "__fake_crate" +version = "0.0.0" + +[[bin]] +name = "__fake_cmd" +path = "main.rs" diff --git a/pre_commit/resources/empty_template_Makefile.PL b/pre_commit/resources/empty_template_Makefile.PL new file mode 100644 index 000000000..ac75fe531 --- /dev/null +++ b/pre_commit/resources/empty_template_Makefile.PL @@ -0,0 +1,6 @@ +use ExtUtils::MakeMaker; + +WriteMakefile( + NAME => "PreCommitDummy", + VERSION => "0.0.1", +); diff --git a/pre_commit/resources/empty_template_environment.yml b/pre_commit/resources/empty_template_environment.yml new file mode 100644 index 000000000..0f29f0c0a --- /dev/null +++ b/pre_commit/resources/empty_template_environment.yml @@ -0,0 +1,9 @@ +channels: + - conda-forge + - defaults +dependencies: + # This cannot be empty as otherwise no environment will be created. + # We're using openssl here as it is available on all system and will + # most likely be always installed anyways. + # See https://github.com/conda/conda/issues/9487 + - openssl diff --git a/pre_commit/resources/empty_template/main.go b/pre_commit/resources/empty_template_main.go similarity index 100% rename from pre_commit/resources/empty_template/main.go rename to pre_commit/resources/empty_template_main.go diff --git a/pre_commit/resources/empty_template_main.rs b/pre_commit/resources/empty_template_main.rs new file mode 100644 index 000000000..f328e4d9d --- /dev/null +++ b/pre_commit/resources/empty_template_main.rs @@ -0,0 +1 @@ +fn main() {} diff --git a/pre_commit/resources/empty_template/package.json b/pre_commit/resources/empty_template_package.json similarity index 100% rename from pre_commit/resources/empty_template/package.json rename to pre_commit/resources/empty_template_package.json diff --git a/pre_commit/resources/empty_template/pre_commit_dummy_package.gemspec b/pre_commit/resources/empty_template_pre_commit_dummy_package.gemspec similarity index 100% rename from pre_commit/resources/empty_template/pre_commit_dummy_package.gemspec rename to pre_commit/resources/empty_template_pre_commit_dummy_package.gemspec diff --git a/pre_commit/resources/empty_template/setup.py b/pre_commit/resources/empty_template_setup.py similarity index 100% rename from pre_commit/resources/empty_template/setup.py rename to pre_commit/resources/empty_template_setup.py diff --git a/pre_commit/resources/hook-tmpl b/pre_commit/resources/hook-tmpl old mode 100644 new mode 100755 index da939ff1a..299144ec7 --- a/pre_commit/resources/hook-tmpl +++ b/pre_commit/resources/hook-tmpl @@ -1,73 +1,44 @@ -#!/usr/bin/env bash -# This is a randomish md5 to identify this script -# 138fd403232d2ddd5efb44317e38bf03 - -pushd `dirname $0` > /dev/null -HERE=`pwd` -popd > /dev/null - -retv=0 -args="" - -ENV_PYTHON='{sys_executable}' -SKIP_ON_MISSING_CONF={skip_on_missing_conf} - -which pre-commit >& /dev/null -WHICH_RETV=$? -"$ENV_PYTHON" -c 'import pre_commit.main' >& /dev/null -ENV_PYTHON_RETV=$? -python -c 'import pre_commit.main' >& /dev/null -PYTHON_RETV=$? - - -if (( - (WHICH_RETV != 0) && - (ENV_PYTHON_RETV != 0) && - (PYTHON_RETV != 0) -)); then - echo '`{hook_type}` not found. Did you forget to activate your virtualenv?' - exit 1 -fi - - -# Run the legacy pre-commit if it exists -if [ -x "$HERE"/{hook_type}.legacy ]; then - "$HERE"/{hook_type}.legacy - if [ $? -ne 0 ]; then - retv=1 - fi -fi - -CONF_FILE=$(git rev-parse --show-toplevel)"/.pre-commit-config.yaml" -if [ ! -f $CONF_FILE ]; then - if [ $SKIP_ON_MISSING_CONF = true ] || [ ! -z $PRE_COMMIT_ALLOW_NO_CONFIG ]; then - echo '`.pre-commit-config.yaml` config file not found. Skipping `pre-commit`.' - exit $retv - else - echo 'No .pre-commit-config.yaml file was found' - echo '- To temporarily silence this, run `PRE_COMMIT_ALLOW_NO_CONFIG=1 git ...`' - echo '- To permanently silence this, install pre-commit with the `--allow-missing-config` option' - echo '- To uninstall pre-commit run `pre-commit uninstall`' - exit 1 - fi -fi - -{pre_push} - -# Run pre-commit -if ((WHICH_RETV == 0)); then - pre-commit $args - PRE_COMMIT_RETV=$? -elif ((ENV_PYTHON_RETV == 0)); then - "$ENV_PYTHON" -m pre_commit.main $args - PRE_COMMIT_RETV=$? -else - python -m pre_commit.main $args - PRE_COMMIT_RETV=$? -fi - -if ((PRE_COMMIT_RETV != 0)); then - retv=1 -fi - -exit $retv +#!/usr/bin/env python3 +# File generated by pre-commit: https://pre-commit.com +# ID: 138fd403232d2ddd5efb44317e38bf03 +import os +import sys + +# we try our best, but the shebang of this script is difficult to determine: +# - macos doesn't ship with python3 +# - windows executables are almost always `python.exe` +# therefore we continue to support python2 for this small script +if sys.version_info < (3, 3): + from distutils.spawn import find_executable as which +else: + from shutil import which + +# work around https://github.com/Homebrew/homebrew-core/issues/30445 +os.environ.pop('__PYVENV_LAUNCHER__', None) + +# start templated +INSTALL_PYTHON = '' +ARGS = ['hook-impl'] +# end templated +ARGS.extend(('--hook-dir', os.path.realpath(os.path.dirname(__file__)))) +ARGS.append('--') +ARGS.extend(sys.argv[1:]) + +DNE = '`pre-commit` not found. Did you forget to activate your virtualenv?' +if os.access(INSTALL_PYTHON, os.X_OK): + CMD = [INSTALL_PYTHON, '-mpre_commit'] +elif which('pre-commit'): + CMD = ['pre-commit'] +else: + raise SystemExit(DNE) + +CMD.extend(ARGS) +if sys.platform == 'win32': # https://bugs.python.org/issue19124 + import subprocess + + if sys.version_info < (3, 7): # https://bugs.python.org/issue25942 + raise SystemExit(subprocess.Popen(CMD).wait()) + else: + raise SystemExit(subprocess.call(CMD)) +else: + os.execvp(CMD[0], CMD) diff --git a/pre_commit/resources/pre-push-tmpl b/pre_commit/resources/pre-push-tmpl deleted file mode 100644 index 81d0dcbed..000000000 --- a/pre_commit/resources/pre-push-tmpl +++ /dev/null @@ -1,30 +0,0 @@ -z40=0000000000000000000000000000000000000000 -while read local_ref local_sha remote_ref remote_sha -do - if [ "$local_sha" != $z40 ]; then - if [ "$remote_sha" = $z40 ]; then - # First ancestor not found in remote - first_ancestor=$(git rev-list --topo-order --reverse "$local_sha" --not --remotes="$1" | head -n 1) - if [ -n "$first_ancestor" ]; then - # Check that the ancestor has at least one parent - git rev-list --max-parents=0 "$local_sha" | grep "$first_ancestor" > /dev/null - if [ $? -ne 0 ]; then - args="run --all-files" - else - source=$(git rev-parse "$first_ancestor"^) - args="run --origin $local_sha --source $source" - fi - fi - else - args="run --origin $local_sha --source $remote_sha" - fi - fi -done - -if [ "$args" != "" ]; then - args="$args --hook-stage push" -else - # If args is empty, then an attempt to push on an empty - # changeset is being made. In this case, just exit cleanly - exit 0 -fi diff --git a/pre_commit/resources/rbenv.tar.gz b/pre_commit/resources/rbenv.tar.gz index 4505e4714..5307b19d6 100644 Binary files a/pre_commit/resources/rbenv.tar.gz and b/pre_commit/resources/rbenv.tar.gz differ diff --git a/pre_commit/resources/ruby-build.tar.gz b/pre_commit/resources/ruby-build.tar.gz index 66107ddf2..4a69a0991 100644 Binary files a/pre_commit/resources/ruby-build.tar.gz and b/pre_commit/resources/ruby-build.tar.gz differ diff --git a/pre_commit/runner.py b/pre_commit/runner.py deleted file mode 100644 index c7455d718..000000000 --- a/pre_commit/runner.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import unicode_literals - -import os.path - -from cached_property import cached_property - -from pre_commit import git -from pre_commit.clientlib import load_config -from pre_commit.repository import Repository -from pre_commit.store import Store - - -class Runner(object): - """A `Runner` represents the execution context of the hooks. Notably the - repository under test. - """ - - def __init__(self, git_root, config_file): - self.git_root = git_root - self.config_file = config_file - - @classmethod - def create(cls, config_file): - """Creates a PreCommitRunner by doing the following: - - Finds the root of the current git repository - - chdir to that directory - """ - root = git.get_root() - os.chdir(root) - return cls(root, config_file) - - @cached_property - def git_dir(self): - return git.get_git_dir(self.git_root) - - @cached_property - def config_file_path(self): - return os.path.join(self.git_root, self.config_file) - - @cached_property - def repositories(self): - """Returns a tuple of the configured repositories.""" - config = load_config(self.config_file_path) - repositories = tuple(Repository.create(x, self.store) for x in config) - for repository in repositories: - repository.require_installed() - return repositories - - def get_hook_path(self, hook_type): - return os.path.join(self.git_dir, 'hooks', hook_type) - - @cached_property - def pre_commit_path(self): - return self.get_hook_path('pre-commit') - - @cached_property - def pre_push_path(self): - return self.get_hook_path('pre-push') - - @cached_property - def cmd_runner(self): - # TODO: remove this and inline runner.store.cmd_runner - return self.store.cmd_runner - - @cached_property - def store(self): - return Store() diff --git a/pre_commit/schema.py b/pre_commit/schema.py deleted file mode 100644 index d34ad7373..000000000 --- a/pre_commit/schema.py +++ /dev/null @@ -1,279 +0,0 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - -import collections -import contextlib -import io -import os.path -import re -import sys - -import six - - -class ValidationError(ValueError): - def __init__(self, error_msg, ctx=None): - super(ValidationError, self).__init__(error_msg) - self.error_msg = error_msg - self.ctx = ctx - - def __str__(self): - out = '\n' - err = self - while err.ctx is not None: - out += '==> {}\n'.format(err.ctx) - err = err.error_msg - out += '=====> {}'.format(err.error_msg) - return out - - -MISSING = collections.namedtuple('Missing', ())() -type(MISSING).__repr__ = lambda self: 'MISSING' - - -@contextlib.contextmanager -def validate_context(msg): - try: - yield - except ValidationError as e: - _, _, tb = sys.exc_info() - six.reraise(ValidationError, ValidationError(e, ctx=msg), tb) - - -@contextlib.contextmanager -def reraise_as(tp): - try: - yield - except ValidationError as e: - _, _, tb = sys.exc_info() - six.reraise(tp, tp(e), tb) - - -def _dct_noop(self, dct): - pass - - -def _check_optional(self, dct): - if self.key not in dct: - return - with validate_context('At key: {}'.format(self.key)): - self.check_fn(dct[self.key]) - - -def _apply_default_optional(self, dct): - dct.setdefault(self.key, self.default) - - -def _remove_default_optional(self, dct): - if dct.get(self.key, MISSING) == self.default: - del dct[self.key] - - -def _require_key(self, dct): - if self.key not in dct: - raise ValidationError('Missing required key: {}'.format(self.key)) - - -def _check_required(self, dct): - _require_key(self, dct) - _check_optional(self, dct) - - -@property -def _check_fn_required_recurse(self): - def check_fn(val): - validate(val, self.schema) - return check_fn - - -def _apply_default_required_recurse(self, dct): - dct[self.key] = apply_defaults(dct[self.key], self.schema) - - -def _remove_default_required_recurse(self, dct): - dct[self.key] = remove_defaults(dct[self.key], self.schema) - - -def _check_conditional(self, dct): - if dct.get(self.condition_key, MISSING) == self.condition_value: - _check_required(self, dct) - elif self.condition_key in dct and self.ensure_absent and self.key in dct: - if isinstance(self.condition_value, Not): - op = 'is' - cond_val = self.condition_value.val - else: - op = 'is not' - cond_val = self.condition_value - raise ValidationError( - 'Expected {key} to be absent when {cond_key} {op} {cond_val!r}, ' - 'found {key}: {val!r}'.format( - key=self.key, - val=dct[self.key], - cond_key=self.condition_key, - op=op, - cond_val=cond_val, - ) - ) - - -Required = collections.namedtuple('Required', ('key', 'check_fn')) -Required.check = _check_required -Required.apply_default = _dct_noop -Required.remove_default = _dct_noop -RequiredRecurse = collections.namedtuple('RequiredRecurse', ('key', 'schema')) -RequiredRecurse.check = _check_required -RequiredRecurse.check_fn = _check_fn_required_recurse -RequiredRecurse.apply_default = _apply_default_required_recurse -RequiredRecurse.remove_default = _remove_default_required_recurse -Optional = collections.namedtuple('Optional', ('key', 'check_fn', 'default')) -Optional.check = _check_optional -Optional.apply_default = _apply_default_optional -Optional.remove_default = _remove_default_optional -OptionalNoDefault = collections.namedtuple( - 'OptionalNoDefault', ('key', 'check_fn'), -) -OptionalNoDefault.check = _check_optional -OptionalNoDefault.apply_default = _dct_noop -OptionalNoDefault.remove_default = _dct_noop -Conditional = collections.namedtuple( - 'Conditional', - ('key', 'check_fn', 'condition_key', 'condition_value', 'ensure_absent'), -) -Conditional.__new__.__defaults__ = (False,) -Conditional.check = _check_conditional -Conditional.apply_default = _dct_noop -Conditional.remove_default = _dct_noop - - -class Map(collections.namedtuple('Map', ('object_name', 'id_key', 'items'))): - __slots__ = () - - def __new__(cls, object_name, id_key, *items): - return super(Map, cls).__new__(cls, object_name, id_key, items) - - def check(self, v): - if not isinstance(v, dict): - raise ValidationError('Expected a {} map but got a {}'.format( - self.object_name, type(v).__name__, - )) - with validate_context('At {}({}={!r})'.format( - self.object_name, self.id_key, v.get(self.id_key, MISSING), - )): - for item in self.items: - item.check(v) - - def apply_defaults(self, v): - ret = v.copy() - for item in self.items: - item.apply_default(ret) - return ret - - def remove_defaults(self, v): - ret = v.copy() - for item in self.items: - item.remove_default(ret) - return ret - - -class Array(collections.namedtuple('Array', ('of',))): - __slots__ = () - - def check(self, v): - check_array(check_any)(v) - if not v: - raise ValidationError( - "Expected at least 1 '{}'".format(self.of.object_name), - ) - for val in v: - validate(val, self.of) - - def apply_defaults(self, v): - return [apply_defaults(val, self.of) for val in v] - - def remove_defaults(self, v): - return [remove_defaults(val, self.of) for val in v] - - -class Not(object): - def __init__(self, val): - self.val = val - - def __eq__(self, other): - return other is not MISSING and other != self.val - - -def check_any(_): - pass - - -def check_type(tp, typename=None): - def check_type_fn(v): - if not isinstance(v, tp): - raise ValidationError( - 'Expected {} got {}'.format( - typename or tp.__name__, type(v).__name__, - ), - ) - return check_type_fn - - -check_bool = check_type(bool) -check_string = check_type(six.string_types, typename='string') - - -def check_regex(v): - try: - re.compile(v) - except re.error: - raise ValidationError('{!r} is not a valid python regex'.format(v)) - - -def check_array(inner_check): - def check_array_fn(v): - if not isinstance(v, (list, tuple)): - raise ValidationError( - 'Expected array but got {!r}'.format(type(v).__name__), - ) - - for i, val in enumerate(v): - with validate_context('At index {}'.format(i)): - inner_check(val) - return check_array_fn - - -def check_and(*fns): - def check(v): - for fn in fns: - fn(v) - return check - - -def validate(v, schema): - schema.check(v) - return v - - -def apply_defaults(v, schema): - return schema.apply_defaults(v) - - -def remove_defaults(v, schema): - return schema.remove_defaults(v) - - -def load_from_filename(filename, schema, load_strategy, exc_tp): - with reraise_as(exc_tp): - if not os.path.exists(filename): - raise ValidationError('{} does not exist'.format(filename)) - - with io.open(filename) as f: - contents = f.read() - - with validate_context('File {}'.format(filename)): - try: - data = load_strategy(contents) - except Exception as e: - raise ValidationError(str(e)) - - validate(data, schema) - return apply_defaults(data, schema) diff --git a/pre_commit/staged_files_only.py b/pre_commit/staged_files_only.py index d6ace66fb..09d323dc7 100644 --- a/pre_commit/staged_files_only.py +++ b/pre_commit/staged_files_only.py @@ -1,63 +1,90 @@ -from __future__ import unicode_literals - import contextlib -import io import logging +import os.path import time +from typing import Generator +from pre_commit import git from pre_commit.util import CalledProcessError +from pre_commit.util import cmd_output +from pre_commit.util import cmd_output_b +from pre_commit.xargs import xargs logger = logging.getLogger('pre_commit') +def _git_apply(patch: str) -> None: + args = ('apply', '--whitespace=nowarn', patch) + try: + cmd_output_b('git', *args) + except CalledProcessError: + # Retry with autocrlf=false -- see #570 + cmd_output_b('git', '-c', 'core.autocrlf=false', *args) + + @contextlib.contextmanager -def staged_files_only(cmd_runner): - """Clear any unstaged changes from the git working directory inside this - context. +def _intent_to_add_cleared() -> Generator[None, None, None]: + intent_to_add = git.intent_to_add_files() + if intent_to_add: + logger.warning('Unstaged intent-to-add files detected.') - Args: - cmd_runner - PrefixedCommandRunner - """ - # Determine if there are unstaged files - retcode, diff_stdout_binary, _ = cmd_runner.run( - ( - 'git', 'diff', '--ignore-submodules', '--binary', '--exit-code', - '--no-color', '--no-ext-diff', - ), + xargs(('git', 'rm', '--cached', '--'), intent_to_add) + try: + yield + finally: + xargs(('git', 'add', '--intent-to-add', '--'), intent_to_add) + else: + yield + + +@contextlib.contextmanager +def _unstaged_changes_cleared(patch_dir: str) -> Generator[None, None, None]: + tree = cmd_output('git', 'write-tree')[1].strip() + retcode, diff_stdout_binary, _ = cmd_output_b( + 'git', 'diff-index', '--ignore-submodules', '--binary', + '--exit-code', '--no-color', '--no-ext-diff', tree, '--', retcode=None, - encoding=None, ) if retcode and diff_stdout_binary.strip(): - patch_filename = cmd_runner.path('patch{}'.format(int(time.time()))) + patch_filename = f'patch{int(time.time())}' + patch_filename = os.path.join(patch_dir, patch_filename) logger.warning('Unstaged files detected.') - logger.info( - 'Stashing unstaged files to {}.'.format(patch_filename), - ) + logger.info(f'Stashing unstaged files to {patch_filename}.') # Save the current unstaged changes as a patch - with io.open(patch_filename, 'wb') as patch_file: + os.makedirs(patch_dir, exist_ok=True) + with open(patch_filename, 'wb') as patch_file: patch_file.write(diff_stdout_binary) # Clear the working directory of unstaged changes - cmd_runner.run(['git', 'checkout', '--', '.']) + cmd_output_b('git', 'checkout', '--', '.') try: yield finally: # Try to apply the patch we saved try: - cmd_runner.run(('git', 'apply', patch_filename), encoding=None) + _git_apply(patch_filename) except CalledProcessError: logger.warning( 'Stashed changes conflicted with hook auto-fixes... ' - 'Rolling back fixes...' + 'Rolling back fixes...', ) # We failed to apply the patch, presumably due to fixes made # by hooks. # Roll back the changes made by hooks. - cmd_runner.run(['git', 'checkout', '--', '.']) - cmd_runner.run(('git', 'apply', patch_filename), encoding=None) - logger.info('Restored changes from {}.'.format(patch_filename)) + cmd_output_b('git', 'checkout', '--', '.') + _git_apply(patch_filename) + logger.info(f'Restored changes from {patch_filename}.') else: # There weren't any staged files so we don't need to do anything # special yield + + +@contextlib.contextmanager +def staged_files_only(patch_dir: str) -> Generator[None, None, None]: + """Clear any unstaged changes from the git working directory inside this + context. + """ + with _intent_to_add_cleared(), _unstaged_changes_cleared(patch_dir): + yield diff --git a/pre_commit/store.py b/pre_commit/store.py index 67564483c..760b37aaf 100644 --- a/pre_commit/store.py +++ b/pre_commit/store.py @@ -1,143 +1,250 @@ -from __future__ import unicode_literals - import contextlib -import io import logging import os.path import sqlite3 import tempfile - -from cached_property import cached_property +from typing import Callable +from typing import Generator +from typing import List +from typing import Optional +from typing import Sequence +from typing import Tuple import pre_commit.constants as C -from pre_commit.prefixed_command_runner import PrefixedCommandRunner +from pre_commit import file_lock +from pre_commit import git +from pre_commit.util import CalledProcessError from pre_commit.util import clean_path_on_failure -from pre_commit.util import cmd_output -from pre_commit.util import copy_tree_to_path -from pre_commit.util import cwd -from pre_commit.util import no_git_env -from pre_commit.util import resource_filename +from pre_commit.util import cmd_output_b +from pre_commit.util import resource_text +from pre_commit.util import rmtree logger = logging.getLogger('pre_commit') -def _get_default_directory(): +def _get_default_directory() -> str: """Returns the default directory for the Store. This is intentionally underscored to indicate that `Store.get_default_directory` is the intended way to get this information. This is also done so `Store.get_default_directory` can be mocked in tests and `_get_default_directory` can be tested. """ - return os.environ.get( - 'PRE_COMMIT_HOME', - os.path.join(os.path.expanduser('~'), '.pre-commit'), + return os.environ.get('PRE_COMMIT_HOME') or os.path.join( + os.environ.get('XDG_CACHE_HOME') or os.path.expanduser('~/.cache'), + 'pre-commit', ) -class Store(object): +class Store: get_default_directory = staticmethod(_get_default_directory) - def __init__(self, directory=None): - if directory is None: - directory = self.get_default_directory() + def __init__(self, directory: Optional[str] = None) -> None: + self.directory = directory or Store.get_default_directory() + self.db_path = os.path.join(self.directory, 'db.db') - self.directory = directory - self.__created = False - - def _write_readme(self): - with io.open(os.path.join(self.directory, 'README'), 'w') as readme: - readme.write( - 'This directory is maintained by the pre-commit project.\n' - 'Learn more: https://github.com/pre-commit/pre-commit\n' - ) + if not os.path.exists(self.directory): + os.makedirs(self.directory, exist_ok=True) + with open(os.path.join(self.directory, 'README'), 'w') as f: + f.write( + 'This directory is maintained by the pre-commit project.\n' + 'Learn more: https://github.com/pre-commit/pre-commit\n', + ) - def _write_sqlite_db(self): - # To avoid a race where someone ^Cs between db creation and execution - # of the CREATE TABLE statement - fd, tmpfile = tempfile.mkstemp(dir=self.directory) - # We'll be managing this file ourselves - os.close(fd) + if os.path.exists(self.db_path): + return + with self.exclusive_lock(): + # Another process may have already completed this work + if os.path.exists(self.db_path): # pragma: no cover (race) + return + # To avoid a race where someone ^Cs between db creation and + # execution of the CREATE TABLE statement + fd, tmpfile = tempfile.mkstemp(dir=self.directory) + # We'll be managing this file ourselves + os.close(fd) + with self.connect(db_path=tmpfile) as db: + db.executescript( + 'CREATE TABLE repos (' + ' repo TEXT NOT NULL,' + ' ref TEXT NOT NULL,' + ' path TEXT NOT NULL,' + ' PRIMARY KEY (repo, ref)' + ');', + ) + self._create_config_table(db) + + # Atomic file move + os.rename(tmpfile, self.db_path) + + @contextlib.contextmanager + def exclusive_lock(self) -> Generator[None, None, None]: + def blocked_cb() -> None: # pragma: no cover (tests are in-process) + logger.info('Locking pre-commit directory') + + with file_lock.lock(os.path.join(self.directory, '.lock'), blocked_cb): + yield + + @contextlib.contextmanager + def connect( + self, + db_path: Optional[str] = None, + ) -> Generator[sqlite3.Connection, None, None]: + db_path = db_path or self.db_path # sqlite doesn't close its fd with its contextmanager >.< # contextlib.closing fixes this. - # See: http://stackoverflow.com/a/28032829/812183 - with contextlib.closing(sqlite3.connect(tmpfile)) as db: - db.executescript( - 'CREATE TABLE repos (' - ' repo CHAR(255) NOT NULL,' - ' ref CHAR(255) NOT NULL,' - ' path CHAR(255) NOT NULL,' - ' PRIMARY KEY (repo, ref)' - ');' - ) + # See: https://stackoverflow.com/a/28032829/812183 + with contextlib.closing(sqlite3.connect(db_path)) as db: + # this creates a transaction + with db: + yield db + + @classmethod + def db_repo_name(cls, repo: str, deps: Sequence[str]) -> str: + if deps: + return f'{repo}:{",".join(sorted(deps))}' + else: + return repo + + def _new_repo( + self, + repo: str, + ref: str, + deps: Sequence[str], + make_strategy: Callable[[str], None], + ) -> str: + repo = self.db_repo_name(repo, deps) + + def _get_result() -> Optional[str]: + # Check if we already exist + with self.connect() as db: + result = db.execute( + 'SELECT path FROM repos WHERE repo = ? AND ref = ?', + (repo, ref), + ).fetchone() + return result[0] if result else None + + result = _get_result() + if result: + return result + with self.exclusive_lock(): + # Another process may have already completed this work + result = _get_result() + if result: # pragma: no cover (race) + return result + + logger.info(f'Initializing environment for {repo}.') + + directory = tempfile.mkdtemp(prefix='repo', dir=self.directory) + with clean_path_on_failure(directory): + make_strategy(directory) + + # Update our db with the created repo + with self.connect() as db: + db.execute( + 'INSERT INTO repos (repo, ref, path) VALUES (?, ?, ?)', + [repo, ref, directory], + ) + return directory - # Atomic file move - os.rename(tmpfile, self.db_path) + def _complete_clone(self, ref: str, git_cmd: Callable[..., None]) -> None: + """Perform a complete clone of a repository and its submodules """ - def _create(self): - if os.path.exists(self.db_path): - return - if not os.path.exists(self.directory): - os.makedirs(self.directory) - self._write_readme() - self._write_sqlite_db() - - def require_created(self): - """Require the pre-commit file store to be created.""" - if not self.__created: - self._create() - self.__created = True - - def _new_repo(self, repo, ref, make_strategy): - self.require_created() - - # Check if we already exist - with sqlite3.connect(self.db_path) as db: - result = db.execute( - 'SELECT path FROM repos WHERE repo = ? AND ref = ?', - [repo, ref], - ).fetchone() - if result: - return result[0] - - logger.info('Initializing environment for {}.'.format(repo)) - - directory = tempfile.mkdtemp(prefix='repo', dir=self.directory) - with clean_path_on_failure(directory): - make_strategy(directory) - - # Update our db with the created repo - with sqlite3.connect(self.db_path) as db: - db.execute( - 'INSERT INTO repos (repo, ref, path) VALUES (?, ?, ?)', - [repo, ref, directory], - ) - return directory + git_cmd('fetch', 'origin', '--tags') + git_cmd('checkout', ref) + git_cmd('submodule', 'update', '--init', '--recursive') + + def _shallow_clone(self, ref: str, git_cmd: Callable[..., None]) -> None: + """Perform a shallow clone of a repository and its submodules """ - def clone(self, repo, ref): + git_config = 'protocol.version=2' + git_cmd('-c', git_config, 'fetch', 'origin', ref, '--depth=1') + git_cmd('checkout', 'FETCH_HEAD') + git_cmd( + '-c', git_config, 'submodule', 'update', '--init', '--recursive', + '--depth=1', + ) + + def clone(self, repo: str, ref: str, deps: Sequence[str] = ()) -> str: """Clone the given url and checkout the specific ref.""" - def clone_strategy(directory): - cmd_output( - 'git', 'clone', '--no-checkout', repo, directory, - env=no_git_env(), - ) - with cwd(directory): - cmd_output('git', 'reset', ref, '--hard', env=no_git_env()) - return self._new_repo(repo, ref, clone_strategy) + def clone_strategy(directory: str) -> None: + git.init_repo(directory, repo) + env = git.no_git_env() + + def _git_cmd(*args: str) -> None: + cmd_output_b('git', *args, cwd=directory, env=env) + + try: + self._shallow_clone(ref, _git_cmd) + except CalledProcessError: + self._complete_clone(ref, _git_cmd) + + return self._new_repo(repo, ref, deps, clone_strategy) + + LOCAL_RESOURCES = ( + 'Cargo.toml', 'main.go', 'main.rs', '.npmignore', 'package.json', + 'pre_commit_dummy_package.gemspec', 'setup.py', 'environment.yml', + 'Makefile.PL', + ) + + def make_local(self, deps: Sequence[str]) -> str: + def make_local_strategy(directory: str) -> None: + for resource in self.LOCAL_RESOURCES: + contents = resource_text(f'empty_template_{resource}') + with open(os.path.join(directory, resource), 'w') as f: + f.write(contents) + + env = git.no_git_env() + + # initialize the git repository so it looks more like cloned repos + def _git_cmd(*args: str) -> None: + cmd_output_b('git', *args, cwd=directory, env=env) + + git.init_repo(directory, '<>') + _git_cmd('add', '.') + git.commit(repo=directory) - def make_local(self, deps): - def make_local_strategy(directory): - copy_tree_to_path(resource_filename('empty_template'), directory) return self._new_repo( - 'local:{}'.format(','.join(sorted(deps))), C.LOCAL_REPO_VERSION, - make_local_strategy, + 'local', C.LOCAL_REPO_VERSION, deps, make_local_strategy, ) - @cached_property - def cmd_runner(self): - return PrefixedCommandRunner(self.directory) + def _create_config_table(self, db: sqlite3.Connection) -> None: + db.executescript( + 'CREATE TABLE IF NOT EXISTS configs (' + ' path TEXT NOT NULL,' + ' PRIMARY KEY (path)' + ');', + ) - @cached_property - def db_path(self): - return os.path.join(self.directory, 'db.db') + def mark_config_used(self, path: str) -> None: + path = os.path.realpath(path) + # don't insert config files that do not exist + if not os.path.exists(path): + return + with self.connect() as db: + # TODO: eventually remove this and only create in _create + self._create_config_table(db) + db.execute('INSERT OR IGNORE INTO configs VALUES (?)', (path,)) + + def select_all_configs(self) -> List[str]: + with self.connect() as db: + self._create_config_table(db) + rows = db.execute('SELECT path FROM configs').fetchall() + return [path for path, in rows] + + def delete_configs(self, configs: List[str]) -> None: + with self.connect() as db: + rows = [(path,) for path in configs] + db.executemany('DELETE FROM configs WHERE path = ?', rows) + + def select_all_repos(self) -> List[Tuple[str, str, str]]: + with self.connect() as db: + return db.execute('SELECT repo, ref, path from repos').fetchall() + + def delete_repo(self, db_repo_name: str, ref: str, path: str) -> None: + with self.connect() as db: + db.execute( + 'DELETE FROM repos WHERE repo = ? and ref = ?', + (db_repo_name, ref), + ) + rmtree(path) diff --git a/pre_commit/util.py b/pre_commit/util.py index 4c3ad421b..2db579a5f 100644 --- a/pre_commit/util.py +++ b/pre_commit/util.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import contextlib import errno import functools @@ -7,52 +5,54 @@ import shutil import stat import subprocess +import sys import tempfile +from types import TracebackType +from typing import Any +from typing import Callable +from typing import Dict +from typing import Generator +from typing import IO +from typing import Optional +from typing import Tuple +from typing import Type +from typing import Union + +import yaml -import pkg_resources -import six - -from pre_commit import five from pre_commit import parse_shebang +if sys.version_info >= (3, 7): # pragma: no cover (PY37+) + from importlib.resources import open_binary + from importlib.resources import read_text +else: # pragma: no cover ( str: + # when python/mypy#1484 is solved, this can be `functools.partial` + return yaml.dump( + o, Dumper=Dumper, default_flow_style=False, indent=4, sort_keys=False, + ) - wrapper._cache = {} - return wrapper +def force_bytes(exc: Any) -> bytes: + with contextlib.suppress(TypeError): + return bytes(exc) + with contextlib.suppress(Exception): + return str(exc).encode() + return f''.encode() @contextlib.contextmanager -def clean_path_on_failure(path): +def clean_path_on_failure(path: str) -> Generator[None, None, None]: """Cleans up the directory on an exceptional failure.""" try: yield @@ -63,26 +63,7 @@ def clean_path_on_failure(path): @contextlib.contextmanager -def noop_context(): - yield - - -def no_git_env(): - # Too many bugs dealing with environment variables and GIT: - # https://github.com/pre-commit/pre-commit/issues/300 - # In git 2.6.3 (maybe others), git exports GIT_WORK_TREE while running - # pre-commit hooks - # In git 1.9.1 (maybe others), git exports GIT_DIR and GIT_INDEX_FILE - # while running pre-commit hooks in submodules. - # GIT_DIR: Causes git clone to clone wrong thing - # GIT_INDEX_FILE: Causes 'error invalid object ...' during commit - return { - k: v for k, v in os.environ.items() if not k.startswith('GIT_') - } - - -@contextlib.contextmanager -def tmpdir(): +def tmpdir() -> Generator[str, None, None]: """Contextmanager to create a temporary directory. It will be cleaned up afterwards. """ @@ -93,133 +74,199 @@ def tmpdir(): rmtree(tempdir) -def resource_filename(filename): - return pkg_resources.resource_filename( - 'pre_commit', - os.path.join('resources', filename), - ) +def resource_bytesio(filename: str) -> IO[bytes]: + return open_binary('pre_commit.resources', filename) -def make_executable(filename): +def resource_text(filename: str) -> str: + return read_text('pre_commit.resources', filename) + + +def make_executable(filename: str) -> None: original_mode = os.stat(filename).st_mode - os.chmod( - filename, - original_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, - ) + new_mode = original_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + os.chmod(filename, new_mode) class CalledProcessError(RuntimeError): - def __init__(self, returncode, cmd, expected_returncode, output=None): - super(CalledProcessError, self).__init__( - returncode, cmd, expected_returncode, output, - ) + def __init__( + self, + returncode: int, + cmd: Tuple[str, ...], + expected_returncode: int, + stdout: bytes, + stderr: Optional[bytes], + ) -> None: + super().__init__(returncode, cmd, expected_returncode, stdout, stderr) self.returncode = returncode self.cmd = cmd self.expected_returncode = expected_returncode - self.output = output - - def to_bytes(self): - output = [] - for maybe_text in self.output: - if maybe_text: - output.append( - b'\n ' + - five.to_bytes(maybe_text).replace(b'\n', b'\n ') - ) + self.stdout = stdout + self.stderr = stderr + + def __bytes__(self) -> bytes: + def _indent_or_none(part: Optional[bytes]) -> bytes: + if part: + return b'\n ' + part.replace(b'\n', b'\n ') else: - output.append(b'(none)') + return b' (none)' return b''.join(( - five.to_bytes( - 'Command: {!r}\n' - 'Return code: {}\n' - 'Expected return code: {}\n'.format( - self.cmd, self.returncode, self.expected_returncode - ) - ), - b'Output: ', output[0], b'\n', - b'Errors: ', output[1], b'\n', + f'command: {self.cmd!r}\n'.encode(), + f'return code: {self.returncode}\n'.encode(), + f'expected return code: {self.expected_returncode}\n'.encode(), + b'stdout:', _indent_or_none(self.stdout), b'\n', + b'stderr:', _indent_or_none(self.stderr), )) - def to_text(self): - return self.to_bytes().decode('UTF-8') + def __str__(self) -> str: + return self.__bytes__().decode() - if six.PY2: # pragma: no cover (py2) - __str__ = to_bytes - __unicode__ = to_text - else: # pragma: no cover (py3) - __bytes__ = to_bytes - __str__ = to_text +def _setdefault_kwargs(kwargs: Dict[str, Any]) -> None: + for arg in ('stdin', 'stdout', 'stderr'): + kwargs.setdefault(arg, subprocess.PIPE) -def cmd_output(*cmd, **kwargs): - retcode = kwargs.pop('retcode', 0) - encoding = kwargs.pop('encoding', 'UTF-8') - __popen = kwargs.pop('__popen', subprocess.Popen) - popen_kwargs = { - 'stdin': subprocess.PIPE, - 'stdout': subprocess.PIPE, - 'stderr': subprocess.PIPE, - } +def _oserror_to_output(e: OSError) -> Tuple[int, bytes, None]: + return 1, force_bytes(e).rstrip(b'\n') + b'\n', None - # py2/py3 on windows are more strict about the types here - cmd = tuple(five.n(arg) for arg in cmd) - kwargs['env'] = { - five.n(key): five.n(value) - for key, value in kwargs.pop('env', {}).items() - } or None + +def cmd_output_b( + *cmd: str, + retcode: Optional[int] = 0, + **kwargs: Any, +) -> Tuple[int, bytes, Optional[bytes]]: + _setdefault_kwargs(kwargs) try: cmd = parse_shebang.normalize_cmd(cmd) except parse_shebang.ExecutableNotFoundError as e: - returncode, stdout, stderr = e.to_output() + returncode, stdout_b, stderr_b = e.to_output() else: - popen_kwargs.update(kwargs) - proc = __popen(cmd, **popen_kwargs) - stdout, stderr = proc.communicate() - returncode = proc.returncode - if encoding is not None and stdout is not None: - stdout = stdout.decode(encoding) - if encoding is not None and stderr is not None: - stderr = stderr.decode(encoding) + try: + proc = subprocess.Popen(cmd, **kwargs) + except OSError as e: + returncode, stdout_b, stderr_b = _oserror_to_output(e) + else: + stdout_b, stderr_b = proc.communicate() + returncode = proc.returncode if retcode is not None and retcode != returncode: - raise CalledProcessError( - returncode, cmd, retcode, output=(stdout, stderr), - ) + raise CalledProcessError(returncode, cmd, retcode, stdout_b, stderr_b) + + return returncode, stdout_b, stderr_b + +def cmd_output(*cmd: str, **kwargs: Any) -> Tuple[int, str, Optional[str]]: + returncode, stdout_b, stderr_b = cmd_output_b(*cmd, **kwargs) + stdout = stdout_b.decode() if stdout_b is not None else None + stderr = stderr_b.decode() if stderr_b is not None else None return returncode, stdout, stderr -def rmtree(path): +if os.name != 'nt': # pragma: win32 no cover + from os import openpty + import termios + + class Pty: + def __init__(self) -> None: + self.r: Optional[int] = None + self.w: Optional[int] = None + + def __enter__(self) -> 'Pty': + self.r, self.w = openpty() + + # tty flags normally change \n to \r\n + attrs = termios.tcgetattr(self.r) + assert isinstance(attrs[1], int) + attrs[1] &= ~(termios.ONLCR | termios.OPOST) + termios.tcsetattr(self.r, termios.TCSANOW, attrs) + + return self + + def close_w(self) -> None: + if self.w is not None: + os.close(self.w) + self.w = None + + def close_r(self) -> None: + assert self.r is not None + os.close(self.r) + self.r = None + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: + self.close_w() + self.close_r() + + def cmd_output_p( + *cmd: str, + retcode: Optional[int] = 0, + **kwargs: Any, + ) -> Tuple[int, bytes, Optional[bytes]]: + assert retcode is None + assert kwargs['stderr'] == subprocess.STDOUT, kwargs['stderr'] + _setdefault_kwargs(kwargs) + + try: + cmd = parse_shebang.normalize_cmd(cmd) + except parse_shebang.ExecutableNotFoundError as e: + return e.to_output() + + with open(os.devnull) as devnull, Pty() as pty: + assert pty.r is not None + kwargs.update({'stdin': devnull, 'stdout': pty.w, 'stderr': pty.w}) + try: + proc = subprocess.Popen(cmd, **kwargs) + except OSError as e: + return _oserror_to_output(e) + + pty.close_w() + + buf = b'' + while True: + try: + bts = os.read(pty.r, 4096) + except OSError as e: + if e.errno == errno.EIO: + bts = b'' + else: + raise + else: + buf += bts + if not bts: + break + + return proc.wait(), buf, None +else: # pragma: no cover + cmd_output_p = cmd_output_b + + +def rmtree(path: str) -> None: """On windows, rmtree fails for readonly dirs.""" - def handle_remove_readonly(func, path, exc): # pragma: no cover (windows) + def handle_remove_readonly( + func: Callable[..., Any], + path: str, + exc: Tuple[Type[OSError], OSError, TracebackType], + ) -> None: excvalue = exc[1] if ( func in (os.rmdir, os.remove, os.unlink) and excvalue.errno == errno.EACCES ): - os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + for p in (path, os.path.dirname(path)): + os.chmod(p, os.stat(p).st_mode | stat.S_IWUSR) func(path) else: raise shutil.rmtree(path, ignore_errors=False, onerror=handle_remove_readonly) -def copy_tree_to_path(src_dir, dest_dir): - """Copies all of the things inside src_dir to an already existing dest_dir. - - This looks eerily similar to shutil.copytree, but copytree has no option - for not creating dest_dir. - """ - names = os.listdir(src_dir) - - for name in names: - srcname = os.path.join(src_dir, name) - destname = os.path.join(dest_dir, name) - - if os.path.isdir(srcname): - shutil.copytree(srcname, destname) - else: - shutil.copy(srcname, destname) +def parse_version(s: str) -> Tuple[int, ...]: + """poor man's version comparison""" + return tuple(int(p) for p in s.split('.')) diff --git a/pre_commit/xargs.py b/pre_commit/xargs.py index eea3acdb9..5235dc650 100644 --- a/pre_commit/xargs.py +++ b/pre_commit/xargs.py @@ -1,41 +1,100 @@ -from __future__ import absolute_import -from __future__ import unicode_literals +import concurrent.futures +import contextlib +import math +import os +import subprocess +import sys +from typing import Any +from typing import Callable +from typing import Generator +from typing import Iterable +from typing import List +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import TypeVar from pre_commit import parse_shebang -from pre_commit.util import cmd_output +from pre_commit.util import cmd_output_b +from pre_commit.util import cmd_output_p +from pre_commit.util import EnvironT +TArg = TypeVar('TArg') +TRet = TypeVar('TRet') -# Limit used previously to avoid "xargs ... Bad file number" on windows -# This is slightly less than the posix mandated minimum -MAX_LENGTH = 4000 + +def _environ_size(_env: Optional[EnvironT] = None) -> int: + environ = _env if _env is not None else getattr(os, 'environb', os.environ) + size = 8 * len(environ) # number of pointers in `envp` + for k, v in environ.items(): + size += len(k) + len(v) + 2 # c strings in `envp` + return size + + +def _get_platform_max_length() -> int: # pragma: no cover (platform specific) + if os.name == 'posix': + maximum = os.sysconf('SC_ARG_MAX') - 2048 - _environ_size() + maximum = max(min(maximum, 2 ** 17), 2 ** 12) + return maximum + elif os.name == 'nt': + return 2 ** 15 - 2048 # UNICODE_STRING max - headroom + else: + # posix minimum + return 2 ** 12 + + +def _command_length(*cmd: str) -> int: + full_cmd = ' '.join(cmd) + + # win32 uses the amount of characters, more details at: + # https://github.com/pre-commit/pre-commit/pull/839 + if sys.platform == 'win32': + return len(full_cmd.encode('utf-16le')) // 2 + else: + return len(full_cmd.encode(sys.getfilesystemencoding())) class ArgumentTooLongError(RuntimeError): pass -def partition(cmd, varargs, _max_length=MAX_LENGTH): +def partition( + cmd: Sequence[str], + varargs: Sequence[str], + target_concurrency: int, + _max_length: Optional[int] = None, +) -> Tuple[Tuple[str, ...], ...]: + _max_length = _max_length or _get_platform_max_length() + + # Generally, we try to partition evenly into at least `target_concurrency` + # partitions, but we don't want a bunch of tiny partitions. + max_args = max(4, math.ceil(len(varargs) / target_concurrency)) + cmd = tuple(cmd) ret = [] - ret_cmd = [] - total_len = len(' '.join(cmd)) + ret_cmd: List[str] = [] # Reversed so arguments are in order varargs = list(reversed(varargs)) + total_length = _command_length(*cmd) + 1 while varargs: arg = varargs.pop() - if total_len + 1 + len(arg) <= _max_length: + arg_length = _command_length(arg) + 1 + if ( + total_length + arg_length <= _max_length and + len(ret_cmd) < max_args + ): ret_cmd.append(arg) - total_len += len(arg) + total_length += arg_length elif not ret_cmd: raise ArgumentTooLongError(arg) else: # We've exceeded the length, yield a command ret.append(cmd + tuple(ret_cmd)) ret_cmd = [] - total_len = len(' '.join(cmd)) + total_length = _command_length(*cmd) + 1 varargs.append(arg) ret.append(cmd + tuple(ret_cmd)) @@ -43,37 +102,56 @@ def partition(cmd, varargs, _max_length=MAX_LENGTH): return tuple(ret) -def xargs(cmd, varargs, **kwargs): +@contextlib.contextmanager +def _thread_mapper(maxsize: int) -> Generator[ + Callable[[Callable[[TArg], TRet], Iterable[TArg]], Iterable[TRet]], + None, None, +]: + if maxsize == 1: + yield map + else: + with concurrent.futures.ThreadPoolExecutor(maxsize) as ex: + yield ex.map + + +def xargs( + cmd: Tuple[str, ...], + varargs: Sequence[str], + *, + color: bool = False, + target_concurrency: int = 1, + _max_length: int = _get_platform_max_length(), + **kwargs: Any, +) -> Tuple[int, bytes]: """A simplified implementation of xargs. - negate: Make nonzero successful and zero a failure + color: Make a pty if on a platform that supports it + target_concurrency: Target number of partitions to run concurrently """ - negate = kwargs.pop('negate', False) + cmd_fn = cmd_output_p if color else cmd_output_b retcode = 0 stdout = b'' - stderr = b'' try: - parse_shebang.normexe(cmd[0]) + cmd = parse_shebang.normalize_cmd(cmd) except parse_shebang.ExecutableNotFoundError as e: - return e.to_output() + return e.to_output()[:2] + + partitions = partition(cmd, varargs, target_concurrency, _max_length) - for run_cmd in partition(cmd, varargs, **kwargs): - proc_retcode, proc_out, proc_err = cmd_output( - *run_cmd, encoding=None, retcode=None + def run_cmd_partition( + run_cmd: Tuple[str, ...], + ) -> Tuple[int, bytes, Optional[bytes]]: + return cmd_fn( + *run_cmd, retcode=None, stderr=subprocess.STDOUT, **kwargs, ) - # This is *slightly* too clever so I'll explain it. - # First the xor boolean table: - # T | F | - # +-------+ - # T | F | T | - # --+-------+ - # F | T | F | - # --+-------+ - # When negate is True, it has the effect of flipping the return code - # Otherwise, the retuncode is unchanged - retcode |= bool(proc_retcode) ^ negate - stdout += proc_out - stderr += proc_err - - return retcode, stdout, stderr + + threads = min(len(partitions), target_concurrency) + with _thread_mapper(threads) as thread_map: + results = thread_map(run_cmd_partition, partitions) + + for proc_retcode, proc_out, _ in results: + retcode = max(retcode, proc_retcode) + stdout += proc_out + + return retcode, stdout diff --git a/requirements-dev.txt b/requirements-dev.txt index e18122267..d6a13dc43 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,10 +1,4 @@ --e . - +covdefaults coverage -flake8 -mock pytest pytest-env - -# setuptools breaks pypy3 with extraneous output -setuptools<18.5 diff --git a/setup.cfg b/setup.cfg index e57d130e3..a02fab181 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,70 @@ -[wheel] +[metadata] +name = pre_commit +version = 2.2.0 +description = A framework for managing and maintaining multi-language pre-commit hooks. +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/pre-commit/pre-commit +author = Anthony Sottile +author_email = asottile@umich.edu +license = MIT +license_file = LICENSE +classifiers = + License :: OSI Approved :: MIT License + Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: Implementation :: CPython + Programming Language :: Python :: Implementation :: PyPy + +[options] +packages = find: +install_requires = + cfgv>=2.0.0 + identify>=1.0.0 + nodeenv>=0.11.1 + pyyaml>=5.1 + toml + virtualenv>=15.2 + importlib-metadata;python_version<"3.8" + importlib-resources;python_version<"3.7" +python_requires = >=3.6.1 + +[options.entry_points] +console_scripts = + pre-commit = pre_commit.main:main + pre-commit-validate-config = pre_commit.clientlib:validate_config_main + pre-commit-validate-manifest = pre_commit.clientlib:validate_manifest_main + +[options.package_data] +pre_commit.resources = + *.tar.gz + empty_template_* + hook-tmpl + +[options.packages.find] +exclude = + tests* + testing* + +[bdist_wheel] universal = True + +[coverage:run] +plugins = covdefaults +omit = pre_commit/resources/* + +[mypy] +check_untyped_defs = true +disallow_any_generics = true +disallow_incomplete_defs = true +disallow_untyped_defs = true +no_implicit_optional = true + +[mypy-testing.*] +disallow_untyped_defs = false + +[mypy-tests.*] +disallow_untyped_defs = false diff --git a/setup.py b/setup.py index 3c94bfa32..8bf1ba938 100644 --- a/setup.py +++ b/setup.py @@ -1,57 +1,2 @@ -from setuptools import find_packages from setuptools import setup - - -setup( - name='pre_commit', - description=( - 'A framework for managing and maintaining multi-language pre-commit ' - 'hooks.' - ), - url='https://github.com/pre-commit/pre-commit', - version='0.15.0', - - author='Anthony Sottile', - author_email='asottile@umich.edu', - - platforms='linux', - classifiers=[ - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - ], - - packages=find_packages(exclude=('tests*', 'testing*')), - package_data={ - 'pre_commit': [ - 'resources/hook-tmpl', - 'resources/pre-push-tmpl', - 'resources/rbenv.tar.gz', - 'resources/ruby-build.tar.gz', - 'resources/ruby-download.tar.gz', - 'resources/empty_template/*', - 'resources/empty_template/.npmignore', - ], - }, - install_requires=[ - 'aspy.yaml', - 'cached-property', - 'identify>=1.0.0', - 'nodeenv>=0.11.1', - 'pyyaml', - 'six', - 'virtualenv', - ], - entry_points={ - 'console_scripts': [ - 'pre-commit = pre_commit.main:main', - 'pre-commit-validate-config = pre_commit.clientlib:validate_config_main', # noqa - 'pre-commit-validate-manifest = pre_commit.clientlib:validate_manifest_main', # noqa - ], - }, -) +setup() diff --git a/testing/auto_namedtuple.py b/testing/auto_namedtuple.py index 02e08fef0..0841094eb 100644 --- a/testing/auto_namedtuple.py +++ b/testing/auto_namedtuple.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import collections diff --git a/testing/fixtures.py b/testing/fixtures.py index dffff4ca0..f7def081f 100644 --- a/testing/fixtures.py +++ b/testing/fixtures.py @@ -1,55 +1,66 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import contextlib -import io import os.path -from collections import OrderedDict +import shutil -from aspy.yaml import ordered_dump -from aspy.yaml import ordered_load +from cfgv import apply_defaults +from cfgv import validate import pre_commit.constants as C +from pre_commit import git from pre_commit.clientlib import CONFIG_SCHEMA from pre_commit.clientlib import load_manifest -from pre_commit.schema import apply_defaults -from pre_commit.schema import validate from pre_commit.util import cmd_output -from pre_commit.util import copy_tree_to_path -from pre_commit.util import cwd -from testing.util import get_head_sha +from pre_commit.util import yaml_dump +from pre_commit.util import yaml_load from testing.util import get_resource_path +from testing.util import git_commit + + +def copy_tree_to_path(src_dir, dest_dir): + """Copies all of the things inside src_dir to an already existing dest_dir. + + This looks eerily similar to shutil.copytree, but copytree has no option + for not creating dest_dir. + """ + names = os.listdir(src_dir) + + for name in names: + srcname = os.path.join(src_dir, name) + destname = os.path.join(dest_dir, name) + + if os.path.isdir(srcname): + shutil.copytree(srcname, destname) + else: + shutil.copy(srcname, destname) def git_dir(tempdir_factory): path = tempdir_factory.get() - with cwd(path): - cmd_output('git', 'init') + cmd_output('git', 'init', path) return path def make_repo(tempdir_factory, repo_source): path = git_dir(tempdir_factory) copy_tree_to_path(get_resource_path(repo_source), path) - with cwd(path): - cmd_output('git', 'add', '.') - cmd_output('git', 'commit', '-m', 'Add hooks') + cmd_output('git', 'add', '.', cwd=path) + git_commit(msg=make_repo.__name__, cwd=path) return path @contextlib.contextmanager -def modify_manifest(path): +def modify_manifest(path, commit=True): """Modify the manifest yielded by this context to write to .pre-commit-hooks.yaml. """ manifest_path = os.path.join(path, C.MANIFEST_FILE) - manifest = ordered_load(io.open(manifest_path).read()) + with open(manifest_path) as f: + manifest = yaml_load(f.read()) yield manifest - with io.open(manifest_path, 'w') as manifest_file: - manifest_file.write(ordered_dump(manifest, **C.YAML_DUMP_KWARGS)) - cmd_output( - 'git', 'commit', '-am', 'update .pre-commit-hooks.yaml', cwd=path, - ) + with open(manifest_path, 'w') as manifest_file: + manifest_file.write(yaml_dump(manifest)) + if commit: + git_commit(msg=modify_manifest.__name__, cwd=path) @contextlib.contextmanager @@ -58,44 +69,43 @@ def modify_config(path='.', commit=True): .pre-commit-config.yaml """ config_path = os.path.join(path, C.CONFIG_FILE) - config = ordered_load(io.open(config_path).read()) + with open(config_path) as f: + config = yaml_load(f.read()) yield config - with io.open(config_path, 'w', encoding='UTF-8') as config_file: - config_file.write(ordered_dump(config, **C.YAML_DUMP_KWARGS)) + with open(config_path, 'w', encoding='UTF-8') as config_file: + config_file.write(yaml_dump(config)) if commit: - cmd_output('git', 'commit', '-am', 'update config', cwd=path) - - -def config_with_local_hooks(): - return OrderedDict(( - ('repo', 'local'), - ('hooks', [OrderedDict(( - ('id', 'do_not_commit'), - ('name', 'Block if "DO NOT COMMIT" is found'), - ('entry', 'DO NOT COMMIT'), - ('language', 'pcre'), - ('files', '^(.*)$'), - ))]) - )) - - -def make_config_from_repo( - repo_path, sha=None, hooks=None, check=True, legacy=False, -): - filename = C.MANIFEST_FILE_LEGACY if legacy else C.MANIFEST_FILE - manifest = load_manifest(os.path.join(repo_path, filename)) - config = OrderedDict(( - ('repo', repo_path), - ('sha', sha or get_head_sha(repo_path)), - ( - 'hooks', - hooks or [OrderedDict((('id', hook['id']),)) for hook in manifest], - ), - )) + git_commit(msg=modify_config.__name__, cwd=path) + + +def sample_local_config(): + return { + 'repo': 'local', + 'hooks': [{ + 'id': 'do_not_commit', + 'name': 'Block if "DO NOT COMMIT" is found', + 'entry': 'DO NOT COMMIT', + 'language': 'pygrep', + }], + } + + +def sample_meta_config(): + return {'repo': 'meta', 'hooks': [{'id': 'check-useless-excludes'}]} + + +def make_config_from_repo(repo_path, rev=None, hooks=None, check=True): + manifest = load_manifest(os.path.join(repo_path, C.MANIFEST_FILE)) + config = { + 'repo': f'file://{repo_path}', + 'rev': rev or git.head_rev(repo_path), + 'hooks': hooks or [{'id': hook['id']} for hook in manifest], + } if check: - wrapped = validate([config], CONFIG_SCHEMA) - config, = apply_defaults(wrapped, CONFIG_SCHEMA) + wrapped = validate({'repos': [config]}, CONFIG_SCHEMA) + wrapped = apply_defaults(wrapped, CONFIG_SCHEMA) + config, = wrapped['repos'] return config else: return config @@ -103,31 +113,29 @@ def make_config_from_repo( def read_config(directory, config_file=C.CONFIG_FILE): config_path = os.path.join(directory, config_file) - config = ordered_load(io.open(config_path).read()) + with open(config_path) as f: + config = yaml_load(f.read()) return config def write_config(directory, config, config_file=C.CONFIG_FILE): - if type(config) is not list: - assert type(config) is OrderedDict - config = [config] - with io.open(os.path.join(directory, config_file), 'w') as outfile: - outfile.write(ordered_dump(config, **C.YAML_DUMP_KWARGS)) + if type(config) is not list and 'repos' not in config: + assert isinstance(config, dict), config + config = {'repos': [config]} + with open(os.path.join(directory, config_file), 'w') as outfile: + outfile.write(yaml_dump(config)) def add_config_to_repo(git_path, config, config_file=C.CONFIG_FILE): write_config(git_path, config, config_file=config_file) - with cwd(git_path): - cmd_output('git', 'add', config_file) - cmd_output('git', 'commit', '-m', 'Add hooks config') + cmd_output('git', 'add', config_file, cwd=git_path) + git_commit(msg=add_config_to_repo.__name__, cwd=git_path) return git_path def remove_config_from_repo(git_path, config_file=C.CONFIG_FILE): - os.unlink(os.path.join(git_path, config_file)) - with cwd(git_path): - cmd_output('git', 'add', config_file) - cmd_output('git', 'commit', '-m', 'Remove hooks config') + cmd_output('git', 'rm', config_file, cwd=git_path) + git_commit(msg=remove_config_from_repo.__name__, cwd=git_path) return git_path diff --git a/testing/gen-languages-all b/testing/gen-languages-all new file mode 100755 index 000000000..6d0b26ff9 --- /dev/null +++ b/testing/gen-languages-all @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import sys + +LANGUAGES = [ + 'conda', 'docker', 'docker_image', 'fail', 'golang', 'node', 'perl', + 'pygrep', 'python', 'python_venv', 'ruby', 'rust', 'script', 'swift', + 'system', +] +FIELDS = [ + 'ENVIRONMENT_DIR', 'get_default_version', 'healthy', 'install_environment', + 'run_hook', +] + + +def main() -> int: + print(f' # BEGIN GENERATED ({sys.argv[0]})') + for lang in LANGUAGES: + parts = [f' {lang!r}: Language(name={lang!r}'] + for k in FIELDS: + parts.append(f', {k}={lang}.{k}') + parts.append('), # noqa: E501') + print(''.join(parts)) + print(' # END GENERATED') + return 0 + + +if __name__ == '__main__': + exit(main()) diff --git a/testing/get-swift.sh b/testing/get-swift.sh new file mode 100755 index 000000000..e205d44e2 --- /dev/null +++ b/testing/get-swift.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# This is a script used in CI to install swift +set -euxo pipefail + +. /etc/lsb-release +if [ "$DISTRIB_CODENAME" = "bionic" ]; then + SWIFT_URL='https://swift.org/builds/swift-5.1.3-release/ubuntu1804/swift-5.1.3-RELEASE/swift-5.1.3-RELEASE-ubuntu18.04.tar.gz' + SWIFT_HASH='ac82ccd773fe3d586fc340814e31e120da1ff695c6a712f6634e9cc720769610' +else + echo "unknown dist: ${DISTRIB_CODENAME}" 1>&2 + exit 1 +fi + +check() { + echo "$SWIFT_HASH $TGZ" | sha256sum --check +} + +TGZ="$HOME/.swift/swift.tar.gz" +mkdir -p "$(dirname "$TGZ")" +if ! check >& /dev/null; then + rm -f "$TGZ" + curl --location --silent --output "$TGZ" "$SWIFT_URL" + check +fi + +mkdir -p /tmp/swift +tar -xf "$TGZ" --strip 1 --directory /tmp/swift diff --git a/testing/resources/arbitrary_bytes_repo/.pre-commit-hooks.yaml b/testing/resources/arbitrary_bytes_repo/.pre-commit-hooks.yaml index 0320f025e..c2aec9b9f 100644 --- a/testing/resources/arbitrary_bytes_repo/.pre-commit-hooks.yaml +++ b/testing/resources/arbitrary_bytes_repo/.pre-commit-hooks.yaml @@ -1,6 +1,5 @@ -- id: python3-hook - name: Python 3 Hook - entry: python3-hook - language: python - language_version: python3.5 +- id: hook + name: hook + entry: ./hook.sh + language: script files: \.py$ diff --git a/testing/resources/arbitrary_bytes_repo/hook.sh b/testing/resources/arbitrary_bytes_repo/hook.sh new file mode 100755 index 000000000..9df0c5a07 --- /dev/null +++ b/testing/resources/arbitrary_bytes_repo/hook.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Intentionally write mixed encoding to the output. This should not crash +# pre-commit and should write bytes to the output. +# 'β˜ƒ'.encode() + 'Β²'.encode('latin1') +echo -e '\xe2\x98\x83\xb2' +# exit 1 to trigger printing +exit 1 diff --git a/testing/resources/arbitrary_bytes_repo/python3_hook/main.py b/testing/resources/arbitrary_bytes_repo/python3_hook/main.py deleted file mode 100644 index c6a5547cb..000000000 --- a/testing/resources/arbitrary_bytes_repo/python3_hook/main.py +++ /dev/null @@ -1,13 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import print_function -from __future__ import unicode_literals - -import sys - - -def func(): - # Intentionally write mixed encoding to the output. This should not crash - # pre-commit and should write bytes to the output. - sys.stdout.buffer.write('β˜ƒ'.encode('UTF-8') + 'Β²'.encode('latin1') + b'\n') - # Return 1 to trigger printing - return 1 diff --git a/testing/resources/arbitrary_bytes_repo/setup.py b/testing/resources/arbitrary_bytes_repo/setup.py deleted file mode 100644 index bf7690c03..000000000 --- a/testing/resources/arbitrary_bytes_repo/setup.py +++ /dev/null @@ -1,11 +0,0 @@ -from setuptools import find_packages -from setuptools import setup - -setup( - name='python3_hook', - version='0.0.0', - packages=find_packages('.'), - entry_points={ - 'console_scripts': ['python3-hook = python3_hook.main:func'], - }, -) diff --git a/testing/resources/conda_hooks_repo/.pre-commit-hooks.yaml b/testing/resources/conda_hooks_repo/.pre-commit-hooks.yaml new file mode 100644 index 000000000..a0d274c23 --- /dev/null +++ b/testing/resources/conda_hooks_repo/.pre-commit-hooks.yaml @@ -0,0 +1,10 @@ +- id: sys-exec + name: sys-exec + entry: python -c 'import os; import sys; print(sys.executable.split(os.path.sep)[-2]) if os.name == "nt" else print(sys.executable.split(os.path.sep)[-3])' + language: conda + files: \.py$ +- id: additional-deps + name: additional-deps + entry: python + language: conda + files: \.py$ diff --git a/testing/resources/conda_hooks_repo/environment.yml b/testing/resources/conda_hooks_repo/environment.yml new file mode 100644 index 000000000..e23c079fd --- /dev/null +++ b/testing/resources/conda_hooks_repo/environment.yml @@ -0,0 +1,6 @@ +channels: + - conda-forge + - defaults +dependencies: + - python + - pip diff --git a/testing/resources/docker_image_hooks_repo/.pre-commit-hooks.yaml b/testing/resources/docker_image_hooks_repo/.pre-commit-hooks.yaml new file mode 100644 index 000000000..1b385aa12 --- /dev/null +++ b/testing/resources/docker_image_hooks_repo/.pre-commit-hooks.yaml @@ -0,0 +1,8 @@ +- id: echo-entrypoint + name: echo (via --entrypoint) + language: docker_image + entry: --entrypoint echo cogniteev/echo +- id: echo-cmd + name: echo (via cmd) + language: docker_image + entry: cogniteev/echo echo diff --git a/testing/resources/legacy_hooks_yaml_repo/hooks.yaml b/testing/resources/legacy_hooks_yaml_repo/hooks.yaml deleted file mode 100644 index b2c347c14..000000000 --- a/testing/resources/legacy_hooks_yaml_repo/hooks.yaml +++ /dev/null @@ -1,5 +0,0 @@ -- id: system-hook-with-spaces - name: System hook with spaces - entry: bash -c 'echo "Hello World"' - language: system - files: \.sh$ diff --git a/testing/resources/manifest_without_foo.yaml b/testing/resources/manifest_without_foo.yaml deleted file mode 100644 index 0220233aa..000000000 --- a/testing/resources/manifest_without_foo.yaml +++ /dev/null @@ -1,5 +0,0 @@ -- id: bar - name: Bar - entry: bar - language: python - files: \.py$ diff --git a/testing/resources/node_0_11_8_hooks_repo/.pre-commit-hooks.yaml b/testing/resources/node_0_11_8_hooks_repo/.pre-commit-hooks.yaml deleted file mode 100644 index 005a1e3b0..000000000 --- a/testing/resources/node_0_11_8_hooks_repo/.pre-commit-hooks.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- id: node-11-8-hook - name: Node 0.11.8 hook - entry: node-11-8-hook - language: node - language_version: 0.11.8 - files: \.js$ diff --git a/testing/resources/node_0_11_8_hooks_repo/package.json b/testing/resources/node_0_11_8_hooks_repo/package.json deleted file mode 100644 index 911a3ed9e..000000000 --- a/testing/resources/node_0_11_8_hooks_repo/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "node-11-8-hook", - "version": "0.0.1", - "bin": {"node-11-8-hook": "./bin/main.js"} -} diff --git a/testing/resources/node_versioned_hooks_repo/.pre-commit-hooks.yaml b/testing/resources/node_versioned_hooks_repo/.pre-commit-hooks.yaml new file mode 100644 index 000000000..e7ad5ea7b --- /dev/null +++ b/testing/resources/node_versioned_hooks_repo/.pre-commit-hooks.yaml @@ -0,0 +1,6 @@ +- id: versioned-node-hook + name: Versioned node hook + entry: versioned-node-hook + language: node + language_version: 9.3.0 + files: \.js$ diff --git a/testing/resources/node_0_11_8_hooks_repo/bin/main.js b/testing/resources/node_versioned_hooks_repo/bin/main.js similarity index 100% rename from testing/resources/node_0_11_8_hooks_repo/bin/main.js rename to testing/resources/node_versioned_hooks_repo/bin/main.js diff --git a/testing/resources/node_versioned_hooks_repo/package.json b/testing/resources/node_versioned_hooks_repo/package.json new file mode 100644 index 000000000..18c7787c7 --- /dev/null +++ b/testing/resources/node_versioned_hooks_repo/package.json @@ -0,0 +1,5 @@ +{ + "name": "versioned-node-hook", + "version": "0.0.1", + "bin": {"versioned-node-hook": "./bin/main.js"} +} diff --git a/testing/resources/not_installable_repo/setup.py b/testing/resources/not_installable_repo/setup.py deleted file mode 100644 index ae5f6338c..000000000 --- a/testing/resources/not_installable_repo/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import print_function -from __future__ import unicode_literals - -import sys - - -def main(): - # Intentionally write mixed encoding to the output. This should not crash - # pre-commit and should write bytes to the output. - sys.stderr.write('β˜ƒ'.encode('UTF-8') + 'Β²'.encode('latin1') + b'\n') - # Return 1 to indicate failures - return 1 - - -if __name__ == '__main__': - exit(main()) diff --git a/testing/resources/pcre_hooks_repo/.pre-commit-hooks.yaml b/testing/resources/pcre_hooks_repo/.pre-commit-hooks.yaml deleted file mode 100644 index 709d8df38..000000000 --- a/testing/resources/pcre_hooks_repo/.pre-commit-hooks.yaml +++ /dev/null @@ -1,16 +0,0 @@ -- id: regex-with-quotes - name: Regex with quotes - entry: "foo'bar" - language: pcre - files: '' -- id: other-regex - name: Other regex - entry: ^\[INFO\] - language: pcre - files: '' -- id: regex-with-grep-args - name: Regex with grep extra arguments - entry: foo.+bar - language: pcre - files: '' - args: [-i] diff --git a/testing/resources/perl_hooks_repo/.gitignore b/testing/resources/perl_hooks_repo/.gitignore new file mode 100644 index 000000000..7af994045 --- /dev/null +++ b/testing/resources/perl_hooks_repo/.gitignore @@ -0,0 +1,7 @@ +/MYMETA.json +/MYMETA.yml +/Makefile +/PreCommitHello-*.tar.* +/PreCommitHello-*/ +/blib/ +/pm_to_blib diff --git a/testing/resources/perl_hooks_repo/.pre-commit-hooks.yaml b/testing/resources/perl_hooks_repo/.pre-commit-hooks.yaml new file mode 100644 index 000000000..11e6f6cd9 --- /dev/null +++ b/testing/resources/perl_hooks_repo/.pre-commit-hooks.yaml @@ -0,0 +1,5 @@ +- id: perl-hook + name: perl example hook + entry: pre-commit-perl-hello + language: perl + files: '' diff --git a/testing/resources/perl_hooks_repo/MANIFEST b/testing/resources/perl_hooks_repo/MANIFEST new file mode 100644 index 000000000..4a20084c6 --- /dev/null +++ b/testing/resources/perl_hooks_repo/MANIFEST @@ -0,0 +1,4 @@ +MANIFEST +Makefile.PL +bin/pre-commit-perl-hello +lib/PreCommitHello.pm diff --git a/testing/resources/perl_hooks_repo/Makefile.PL b/testing/resources/perl_hooks_repo/Makefile.PL new file mode 100644 index 000000000..6c70e1071 --- /dev/null +++ b/testing/resources/perl_hooks_repo/Makefile.PL @@ -0,0 +1,10 @@ +use strict; +use warnings; + +use ExtUtils::MakeMaker; + +WriteMakefile( + NAME => "PreCommitHello", + VERSION_FROM => "lib/PreCommitHello.pm", + EXE_FILES => [qw(bin/pre-commit-perl-hello)], +); diff --git a/testing/resources/perl_hooks_repo/bin/pre-commit-perl-hello b/testing/resources/perl_hooks_repo/bin/pre-commit-perl-hello new file mode 100755 index 000000000..9474009a1 --- /dev/null +++ b/testing/resources/perl_hooks_repo/bin/pre-commit-perl-hello @@ -0,0 +1,7 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use PreCommitHello; + +PreCommitHello::hello(); diff --git a/testing/resources/perl_hooks_repo/lib/PreCommitHello.pm b/testing/resources/perl_hooks_repo/lib/PreCommitHello.pm new file mode 100644 index 000000000..c76521cea --- /dev/null +++ b/testing/resources/perl_hooks_repo/lib/PreCommitHello.pm @@ -0,0 +1,12 @@ +package PreCommitHello; + +use strict; +use warnings; + +our $VERSION = "0.1.0"; + +sub hello { + print "Hello from perl-commit Perl!\n"; +} + +1; diff --git a/testing/resources/python3_hooks_repo/.pre-commit-hooks.yaml b/testing/resources/python3_hooks_repo/.pre-commit-hooks.yaml index 0320f025e..2c2370092 100644 --- a/testing/resources/python3_hooks_repo/.pre-commit-hooks.yaml +++ b/testing/resources/python3_hooks_repo/.pre-commit-hooks.yaml @@ -2,5 +2,5 @@ name: Python 3 Hook entry: python3-hook language: python - language_version: python3.5 + language_version: python3 files: \.py$ diff --git a/testing/resources/python_hooks_repo/foo/main.py b/testing/resources/python3_hooks_repo/py3_hook.py similarity index 61% rename from testing/resources/python_hooks_repo/foo/main.py rename to testing/resources/python3_hooks_repo/py3_hook.py index 78c2c0f74..8c9cda4c6 100644 --- a/testing/resources/python_hooks_repo/foo/main.py +++ b/testing/resources/python3_hooks_repo/py3_hook.py @@ -1,9 +1,8 @@ -from __future__ import print_function - import sys -def func(): +def main(): + print(sys.version_info[0]) print(repr(sys.argv[1:])) print('Hello World') return 0 diff --git a/testing/resources/python3_hooks_repo/python3_hook/main.py b/testing/resources/python3_hooks_repo/python3_hook/main.py deleted file mode 100644 index 117c7969d..000000000 --- a/testing/resources/python3_hooks_repo/python3_hook/main.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import print_function - -import sys - - -def func(): - print('{}.{}'.format(*sys.version_info[:2])) - print(repr(sys.argv[1:])) - print('Hello World') - return 0 diff --git a/testing/resources/python3_hooks_repo/setup.py b/testing/resources/python3_hooks_repo/setup.py index bf7690c03..9125dc1df 100644 --- a/testing/resources/python3_hooks_repo/setup.py +++ b/testing/resources/python3_hooks_repo/setup.py @@ -1,11 +1,8 @@ -from setuptools import find_packages from setuptools import setup setup( name='python3_hook', version='0.0.0', - packages=find_packages('.'), - entry_points={ - 'console_scripts': ['python3-hook = python3_hook.main:func'], - }, + py_modules=['py3_hook'], + entry_points={'console_scripts': ['python3-hook = py3_hook:main']}, ) diff --git a/testing/resources/python_hooks_repo/foo.py b/testing/resources/python_hooks_repo/foo.py new file mode 100644 index 000000000..9c4368e20 --- /dev/null +++ b/testing/resources/python_hooks_repo/foo.py @@ -0,0 +1,7 @@ +import sys + + +def main(): + print(repr(sys.argv[1:])) + print('Hello World') + return 0 diff --git a/testing/resources/python_hooks_repo/setup.py b/testing/resources/python_hooks_repo/setup.py index 556dd8f50..0559271ee 100644 --- a/testing/resources/python_hooks_repo/setup.py +++ b/testing/resources/python_hooks_repo/setup.py @@ -1,11 +1,8 @@ -from setuptools import find_packages from setuptools import setup setup( - name='Foo', + name='foo', version='0.0.0', - packages=find_packages('.'), - entry_points={ - 'console_scripts': ['foo = foo.main:func'], - }, + py_modules=['foo'], + entry_points={'console_scripts': ['foo = foo:main']}, ) diff --git a/testing/resources/not_installable_repo/.pre-commit-hooks.yaml b/testing/resources/python_venv_hooks_repo/.pre-commit-hooks.yaml similarity index 52% rename from testing/resources/not_installable_repo/.pre-commit-hooks.yaml rename to testing/resources/python_venv_hooks_repo/.pre-commit-hooks.yaml index 48c1f9efe..a666ed87a 100644 --- a/testing/resources/not_installable_repo/.pre-commit-hooks.yaml +++ b/testing/resources/python_venv_hooks_repo/.pre-commit-hooks.yaml @@ -1,6 +1,5 @@ - id: foo name: Foo entry: foo - language: python - language_version: python2.7 + language: python_venv files: \.py$ diff --git a/testing/resources/python_venv_hooks_repo/foo.py b/testing/resources/python_venv_hooks_repo/foo.py new file mode 100644 index 000000000..9c4368e20 --- /dev/null +++ b/testing/resources/python_venv_hooks_repo/foo.py @@ -0,0 +1,7 @@ +import sys + + +def main(): + print(repr(sys.argv[1:])) + print('Hello World') + return 0 diff --git a/testing/resources/python_venv_hooks_repo/setup.py b/testing/resources/python_venv_hooks_repo/setup.py new file mode 100644 index 000000000..0559271ee --- /dev/null +++ b/testing/resources/python_venv_hooks_repo/setup.py @@ -0,0 +1,8 @@ +from setuptools import setup + +setup( + name='foo', + version='0.0.0', + py_modules=['foo'], + entry_points={'console_scripts': ['foo = foo:main']}, +) diff --git a/testing/resources/ruby_versioned_hooks_repo/.pre-commit-hooks.yaml b/testing/resources/ruby_versioned_hooks_repo/.pre-commit-hooks.yaml index fcba780fd..63e1dd4c6 100644 --- a/testing/resources/ruby_versioned_hooks_repo/.pre-commit-hooks.yaml +++ b/testing/resources/ruby_versioned_hooks_repo/.pre-commit-hooks.yaml @@ -2,5 +2,5 @@ name: Ruby Hook entry: ruby_hook language: ruby - language_version: 2.1.5 + language_version: 2.5.1 files: \.rb$ diff --git a/testing/resources/rust_hooks_repo/.pre-commit-hooks.yaml b/testing/resources/rust_hooks_repo/.pre-commit-hooks.yaml new file mode 100644 index 000000000..df1269ff8 --- /dev/null +++ b/testing/resources/rust_hooks_repo/.pre-commit-hooks.yaml @@ -0,0 +1,5 @@ +- id: rust-hook + name: rust example hook + entry: rust-hello-world + language: rust + files: '' diff --git a/testing/resources/rust_hooks_repo/Cargo.lock b/testing/resources/rust_hooks_repo/Cargo.lock new file mode 100644 index 000000000..36fbfda2b --- /dev/null +++ b/testing/resources/rust_hooks_repo/Cargo.lock @@ -0,0 +1,3 @@ +[[package]] +name = "rust-hello-world" +version = "0.1.0" diff --git a/testing/resources/rust_hooks_repo/Cargo.toml b/testing/resources/rust_hooks_repo/Cargo.toml new file mode 100644 index 000000000..cd83b4358 --- /dev/null +++ b/testing/resources/rust_hooks_repo/Cargo.toml @@ -0,0 +1,3 @@ +[package] +name = "rust-hello-world" +version = "0.1.0" diff --git a/testing/resources/rust_hooks_repo/src/main.rs b/testing/resources/rust_hooks_repo/src/main.rs new file mode 100644 index 000000000..ad379d6ea --- /dev/null +++ b/testing/resources/rust_hooks_repo/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("hello world"); +} diff --git a/testing/resources/stdout_stderr_repo/.pre-commit-hooks.yaml b/testing/resources/stdout_stderr_repo/.pre-commit-hooks.yaml new file mode 100644 index 000000000..6800d2593 --- /dev/null +++ b/testing/resources/stdout_stderr_repo/.pre-commit-hooks.yaml @@ -0,0 +1,8 @@ +- id: stdout-stderr + name: stdout-stderr + language: script + entry: ./stdout-stderr-entry +- id: tty-check + name: tty-check + language: script + entry: ./tty-check-entry diff --git a/testing/resources/stdout_stderr_repo/stdout-stderr-entry b/testing/resources/stdout_stderr_repo/stdout-stderr-entry new file mode 100755 index 000000000..7563df53c --- /dev/null +++ b/testing/resources/stdout_stderr_repo/stdout-stderr-entry @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +echo 0 +echo 1 1>&2 +echo 2 +echo 3 1>&2 +echo 4 +echo 5 1>&2 diff --git a/testing/resources/stdout_stderr_repo/tty-check-entry b/testing/resources/stdout_stderr_repo/tty-check-entry new file mode 100755 index 000000000..01a9d3883 --- /dev/null +++ b/testing/resources/stdout_stderr_repo/tty-check-entry @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +t() { + if [ -t "$1" ]; then + echo "$2: True" + else + echo "$2: False" + fi +} +t 0 stdin +t 1 stdout +t 2 stderr diff --git a/testing/resources/swift_hooks_repo/Package.swift b/testing/resources/swift_hooks_repo/Package.swift index 6e02c188a..04976d3ff 100644 --- a/testing/resources/swift_hooks_repo/Package.swift +++ b/testing/resources/swift_hooks_repo/Package.swift @@ -1,5 +1,7 @@ +// swift-tools-version:5.0 import PackageDescription let package = Package( - name: "swift_hooks_repo" + name: "swift_hooks_repo", + targets: [.target(name: "swift_hooks_repo")] ) diff --git a/testing/resources/swift_hooks_repo/Sources/main.swift b/testing/resources/swift_hooks_repo/Sources/swift_hooks_repo/main.swift similarity index 100% rename from testing/resources/swift_hooks_repo/Sources/main.swift rename to testing/resources/swift_hooks_repo/Sources/swift_hooks_repo/main.swift diff --git a/testing/resources/valid_yaml_but_invalid_config.yaml b/testing/resources/valid_yaml_but_invalid_config.yaml deleted file mode 100644 index 2ed187b2d..000000000 --- a/testing/resources/valid_yaml_but_invalid_config.yaml +++ /dev/null @@ -1,5 +0,0 @@ -- repo: git@github.com:pre-commit/pre-commit-hooks - hooks: - - id: pyflakes - - id: jslint - - id: trim_trailing_whitespace diff --git a/testing/resources/valid_yaml_but_invalid_manifest.yaml b/testing/resources/valid_yaml_but_invalid_manifest.yaml deleted file mode 100644 index 20e9ff3fe..000000000 --- a/testing/resources/valid_yaml_but_invalid_manifest.yaml +++ /dev/null @@ -1 +0,0 @@ -foo: bar diff --git a/testing/test_symlink b/testing/test_symlink deleted file mode 120000 index ee1f6cb7f..000000000 --- a/testing/test_symlink +++ /dev/null @@ -1 +0,0 @@ -does_not_exist \ No newline at end of file diff --git a/testing/util.py b/testing/util.py index 4d752f3e9..439bee794 100644 --- a/testing/util.py +++ b/testing/util.py @@ -1,14 +1,13 @@ -from __future__ import unicode_literals - +import contextlib import os.path +import subprocess import pytest from pre_commit import parse_shebang from pre_commit.languages.docker import docker_is_running -from pre_commit.languages.pcre import GREP from pre_commit.util import cmd_output -from pre_commit.util import cwd +from testing.auto_namedtuple import auto_namedtuple TESTING_DIR = os.path.abspath(os.path.dirname(__file__)) @@ -18,57 +17,97 @@ def get_resource_path(path): return os.path.join(TESTING_DIR, 'resources', path) -def get_head_sha(dir): - with cwd(dir): - return cmd_output('git', 'rev-parse', 'HEAD')[1].strip() - - -def cmd_output_mocked_pre_commit_home(*args, **kwargs): - # keyword-only argument - tempdir_factory = kwargs.pop('tempdir_factory') - pre_commit_home = kwargs.pop('pre_commit_home', tempdir_factory.get()) +def cmd_output_mocked_pre_commit_home( + *args, tempdir_factory, pre_commit_home=None, env=None, **kwargs, +): + if pre_commit_home is None: + pre_commit_home = tempdir_factory.get() + env = env if env is not None else os.environ + kwargs.setdefault('stderr', subprocess.STDOUT) # Don't want to write to the home directory - env = dict(kwargs.pop('env', os.environ), PRE_COMMIT_HOME=pre_commit_home) - return cmd_output(*args, env=env, **kwargs) + env = dict(env, PRE_COMMIT_HOME=pre_commit_home) + ret, out, _ = cmd_output(*args, env=env, **kwargs) + return ret, out.replace('\r\n', '\n'), None skipif_cant_run_docker = pytest.mark.skipif( - docker_is_running() is False, - reason='Docker isn\'t running or can\'t be accessed' + os.name == 'nt' or not docker_is_running(), + reason="Docker isn't running or can't be accessed", ) - -skipif_slowtests_false = pytest.mark.skipif( - os.environ.get('slowtests') == 'false', - reason='slowtests=false', -) - skipif_cant_run_swift = pytest.mark.skipif( parse_shebang.find_executable('swift') is None, - reason='swift isn\'t installed or can\'t be found' + reason="swift isn't installed or can't be found", ) - xfailif_windows_no_ruby = pytest.mark.xfail( os.name == 'nt', reason='Ruby support not yet implemented on windows.', ) - -xfailif_windows_no_node = pytest.mark.xfail( - os.name == 'nt', - reason='Node support not yet implemented on windows.', -) +xfailif_windows = pytest.mark.xfail(os.name == 'nt', reason='windows') -def platform_supports_pcre(): - output = cmd_output(GREP, '-P', "name='pre", 'setup.py', retcode=None) - return output[0] == 0 and "name='pre_commit'," in output[1] +def supports_venv(): # pragma: no cover (platform specific) + try: + __import__('ensurepip') + __import__('venv') + return True + except ImportError: + return False -xfailif_no_pcre_support = pytest.mark.xfail( - not platform_supports_pcre(), - reason='grep -P is not supported on this platform', +xfailif_no_venv = pytest.mark.xfail( + not supports_venv(), reason='Does not support venv module', ) -xfailif_no_symlink = pytest.mark.xfail( - not hasattr(os, 'symlink'), - reason='Symlink is not supported on this platform', -) + +def run_opts( + all_files=False, + files=(), + color=False, + verbose=False, + hook=None, + from_ref='', + to_ref='', + remote_name='', + remote_url='', + hook_stage='commit', + show_diff_on_failure=False, + commit_msg_filename='', + checkout_type='', +): + # These are mutually exclusive + assert not (all_files and files) + return auto_namedtuple( + all_files=all_files, + files=files, + color=color, + verbose=verbose, + hook=hook, + from_ref=from_ref, + to_ref=to_ref, + remote_name=remote_name, + remote_url=remote_url, + hook_stage=hook_stage, + show_diff_on_failure=show_diff_on_failure, + commit_msg_filename=commit_msg_filename, + checkout_type=checkout_type, + ) + + +@contextlib.contextmanager +def cwd(path): + original_cwd = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(original_cwd) + + +def git_commit(*args, fn=cmd_output, msg='commit!', **kwargs): + kwargs.setdefault('stderr', subprocess.STDOUT) + + cmd = ('git', 'commit', '--allow-empty', '--no-gpg-sign', '-a') + args + if msg is not None: # allow skipping `-m` with `msg=None` + cmd += ('-m', msg) + ret, out, _ = fn(*cmd, **kwargs) + return ret, out.replace('\r\n', '\n') diff --git a/tests/clientlib_test.py b/tests/clientlib_test.py index 65209a644..c48adbde9 100644 --- a/tests/clientlib_test.py +++ b/tests/clientlib_test.py @@ -1,151 +1,97 @@ -from __future__ import unicode_literals +import logging +import cfgv import pytest -from pre_commit import schema -from pre_commit.clientlib import check_language +import pre_commit.constants as C from pre_commit.clientlib import check_type_tag from pre_commit.clientlib import CONFIG_HOOK_DICT +from pre_commit.clientlib import CONFIG_REPO_DICT from pre_commit.clientlib import CONFIG_SCHEMA -from pre_commit.clientlib import is_local_repo +from pre_commit.clientlib import DEFAULT_LANGUAGE_VERSION from pre_commit.clientlib import MANIFEST_SCHEMA +from pre_commit.clientlib import MigrateShaToRev from pre_commit.clientlib import validate_config_main from pre_commit.clientlib import validate_manifest_main -from testing.util import get_resource_path +from testing.fixtures import sample_local_config def is_valid_according_to_schema(obj, obj_schema): try: - schema.validate(obj, obj_schema) + cfgv.validate(obj, obj_schema) return True - except schema.ValidationError: + except cfgv.ValidationError: return False -@pytest.mark.parametrize('value', ('not a language', 'python3')) -def test_check_language_failures(value): - with pytest.raises(schema.ValidationError): - check_language(value) - - @pytest.mark.parametrize('value', ('definitely-not-a-tag', 'fiel')) def test_check_type_tag_failures(value): - with pytest.raises(schema.ValidationError): + with pytest.raises(cfgv.ValidationError): check_type_tag(value) -@pytest.mark.parametrize('value', ('python', 'node', 'pcre')) -def test_check_language_ok(value): - check_language(value) - - -def test_is_local_repo(): - assert is_local_repo({'repo': 'local'}) - - @pytest.mark.parametrize( - ('args', 'expected_output'), - ( - (['.pre-commit-config.yaml'], 0), - (['non_existent_file.yaml'], 1), - ([get_resource_path('valid_yaml_but_invalid_config.yaml')], 1), - ([get_resource_path('non_parseable_yaml_file.notyaml')], 1), + ('config_obj', 'expected'), ( + ( + { + 'repos': [{ + 'repo': 'git@github.com:pre-commit/pre-commit-hooks', + 'rev': 'cd74dc150c142c3be70b24eaf0b02cae9d235f37', + 'hooks': [{'id': 'pyflakes', 'files': '\\.py$'}], + }], + }, + True, + ), + ( + { + 'repos': [{ + 'repo': 'git@github.com:pre-commit/pre-commit-hooks', + 'rev': 'cd74dc150c142c3be70b24eaf0b02cae9d235f37', + 'hooks': [ + { + 'id': 'pyflakes', + 'files': '\\.py$', + 'args': ['foo', 'bar', 'baz'], + }, + ], + }], + }, + True, + ), + ( + { + 'repos': [{ + 'repo': 'git@github.com:pre-commit/pre-commit-hooks', + 'rev': 'cd74dc150c142c3be70b24eaf0b02cae9d235f37', + 'hooks': [ + { + 'id': 'pyflakes', + 'files': '\\.py$', + # Exclude pattern must be a string + 'exclude': 0, + 'args': ['foo', 'bar', 'baz'], + }, + ], + }], + }, + False, + ), ), ) -def test_validate_config_main(args, expected_output): - assert validate_config_main(args) == expected_output - - -@pytest.mark.parametrize(('config_obj', 'expected'), ( - ([], False), - ( - [{ - 'repo': 'git@github.com:pre-commit/pre-commit-hooks', - 'sha': 'cd74dc150c142c3be70b24eaf0b02cae9d235f37', - 'hooks': [{'id': 'pyflakes', 'files': '\\.py$'}], - }], - True, - ), - ( - [{ - 'repo': 'git@github.com:pre-commit/pre-commit-hooks', - 'sha': 'cd74dc150c142c3be70b24eaf0b02cae9d235f37', - 'hooks': [ - { - 'id': 'pyflakes', - 'files': '\\.py$', - 'args': ['foo', 'bar', 'baz'], - }, - ], - }], - True, - ), - ( - [{ - 'repo': 'git@github.com:pre-commit/pre-commit-hooks', - 'sha': 'cd74dc150c142c3be70b24eaf0b02cae9d235f37', - 'hooks': [ - { - 'id': 'pyflakes', - 'files': '\\.py$', - # Exclude pattern must be a string - 'exclude': 0, - 'args': ['foo', 'bar', 'baz'], - }, - ], - }], - False, - ), -)) def test_config_valid(config_obj, expected): ret = is_valid_according_to_schema(config_obj, CONFIG_SCHEMA) assert ret is expected -@pytest.mark.parametrize('config_obj', ( - [{ - 'repo': 'local', - 'sha': 'foo', - 'hooks': [{ - 'id': 'do_not_commit', - 'name': 'Block if "DO NOT COMMIT" is found', - 'entry': 'DO NOT COMMIT', - 'language': 'pcre', - 'files': '^(.*)$', - }], - }], -)) -def test_config_with_local_hooks_definition_fails(config_obj): - with pytest.raises(schema.ValidationError): - schema.validate(config_obj, CONFIG_SCHEMA) - - -@pytest.mark.parametrize('config_obj', ( - [{ - 'repo': 'local', - 'hooks': [{ - 'id': 'arg-per-line', - 'name': 'Args per line hook', - 'entry': 'bin/hook.sh', - 'language': 'script', - 'files': '', - 'args': ['hello', 'world'], - }], - }], - [{ - 'repo': 'local', - 'hooks': [{ - 'id': 'arg-per-line', - 'name': 'Args per line hook', - 'entry': 'bin/hook.sh', - 'language': 'script', - 'files': '', - 'args': ['hello', 'world'], - }] - }], -)) -def test_config_with_local_hooks_definition_passes(config_obj): - schema.validate(config_obj, CONFIG_SCHEMA) +def test_local_hooks_with_rev_fails(): + config_obj = {'repos': [dict(sample_local_config(), rev='foo')]} + with pytest.raises(cfgv.ValidationError): + cfgv.validate(config_obj, CONFIG_SCHEMA) + + +def test_config_with_local_hooks_definition_passes(): + config_obj = {'repos': [sample_local_config()]} + cfgv.validate(config_obj, CONFIG_SCHEMA) def test_config_schema_does_not_contain_defaults(): @@ -153,33 +99,88 @@ def test_config_schema_does_not_contain_defaults(): will clobber potentially useful values in the backing manifest. #227 """ for item in CONFIG_HOOK_DICT.items: - assert not isinstance(item, schema.Optional) + assert not isinstance(item, cfgv.Optional) -@pytest.mark.parametrize( - ('args', 'expected_output'), - ( - (['.pre-commit-hooks.yaml'], 0), - (['non_existent_file.yaml'], 1), - ([get_resource_path('valid_yaml_but_invalid_manifest.yaml')], 1), - ([get_resource_path('non_parseable_yaml_file.notyaml')], 1), - ), -) -def test_validate_manifest_main(args, expected_output): - assert validate_manifest_main(args) == expected_output +def test_validate_manifest_main_ok(): + assert not validate_manifest_main(('.pre-commit-hooks.yaml',)) + + +def test_validate_config_main_ok(): + assert not validate_config_main(('.pre-commit-config.yaml',)) + + +def test_validate_config_old_list_format_ok(tmpdir): + f = tmpdir.join('cfg.yaml') + f.write('- {repo: meta, hooks: [{id: identity}]}') + assert not validate_config_main((f.strpath,)) + + +def test_validate_warn_on_unknown_keys_at_repo_level(tmpdir, caplog): + f = tmpdir.join('cfg.yaml') + f.write( + '- repo: https://gitlab.com/pycqa/flake8\n' + ' rev: 3.7.7\n' + ' hooks:\n' + ' - id: flake8\n' + ' args: [--some-args]\n', + ) + ret_val = validate_config_main((f.strpath,)) + assert not ret_val + assert caplog.record_tuples == [ + ( + 'pre_commit', + logging.WARNING, + 'Unexpected key(s) present on https://gitlab.com/pycqa/flake8: ' + 'args', + ), + ] + + +def test_validate_warn_on_unknown_keys_at_top_level(tmpdir, caplog): + f = tmpdir.join('cfg.yaml') + f.write( + 'repos:\n' + '- repo: https://gitlab.com/pycqa/flake8\n' + ' rev: 3.7.7\n' + ' hooks:\n' + ' - id: flake8\n' + 'foo:\n' + ' id: 1.0.0\n', + ) + ret_val = validate_config_main((f.strpath,)) + assert not ret_val + assert caplog.record_tuples == [ + ( + 'pre_commit', + logging.WARNING, + 'Unexpected key(s) present at root: foo', + ), + ] + + +@pytest.mark.parametrize('fn', (validate_config_main, validate_manifest_main)) +def test_mains_not_ok(tmpdir, fn): + not_yaml = tmpdir.join('f.notyaml') + not_yaml.write('{') + not_schema = tmpdir.join('notconfig.yaml') + not_schema.write('{}') + + assert fn(('does-not-exist',)) + assert fn((not_yaml.strpath,)) + assert fn((not_schema.strpath,)) @pytest.mark.parametrize( ('manifest_obj', 'expected'), ( - ([], False), ( [{ 'id': 'a', 'name': 'b', 'entry': 'c', 'language': 'python', - 'files': r'\.py$' + 'files': r'\.py$', }], True, ), @@ -196,8 +197,6 @@ def test_validate_manifest_main(args, expected_output): ), ( # A regression in 0.13.5: always_run and files are permissible - # together (but meaningless). In a future version upgrade this to - # an error [{ 'id': 'a', 'name': 'b', @@ -208,8 +207,107 @@ def test_validate_manifest_main(args, expected_output): }], True, ), - ) + ), ) def test_valid_manifests(manifest_obj, expected): ret = is_valid_according_to_schema(manifest_obj, MANIFEST_SCHEMA) assert ret is expected + + +@pytest.mark.parametrize( + 'dct', + ( + {'repo': 'local'}, {'repo': 'meta'}, + {'repo': 'wat', 'sha': 'wat'}, {'repo': 'wat', 'rev': 'wat'}, + ), +) +def test_migrate_sha_to_rev_ok(dct): + MigrateShaToRev().check(dct) + + +def test_migrate_sha_to_rev_dont_specify_both(): + with pytest.raises(cfgv.ValidationError) as excinfo: + MigrateShaToRev().check({'repo': 'a', 'sha': 'b', 'rev': 'c'}) + msg, = excinfo.value.args + assert msg == 'Cannot specify both sha and rev' + + +@pytest.mark.parametrize( + 'dct', + ( + {'repo': 'a'}, + {'repo': 'meta', 'sha': 'a'}, {'repo': 'meta', 'rev': 'a'}, + ), +) +def test_migrate_sha_to_rev_conditional_check_failures(dct): + with pytest.raises(cfgv.ValidationError): + MigrateShaToRev().check(dct) + + +def test_migrate_to_sha_apply_default(): + dct = {'repo': 'a', 'sha': 'b'} + MigrateShaToRev().apply_default(dct) + assert dct == {'repo': 'a', 'rev': 'b'} + + +def test_migrate_to_sha_ok(): + dct = {'repo': 'a', 'rev': 'b'} + MigrateShaToRev().apply_default(dct) + assert dct == {'repo': 'a', 'rev': 'b'} + + +@pytest.mark.parametrize( + 'config_repo', + ( + # i-dont-exist isn't a valid hook + {'repo': 'meta', 'hooks': [{'id': 'i-dont-exist'}]}, + # invalid to set a language for a meta hook + {'repo': 'meta', 'hooks': [{'id': 'identity', 'language': 'python'}]}, + # name override must be string + {'repo': 'meta', 'hooks': [{'id': 'identity', 'name': False}]}, + ), +) +def test_meta_hook_invalid(config_repo): + with pytest.raises(cfgv.ValidationError): + cfgv.validate(config_repo, CONFIG_REPO_DICT) + + +@pytest.mark.parametrize( + 'mapping', + ( + # invalid language key + {'pony': '1.0'}, + # not a string for version + {'python': 3}, + ), +) +def test_default_language_version_invalid(mapping): + with pytest.raises(cfgv.ValidationError): + cfgv.validate(mapping, DEFAULT_LANGUAGE_VERSION) + + +def test_minimum_pre_commit_version_failing(): + with pytest.raises(cfgv.ValidationError) as excinfo: + cfg = {'repos': [], 'minimum_pre_commit_version': '999'} + cfgv.validate(cfg, CONFIG_SCHEMA) + assert str(excinfo.value) == ( + f'\n' + f'==> At Config()\n' + f'==> At key: minimum_pre_commit_version\n' + f'=====> pre-commit version 999 is required but version {C.VERSION} ' + f'is installed. Perhaps run `pip install --upgrade pre-commit`.' + ) + + +def test_minimum_pre_commit_version_passing(): + cfg = {'repos': [], 'minimum_pre_commit_version': '0'} + cfgv.validate(cfg, CONFIG_SCHEMA) + + +@pytest.mark.parametrize('schema', (CONFIG_SCHEMA, CONFIG_REPO_DICT)) +def test_warn_additional(schema): + allowed_keys = {item.key for item in schema.items if hasattr(item, 'key')} + warn_additional, = [ + x for x in schema.items if isinstance(x, cfgv.WarnAdditionalKeys) + ] + assert allowed_keys == set(warn_additional.keys) diff --git a/tests/color_test.py b/tests/color_test.py index 4fb7676ad..98b39c1e1 100644 --- a/tests/color_test.py +++ b/tests/color_test.py @@ -1,20 +1,20 @@ -from __future__ import unicode_literals - import sys +from unittest import mock -import mock import pytest +from pre_commit import envcontext from pre_commit.color import format_color from pre_commit.color import GREEN -from pre_commit.color import InvalidColorSetting from pre_commit.color import use_color -@pytest.mark.parametrize(('in_text', 'in_color', 'in_use_color', 'expected'), ( - ('foo', GREEN, True, '{}foo\033[0m'.format(GREEN)), - ('foo', GREEN, False, 'foo'), -)) +@pytest.mark.parametrize( + ('in_text', 'in_color', 'in_use_color', 'expected'), ( + ('foo', GREEN, True, f'{GREEN}foo\033[m'), + ('foo', GREEN, False, 'foo'), + ), +) def test_format_color(in_text, in_color, in_use_color, expected): ret = format_color(in_text, in_color, in_use_color) assert ret == expected @@ -33,11 +33,27 @@ def test_use_color_no_tty(): assert use_color('auto') is False -def test_use_color_tty(): +def test_use_color_tty_with_color_support(): + with mock.patch.object(sys.stdout, 'isatty', return_value=True): + with mock.patch('pre_commit.color.terminal_supports_color', True): + with envcontext.envcontext((('TERM', envcontext.UNSET),)): + assert use_color('auto') is True + + +def test_use_color_tty_without_color_support(): + with mock.patch.object(sys.stdout, 'isatty', return_value=True): + with mock.patch('pre_commit.color.terminal_supports_color', False): + with envcontext.envcontext((('TERM', envcontext.UNSET),)): + assert use_color('auto') is False + + +def test_use_color_dumb_term(): with mock.patch.object(sys.stdout, 'isatty', return_value=True): - assert use_color('auto') is True + with mock.patch('pre_commit.color.terminal_supports_color', True): + with envcontext.envcontext((('TERM', 'dumb'),)): + assert use_color('auto') is False def test_use_color_raises_if_given_shenanigans(): - with pytest.raises(InvalidColorSetting): + with pytest.raises(ValueError): use_color('herpaderp') diff --git a/tests/commands/autoupdate_test.py b/tests/commands/autoupdate_test.py index 550946b6f..2c7b2f1fa 100644 --- a/tests/commands/autoupdate_test.py +++ b/tests/commands/autoupdate_test.py @@ -1,59 +1,137 @@ -from __future__ import unicode_literals - -import shutil -from collections import OrderedDict +import shlex import pytest import pre_commit.constants as C -from pre_commit.clientlib import load_config -from pre_commit.commands.autoupdate import _update_repo +from pre_commit import git +from pre_commit.commands.autoupdate import _check_hooks_still_exist_at_rev from pre_commit.commands.autoupdate import autoupdate from pre_commit.commands.autoupdate import RepositoryCannotBeUpdatedError -from pre_commit.runner import Runner +from pre_commit.commands.autoupdate import RevInfo from pre_commit.util import cmd_output -from pre_commit.util import cwd from testing.auto_namedtuple import auto_namedtuple from testing.fixtures import add_config_to_repo -from testing.fixtures import config_with_local_hooks -from testing.fixtures import git_dir from testing.fixtures import make_config_from_repo from testing.fixtures import make_repo +from testing.fixtures import modify_manifest +from testing.fixtures import read_config +from testing.fixtures import sample_local_config from testing.fixtures import write_config -from testing.util import get_head_sha -from testing.util import get_resource_path +from testing.util import git_commit -@pytest.yield_fixture -def up_to_date_repo(tempdir_factory): +@pytest.fixture +def up_to_date(tempdir_factory): yield make_repo(tempdir_factory, 'python_hooks_repo') -def test_up_to_date_repo(up_to_date_repo, runner_with_mocked_store): - config = make_config_from_repo(up_to_date_repo) - input_sha = config['sha'] - ret = _update_repo(config, runner_with_mocked_store, tags_only=False) - assert ret['sha'] == input_sha +@pytest.fixture +def out_of_date(tempdir_factory): + path = make_repo(tempdir_factory, 'python_hooks_repo') + original_rev = git.head_rev(path) + git_commit(cwd=path) + head_rev = git.head_rev(path) -def test_autoupdate_up_to_date_repo( - up_to_date_repo, in_tmpdir, mock_out_store_directory, -): - # Write out the config - config = make_config_from_repo(up_to_date_repo, check=False) - write_config('.', config) + yield auto_namedtuple( + path=path, original_rev=original_rev, head_rev=head_rev, + ) - before = open(C.CONFIG_FILE).read() - assert '^$' not in before - ret = autoupdate(Runner('.', C.CONFIG_FILE), tags_only=False) - after = open(C.CONFIG_FILE).read() - assert ret == 0 - assert before == after +@pytest.fixture +def tagged(out_of_date): + cmd_output('git', 'tag', 'v1.2.3', cwd=out_of_date.path) + yield out_of_date -def test_autoupdate_old_revision_broken( - tempdir_factory, in_tmpdir, mock_out_store_directory, -): + +@pytest.fixture +def hook_disappearing(tempdir_factory): + path = make_repo(tempdir_factory, 'python_hooks_repo') + original_rev = git.head_rev(path) + + with modify_manifest(path) as manifest: + manifest[0]['id'] = 'bar' + + yield auto_namedtuple(path=path, original_rev=original_rev) + + +def test_rev_info_from_config(): + info = RevInfo.from_config({'repo': 'repo/path', 'rev': 'v1.2.3'}) + assert info == RevInfo('repo/path', 'v1.2.3', None) + + +def test_rev_info_update_up_to_date_repo(up_to_date): + config = make_config_from_repo(up_to_date) + info = RevInfo.from_config(config) + new_info = info.update(tags_only=False, freeze=False) + assert info == new_info + + +def test_rev_info_update_out_of_date_repo(out_of_date): + config = make_config_from_repo( + out_of_date.path, rev=out_of_date.original_rev, + ) + info = RevInfo.from_config(config) + new_info = info.update(tags_only=False, freeze=False) + assert new_info.rev == out_of_date.head_rev + + +def test_rev_info_update_non_master_default_branch(out_of_date): + # change the default branch to be not-master + cmd_output('git', '-C', out_of_date.path, 'branch', '-m', 'dev') + test_rev_info_update_out_of_date_repo(out_of_date) + + +def test_rev_info_update_tags_even_if_not_tags_only(tagged): + config = make_config_from_repo(tagged.path, rev=tagged.original_rev) + info = RevInfo.from_config(config) + new_info = info.update(tags_only=False, freeze=False) + assert new_info.rev == 'v1.2.3' + + +def test_rev_info_update_tags_only_does_not_pick_tip(tagged): + git_commit(cwd=tagged.path) + config = make_config_from_repo(tagged.path, rev=tagged.original_rev) + info = RevInfo.from_config(config) + new_info = info.update(tags_only=True, freeze=False) + assert new_info.rev == 'v1.2.3' + + +def test_rev_info_update_freeze_tag(tagged): + git_commit(cwd=tagged.path) + config = make_config_from_repo(tagged.path, rev=tagged.original_rev) + info = RevInfo.from_config(config) + new_info = info.update(tags_only=True, freeze=True) + assert new_info.rev == tagged.head_rev + assert new_info.frozen == 'v1.2.3' + + +def test_rev_info_update_does_not_freeze_if_already_sha(out_of_date): + config = make_config_from_repo( + out_of_date.path, rev=out_of_date.original_rev, + ) + info = RevInfo.from_config(config) + new_info = info.update(tags_only=True, freeze=True) + assert new_info.rev == out_of_date.head_rev + assert new_info.frozen is None + + +def test_autoupdate_up_to_date_repo(up_to_date, tmpdir, store): + contents = ( + f'repos:\n' + f'- repo: {up_to_date}\n' + f' rev: {git.head_rev(up_to_date)}\n' + f' hooks:\n' + f' - id: foo\n' + ) + cfg = tmpdir.join(C.CONFIG_FILE) + cfg.write(contents) + + assert autoupdate(str(cfg), store, freeze=False, tags_only=False) == 0 + assert cfg.read() == contents + + +def test_autoupdate_old_revision_broken(tempdir_factory, in_tmpdir, store): """In $FUTURE_VERSION, hooks.yaml will no longer be supported. This asserts that when that day comes, pre-commit will be able to autoupdate despite not being able to read hooks.yaml in that repository. @@ -61,177 +139,299 @@ def test_autoupdate_old_revision_broken( path = make_repo(tempdir_factory, 'python_hooks_repo') config = make_config_from_repo(path, check=False) - with cwd(path): - cmd_output('git', 'mv', C.MANIFEST_FILE, 'nope.yaml') - cmd_output('git', 'commit', '-m', 'simulate old repo') - # Assume this is the revision the user's old repository was at - rev = get_head_sha(path) - cmd_output('git', 'mv', 'nope.yaml', C.MANIFEST_FILE) - cmd_output('git', 'commit', '-m', 'move hooks file') - update_rev = get_head_sha(path) + cmd_output('git', 'mv', C.MANIFEST_FILE, 'nope.yaml', cwd=path) + git_commit(cwd=path) + # Assume this is the revision the user's old repository was at + rev = git.head_rev(path) + cmd_output('git', 'mv', 'nope.yaml', C.MANIFEST_FILE, cwd=path) + git_commit(cwd=path) + update_rev = git.head_rev(path) - config['sha'] = rev + config['rev'] = rev write_config('.', config) - before = open(C.CONFIG_FILE).read() - ret = autoupdate(Runner('.', C.CONFIG_FILE), tags_only=False) - after = open(C.CONFIG_FILE).read() - assert ret == 0 + with open(C.CONFIG_FILE) as f: + before = f.read() + assert autoupdate(C.CONFIG_FILE, store, freeze=False, tags_only=False) == 0 + with open(C.CONFIG_FILE) as f: + after = f.read() assert before != after assert update_rev in after -@pytest.yield_fixture -def out_of_date_repo(tempdir_factory): - path = make_repo(tempdir_factory, 'python_hooks_repo') - original_sha = get_head_sha(path) - - # Make a commit - with cwd(path): - cmd_output('git', 'commit', '--allow-empty', '-m', 'foo') - head_sha = get_head_sha(path) - - yield auto_namedtuple( - path=path, original_sha=original_sha, head_sha=head_sha, +def test_autoupdate_out_of_date_repo(out_of_date, tmpdir, store): + fmt = ( + 'repos:\n' + '- repo: {}\n' + ' rev: {}\n' + ' hooks:\n' + ' - id: foo\n' ) + cfg = tmpdir.join(C.CONFIG_FILE) + cfg.write(fmt.format(out_of_date.path, out_of_date.original_rev)) + + assert autoupdate(str(cfg), store, freeze=False, tags_only=False) == 0 + assert cfg.read() == fmt.format(out_of_date.path, out_of_date.head_rev) + + +def test_autoupdate_only_one_to_update(up_to_date, out_of_date, tmpdir, store): + fmt = ( + 'repos:\n' + '- repo: {}\n' + ' rev: {}\n' + ' hooks:\n' + ' - id: foo\n' + '- repo: {}\n' + ' rev: {}\n' + ' hooks:\n' + ' - id: foo\n' + ) + cfg = tmpdir.join(C.CONFIG_FILE) + before = fmt.format( + up_to_date, git.head_rev(up_to_date), + out_of_date.path, out_of_date.original_rev, + ) + cfg.write(before) - -def test_out_of_date_repo(out_of_date_repo, runner_with_mocked_store): - config = make_config_from_repo( - out_of_date_repo.path, sha=out_of_date_repo.original_sha, + assert autoupdate(str(cfg), store, freeze=False, tags_only=False) == 0 + assert cfg.read() == fmt.format( + up_to_date, git.head_rev(up_to_date), + out_of_date.path, out_of_date.head_rev, ) - ret = _update_repo(config, runner_with_mocked_store, tags_only=False) - assert ret['sha'] != out_of_date_repo.original_sha - assert ret['sha'] == out_of_date_repo.head_sha -def test_autoupdate_out_of_date_repo( - out_of_date_repo, in_tmpdir, mock_out_store_directory +def test_autoupdate_out_of_date_repo_with_correct_repo_name( + out_of_date, in_tmpdir, store, ): - # Write out the config - config = make_config_from_repo( - out_of_date_repo.path, sha=out_of_date_repo.original_sha, check=False, + stale_config = make_config_from_repo( + out_of_date.path, rev=out_of_date.original_rev, check=False, ) + local_config = sample_local_config() + config = {'repos': [stale_config, local_config]} write_config('.', config) - before = open(C.CONFIG_FILE).read() - ret = autoupdate(Runner('.', C.CONFIG_FILE), tags_only=False) - after = open(C.CONFIG_FILE).read() + with open(C.CONFIG_FILE) as f: + before = f.read() + repo_name = f'file://{out_of_date.path}' + ret = autoupdate( + C.CONFIG_FILE, store, freeze=False, tags_only=False, + repos=(repo_name,), + ) + with open(C.CONFIG_FILE) as f: + after = f.read() assert ret == 0 assert before != after - # Make sure we don't add defaults - assert 'exclude' not in after - assert out_of_date_repo.head_sha in after - - -@pytest.yield_fixture -def tagged_repo(out_of_date_repo): - with cwd(out_of_date_repo.path): - cmd_output('git', 'tag', 'v1.2.3') - yield out_of_date_repo + assert out_of_date.head_rev in after + assert 'local' in after -def test_autoupdate_tagged_repo( - tagged_repo, in_tmpdir, mock_out_store_directory, +def test_autoupdate_out_of_date_repo_with_wrong_repo_name( + out_of_date, in_tmpdir, store, ): config = make_config_from_repo( - tagged_repo.path, sha=tagged_repo.original_sha, + out_of_date.path, rev=out_of_date.original_rev, check=False, ) write_config('.', config) - ret = autoupdate(Runner('.', C.CONFIG_FILE), tags_only=False) + with open(C.CONFIG_FILE) as f: + before = f.read() + # It will not update it, because the name doesn't match + ret = autoupdate( + C.CONFIG_FILE, store, freeze=False, tags_only=False, + repos=('dne',), + ) + with open(C.CONFIG_FILE) as f: + after = f.read() assert ret == 0 - assert 'v1.2.3' in open(C.CONFIG_FILE).read() + assert before == after -@pytest.yield_fixture -def tagged_repo_with_more_commits(tagged_repo): - with cwd(tagged_repo.path): - cmd_output('git', 'commit', '--allow-empty', '-m', 'commit!') - yield tagged_repo +def test_does_not_reformat(tmpdir, out_of_date, store): + fmt = ( + 'repos:\n' + '- repo: {}\n' + ' rev: {} # definitely the version I want!\n' + ' hooks:\n' + ' - id: foo\n' + ' # These args are because reasons!\n' + ' args: [foo, bar, baz]\n' + ) + cfg = tmpdir.join(C.CONFIG_FILE) + cfg.write(fmt.format(out_of_date.path, out_of_date.original_rev)) + assert autoupdate(str(cfg), store, freeze=False, tags_only=False) == 0 + expected = fmt.format(out_of_date.path, out_of_date.head_rev) + assert cfg.read() == expected -def test_autoupdate_tags_only( - tagged_repo_with_more_commits, in_tmpdir, mock_out_store_directory, -): - config = make_config_from_repo( - tagged_repo_with_more_commits.path, - sha=tagged_repo_with_more_commits.original_sha, + +def test_loses_formatting_when_not_detectable(out_of_date, store, tmpdir): + """A best-effort attempt is made at updating rev without rewriting + formatting. When the original formatting cannot be detected, this + is abandoned. + """ + config = ( + 'repos: [\n' + ' {{\n' + ' repo: {}, rev: {},\n' + ' hooks: [\n' + ' # A comment!\n' + ' {{id: foo}},\n' + ' ],\n' + ' }}\n' + ']\n'.format( + shlex.quote(out_of_date.path), out_of_date.original_rev, + ) ) + cfg = tmpdir.join(C.CONFIG_FILE) + cfg.write(config) + + assert autoupdate(str(cfg), store, freeze=False, tags_only=False) == 0 + expected = ( + f'repos:\n' + f'- repo: {out_of_date.path}\n' + f' rev: {out_of_date.head_rev}\n' + f' hooks:\n' + f' - id: foo\n' + ) + assert cfg.read() == expected + + +def test_autoupdate_tagged_repo(tagged, in_tmpdir, store): + config = make_config_from_repo(tagged.path, rev=tagged.original_rev) write_config('.', config) - ret = autoupdate(Runner('.', C.CONFIG_FILE), tags_only=True) - assert ret == 0 - assert 'v1.2.3' in open(C.CONFIG_FILE).read() + assert autoupdate(C.CONFIG_FILE, store, freeze=False, tags_only=False) == 0 + with open(C.CONFIG_FILE) as f: + assert 'v1.2.3' in f.read() -@pytest.yield_fixture -def hook_disappearing_repo(tempdir_factory): - path = make_repo(tempdir_factory, 'python_hooks_repo') - original_sha = get_head_sha(path) +def test_autoupdate_freeze(tagged, in_tmpdir, store): + config = make_config_from_repo(tagged.path, rev=tagged.original_rev) + write_config('.', config) - with cwd(path): - shutil.copy( - get_resource_path('manifest_without_foo.yaml'), - C.MANIFEST_FILE, - ) - cmd_output('git', 'add', '.') - cmd_output('git', 'commit', '-m', 'Remove foo') + assert autoupdate(C.CONFIG_FILE, store, freeze=True, tags_only=False) == 0 + with open(C.CONFIG_FILE) as f: + expected = f'rev: {tagged.head_rev} # frozen: v1.2.3' + assert expected in f.read() - yield auto_namedtuple(path=path, original_sha=original_sha) + # if we un-freeze it should remove the frozen comment + assert autoupdate(C.CONFIG_FILE, store, freeze=False, tags_only=False) == 0 + with open(C.CONFIG_FILE) as f: + assert 'rev: v1.2.3\n' in f.read() -def test_hook_disppearing_repo_raises( - hook_disappearing_repo, runner_with_mocked_store -): +def test_autoupdate_tags_only(tagged, in_tmpdir, store): + # add some commits after the tag + git_commit(cwd=tagged.path) + + config = make_config_from_repo(tagged.path, rev=tagged.original_rev) + write_config('.', config) + + assert autoupdate(C.CONFIG_FILE, store, freeze=False, tags_only=True) == 0 + with open(C.CONFIG_FILE) as f: + assert 'v1.2.3' in f.read() + + +def test_autoupdate_latest_no_config(out_of_date, in_tmpdir, store): config = make_config_from_repo( - hook_disappearing_repo.path, - sha=hook_disappearing_repo.original_sha, - hooks=[OrderedDict((('id', 'foo'),))], + out_of_date.path, rev=out_of_date.original_rev, ) - with pytest.raises(RepositoryCannotBeUpdatedError): - _update_repo(config, runner_with_mocked_store, tags_only=False) + write_config('.', config) + cmd_output('git', 'rm', '-r', ':/', cwd=out_of_date.path) + git_commit(cwd=out_of_date.path) -def test_autoupdate_hook_disappearing_repo( - hook_disappearing_repo, in_tmpdir, mock_out_store_directory -): + assert autoupdate(C.CONFIG_FILE, store, freeze=False, tags_only=False) == 1 + with open(C.CONFIG_FILE) as f: + assert out_of_date.original_rev in f.read() + + +def test_hook_disppearing_repo_raises(hook_disappearing, store): config = make_config_from_repo( - hook_disappearing_repo.path, - sha=hook_disappearing_repo.original_sha, - hooks=[OrderedDict((('id', 'foo'),))], - check=False, + hook_disappearing.path, + rev=hook_disappearing.original_rev, + hooks=[{'id': 'foo'}], ) - write_config('.', config) + info = RevInfo.from_config(config).update(tags_only=False, freeze=False) + with pytest.raises(RepositoryCannotBeUpdatedError): + _check_hooks_still_exist_at_rev(config, info, store) - before = open(C.CONFIG_FILE).read() - ret = autoupdate(Runner('.', C.CONFIG_FILE), tags_only=False) - after = open(C.CONFIG_FILE).read() - assert ret == 1 - assert before == after + +def test_autoupdate_hook_disappearing_repo(hook_disappearing, tmpdir, store): + contents = ( + f'repos:\n' + f'- repo: {hook_disappearing.path}\n' + f' rev: {hook_disappearing.original_rev}\n' + f' hooks:\n' + f' - id: foo\n' + ) + cfg = tmpdir.join(C.CONFIG_FILE) + cfg.write(contents) + + assert autoupdate(str(cfg), store, freeze=False, tags_only=False) == 1 + assert cfg.read() == contents -def test_autoupdate_local_hooks(tempdir_factory): - git_path = git_dir(tempdir_factory) - config = config_with_local_hooks() - path = add_config_to_repo(git_path, config) - runner = Runner(path, C.CONFIG_FILE) - assert autoupdate(runner, tags_only=False) == 0 - new_config_writen = load_config(runner.config_file_path) - assert len(new_config_writen) == 1 - assert new_config_writen[0] == config +def test_autoupdate_local_hooks(in_git_dir, store): + config = sample_local_config() + add_config_to_repo('.', config) + assert autoupdate(C.CONFIG_FILE, store, freeze=False, tags_only=False) == 0 + new_config_writen = read_config('.') + assert len(new_config_writen['repos']) == 1 + assert new_config_writen['repos'][0] == config def test_autoupdate_local_hooks_with_out_of_date_repo( - out_of_date_repo, in_tmpdir, mock_out_store_directory + out_of_date, in_tmpdir, store, ): stale_config = make_config_from_repo( - out_of_date_repo.path, sha=out_of_date_repo.original_sha, check=False, + out_of_date.path, rev=out_of_date.original_rev, check=False, ) - local_config = config_with_local_hooks() - config = [local_config, stale_config] + local_config = sample_local_config() + config = {'repos': [local_config, stale_config]} write_config('.', config) - runner = Runner('.', C.CONFIG_FILE) - assert autoupdate(runner, tags_only=False) == 0 - new_config_writen = load_config(runner.config_file_path) - assert len(new_config_writen) == 2 - assert new_config_writen[0] == local_config + assert autoupdate(C.CONFIG_FILE, store, freeze=False, tags_only=False) == 0 + new_config_writen = read_config('.') + assert len(new_config_writen['repos']) == 2 + assert new_config_writen['repos'][0] == local_config + + +def test_autoupdate_meta_hooks(tmpdir, store): + cfg = tmpdir.join(C.CONFIG_FILE) + cfg.write( + 'repos:\n' + '- repo: meta\n' + ' hooks:\n' + ' - id: check-useless-excludes\n', + ) + assert autoupdate(str(cfg), store, freeze=False, tags_only=True) == 0 + assert cfg.read() == ( + 'repos:\n' + '- repo: meta\n' + ' hooks:\n' + ' - id: check-useless-excludes\n' + ) + + +def test_updates_old_format_to_new_format(tmpdir, capsys, store): + cfg = tmpdir.join(C.CONFIG_FILE) + cfg.write( + '- repo: local\n' + ' hooks:\n' + ' - id: foo\n' + ' name: foo\n' + ' entry: ./bin/foo.sh\n' + ' language: script\n', + ) + assert autoupdate(str(cfg), store, freeze=False, tags_only=True) == 0 + contents = cfg.read() + assert contents == ( + 'repos:\n' + '- repo: local\n' + ' hooks:\n' + ' - id: foo\n' + ' name: foo\n' + ' entry: ./bin/foo.sh\n' + ' language: script\n' + ) + out, _ = capsys.readouterr() + assert out == 'Configuration has been migrated.\n' diff --git a/tests/commands/clean_test.py b/tests/commands/clean_test.py index bdbdc9987..955a6bc4e 100644 --- a/tests/commands/clean_test.py +++ b/tests/commands/clean_test.py @@ -1,20 +1,33 @@ -from __future__ import unicode_literals - import os.path +from unittest import mock + +import pytest from pre_commit.commands.clean import clean -from pre_commit.util import rmtree -def test_clean(runner_with_mocked_store): - assert os.path.exists(runner_with_mocked_store.store.directory) - clean(runner_with_mocked_store) - assert not os.path.exists(runner_with_mocked_store.store.directory) +@pytest.fixture(autouse=True) +def fake_old_dir(tempdir_factory): + fake_old_dir = tempdir_factory.get() + + def _expanduser(path, *args, **kwargs): + assert path == '~/.pre-commit' + return fake_old_dir + + with mock.patch.object(os.path, 'expanduser', side_effect=_expanduser): + yield fake_old_dir + + +def test_clean(store, fake_old_dir): + assert os.path.exists(fake_old_dir) + assert os.path.exists(store.directory) + clean(store) + assert not os.path.exists(fake_old_dir) + assert not os.path.exists(store.directory) -def test_clean_empty(runner_with_mocked_store): - """Make sure clean succeeds when we the directory doesn't exist.""" - rmtree(runner_with_mocked_store.store.directory) - assert not os.path.exists(runner_with_mocked_store.store.directory) - clean(runner_with_mocked_store) - assert not os.path.exists(runner_with_mocked_store.store.directory) +def test_clean_idempotent(store): + clean(store) + assert not os.path.exists(store.directory) + clean(store) + assert not os.path.exists(store.directory) diff --git a/tests/commands/gc_test.py b/tests/commands/gc_test.py new file mode 100644 index 000000000..02b36945b --- /dev/null +++ b/tests/commands/gc_test.py @@ -0,0 +1,161 @@ +import os + +import pre_commit.constants as C +from pre_commit import git +from pre_commit.clientlib import load_config +from pre_commit.commands.autoupdate import autoupdate +from pre_commit.commands.gc import gc +from pre_commit.commands.install_uninstall import install_hooks +from pre_commit.repository import all_hooks +from testing.fixtures import make_config_from_repo +from testing.fixtures import make_repo +from testing.fixtures import modify_config +from testing.fixtures import sample_local_config +from testing.fixtures import sample_meta_config +from testing.fixtures import write_config +from testing.util import git_commit + + +def _repo_count(store): + return len(store.select_all_repos()) + + +def _config_count(store): + return len(store.select_all_configs()) + + +def _remove_config_assert_cleared(store, cap_out): + os.remove(C.CONFIG_FILE) + assert not gc(store) + assert _config_count(store) == 0 + assert _repo_count(store) == 0 + assert cap_out.get().splitlines()[-1] == '1 repo(s) removed.' + + +def test_gc(tempdir_factory, store, in_git_dir, cap_out): + path = make_repo(tempdir_factory, 'script_hooks_repo') + old_rev = git.head_rev(path) + git_commit(cwd=path) + + write_config('.', make_config_from_repo(path, rev=old_rev)) + store.mark_config_used(C.CONFIG_FILE) + + # update will clone both the old and new repo, making the old one gc-able + install_hooks(C.CONFIG_FILE, store) + assert not autoupdate(C.CONFIG_FILE, store, freeze=False, tags_only=False) + + assert _config_count(store) == 1 + assert _repo_count(store) == 2 + assert not gc(store) + assert _config_count(store) == 1 + assert _repo_count(store) == 1 + assert cap_out.get().splitlines()[-1] == '1 repo(s) removed.' + + _remove_config_assert_cleared(store, cap_out) + + +def test_gc_repo_not_cloned(tempdir_factory, store, in_git_dir, cap_out): + path = make_repo(tempdir_factory, 'script_hooks_repo') + write_config('.', make_config_from_repo(path)) + store.mark_config_used(C.CONFIG_FILE) + + assert _config_count(store) == 1 + assert _repo_count(store) == 0 + assert not gc(store) + assert _config_count(store) == 1 + assert _repo_count(store) == 0 + assert cap_out.get().splitlines()[-1] == '0 repo(s) removed.' + + +def test_gc_meta_repo_does_not_crash(store, in_git_dir, cap_out): + write_config('.', sample_meta_config()) + store.mark_config_used(C.CONFIG_FILE) + assert not gc(store) + assert cap_out.get().splitlines()[-1] == '0 repo(s) removed.' + + +def test_gc_local_repo_does_not_crash(store, in_git_dir, cap_out): + write_config('.', sample_local_config()) + store.mark_config_used(C.CONFIG_FILE) + assert not gc(store) + assert cap_out.get().splitlines()[-1] == '0 repo(s) removed.' + + +def test_gc_unused_local_repo_with_env(store, in_git_dir, cap_out): + config = { + 'repo': 'local', + 'hooks': [{ + 'id': 'flake8', 'name': 'flake8', 'entry': 'flake8', + # a `language: python` local hook will create an environment + 'types': ['python'], 'language': 'python', + }], + } + write_config('.', config) + store.mark_config_used(C.CONFIG_FILE) + + # this causes the repositories to be created + all_hooks(load_config(C.CONFIG_FILE), store) + + assert _config_count(store) == 1 + assert _repo_count(store) == 1 + assert not gc(store) + assert _config_count(store) == 1 + assert _repo_count(store) == 1 + assert cap_out.get().splitlines()[-1] == '0 repo(s) removed.' + + _remove_config_assert_cleared(store, cap_out) + + +def test_gc_config_with_missing_hook( + tempdir_factory, store, in_git_dir, cap_out, +): + path = make_repo(tempdir_factory, 'script_hooks_repo') + write_config('.', make_config_from_repo(path)) + store.mark_config_used(C.CONFIG_FILE) + # to trigger a clone + all_hooks(load_config(C.CONFIG_FILE), store) + + with modify_config() as config: + # add a hook which does not exist, make sure we don't crash + config['repos'][0]['hooks'].append({'id': 'does-not-exist'}) + + assert _config_count(store) == 1 + assert _repo_count(store) == 1 + assert not gc(store) + assert _config_count(store) == 1 + assert _repo_count(store) == 1 + assert cap_out.get().splitlines()[-1] == '0 repo(s) removed.' + + _remove_config_assert_cleared(store, cap_out) + + +def test_gc_deletes_invalid_configs(store, in_git_dir, cap_out): + config = {'i am': 'invalid'} + write_config('.', config) + store.mark_config_used(C.CONFIG_FILE) + + assert _config_count(store) == 1 + assert not gc(store) + assert _config_count(store) == 0 + assert cap_out.get().splitlines()[-1] == '0 repo(s) removed.' + + +def test_invalid_manifest_gcd(tempdir_factory, store, in_git_dir, cap_out): + # clean up repos from old pre-commit versions + path = make_repo(tempdir_factory, 'script_hooks_repo') + write_config('.', make_config_from_repo(path)) + store.mark_config_used(C.CONFIG_FILE) + + # trigger a clone + install_hooks(C.CONFIG_FILE, store) + + # we'll "break" the manifest to simulate an old version clone + (_, _, path), = store.select_all_repos() + os.remove(os.path.join(path, C.MANIFEST_FILE)) + + assert _config_count(store) == 1 + assert _repo_count(store) == 1 + assert not gc(store) + assert _config_count(store) == 1 + assert _repo_count(store) == 0 + assert cap_out.get().splitlines()[-1] == '1 repo(s) removed.' diff --git a/tests/commands/hook_impl_test.py b/tests/commands/hook_impl_test.py new file mode 100644 index 000000000..032fa8fa8 --- /dev/null +++ b/tests/commands/hook_impl_test.py @@ -0,0 +1,235 @@ +import subprocess +import sys +from unittest import mock + +import pytest + +import pre_commit.constants as C +from pre_commit import git +from pre_commit.commands import hook_impl +from pre_commit.envcontext import envcontext +from pre_commit.util import cmd_output +from pre_commit.util import make_executable +from testing.fixtures import git_dir +from testing.fixtures import sample_local_config +from testing.fixtures import write_config +from testing.util import cwd +from testing.util import git_commit + + +def test_validate_config_file_exists(tmpdir): + cfg = tmpdir.join(C.CONFIG_FILE).ensure() + hook_impl._validate_config(0, cfg, True) + + +def test_validate_config_missing(capsys): + with pytest.raises(SystemExit) as excinfo: + hook_impl._validate_config(123, 'DNE.yaml', False) + ret, = excinfo.value.args + assert ret == 1 + assert capsys.readouterr().out == ( + 'No DNE.yaml file was found\n' + '- To temporarily silence this, run ' + '`PRE_COMMIT_ALLOW_NO_CONFIG=1 git ...`\n' + '- To permanently silence this, install pre-commit with the ' + '--allow-missing-config option\n' + '- To uninstall pre-commit run `pre-commit uninstall`\n' + ) + + +def test_validate_config_skip_missing_config(capsys): + with pytest.raises(SystemExit) as excinfo: + hook_impl._validate_config(123, 'DNE.yaml', True) + ret, = excinfo.value.args + assert ret == 123 + expected = '`DNE.yaml` config file not found. Skipping `pre-commit`.\n' + assert capsys.readouterr().out == expected + + +def test_validate_config_skip_via_env_variable(capsys): + with pytest.raises(SystemExit) as excinfo: + with envcontext((('PRE_COMMIT_ALLOW_NO_CONFIG', '1'),)): + hook_impl._validate_config(0, 'DNE.yaml', False) + ret, = excinfo.value.args + assert ret == 0 + expected = '`DNE.yaml` config file not found. Skipping `pre-commit`.\n' + assert capsys.readouterr().out == expected + + +def test_run_legacy_does_not_exist(tmpdir): + retv, stdin = hook_impl._run_legacy('pre-commit', tmpdir, ()) + assert (retv, stdin) == (0, b'') + + +def test_run_legacy_executes_legacy_script(tmpdir, capfd): + hook = tmpdir.join('pre-commit.legacy') + hook.write('#!/usr/bin/env bash\necho hi "$@"\nexit 1\n') + make_executable(hook) + retv, stdin = hook_impl._run_legacy('pre-commit', tmpdir, ('arg1', 'arg2')) + assert capfd.readouterr().out.strip() == 'hi arg1 arg2' + assert (retv, stdin) == (1, b'') + + +def test_run_legacy_pre_push_returns_stdin(tmpdir): + with mock.patch.object(sys.stdin.buffer, 'read', return_value=b'stdin'): + retv, stdin = hook_impl._run_legacy('pre-push', tmpdir, ()) + assert (retv, stdin) == (0, b'stdin') + + +def test_run_legacy_recursive(tmpdir): + hook = tmpdir.join('pre-commit.legacy').ensure() + make_executable(hook) + + # simulate a call being recursive + def call(*_, **__): + return hook_impl._run_legacy('pre-commit', tmpdir, ()) + + with mock.patch.object(subprocess, 'run', call): + with pytest.raises(SystemExit): + call() + + +def test_run_ns_pre_commit(): + ns = hook_impl._run_ns('pre-commit', True, (), b'') + assert ns is not None + assert ns.hook_stage == 'commit' + assert ns.color is True + + +def test_run_ns_commit_msg(): + ns = hook_impl._run_ns('commit-msg', False, ('.git/COMMIT_MSG',), b'') + assert ns is not None + assert ns.hook_stage == 'commit-msg' + assert ns.color is False + assert ns.commit_msg_filename == '.git/COMMIT_MSG' + + +def test_run_ns_post_checkout(): + ns = hook_impl._run_ns('post-checkout', True, ('a', 'b', 'c'), b'') + assert ns is not None + assert ns.hook_stage == 'post-checkout' + assert ns.color is True + assert ns.from_ref == 'a' + assert ns.to_ref == 'b' + assert ns.checkout_type == 'c' + + +@pytest.fixture +def push_example(tempdir_factory): + src = git_dir(tempdir_factory) + git_commit(cwd=src) + src_head = git.head_rev(src) + + clone = tempdir_factory.get() + cmd_output('git', 'clone', src, clone) + git_commit(cwd=clone) + clone_head = git.head_rev(clone) + return (src, src_head, clone, clone_head) + + +def test_run_ns_pre_push_updating_branch(push_example): + src, src_head, clone, clone_head = push_example + + with cwd(clone): + args = ('origin', src) + stdin = f'HEAD {clone_head} refs/heads/b {src_head}\n'.encode() + ns = hook_impl._run_ns('pre-push', False, args, stdin) + + assert ns is not None + assert ns.hook_stage == 'push' + assert ns.color is False + assert ns.remote_name == 'origin' + assert ns.remote_url == src + assert ns.from_ref == src_head + assert ns.to_ref == clone_head + assert ns.all_files is False + + +def test_run_ns_pre_push_new_branch(push_example): + src, src_head, clone, clone_head = push_example + + with cwd(clone): + args = ('origin', src) + stdin = f'HEAD {clone_head} refs/heads/b {hook_impl.Z40}\n'.encode() + ns = hook_impl._run_ns('pre-push', False, args, stdin) + + assert ns is not None + assert ns.from_ref == src_head + assert ns.to_ref == clone_head + + +def test_run_ns_pre_push_new_branch_existing_rev(push_example): + src, src_head, clone, _ = push_example + + with cwd(clone): + args = ('origin', src) + stdin = f'HEAD {src_head} refs/heads/b2 {hook_impl.Z40}\n'.encode() + ns = hook_impl._run_ns('pre-push', False, args, stdin) + + assert ns is None + + +def test_pushing_orphan_branch(push_example): + src, src_head, clone, _ = push_example + + cmd_output('git', 'checkout', '--orphan', 'b2', cwd=clone) + git_commit(cwd=clone, msg='something else to get unique hash') + clone_rev = git.head_rev(clone) + + with cwd(clone): + args = ('origin', src) + stdin = f'HEAD {clone_rev} refs/heads/b2 {hook_impl.Z40}\n'.encode() + ns = hook_impl._run_ns('pre-push', False, args, stdin) + + assert ns is not None + assert ns.all_files is True + + +def test_run_ns_pre_push_deleting_branch(push_example): + src, src_head, clone, _ = push_example + + with cwd(clone): + args = ('origin', src) + stdin = f'(delete) {hook_impl.Z40} refs/heads/b {src_head}'.encode() + ns = hook_impl._run_ns('pre-push', False, args, stdin) + + assert ns is None + + +def test_hook_impl_main_noop_pre_push(cap_out, store, push_example): + src, src_head, clone, _ = push_example + + stdin = f'(delete) {hook_impl.Z40} refs/heads/b {src_head}'.encode() + with mock.patch.object(sys.stdin.buffer, 'read', return_value=stdin): + with cwd(clone): + write_config('.', sample_local_config()) + ret = hook_impl.hook_impl( + store, + config=C.CONFIG_FILE, + color=False, + hook_type='pre-push', + hook_dir='.git/hooks', + skip_on_missing_config=False, + args=('origin', src), + ) + assert ret == 0 + assert cap_out.get() == '' + + +def test_hook_impl_main_runs_hooks(cap_out, tempdir_factory, store): + with cwd(git_dir(tempdir_factory)): + write_config('.', sample_local_config()) + ret = hook_impl.hook_impl( + store, + config=C.CONFIG_FILE, + color=False, + hook_type='pre-commit', + hook_dir='.git/hooks', + skip_on_missing_config=False, + args=(), + ) + assert ret == 0 + expected = '''\ +Block if "DO NOT COMMIT" is found....................(no files to check)Skipped +''' + assert cap_out.get() == expected diff --git a/tests/commands/init_templatedir_test.py b/tests/commands/init_templatedir_test.py new file mode 100644 index 000000000..d14a171f6 --- /dev/null +++ b/tests/commands/init_templatedir_test.py @@ -0,0 +1,92 @@ +import os.path +from unittest import mock + +import pre_commit.constants as C +from pre_commit.commands.init_templatedir import init_templatedir +from pre_commit.envcontext import envcontext +from pre_commit.util import cmd_output +from testing.fixtures import git_dir +from testing.fixtures import make_consuming_repo +from testing.util import cmd_output_mocked_pre_commit_home +from testing.util import cwd +from testing.util import git_commit + + +def test_init_templatedir(tmpdir, tempdir_factory, store, cap_out): + target = str(tmpdir.join('tmpl')) + init_templatedir(C.CONFIG_FILE, store, target, hook_types=['pre-commit']) + lines = cap_out.get().splitlines() + assert lines[0].startswith('pre-commit installed at ') + assert lines[1] == ( + '[WARNING] `init.templateDir` not set to the target directory' + ) + assert lines[2].startswith( + '[WARNING] maybe `git config --global init.templateDir', + ) + + with envcontext((('GIT_TEMPLATE_DIR', target),)): + path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') + + with cwd(path): + retcode, output = git_commit( + fn=cmd_output_mocked_pre_commit_home, + tempdir_factory=tempdir_factory, + ) + assert retcode == 0 + assert 'Bash hook....' in output + + +def test_init_templatedir_already_set(tmpdir, tempdir_factory, store, cap_out): + target = str(tmpdir.join('tmpl')) + tmp_git_dir = git_dir(tempdir_factory) + with cwd(tmp_git_dir): + cmd_output('git', 'config', 'init.templateDir', target) + init_templatedir( + C.CONFIG_FILE, store, target, hook_types=['pre-commit'], + ) + + lines = cap_out.get().splitlines() + assert len(lines) == 1 + assert lines[0].startswith('pre-commit installed at') + + +def test_init_templatedir_not_set(tmpdir, store, cap_out): + # set HOME to ignore the current `.gitconfig` + with envcontext((('HOME', str(tmpdir)),)): + with tmpdir.join('tmpl').ensure_dir().as_cwd(): + # we have not set init.templateDir so this should produce a warning + init_templatedir( + C.CONFIG_FILE, store, '.', hook_types=['pre-commit'], + ) + + lines = cap_out.get().splitlines() + assert len(lines) == 3 + assert lines[1] == ( + '[WARNING] `init.templateDir` not set to the target directory' + ) + + +def test_init_templatedir_expanduser(tmpdir, tempdir_factory, store, cap_out): + target = str(tmpdir.join('tmpl')) + tmp_git_dir = git_dir(tempdir_factory) + with cwd(tmp_git_dir): + cmd_output('git', 'config', 'init.templateDir', '~/templatedir') + with mock.patch.object(os.path, 'expanduser', return_value=target): + init_templatedir( + C.CONFIG_FILE, store, target, hook_types=['pre-commit'], + ) + + lines = cap_out.get().splitlines() + assert len(lines) == 1 + assert lines[0].startswith('pre-commit installed at') + + +def test_init_templatedir_hookspath_set(tmpdir, tempdir_factory, store): + target = tmpdir.join('tmpl') + tmp_git_dir = git_dir(tempdir_factory) + with cwd(tmp_git_dir): + cmd_output('git', 'config', '--local', 'core.hooksPath', 'hooks') + init_templatedir( + C.CONFIG_FILE, store, target, hook_types=['pre-commit'], + ) + assert target.join('hooks/pre-commit').exists() diff --git a/tests/commands/install_uninstall_test.py b/tests/commands/install_uninstall_test.py index ad8d2456d..66b91903b 100644 --- a/tests/commands/install_uninstall_test.py +++ b/tests/commands/install_uninstall_test.py @@ -1,34 +1,30 @@ -# -*- coding: UTF-8 -*- -from __future__ import absolute_import -from __future__ import unicode_literals - -import io import os.path import re -import shutil -import subprocess import sys - -import mock +from unittest import mock import pre_commit.constants as C +from pre_commit import git +from pre_commit.commands import install_uninstall from pre_commit.commands.install_uninstall import CURRENT_HASH from pre_commit.commands.install_uninstall import install from pre_commit.commands.install_uninstall import install_hooks from pre_commit.commands.install_uninstall import is_our_script from pre_commit.commands.install_uninstall import PRIOR_HASHES +from pre_commit.commands.install_uninstall import shebang from pre_commit.commands.install_uninstall import uninstall -from pre_commit.runner import Runner +from pre_commit.parse_shebang import find_executable from pre_commit.util import cmd_output -from pre_commit.util import cwd from pre_commit.util import make_executable -from pre_commit.util import mkdirp -from pre_commit.util import resource_filename +from pre_commit.util import resource_text +from testing.fixtures import add_config_to_repo from testing.fixtures import git_dir from testing.fixtures import make_consuming_repo from testing.fixtures import remove_config_from_repo +from testing.fixtures import write_config from testing.util import cmd_output_mocked_pre_commit_home -from testing.util import xfailif_no_symlink +from testing.util import cwd +from testing.util import git_commit def test_is_not_script(): @@ -36,178 +32,214 @@ def test_is_not_script(): def test_is_script(): - assert is_our_script(resource_filename('hook-tmpl')) + assert is_our_script('pre_commit/resources/hook-tmpl') def test_is_previous_pre_commit(tmpdir): f = tmpdir.join('foo') - f.write(PRIOR_HASHES[0] + '\n') + f.write(f'{PRIOR_HASHES[0]}\n') assert is_our_script(f.strpath) -def test_install_pre_commit(tempdir_factory): - path = git_dir(tempdir_factory) - runner = Runner(path, C.CONFIG_FILE) - ret = install(runner) - assert ret == 0 - assert os.path.exists(runner.pre_commit_path) - pre_commit_contents = io.open(runner.pre_commit_path).read() - pre_commit_script = resource_filename('hook-tmpl') - expected_contents = io.open(pre_commit_script).read().format( - sys_executable=sys.executable, - hook_type='pre-commit', - pre_push='', - skip_on_missing_conf='false', - ) - assert pre_commit_contents == expected_contents - assert os.access(runner.pre_commit_path, os.X_OK) - - ret = install(runner, hook_type='pre-push') - assert ret == 0 - assert os.path.exists(runner.pre_push_path) - pre_push_contents = io.open(runner.pre_push_path).read() - pre_push_tmpl = resource_filename('pre-push-tmpl') - pre_push_template_contents = io.open(pre_push_tmpl).read() - expected_contents = io.open(pre_commit_script).read().format( - sys_executable=sys.executable, - hook_type='pre-push', - pre_push=pre_push_template_contents, - skip_on_missing_conf='false', - ) - assert pre_push_contents == expected_contents +def patch_platform(platform): + return mock.patch.object(sys, 'platform', platform) -def test_install_hooks_directory_not_present(tempdir_factory): - path = git_dir(tempdir_factory) +def patch_lookup_path(path): + return mock.patch.object(install_uninstall, 'POSIX_SEARCH_PATH', path) + + +def patch_sys_exe(exe): + return mock.patch.object(install_uninstall, 'SYS_EXE', exe) + + +def test_shebang_windows(): + with patch_platform('win32'), patch_sys_exe('python.exe'): + assert shebang() == '#!/usr/bin/env python.exe' + + +def test_shebang_posix_not_on_path(): + with patch_platform('posix'), patch_lookup_path(()): + with patch_sys_exe('python3.6'): + assert shebang() == '#!/usr/bin/env python3.6' + + +def test_shebang_posix_on_path(tmpdir): + exe = tmpdir.join(f'python{sys.version_info[0]}').ensure() + make_executable(exe) + + with patch_platform('posix'), patch_lookup_path((tmpdir.strpath,)): + with patch_sys_exe('python'): + assert shebang() == f'#!/usr/bin/env python{sys.version_info[0]}' + + +def test_install_pre_commit(in_git_dir, store): + assert not install(C.CONFIG_FILE, store, hook_types=['pre-commit']) + assert os.access(in_git_dir.join('.git/hooks/pre-commit').strpath, os.X_OK) + + assert not install(C.CONFIG_FILE, store, hook_types=['pre-push']) + assert os.access(in_git_dir.join('.git/hooks/pre-push').strpath, os.X_OK) + + +def test_install_hooks_directory_not_present(in_git_dir, store): # Simulate some git clients which don't make .git/hooks #234 - hooks = os.path.join(path, '.git', 'hooks') - if os.path.exists(hooks): # pragma: no cover (latest git) - shutil.rmtree(hooks) - runner = Runner(path, C.CONFIG_FILE) - install(runner) - assert os.path.exists(runner.pre_commit_path) - - -@xfailif_no_symlink -def test_install_hooks_dead_symlink( - tempdir_factory, -): # pragma: no cover (non-windows) - path = git_dir(tempdir_factory) - runner = Runner(path, C.CONFIG_FILE) - mkdirp(os.path.dirname(runner.pre_commit_path)) - os.symlink('/fake/baz', os.path.join(path, '.git', 'hooks', 'pre-commit')) - install(runner) - assert os.path.exists(runner.pre_commit_path) + if in_git_dir.join('.git/hooks').exists(): # pragma: no cover (odd git) + in_git_dir.join('.git/hooks').remove() + install(C.CONFIG_FILE, store, hook_types=['pre-commit']) + assert in_git_dir.join('.git/hooks/pre-commit').exists() -def test_uninstall_does_not_blow_up_when_not_there(tempdir_factory): - path = git_dir(tempdir_factory) - runner = Runner(path, C.CONFIG_FILE) - ret = uninstall(runner) - assert ret == 0 +def test_install_multiple_hooks_at_once(in_git_dir, store): + install(C.CONFIG_FILE, store, hook_types=['pre-commit', 'pre-push']) + assert in_git_dir.join('.git/hooks/pre-commit').exists() + assert in_git_dir.join('.git/hooks/pre-push').exists() + uninstall(hook_types=['pre-commit', 'pre-push']) + assert not in_git_dir.join('.git/hooks/pre-commit').exists() + assert not in_git_dir.join('.git/hooks/pre-push').exists() -def test_uninstall(tempdir_factory): - path = git_dir(tempdir_factory) - runner = Runner(path, C.CONFIG_FILE) - assert not os.path.exists(runner.pre_commit_path) - install(runner) - assert os.path.exists(runner.pre_commit_path) - uninstall(runner) - assert not os.path.exists(runner.pre_commit_path) +def test_install_refuses_core_hookspath(in_git_dir, store): + cmd_output('git', 'config', '--local', 'core.hooksPath', 'hooks') + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) + + +def test_install_hooks_dead_symlink(in_git_dir, store): + hook = in_git_dir.join('.git/hooks').ensure_dir().join('pre-commit') + os.symlink('/fake/baz', hook.strpath) + install(C.CONFIG_FILE, store, hook_types=['pre-commit']) + assert hook.exists() + + +def test_uninstall_does_not_blow_up_when_not_there(in_git_dir): + assert uninstall(hook_types=['pre-commit']) == 0 + + +def test_uninstall(in_git_dir, store): + assert not in_git_dir.join('.git/hooks/pre-commit').exists() + install(C.CONFIG_FILE, store, hook_types=['pre-commit']) + assert in_git_dir.join('.git/hooks/pre-commit').exists() + uninstall(hook_types=['pre-commit']) + assert not in_git_dir.join('.git/hooks/pre-commit').exists() def _get_commit_output(tempdir_factory, touch_file='foo', **kwargs): - cmd_output('touch', touch_file) + open(touch_file, 'a').close() cmd_output('git', 'add', touch_file) - return cmd_output_mocked_pre_commit_home( - 'git', 'commit', '-am', 'Commit!', '--allow-empty', - # git commit puts pre-commit to stderr - stderr=subprocess.STDOUT, + return git_commit( + fn=cmd_output_mocked_pre_commit_home, retcode=None, tempdir_factory=tempdir_factory, - **kwargs - )[:2] + **kwargs, + ) # osx does this different :( FILES_CHANGED = ( r'(' - r' 1 file changed, 0 insertions\(\+\), 0 deletions\(-\)\r?\n' + r' 1 file changed, 0 insertions\(\+\), 0 deletions\(-\)\n' r'|' - r' 0 files changed\r?\n' + r' 0 files changed\n' r')' ) NORMAL_PRE_COMMIT_RUN = re.compile( - r'^\[INFO\] Initializing environment for .+\.\r?\n' - r'Bash hook\.+Passed\r?\n' - r'\[master [a-f0-9]{7}\] Commit!\r?\n' + - FILES_CHANGED + - r' create mode 100644 foo\r?\n$' + fr'^\[INFO\] Initializing environment for .+\.\n' + fr'Bash hook\.+Passed\n' + fr'\[master [a-f0-9]{{7}}\] commit!\n' + fr'{FILES_CHANGED}' + fr' create mode 100644 foo\n$', ) -def test_install_pre_commit_and_run(tempdir_factory): +def test_install_pre_commit_and_run(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - assert install(Runner(path, C.CONFIG_FILE)) == 0 + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 ret, output = _get_commit_output(tempdir_factory) assert ret == 0 assert NORMAL_PRE_COMMIT_RUN.match(output) -def test_install_in_submodule_and_run(tempdir_factory): +def test_install_pre_commit_and_run_custom_path(tempdir_factory, store): + path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') + with cwd(path): + cmd_output('git', 'mv', C.CONFIG_FILE, 'custom.yaml') + git_commit(cwd=path) + assert install('custom.yaml', store, hook_types=['pre-commit']) == 0 + + ret, output = _get_commit_output(tempdir_factory) + assert ret == 0 + assert NORMAL_PRE_COMMIT_RUN.match(output) + + +def test_install_in_submodule_and_run(tempdir_factory, store): src_path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') parent_path = git_dir(tempdir_factory) - with cwd(parent_path): - cmd_output('git', 'submodule', 'add', src_path, 'sub') - cmd_output('git', 'commit', '-m', 'foo') + cmd_output('git', 'submodule', 'add', src_path, 'sub', cwd=parent_path) + git_commit(cwd=parent_path) sub_pth = os.path.join(parent_path, 'sub') with cwd(sub_pth): - assert install(Runner(sub_pth, C.CONFIG_FILE)) == 0 + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 ret, output = _get_commit_output(tempdir_factory) assert ret == 0 assert NORMAL_PRE_COMMIT_RUN.match(output) -def test_commit_am(tempdir_factory): +def test_install_in_worktree_and_run(tempdir_factory, store): + src_path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') + path = tempdir_factory.get() + cmd_output('git', '-C', src_path, 'branch', '-m', 'notmaster') + cmd_output('git', '-C', src_path, 'worktree', 'add', path, '-b', 'master') + + with cwd(path): + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 + ret, output = _get_commit_output(tempdir_factory) + assert ret == 0 + assert NORMAL_PRE_COMMIT_RUN.match(output) + + +def test_commit_am(tempdir_factory, store): """Regression test for #322.""" path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): # Make an unstaged change open('unstaged', 'w').close() cmd_output('git', 'add', '.') - cmd_output('git', 'commit', '-m', 'foo') - with io.open('unstaged', 'w') as foo_file: + git_commit(cwd=path) + with open('unstaged', 'w') as foo_file: foo_file.write('Oh hai') - assert install(Runner(path, C.CONFIG_FILE)) == 0 + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 ret, output = _get_commit_output(tempdir_factory) assert ret == 0 -def test_unicode_merge_commit_message(tempdir_factory): +def test_unicode_merge_commit_message(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - assert install(Runner(path, C.CONFIG_FILE)) == 0 + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 cmd_output('git', 'checkout', 'master', '-b', 'foo') - cmd_output('git', 'commit', '--allow-empty', '-m', 'branch2') + git_commit('-n', cwd=path) cmd_output('git', 'checkout', 'master') cmd_output('git', 'merge', 'foo', '--no-ff', '--no-commit', '-m', 'β˜ƒ') # Used to crash - cmd_output('git', 'commit', '--no-edit') + git_commit( + '--no-edit', + msg=None, + fn=cmd_output_mocked_pre_commit_home, + tempdir_factory=tempdir_factory, + ) -def test_install_idempotent(tempdir_factory): +def test_install_idempotent(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - assert install(Runner(path, C.CONFIG_FILE)) == 0 - assert install(Runner(path, C.CONFIG_FILE)) == 0 + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 ret, output = _get_commit_output(tempdir_factory) assert ret == 0 @@ -216,23 +248,37 @@ def test_install_idempotent(tempdir_factory): def _path_without_us(): # Choose a path which *probably* doesn't include us - return os.pathsep.join([ - x for x in os.environ['PATH'].split(os.pathsep) - if x.lower() != os.path.dirname(sys.executable).lower() - ]) - - -def test_environment_not_sourced(tempdir_factory): + env = dict(os.environ) + exe = find_executable('pre-commit', _environ=env) + while exe: + parts = env['PATH'].split(os.pathsep) + after = [x for x in parts if x.lower() != os.path.dirname(exe).lower()] + if parts == after: + raise AssertionError(exe, parts) + env['PATH'] = os.pathsep.join(after) + exe = find_executable('pre-commit', _environ=env) + return env['PATH'] + + +def test_environment_not_sourced(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - # Patch the executable to simulate rming virtualenv - with mock.patch.object(sys, 'executable', '/bin/false'): - assert install(Runner(path, C.CONFIG_FILE)) == 0 + assert not install(C.CONFIG_FILE, store, hook_types=['pre-commit']) + # simulate deleting the virtualenv by rewriting the exe + hook = os.path.join(path, '.git/hooks/pre-commit') + with open(hook) as f: + src = f.read() + src = re.sub( + '\nINSTALL_PYTHON =.*\n', + '\nINSTALL_PYTHON = "/dne"\n', + src, + ) + with open(hook, 'w') as f: + f.write(src) # Use a specific homedir to ignore --user installs homedir = tempdir_factory.get() - ret, stdout, stderr = cmd_output( - 'git', 'commit', '--allow-empty', '-m', 'foo', + ret, out = git_commit( env={ 'HOME': homedir, 'PATH': _path_without_us(), @@ -245,28 +291,28 @@ def test_environment_not_sourced(tempdir_factory): retcode=None, ) assert ret == 1 - assert stdout == '' - assert stderr == ( + assert out == ( '`pre-commit` not found. ' 'Did you forget to activate your virtualenv?\n' ) FAILING_PRE_COMMIT_RUN = re.compile( - r'^\[INFO\] Initializing environment for .+\.\r?\n' - r'Failing hook\.+Failed\r?\n' - r'hookid: failing_hook\r?\n' - r'\r?\n' - r'Fail\r?\n' - r'foo\r?\n' - r'\r?\n$' + r'^\[INFO\] Initializing environment for .+\.\n' + r'Failing hook\.+Failed\n' + r'- hook id: failing_hook\n' + r'- exit code: 1\n' + r'\n' + r'Fail\n' + r'foo\n' + r'\n$', ) -def test_failing_hooks_returns_nonzero(tempdir_factory): +def test_failing_hooks_returns_nonzero(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'failing_hook_repo') with cwd(path): - assert install(Runner(path, C.CONFIG_FILE)) == 0 + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 ret, output = _get_commit_output(tempdir_factory) assert ret == 1 @@ -274,23 +320,24 @@ def test_failing_hooks_returns_nonzero(tempdir_factory): EXISTING_COMMIT_RUN = re.compile( - r'^legacy hook\r?\n' - r'\[master [a-f0-9]{7}\] Commit!\r?\n' + - FILES_CHANGED + - r' create mode 100644 baz\r?\n$' + fr'^legacy hook\n' + fr'\[master [a-f0-9]{{7}}\] commit!\n' + fr'{FILES_CHANGED}' + fr' create mode 100644 baz\n$', ) -def test_install_existing_hooks_no_overwrite(tempdir_factory): +def _write_legacy_hook(path): + os.makedirs(os.path.join(path, '.git/hooks'), exist_ok=True) + with open(os.path.join(path, '.git/hooks/pre-commit'), 'w') as f: + f.write(f'{shebang()}\nprint("legacy hook")\n') + make_executable(f.name) + + +def test_install_existing_hooks_no_overwrite(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - runner = Runner(path, C.CONFIG_FILE) - - # Write out an "old" hook - mkdirp(os.path.dirname(runner.pre_commit_path)) - with io.open(runner.pre_commit_path, 'w') as hook_file: - hook_file.write('#!/usr/bin/env bash\necho "legacy hook"\n') - make_executable(runner.pre_commit_path) + _write_legacy_hook(path) # Make sure we installed the "old" hook correctly ret, output = _get_commit_output(tempdir_factory, touch_file='baz') @@ -298,7 +345,7 @@ def test_install_existing_hooks_no_overwrite(tempdir_factory): assert EXISTING_COMMIT_RUN.match(output) # Now install pre-commit (no-overwrite) - assert install(runner) == 0 + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 # We should run both the legacy and pre-commit hooks ret, output = _get_commit_output(tempdir_factory) @@ -307,20 +354,24 @@ def test_install_existing_hooks_no_overwrite(tempdir_factory): assert NORMAL_PRE_COMMIT_RUN.match(output[len('legacy hook\n'):]) -def test_install_existing_hook_no_overwrite_idempotent(tempdir_factory): +def test_legacy_overwriting_legacy_hook(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - runner = Runner(path, C.CONFIG_FILE) + _write_legacy_hook(path) + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 + _write_legacy_hook(path) + # this previously crashed on windows. See #1010 + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 - # Write out an "old" hook - mkdirp(os.path.dirname(runner.pre_commit_path)) - with io.open(runner.pre_commit_path, 'w') as hook_file: - hook_file.write('#!/usr/bin/env bash\necho "legacy hook"\n') - make_executable(runner.pre_commit_path) + +def test_install_existing_hook_no_overwrite_idempotent(tempdir_factory, store): + path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') + with cwd(path): + _write_legacy_hook(path) # Install twice - assert install(runner) == 0 - assert install(runner) == 0 + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 # We should run both the legacy and pre-commit hooks ret, output = _get_commit_output(tempdir_factory) @@ -330,24 +381,22 @@ def test_install_existing_hook_no_overwrite_idempotent(tempdir_factory): FAIL_OLD_HOOK = re.compile( - r'fail!\r?\n' - r'\[INFO\] Initializing environment for .+\.\r?\n' - r'Bash hook\.+Passed\r?\n' + r'fail!\n' + r'\[INFO\] Initializing environment for .+\.\n' + r'Bash hook\.+Passed\n', ) -def test_failing_existing_hook_returns_1(tempdir_factory): +def test_failing_existing_hook_returns_1(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - runner = Runner(path, C.CONFIG_FILE) - # Write out a failing "old" hook - mkdirp(os.path.dirname(runner.pre_commit_path)) - with io.open(runner.pre_commit_path, 'w') as hook_file: - hook_file.write('#!/usr/bin/env bash\necho "fail!"\nexit 1\n') - make_executable(runner.pre_commit_path) + os.makedirs(os.path.join(path, '.git/hooks'), exist_ok=True) + with open(os.path.join(path, '.git/hooks/pre-commit'), 'w') as f: + f.write('#!/usr/bin/env bash\necho "fail!"\nexit 1\n') + make_executable(f.name) - assert install(runner) == 0 + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 # We should get a failure from the legacy hook ret, output = _get_commit_output(tempdir_factory) @@ -355,48 +404,39 @@ def test_failing_existing_hook_returns_1(tempdir_factory): assert FAIL_OLD_HOOK.match(output) -def test_install_overwrite_no_existing_hooks(tempdir_factory): +def test_install_overwrite_no_existing_hooks(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - assert install(Runner(path, C.CONFIG_FILE), overwrite=True) == 0 + assert not install( + C.CONFIG_FILE, store, hook_types=['pre-commit'], overwrite=True, + ) ret, output = _get_commit_output(tempdir_factory) assert ret == 0 assert NORMAL_PRE_COMMIT_RUN.match(output) -def test_install_overwrite(tempdir_factory): +def test_install_overwrite(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - runner = Runner(path, C.CONFIG_FILE) - - # Write out the "old" hook - mkdirp(os.path.dirname(runner.pre_commit_path)) - with io.open(runner.pre_commit_path, 'w') as hook_file: - hook_file.write('#!/usr/bin/env bash\necho "legacy hook"\n') - make_executable(runner.pre_commit_path) - - assert install(runner, overwrite=True) == 0 + _write_legacy_hook(path) + assert not install( + C.CONFIG_FILE, store, hook_types=['pre-commit'], overwrite=True, + ) ret, output = _get_commit_output(tempdir_factory) assert ret == 0 assert NORMAL_PRE_COMMIT_RUN.match(output) -def test_uninstall_restores_legacy_hooks(tempdir_factory): +def test_uninstall_restores_legacy_hooks(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - runner = Runner(path, C.CONFIG_FILE) - - # Write out an "old" hook - mkdirp(os.path.dirname(runner.pre_commit_path)) - with io.open(runner.pre_commit_path, 'w') as hook_file: - hook_file.write('#!/usr/bin/env bash\necho "legacy hook"\n') - make_executable(runner.pre_commit_path) + _write_legacy_hook(path) # Now install and uninstall pre-commit - assert install(runner) == 0 - assert uninstall(runner) == 0 + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 + assert uninstall(hook_types=['pre-commit']) == 0 # Make sure we installed the "old" hook correctly ret, output = _get_commit_output(tempdir_factory, touch_file='baz') @@ -404,87 +444,75 @@ def test_uninstall_restores_legacy_hooks(tempdir_factory): assert EXISTING_COMMIT_RUN.match(output) -def test_replace_old_commit_script(tempdir_factory): +def test_replace_old_commit_script(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - runner = Runner(path, C.CONFIG_FILE) - # Install a script that looks like our old script - pre_commit_contents = io.open( - resource_filename('hook-tmpl'), - ).read() + pre_commit_contents = resource_text('hook-tmpl') new_contents = pre_commit_contents.replace( CURRENT_HASH, PRIOR_HASHES[-1], ) - mkdirp(os.path.dirname(runner.pre_commit_path)) - with io.open(runner.pre_commit_path, 'w') as pre_commit_file: - pre_commit_file.write(new_contents) - make_executable(runner.pre_commit_path) + os.makedirs(os.path.join(path, '.git/hooks'), exist_ok=True) + with open(os.path.join(path, '.git/hooks/pre-commit'), 'w') as f: + f.write(new_contents) + make_executable(f.name) # Install normally - assert install(runner) == 0 + assert install(C.CONFIG_FILE, store, hook_types=['pre-commit']) == 0 ret, output = _get_commit_output(tempdir_factory) assert ret == 0 assert NORMAL_PRE_COMMIT_RUN.match(output) -def test_uninstall_doesnt_remove_not_our_hooks(tempdir_factory): - path = git_dir(tempdir_factory) - with cwd(path): - runner = Runner(path, C.CONFIG_FILE) - mkdirp(os.path.dirname(runner.pre_commit_path)) - with io.open(runner.pre_commit_path, 'w') as pre_commit_file: - pre_commit_file.write('#!/usr/bin/env bash\necho 1\n') - make_executable(runner.pre_commit_path) +def test_uninstall_doesnt_remove_not_our_hooks(in_git_dir): + pre_commit = in_git_dir.join('.git/hooks').ensure_dir().join('pre-commit') + pre_commit.write('#!/usr/bin/env bash\necho 1\n') + make_executable(pre_commit.strpath) - assert uninstall(runner) == 0 + assert uninstall(hook_types=['pre-commit']) == 0 - assert os.path.exists(runner.pre_commit_path) + assert pre_commit.exists() PRE_INSTALLED = re.compile( - r'Bash hook\.+Passed\r?\n' - r'\[master [a-f0-9]{7}\] Commit!\r?\n' + - FILES_CHANGED + - r' create mode 100644 foo\r?\n$' + fr'Bash hook\.+Passed\n' + fr'\[master [a-f0-9]{{7}}\] commit!\n' + fr'{FILES_CHANGED}' + fr' create mode 100644 foo\n$', ) -def test_installs_hooks_with_hooks_True( - tempdir_factory, - mock_out_store_directory, -): +def test_installs_hooks_with_hooks_True(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - install(Runner(path, C.CONFIG_FILE), hooks=True) + install(C.CONFIG_FILE, store, hook_types=['pre-commit'], hooks=True) ret, output = _get_commit_output( - tempdir_factory, pre_commit_home=mock_out_store_directory, + tempdir_factory, pre_commit_home=store.directory, ) assert ret == 0 assert PRE_INSTALLED.match(output) -def test_install_hooks_command(tempdir_factory, mock_out_store_directory): +def test_install_hooks_command(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - runner = Runner(path, C.CONFIG_FILE) - install(runner) - install_hooks(runner) + install(C.CONFIG_FILE, store, hook_types=['pre-commit']) + install_hooks(C.CONFIG_FILE, store) ret, output = _get_commit_output( - tempdir_factory, pre_commit_home=mock_out_store_directory, + tempdir_factory, pre_commit_home=store.directory, ) assert ret == 0 assert PRE_INSTALLED.match(output) -def test_installed_from_venv(tempdir_factory): +def test_installed_from_venv(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - install(Runner(path, C.CONFIG_FILE)) + install(C.CONFIG_FILE, store, hook_types=['pre-commit']) # No environment so pre-commit is not on the path when running! # Should still pick up the python from when we installed ret, output = _get_commit_output( @@ -508,77 +536,341 @@ def test_installed_from_venv(tempdir_factory): assert NORMAL_PRE_COMMIT_RUN.match(output) -def _get_push_output(tempdir_factory): +def _get_push_output(tempdir_factory, remote='origin', opts=()): return cmd_output_mocked_pre_commit_home( - 'git', 'push', 'origin', 'HEAD:new_branch', - # git push puts pre-commit to stderr - stderr=subprocess.STDOUT, + 'git', 'push', remote, 'HEAD:new_branch', *opts, tempdir_factory=tempdir_factory, retcode=None, )[:2] -def test_pre_push_integration_failing(tempdir_factory): +def test_pre_push_integration_failing(tempdir_factory, store): upstream = make_consuming_repo(tempdir_factory, 'failing_hook_repo') path = tempdir_factory.get() cmd_output('git', 'clone', upstream, path) with cwd(path): - install(Runner(path, C.CONFIG_FILE), hook_type='pre-push') + install(C.CONFIG_FILE, store, hook_types=['pre-push']) # commit succeeds because pre-commit is only installed for pre-push assert _get_commit_output(tempdir_factory)[0] == 0 + assert _get_commit_output(tempdir_factory, touch_file='zzz')[0] == 0 retc, output = _get_push_output(tempdir_factory) assert retc == 1 assert 'Failing hook' in output assert 'Failed' in output - assert 'hookid: failing_hook' in output + assert 'foo zzz' in output # both filenames should be printed + assert 'hook id: failing_hook' in output + + +def test_pre_push_integration_accepted(tempdir_factory, store): + upstream = make_consuming_repo(tempdir_factory, 'script_hooks_repo') + path = tempdir_factory.get() + cmd_output('git', 'clone', upstream, path) + with cwd(path): + install(C.CONFIG_FILE, store, hook_types=['pre-push']) + assert _get_commit_output(tempdir_factory)[0] == 0 + + retc, output = _get_push_output(tempdir_factory) + assert retc == 0 + assert 'Bash hook' in output + assert 'Passed' in output -def test_pre_push_integration_accepted(tempdir_factory): +def test_pre_push_force_push_without_fetch(tempdir_factory, store): upstream = make_consuming_repo(tempdir_factory, 'script_hooks_repo') + path1 = tempdir_factory.get() + path2 = tempdir_factory.get() + cmd_output('git', 'clone', upstream, path1) + cmd_output('git', 'clone', upstream, path2) + with cwd(path1): + assert _get_commit_output(tempdir_factory)[0] == 0 + assert _get_push_output(tempdir_factory)[0] == 0 + + with cwd(path2): + install(C.CONFIG_FILE, store, hook_types=['pre-push']) + assert _get_commit_output(tempdir_factory, msg='force!')[0] == 0 + + retc, output = _get_push_output(tempdir_factory, opts=('--force',)) + assert retc == 0 + assert 'Bash hook' in output + assert 'Passed' in output + + +def test_pre_push_new_upstream(tempdir_factory, store): + upstream = make_consuming_repo(tempdir_factory, 'script_hooks_repo') + upstream2 = git_dir(tempdir_factory) path = tempdir_factory.get() cmd_output('git', 'clone', upstream, path) with cwd(path): - install(Runner(path, C.CONFIG_FILE), hook_type='pre-push') + install(C.CONFIG_FILE, store, hook_types=['pre-push']) assert _get_commit_output(tempdir_factory)[0] == 0 + cmd_output('git', 'remote', 'rename', 'origin', 'upstream') + cmd_output('git', 'remote', 'add', 'origin', upstream2) retc, output = _get_push_output(tempdir_factory) assert retc == 0 assert 'Bash hook' in output assert 'Passed' in output -def test_pre_push_integration_empty_push(tempdir_factory): +def test_pre_push_environment_variables(tempdir_factory, store): + config = { + 'repo': 'local', + 'hooks': [ + { + 'id': 'print-remote-info', + 'name': 'print remote info', + 'entry': 'bash -c "echo remote: $PRE_COMMIT_REMOTE_NAME"', + 'language': 'system', + 'verbose': True, + }, + ], + } + + upstream = git_dir(tempdir_factory) + clone = tempdir_factory.get() + cmd_output('git', 'clone', upstream, clone) + add_config_to_repo(clone, config) + with cwd(clone): + install(C.CONFIG_FILE, store, hook_types=['pre-push']) + + cmd_output('git', 'remote', 'rename', 'origin', 'origin2') + retc, output = _get_push_output(tempdir_factory, remote='origin2') + assert retc == 0 + assert '\nremote: origin2\n' in output + + +def test_pre_push_integration_empty_push(tempdir_factory, store): upstream = make_consuming_repo(tempdir_factory, 'script_hooks_repo') path = tempdir_factory.get() cmd_output('git', 'clone', upstream, path) with cwd(path): - install(Runner(path, C.CONFIG_FILE), hook_type='pre-push') + install(C.CONFIG_FILE, store, hook_types=['pre-push']) _get_push_output(tempdir_factory) retc, output = _get_push_output(tempdir_factory) assert output == 'Everything up-to-date\n' assert retc == 0 -def test_install_disallow_mising_config(tempdir_factory): +def test_pre_push_legacy(tempdir_factory, store): + upstream = make_consuming_repo(tempdir_factory, 'script_hooks_repo') + path = tempdir_factory.get() + cmd_output('git', 'clone', upstream, path) + with cwd(path): + os.makedirs(os.path.join(path, '.git/hooks'), exist_ok=True) + with open(os.path.join(path, '.git/hooks/pre-push'), 'w') as f: + f.write( + '#!/usr/bin/env bash\n' + 'set -eu\n' + 'read lr ls rr rs\n' + 'test -n "$lr" -a -n "$ls" -a -n "$rr" -a -n "$rs"\n' + 'echo legacy\n', + ) + make_executable(f.name) + + install(C.CONFIG_FILE, store, hook_types=['pre-push']) + assert _get_commit_output(tempdir_factory)[0] == 0 + + retc, output = _get_push_output(tempdir_factory) + assert retc == 0 + first_line, _, third_line = output.splitlines()[:3] + assert first_line == 'legacy' + assert third_line.startswith('Bash hook') + assert third_line.endswith('Passed') + + +def test_commit_msg_integration_failing( + commit_msg_repo, tempdir_factory, store, +): + install(C.CONFIG_FILE, store, hook_types=['commit-msg']) + retc, out = _get_commit_output(tempdir_factory) + assert retc == 1 + assert out == '''\ +Must have "Signed off by:"...............................................Failed +- hook id: must-have-signoff +- exit code: 1 +''' + + +def test_commit_msg_integration_passing( + commit_msg_repo, tempdir_factory, store, +): + install(C.CONFIG_FILE, store, hook_types=['commit-msg']) + msg = 'Hi\nSigned off by: me, lol' + retc, out = _get_commit_output(tempdir_factory, msg=msg) + assert retc == 0 + first_line = out.splitlines()[0] + assert first_line.startswith('Must have "Signed off by:"...') + assert first_line.endswith('...Passed') + + +def test_commit_msg_legacy(commit_msg_repo, tempdir_factory, store): + hook_path = os.path.join(commit_msg_repo, '.git/hooks/commit-msg') + os.makedirs(os.path.dirname(hook_path), exist_ok=True) + with open(hook_path, 'w') as hook_file: + hook_file.write( + '#!/usr/bin/env bash\n' + 'set -eu\n' + 'test -e "$1"\n' + 'echo legacy\n', + ) + make_executable(hook_path) + + install(C.CONFIG_FILE, store, hook_types=['commit-msg']) + + msg = 'Hi\nSigned off by: asottile' + retc, out = _get_commit_output(tempdir_factory, msg=msg) + assert retc == 0 + first_line, second_line = out.splitlines()[:2] + assert first_line == 'legacy' + assert second_line.startswith('Must have "Signed off by:"...') + + +def test_post_checkout_integration(tempdir_factory, store): + path = git_dir(tempdir_factory) + config = [ + { + 'repo': 'local', + 'hooks': [{ + 'id': 'post-checkout', + 'name': 'Post checkout', + 'entry': 'bash -c "echo ${PRE_COMMIT_TO_REF}"', + 'language': 'system', + 'always_run': True, + 'verbose': True, + 'stages': ['post-checkout'], + }], + }, + {'repo': 'meta', 'hooks': [{'id': 'identity'}]}, + ] + write_config(path, config) + with cwd(path): + cmd_output('git', 'add', '.') + git_commit() + + # add a file only on `feature`, it should not be passed to hooks + cmd_output('git', 'checkout', '-b', 'feature') + open('some_file', 'a').close() + cmd_output('git', 'add', '.') + git_commit() + cmd_output('git', 'checkout', 'master') + + install(C.CONFIG_FILE, store, hook_types=['post-checkout']) + retc, _, stderr = cmd_output('git', 'checkout', 'feature') + assert stderr is not None + assert retc == 0 + assert git.head_rev(path) in stderr + assert 'some_file' not in stderr + + +def test_prepare_commit_msg_integration_failing( + failing_prepare_commit_msg_repo, tempdir_factory, store, +): + install(C.CONFIG_FILE, store, hook_types=['prepare-commit-msg']) + retc, out = _get_commit_output(tempdir_factory) + assert retc == 1 + assert out == '''\ +Add "Signed off by:".....................................................Failed +- hook id: add-signoff +- exit code: 1 +''' + + +def test_prepare_commit_msg_integration_passing( + prepare_commit_msg_repo, tempdir_factory, store, +): + install(C.CONFIG_FILE, store, hook_types=['prepare-commit-msg']) + retc, out = _get_commit_output(tempdir_factory, msg='Hi') + assert retc == 0 + first_line = out.splitlines()[0] + assert first_line.startswith('Add "Signed off by:"...') + assert first_line.endswith('...Passed') + commit_msg_path = os.path.join( + prepare_commit_msg_repo, '.git/COMMIT_EDITMSG', + ) + with open(commit_msg_path) as f: + assert 'Signed off by: ' in f.read() + + +def test_prepare_commit_msg_legacy( + prepare_commit_msg_repo, tempdir_factory, store, +): + hook_path = os.path.join( + prepare_commit_msg_repo, '.git/hooks/prepare-commit-msg', + ) + os.makedirs(os.path.dirname(hook_path), exist_ok=True) + with open(hook_path, 'w') as hook_file: + hook_file.write( + '#!/usr/bin/env bash\n' + 'set -eu\n' + 'test -e "$1"\n' + 'echo legacy\n', + ) + make_executable(hook_path) + + install(C.CONFIG_FILE, store, hook_types=['prepare-commit-msg']) + + retc, out = _get_commit_output(tempdir_factory, msg='Hi') + assert retc == 0 + first_line, second_line = out.splitlines()[:2] + assert first_line == 'legacy' + assert second_line.startswith('Add "Signed off by:"...') + commit_msg_path = os.path.join( + prepare_commit_msg_repo, '.git/COMMIT_EDITMSG', + ) + with open(commit_msg_path) as f: + assert 'Signed off by: ' in f.read() + + +def test_pre_merge_commit_integration(tempdir_factory, store): + expected = re.compile( + r'^\[INFO\] Initializing environment for .+\n' + r'Bash hook\.+Passed\n' + r"Merge made by the 'recursive' strategy.\n" + r' foo \| 0\n' + r' 1 file changed, 0 insertions\(\+\), 0 deletions\(-\)\n' + r' create mode 100644 foo\n$', + ) + path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - runner = Runner(path, C.CONFIG_FILE) + ret = install(C.CONFIG_FILE, store, hook_types=['pre-merge-commit']) + assert ret == 0 + cmd_output('git', 'checkout', 'master', '-b', 'feature') + _get_commit_output(tempdir_factory) + cmd_output('git', 'checkout', 'master') + ret, output, _ = cmd_output_mocked_pre_commit_home( + 'git', 'merge', '--no-ff', '--no-edit', 'feature', + tempdir_factory=tempdir_factory, + ) + assert ret == 0 + assert expected.match(output) + + +def test_install_disallow_missing_config(tempdir_factory, store): + path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') + with cwd(path): remove_config_from_repo(path) - assert install(runner, overwrite=True, skip_on_missing_conf=False) == 0 + ret = install( + C.CONFIG_FILE, store, hook_types=['pre-commit'], + overwrite=True, skip_on_missing_config=False, + ) + assert ret == 0 ret, output = _get_commit_output(tempdir_factory) assert ret == 1 -def test_install_allow_mising_config(tempdir_factory): +def test_install_allow_missing_config(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - runner = Runner(path, C.CONFIG_FILE) - remove_config_from_repo(path) - assert install(runner, overwrite=True, skip_on_missing_conf=True) == 0 + ret = install( + C.CONFIG_FILE, store, hook_types=['pre-commit'], + overwrite=True, skip_on_missing_config=True, + ) + assert ret == 0 ret, output = _get_commit_output(tempdir_factory) assert ret == 0 @@ -589,13 +881,15 @@ def test_install_allow_mising_config(tempdir_factory): assert expected in output -def test_install_temporarily_allow_mising_config(tempdir_factory): +def test_install_temporarily_allow_mising_config(tempdir_factory, store): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(path): - runner = Runner(path, C.CONFIG_FILE) - remove_config_from_repo(path) - assert install(runner, overwrite=True, skip_on_missing_conf=False) == 0 + ret = install( + C.CONFIG_FILE, store, hook_types=['pre-commit'], + overwrite=True, skip_on_missing_config=False, + ) + assert ret == 0 env = dict(os.environ, PRE_COMMIT_ALLOW_NO_CONFIG='1') ret, output = _get_commit_output(tempdir_factory, env=env) diff --git a/tests/commands/migrate_config_test.py b/tests/commands/migrate_config_test.py new file mode 100644 index 000000000..efc0d1cb4 --- /dev/null +++ b/tests/commands/migrate_config_test.py @@ -0,0 +1,156 @@ +import pytest + +import pre_commit.constants as C +from pre_commit.commands.migrate_config import _indent +from pre_commit.commands.migrate_config import migrate_config + + +@pytest.mark.parametrize( + ('s', 'expected'), + ( + ('', ''), + ('a', ' a'), + ('foo\nbar', ' foo\n bar'), + ('foo\n\nbar\n', ' foo\n\n bar\n'), + ('\n\n\n', '\n\n\n'), + ), +) +def test_indent(s, expected): + assert _indent(s) == expected + + +def test_migrate_config_normal_format(tmpdir, capsys): + cfg = tmpdir.join(C.CONFIG_FILE) + cfg.write( + '- repo: local\n' + ' hooks:\n' + ' - id: foo\n' + ' name: foo\n' + ' entry: ./bin/foo.sh\n' + ' language: script\n', + ) + with tmpdir.as_cwd(): + assert not migrate_config(C.CONFIG_FILE) + out, _ = capsys.readouterr() + assert out == 'Configuration has been migrated.\n' + contents = cfg.read() + assert contents == ( + 'repos:\n' + '- repo: local\n' + ' hooks:\n' + ' - id: foo\n' + ' name: foo\n' + ' entry: ./bin/foo.sh\n' + ' language: script\n' + ) + + +def test_migrate_config_document_marker(tmpdir): + cfg = tmpdir.join(C.CONFIG_FILE) + cfg.write( + '# comment\n' + '\n' + '---\n' + '- repo: local\n' + ' hooks:\n' + ' - id: foo\n' + ' name: foo\n' + ' entry: ./bin/foo.sh\n' + ' language: script\n', + ) + with tmpdir.as_cwd(): + assert not migrate_config(C.CONFIG_FILE) + contents = cfg.read() + assert contents == ( + '# comment\n' + '\n' + '---\n' + 'repos:\n' + '- repo: local\n' + ' hooks:\n' + ' - id: foo\n' + ' name: foo\n' + ' entry: ./bin/foo.sh\n' + ' language: script\n' + ) + + +def test_migrate_config_list_literal(tmpdir): + cfg = tmpdir.join(C.CONFIG_FILE) + cfg.write( + '[{\n' + ' repo: local,\n' + ' hooks: [{\n' + ' id: foo, name: foo, entry: ./bin/foo.sh,\n' + ' language: script,\n' + ' }]\n' + '}]', + ) + with tmpdir.as_cwd(): + assert not migrate_config(C.CONFIG_FILE) + contents = cfg.read() + assert contents == ( + 'repos:\n' + ' [{\n' + ' repo: local,\n' + ' hooks: [{\n' + ' id: foo, name: foo, entry: ./bin/foo.sh,\n' + ' language: script,\n' + ' }]\n' + ' }]' + ) + + +def test_already_migrated_configuration_noop(tmpdir, capsys): + contents = ( + 'repos:\n' + '- repo: local\n' + ' hooks:\n' + ' - id: foo\n' + ' name: foo\n' + ' entry: ./bin/foo.sh\n' + ' language: script\n' + ) + cfg = tmpdir.join(C.CONFIG_FILE) + cfg.write(contents) + with tmpdir.as_cwd(): + assert not migrate_config(C.CONFIG_FILE) + out, _ = capsys.readouterr() + assert out == 'Configuration is already migrated.\n' + assert cfg.read() == contents + + +def test_migrate_config_sha_to_rev(tmpdir): + contents = ( + 'repos:\n' + '- repo: https://github.com/pre-commit/pre-commit-hooks\n' + ' sha: v1.2.0\n' + ' hooks: []\n' + '- repo: https://github.com/pre-commit/pre-commit-hooks\n' + ' sha: v1.2.0\n' + ' hooks: []\n' + ) + cfg = tmpdir.join(C.CONFIG_FILE) + cfg.write(contents) + with tmpdir.as_cwd(): + assert not migrate_config(C.CONFIG_FILE) + contents = cfg.read() + assert contents == ( + 'repos:\n' + '- repo: https://github.com/pre-commit/pre-commit-hooks\n' + ' rev: v1.2.0\n' + ' hooks: []\n' + '- repo: https://github.com/pre-commit/pre-commit-hooks\n' + ' rev: v1.2.0\n' + ' hooks: []\n' + ) + + +@pytest.mark.parametrize('contents', ('', '\n')) +def test_empty_configuration_file_user_error(tmpdir, contents): + cfg = tmpdir.join(C.CONFIG_FILE) + cfg.write(contents) + with tmpdir.as_cwd(): + assert not migrate_config(C.CONFIG_FILE) + # even though the config is invalid, this should be a noop + assert cfg.read() == contents diff --git a/tests/commands/run_test.py b/tests/commands/run_test.py index 1643cbb81..f8e882367 100644 --- a/tests/commands/run_test.py +++ b/tests/commands/run_test.py @@ -1,114 +1,157 @@ -# -*- coding: UTF-8 -*- -from __future__ import unicode_literals - -import io import os.path -import subprocess +import shlex import sys -from collections import OrderedDict +import time +from unittest import mock -import mock import pytest import pre_commit.constants as C +from pre_commit import color from pre_commit.commands.install_uninstall import install from pre_commit.commands.run import _compute_cols +from pre_commit.commands.run import _full_msg from pre_commit.commands.run import _get_skips from pre_commit.commands.run import _has_unmerged_paths -from pre_commit.commands.run import get_changed_files +from pre_commit.commands.run import _start_msg +from pre_commit.commands.run import Classifier +from pre_commit.commands.run import filter_by_include_exclude from pre_commit.commands.run import run -from pre_commit.runner import Runner from pre_commit.util import cmd_output -from pre_commit.util import cwd +from pre_commit.util import EnvironT from pre_commit.util import make_executable from testing.auto_namedtuple import auto_namedtuple from testing.fixtures import add_config_to_repo +from testing.fixtures import git_dir from testing.fixtures import make_consuming_repo from testing.fixtures import modify_config from testing.fixtures import read_config +from testing.fixtures import sample_meta_config +from testing.fixtures import write_config from testing.util import cmd_output_mocked_pre_commit_home +from testing.util import cwd +from testing.util import git_commit +from testing.util import run_opts + + +def test_start_msg(): + ret = _start_msg(start='start', end_len=5, cols=15) + # 4 dots: 15 - 5 - 5 - 1 + assert ret == 'start....' + + +def test_full_msg(): + ret = _full_msg( + start='start', + end_msg='end', + end_color='', + use_color=False, + cols=15, + ) + # 6 dots: 15 - 5 - 3 - 1 + assert ret == 'start......end\n' + + +def test_full_msg_with_color(): + ret = _full_msg( + start='start', + end_msg='end', + end_color=color.RED, + use_color=True, + cols=15, + ) + # 6 dots: 15 - 5 - 3 - 1 + assert ret == f'start......{color.RED}end{color.NORMAL}\n' + + +def test_full_msg_with_postfix(): + ret = _full_msg( + start='start', + postfix='post ', + end_msg='end', + end_color='', + use_color=False, + cols=20, + ) + # 6 dots: 20 - 5 - 5 - 3 - 1 + assert ret == 'start......post end\n' + + +def test_full_msg_postfix_not_colored(): + ret = _full_msg( + start='start', + postfix='post ', + end_msg='end', + end_color=color.RED, + use_color=True, + cols=20, + ) + # 6 dots: 20 - 5 - 5 - 3 - 1 + assert ret == f'start......post {color.RED}end{color.NORMAL}\n' -@pytest.yield_fixture +@pytest.fixture def repo_with_passing_hook(tempdir_factory): git_path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(git_path): yield git_path -@pytest.yield_fixture +@pytest.fixture def repo_with_failing_hook(tempdir_factory): git_path = make_consuming_repo(tempdir_factory, 'failing_hook_repo') with cwd(git_path): yield git_path +@pytest.fixture +def aliased_repo(tempdir_factory): + git_path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') + with cwd(git_path): + with modify_config() as config: + config['repos'][0]['hooks'].append( + {'id': 'bash_hook', 'alias': 'foo_bash'}, + ) + stage_a_file() + yield git_path + + def stage_a_file(filename='foo.py'): open(filename, 'a').close() cmd_output('git', 'add', filename) -def _get_opts( - all_files=False, - files=(), - color=False, - verbose=False, - hook=None, - no_stash=False, - origin='', - source='', - allow_unstaged_config=False, - hook_stage='commit', - show_diff_on_failure=False, -): - # These are mutually exclusive - assert not (all_files and files) - return auto_namedtuple( - all_files=all_files, - files=files, - color=color, - verbose=verbose, - hook=hook, - no_stash=no_stash, - origin=origin, - source=source, - allow_unstaged_config=allow_unstaged_config, - hook_stage=hook_stage, - show_diff_on_failure=show_diff_on_failure, - ) - - -def _do_run(cap_out, repo, args, environ={}, config_file=C.CONFIG_FILE): - runner = Runner(repo, config_file) - with cwd(runner.git_root): # replicates Runner.create behaviour - ret = run(runner, args, environ=environ) +def _do_run(cap_out, store, repo, args, environ={}, config_file=C.CONFIG_FILE): + with cwd(repo): # replicates `main._adjust_args_and_chdir` behaviour + ret = run(config_file, store, args, environ=environ) printed = cap_out.get_bytes() return ret, printed -def _test_run(cap_out, repo, opts, expected_outputs, expected_ret, stage, - config_file=C.CONFIG_FILE): +def _test_run( + cap_out, store, repo, opts, expected_outputs, expected_ret, stage, + config_file=C.CONFIG_FILE, +): if stage: stage_a_file() - args = _get_opts(**opts) - ret, printed = _do_run(cap_out, repo, args, config_file=config_file) + args = run_opts(**opts) + ret, printed = _do_run(cap_out, store, repo, args, config_file=config_file) assert ret == expected_ret, (ret, expected_ret, printed) for expected_output_part in expected_outputs: assert expected_output_part in printed -def test_run_all_hooks_failing( - cap_out, repo_with_failing_hook, mock_out_store_directory, -): +def test_run_all_hooks_failing(cap_out, store, repo_with_failing_hook): _test_run( cap_out, + store, repo_with_failing_hook, {}, ( b'Failing hook', b'Failed', - b'hookid: failing_hook', + b'hook id: failing_hook', b'Fail\nfoo.py\n', ), expected_ret=1, @@ -116,17 +159,15 @@ def test_run_all_hooks_failing( ) -def test_arbitrary_bytes_hook( - cap_out, tempdir_factory, mock_out_store_directory, -): +def test_arbitrary_bytes_hook(cap_out, store, tempdir_factory): git_path = make_consuming_repo(tempdir_factory, 'arbitrary_bytes_repo') with cwd(git_path): - _test_run(cap_out, git_path, {}, (b'\xe2\x98\x83\xb2\n',), 1, True) + _test_run( + cap_out, store, git_path, {}, (b'\xe2\x98\x83\xb2\n',), 1, True, + ) -def test_hook_that_modifies_but_returns_zero( - cap_out, tempdir_factory, mock_out_store_directory, -): +def test_hook_that_modifies_but_returns_zero(cap_out, store, tempdir_factory): git_path = make_consuming_repo( tempdir_factory, 'modified_file_returns_zero_repo', ) @@ -134,57 +175,135 @@ def test_hook_that_modifies_but_returns_zero( stage_a_file('bar.py') _test_run( cap_out, + store, git_path, {}, ( # The first should fail b'Failed', # With a modified file (default message + the hook's output) - b'Files were modified by this hook. Additional output:\n\n' + b'- files were modified by this hook\n\n' b'Modified: foo.py', # The next hook should pass despite the first modifying b'Passed', # The next hook should fail b'Failed', # bar.py was modified, but provides no additional output - b'Files were modified by this hook.\n', + b'- files were modified by this hook\n', ), 1, True, ) -def test_types_hook_repository( - cap_out, tempdir_factory, mock_out_store_directory, -): +def test_types_hook_repository(cap_out, store, tempdir_factory): git_path = make_consuming_repo(tempdir_factory, 'types_repo') with cwd(git_path): stage_a_file('bar.py') stage_a_file('bar.notpy') - ret, printed = _do_run(cap_out, git_path, _get_opts()) + ret, printed = _do_run(cap_out, store, git_path, run_opts()) assert ret == 1 assert b'bar.py' in printed assert b'bar.notpy' not in printed -def test_exclude_types_hook_repository( - cap_out, tempdir_factory, mock_out_store_directory, -): +def test_exclude_types_hook_repository(cap_out, store, tempdir_factory): git_path = make_consuming_repo(tempdir_factory, 'exclude_types_repo') with cwd(git_path): - with io.open('exe', 'w') as exe: + with open('exe', 'w') as exe: exe.write('#!/usr/bin/env python3\n') make_executable('exe') cmd_output('git', 'add', 'exe') stage_a_file('bar.py') - ret, printed = _do_run(cap_out, git_path, _get_opts()) + ret, printed = _do_run(cap_out, store, git_path, run_opts()) assert ret == 1 assert b'bar.py' in printed assert b'exe' not in printed +def test_global_exclude(cap_out, store, in_git_dir): + config = { + 'exclude': r'^foo\.py$', + 'repos': [{'repo': 'meta', 'hooks': [{'id': 'identity'}]}], + } + write_config('.', config) + open('foo.py', 'a').close() + open('bar.py', 'a').close() + cmd_output('git', 'add', '.') + opts = run_opts(verbose=True) + ret, printed = _do_run(cap_out, store, str(in_git_dir), opts) + assert ret == 0 + # Does not contain foo.py since it was excluded + assert printed.startswith(f'identity{"." * 65}Passed\n'.encode()) + assert printed.endswith(b'\n\n.pre-commit-config.yaml\nbar.py\n\n') + + +def test_global_files(cap_out, store, in_git_dir): + config = { + 'files': r'^bar\.py$', + 'repos': [{'repo': 'meta', 'hooks': [{'id': 'identity'}]}], + } + write_config('.', config) + open('foo.py', 'a').close() + open('bar.py', 'a').close() + cmd_output('git', 'add', '.') + opts = run_opts(verbose=True) + ret, printed = _do_run(cap_out, store, str(in_git_dir), opts) + assert ret == 0 + # Does not contain foo.py since it was excluded + assert printed.startswith(f'identity{"." * 65}Passed\n'.encode()) + assert printed.endswith(b'\n\nbar.py\n\n') + + +@pytest.mark.parametrize( + ('t1', 't2', 'expected'), + ( + (1.234, 2., b'\n- duration: 0.77s\n'), + (1., 1., b'\n- duration: 0s\n'), + ), +) +def test_verbose_duration(cap_out, store, in_git_dir, t1, t2, expected): + write_config('.', {'repo': 'meta', 'hooks': [{'id': 'identity'}]}) + cmd_output('git', 'add', '.') + opts = run_opts(verbose=True) + with mock.patch.object(time, 'time', side_effect=(t1, t2)): + ret, printed = _do_run(cap_out, store, str(in_git_dir), opts) + assert ret == 0 + assert expected in printed + + +@pytest.mark.parametrize( + ('args', 'expected_out'), + [ + ( + { + 'show_diff_on_failure': True, + }, + b'All changes made by hooks:', + ), + ( + { + 'show_diff_on_failure': True, + 'color': True, + }, + b'All changes made by hooks:', + ), + ( + { + 'show_diff_on_failure': True, + 'all_files': True, + }, + b'reproduce locally with: pre-commit run --all-files', + ), + ], +) def test_show_diff_on_failure( - capfd, cap_out, tempdir_factory, mock_out_store_directory, + args, + expected_out, + capfd, + cap_out, + store, + tempdir_factory, ): git_path = make_consuming_repo( tempdir_factory, 'modified_file_returns_zero_repo', @@ -192,9 +311,9 @@ def test_show_diff_on_failure( with cwd(git_path): stage_a_file('bar.py') _test_run( - cap_out, git_path, {'show_diff_on_failure': True}, + cap_out, store, git_path, args, # we're only testing the output after running - (), 1, True, + expected_out, 1, True, ) out, _ = capfd.readouterr() assert 'diff --git' in out @@ -206,7 +325,18 @@ def test_show_diff_on_failure( ({}, (b'Bash hook', b'Passed'), 0, True), ({'verbose': True}, (b'foo.py\nHello World',), 0, True), ({'hook': 'bash_hook'}, (b'Bash hook', b'Passed'), 0, True), - ({'hook': 'nope'}, (b'No hook with id `nope`',), 1, True), + ( + {'hook': 'nope'}, + (b'No hook with id `nope` in stage `commit`',), + 1, + True, + ), + ( + {'hook': 'nope', 'hook_stage': 'push'}, + (b'No hook with id `nope` in stage `push`',), + 1, + True, + ), ( {'all_files': True, 'verbose': True}, (b'foo.py',), @@ -220,19 +350,20 @@ def test_show_diff_on_failure( True, ), ({}, (b'Bash hook', b'(no files to check)', b'Skipped'), 0, False), - ) + ), ) def test_run( cap_out, + store, repo_with_passing_hook, options, outputs, expected_ret, stage, - mock_out_store_directory, ): _test_run( cap_out, + store, repo_with_passing_hook, options, outputs, @@ -241,12 +372,7 @@ def test_run( ) -def test_run_output_logfile( - cap_out, - tempdir_factory, - mock_out_store_directory, -): - +def test_run_output_logfile(cap_out, store, tempdir_factory): expected_output = ( b'This is STDOUT output\n', b'This is STDERR output\n', @@ -256,10 +382,11 @@ def test_run_output_logfile( with cwd(git_path): _test_run( cap_out, + store, git_path, {}, expected_output, expected_ret=1, - stage=True + stage=True, ) logfile_path = os.path.join(git_path, 'test.log') assert os.path.exists(logfile_path) @@ -270,13 +397,12 @@ def test_run_output_logfile( assert expected_output_part in logfile_content -def test_always_run( - cap_out, repo_with_passing_hook, mock_out_store_directory, -): +def test_always_run(cap_out, store, repo_with_passing_hook): with modify_config() as config: - config[0]['hooks'][0]['always_run'] = True + config['repos'][0]['hooks'][0]['always_run'] = True _test_run( cap_out, + store, repo_with_passing_hook, {}, (b'Bash hook', b'Passed'), @@ -285,112 +411,98 @@ def test_always_run( ) -def test_always_run_alt_config( - cap_out, repo_with_passing_hook, mock_out_store_directory, -): +def test_always_run_alt_config(cap_out, store, repo_with_passing_hook): repo_root = '.' config = read_config(repo_root) - config[0]['hooks'][0]['always_run'] = True + config['repos'][0]['hooks'][0]['always_run'] = True alt_config_file = 'alternate_config.yaml' add_config_to_repo(repo_root, config, config_file=alt_config_file) _test_run( cap_out, + store, repo_with_passing_hook, {}, (b'Bash hook', b'Passed'), 0, stage=False, - config_file=alt_config_file + config_file=alt_config_file, ) -@pytest.mark.parametrize( - ('origin', 'source', 'expect_failure'), - ( - ('master', 'master', False), - ('master', '', True), - ('', 'master', True), +def test_hook_verbose_enabled(cap_out, store, repo_with_passing_hook): + with modify_config() as config: + config['repos'][0]['hooks'][0]['always_run'] = True + config['repos'][0]['hooks'][0]['verbose'] = True + + _test_run( + cap_out, + store, + repo_with_passing_hook, + {}, + (b'Hello World',), + 0, + stage=False, ) -) -def test_origin_source_error_msg( - repo_with_passing_hook, origin, source, expect_failure, - mock_out_store_directory, cap_out, -): - args = _get_opts(origin=origin, source=source) - ret, printed = _do_run(cap_out, repo_with_passing_hook, args) - warning_msg = b'Specify both --origin and --source.' - if expect_failure: - assert ret == 1 - assert warning_msg in printed - else: - assert ret == 0 - assert warning_msg not in printed @pytest.mark.parametrize( - ('no_stash', 'all_files', 'expect_stash'), - ( - (True, True, False), - (True, False, False), - (False, True, False), - (False, False, True), - ), + ('from_ref', 'to_ref'), (('master', ''), ('', 'master')), ) -def test_no_stash( - cap_out, - repo_with_passing_hook, - no_stash, - all_files, - expect_stash, - mock_out_store_directory, +def test_from_ref_to_ref_error_msg_error( + cap_out, store, repo_with_passing_hook, from_ref, to_ref, ): - stage_a_file() - # Make unstaged changes - with open('foo.py', 'w') as foo_file: - foo_file.write('import os\n') + args = run_opts(from_ref=from_ref, to_ref=to_ref) + ret, printed = _do_run(cap_out, store, repo_with_passing_hook, args) + assert ret == 1 + assert b'Specify both --from-ref and --to-ref.' in printed - args = _get_opts(no_stash=no_stash, all_files=all_files) - ret, printed = _do_run(cap_out, repo_with_passing_hook, args) + +def test_all_push_options_ok(cap_out, store, repo_with_passing_hook): + args = run_opts( + from_ref='master', to_ref='master', + remote_name='origin', remote_url='https://example.com/repo', + ) + ret, printed = _do_run(cap_out, store, repo_with_passing_hook, args) assert ret == 0 - warning_msg = b'[WARNING] Unstaged files detected.' - if expect_stash: - assert warning_msg in printed - else: - assert warning_msg not in printed + assert b'Specify both --from-ref and --to-ref.' not in printed -@pytest.mark.parametrize(('output', 'expected'), (('some', True), ('', False))) -def test_has_unmerged_paths(output, expected): - mock_runner = mock.Mock() - mock_runner.cmd_runner.run.return_value = (1, output, '') - assert _has_unmerged_paths(mock_runner) is expected +def test_checkout_type(cap_out, store, repo_with_passing_hook): + args = run_opts(from_ref='', to_ref='', checkout_type='1') + environ: EnvironT = {} + ret, printed = _do_run( + cap_out, store, repo_with_passing_hook, args, environ, + ) + assert environ['PRE_COMMIT_CHECKOUT_TYPE'] == '1' + + +def test_has_unmerged_paths(in_merge_conflict): + assert _has_unmerged_paths() is True + cmd_output('git', 'add', '.') + assert _has_unmerged_paths() is False -def test_merge_conflict(cap_out, in_merge_conflict, mock_out_store_directory): - ret, printed = _do_run(cap_out, in_merge_conflict, _get_opts()) +def test_merge_conflict(cap_out, store, in_merge_conflict): + ret, printed = _do_run(cap_out, store, in_merge_conflict, run_opts()) assert ret == 1 assert b'Unmerged files. Resolve before committing.' in printed -def test_merge_conflict_modified( - cap_out, in_merge_conflict, mock_out_store_directory, -): +def test_merge_conflict_modified(cap_out, store, in_merge_conflict): # Touch another file so we have unstaged non-conflicting things assert os.path.exists('dummy') with open('dummy', 'w') as dummy_file: dummy_file.write('bar\nbaz\n') - ret, printed = _do_run(cap_out, in_merge_conflict, _get_opts()) + ret, printed = _do_run(cap_out, store, in_merge_conflict, run_opts()) assert ret == 1 assert b'Unmerged files. Resolve before committing.' in printed -def test_merge_conflict_resolved( - cap_out, in_merge_conflict, mock_out_store_directory, -): +def test_merge_conflict_resolved(cap_out, store, in_merge_conflict): cmd_output('git', 'add', '.') - ret, printed = _do_run(cap_out, in_merge_conflict, _get_opts()) + ret, printed = _do_run(cap_out, store, in_merge_conflict, run_opts()) for msg in ( b'Checking merge-conflict files only.', b'Bash hook', b'Passed', ): @@ -398,20 +510,21 @@ def test_merge_conflict_resolved( @pytest.mark.parametrize( - ('hooks', 'verbose', 'expected'), + ('hooks', 'expected'), ( - ([], True, 80), - ([{'id': 'a', 'name': 'a' * 51}], False, 81), - ([{'id': 'a', 'name': 'a' * 51}], True, 85), + ([], 80), + ([auto_namedtuple(id='a', name='a' * 51)], 81), ( - [{'id': 'a', 'name': 'a' * 51}, {'id': 'b', 'name': 'b' * 52}], - False, + [ + auto_namedtuple(id='a', name='a' * 51), + auto_namedtuple(id='b', name='b' * 52), + ], 82, ), ), ) -def test_compute_cols(hooks, verbose, expected): - assert _compute_cols(hooks, verbose) == expected +def test_compute_cols(hooks, expected): + assert _compute_cols(hooks) == expected @pytest.mark.parametrize( @@ -431,51 +544,75 @@ def test_get_skips(environ, expected_output): assert ret == expected_output -def test_skip_hook(cap_out, repo_with_passing_hook, mock_out_store_directory): +def test_skip_hook(cap_out, store, repo_with_passing_hook): ret, printed = _do_run( - cap_out, repo_with_passing_hook, _get_opts(), {'SKIP': 'bash_hook'}, + cap_out, store, repo_with_passing_hook, run_opts(), + {'SKIP': 'bash_hook'}, ) for msg in (b'Bash hook', b'Skipped'): assert msg in printed +def test_skip_aliased_hook(cap_out, store, aliased_repo): + ret, printed = _do_run( + cap_out, store, aliased_repo, + run_opts(hook='foo_bash'), + {'SKIP': 'foo_bash'}, + ) + assert ret == 0 + # Only the aliased hook runs and is skipped + for msg in (b'Bash hook', b'Skipped'): + assert printed.count(msg) == 1 + + def test_hook_id_not_in_non_verbose_output( - cap_out, repo_with_passing_hook, mock_out_store_directory, + cap_out, store, repo_with_passing_hook, ): ret, printed = _do_run( - cap_out, repo_with_passing_hook, _get_opts(verbose=False), + cap_out, store, repo_with_passing_hook, run_opts(verbose=False), ) assert b'[bash_hook]' not in printed -def test_hook_id_in_verbose_output( - cap_out, repo_with_passing_hook, mock_out_store_directory, -): +def test_hook_id_in_verbose_output(cap_out, store, repo_with_passing_hook): ret, printed = _do_run( - cap_out, repo_with_passing_hook, _get_opts(verbose=True), + cap_out, store, repo_with_passing_hook, run_opts(verbose=True), ) - assert b'[bash_hook] Bash hook' in printed + assert b'- hook id: bash_hook' in printed -def test_multiple_hooks_same_id( - cap_out, repo_with_passing_hook, mock_out_store_directory, -): +def test_multiple_hooks_same_id(cap_out, store, repo_with_passing_hook): with cwd(repo_with_passing_hook): # Add bash hook on there again with modify_config() as config: - config[0]['hooks'].append({'id': 'bash_hook'}) + config['repos'][0]['hooks'].append({'id': 'bash_hook'}) stage_a_file() - ret, output = _do_run(cap_out, repo_with_passing_hook, _get_opts()) + ret, output = _do_run(cap_out, store, repo_with_passing_hook, run_opts()) assert ret == 0 assert output.count(b'Bash hook') == 2 -def test_non_ascii_hook_id( - repo_with_passing_hook, mock_out_store_directory, tempdir_factory, -): +def test_aliased_hook_run(cap_out, store, aliased_repo): + ret, output = _do_run( + cap_out, store, aliased_repo, + run_opts(verbose=True, hook='bash_hook'), + ) + assert ret == 0 + # Both hooks will run since they share the same ID + assert output.count(b'Bash hook') == 2 + + ret, output = _do_run( + cap_out, store, aliased_repo, + run_opts(verbose=True, hook='foo_bash'), + ) + assert ret == 0 + # Only the aliased hook runs + assert output.count(b'Bash hook') == 1 + + +def test_non_ascii_hook_id(repo_with_passing_hook, tempdir_factory): with cwd(repo_with_passing_hook): - install(Runner(repo_with_passing_hook, C.CONFIG_FILE)) _, stdout, _ = cmd_output_mocked_pre_commit_home( sys.executable, '-m', 'pre_commit.main', 'run', 'β˜ƒ', retcode=None, tempdir_factory=tempdir_factory, @@ -485,180 +622,164 @@ def test_non_ascii_hook_id( assert 'UnicodeEncodeError' not in stdout -def test_stdout_write_bug_py26( - repo_with_failing_hook, mock_out_store_directory, tempdir_factory, -): +def test_stdout_write_bug_py26(repo_with_failing_hook, store, tempdir_factory): with cwd(repo_with_failing_hook): with modify_config() as config: - config[0]['hooks'][0]['args'] = ['β˜ƒ'] + config['repos'][0]['hooks'][0]['args'] = ['β˜ƒ'] stage_a_file() - install(Runner(repo_with_failing_hook, C.CONFIG_FILE)) + install(C.CONFIG_FILE, store, hook_types=['pre-commit']) # Have to use subprocess because pytest monkeypatches sys.stdout - _, stdout, _ = cmd_output_mocked_pre_commit_home( - 'git', 'commit', '-m', 'Commit!', - # git commit puts pre-commit to stderr - stderr=subprocess.STDOUT, - retcode=None, + _, out = git_commit( + fn=cmd_output_mocked_pre_commit_home, tempdir_factory=tempdir_factory, - ) - assert 'UnicodeEncodeError' not in stdout - # Doesn't actually happen, but a reasonable assertion - assert 'UnicodeDecodeError' not in stdout - - -def test_hook_install_failure(mock_out_store_directory, tempdir_factory): - git_path = make_consuming_repo(tempdir_factory, 'not_installable_repo') - with cwd(git_path): - install(Runner(git_path, C.CONFIG_FILE)) - - _, stdout, _ = cmd_output_mocked_pre_commit_home( - 'git', 'commit', '-m', 'Commit!', - # git commit puts pre-commit to stderr - stderr=subprocess.STDOUT, retcode=None, - encoding=None, - tempdir_factory=tempdir_factory, ) - assert b'UnicodeDecodeError' not in stdout + assert 'UnicodeEncodeError' not in out # Doesn't actually happen, but a reasonable assertion - assert b'UnicodeEncodeError' not in stdout - - # Sanity check our output - assert ( - b'An unexpected error has occurred: CalledProcessError: ' in - stdout - ) - assert 'β˜ƒ'.encode('UTF-8') + 'Β²'.encode('latin1') in stdout - - -def test_get_changed_files(): - files = get_changed_files( - '78c682a1d13ba20e7cb735313b9314a74365cd3a', - '3387edbb1288a580b37fe25225aa0b856b18ad1a', - ) - assert files == ['CHANGELOG.md', 'setup.py'] - - # files changed in source but not in origin should not be returned - files = get_changed_files('HEAD~10', 'HEAD') - assert files == [] + assert 'UnicodeDecodeError' not in out -def test_lots_of_files(mock_out_store_directory, tempdir_factory): +def test_lots_of_files(store, tempdir_factory): # windows xargs seems to have a bug, here's a regression test for # our workaround - git_path = make_consuming_repo(tempdir_factory, 'python_hooks_repo') + git_path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') with cwd(git_path): # Override files so we run against them with modify_config() as config: - config[0]['hooks'][0]['files'] = '' + config['repos'][0]['hooks'][0]['files'] = '' # Write a crap ton of files for i in range(400): - filename = '{}{}'.format('a' * 100, i) - open(filename, 'w').close() + open(f'{"a" * 100}{i}', 'w').close() - cmd_output('bash', '-c', 'git add .') - install(Runner(git_path, C.CONFIG_FILE)) + cmd_output('git', 'add', '.') + install(C.CONFIG_FILE, store, hook_types=['pre-commit']) - cmd_output_mocked_pre_commit_home( - 'git', 'commit', '-m', 'Commit!', - # git commit puts pre-commit to stderr - stderr=subprocess.STDOUT, + git_commit( + fn=cmd_output_mocked_pre_commit_home, tempdir_factory=tempdir_factory, ) -@pytest.mark.parametrize( - ('hook_stage', 'stage_for_first_hook', 'stage_for_second_hook', - 'expected_output'), - ( - ('push', ['commit'], ['commit'], [b'', b'']), - ('push', ['commit', 'push'], ['commit', 'push'], - [b'hook 1', b'hook 2']), - ('push', [], [], [b'hook 1', b'hook 2']), - ('push', [], ['commit'], [b'hook 1', b'']), - ('push', ['push'], ['commit'], [b'hook 1', b'']), - ('push', ['commit'], ['push'], [b'', b'hook 2']), - ('commit', ['commit', 'push'], ['commit', 'push'], - [b'hook 1', b'hook 2']), - ('commit', ['commit'], ['commit'], [b'hook 1', b'hook 2']), - ('commit', [], [], [b'hook 1', b'hook 2']), - ('commit', [], ['commit'], [b'hook 1', b'hook 2']), - ('commit', ['push'], ['commit'], [b'', b'hook 2']), - ('commit', ['commit'], ['push'], [b'hook 1', b'']), - ) -) -def test_local_hook_for_stages( - cap_out, - repo_with_passing_hook, mock_out_store_directory, - stage_for_first_hook, - stage_for_second_hook, - hook_stage, - expected_output, -): - config = OrderedDict(( - ('repo', 'local'), - ('hooks', (OrderedDict(( - ('id', 'flake8'), - ('name', 'hook 1'), - ('entry', 'python -m flake8.__main__'), - ('language', 'system'), - ('files', r'\.py$'), - ('stages', stage_for_first_hook) - )), OrderedDict(( - ('id', 'do_not_commit'), - ('name', 'hook 2'), - ('entry', 'DO NOT COMMIT'), - ('language', 'pcre'), - ('files', '^(.*)$'), - ('stages', stage_for_second_hook) - )))) - )) +def test_stages(cap_out, store, repo_with_passing_hook): + config = { + 'repo': 'local', + 'hooks': [ + { + 'id': f'do-not-commit-{i}', + 'name': f'hook {i}', + 'entry': 'DO NOT COMMIT', + 'language': 'pygrep', + 'stages': [stage], + } + for i, stage in enumerate(('commit', 'push', 'manual'), 1) + ], + } add_config_to_repo(repo_with_passing_hook, config) - with io.open('dummy.py', 'w') as staged_file: - staged_file.write('"""TODO: something"""\n') - cmd_output('git', 'add', 'dummy.py') + stage_a_file() + + def _run_for_stage(stage): + args = run_opts(hook_stage=stage) + ret, printed = _do_run(cap_out, store, repo_with_passing_hook, args) + assert not ret, (ret, printed) + # this test should only run one hook + assert printed.count(b'hook ') == 1 + return printed + + assert _run_for_stage('commit').startswith(b'hook 1...') + assert _run_for_stage('push').startswith(b'hook 2...') + assert _run_for_stage('manual').startswith(b'hook 3...') + + +def test_commit_msg_hook(cap_out, store, commit_msg_repo): + filename = '.git/COMMIT_EDITMSG' + with open(filename, 'w') as f: + f.write('This is the commit message') _test_run( cap_out, - repo_with_passing_hook, - {'hook_stage': hook_stage}, - expected_outputs=expected_output, - expected_ret=0, - stage=False + store, + commit_msg_repo, + {'hook_stage': 'commit-msg', 'commit_msg_filename': filename}, + expected_outputs=[b'Must have "Signed off by:"', b'Failed'], + expected_ret=1, + stage=False, ) -def test_local_hook_passes( - cap_out, repo_with_passing_hook, mock_out_store_directory, -): - config = OrderedDict(( - ('repo', 'local'), - ('hooks', (OrderedDict(( - ('id', 'flake8'), - ('name', 'flake8'), - ('entry', 'python -m flake8.__main__'), - ('language', 'system'), - ('files', r'\.py$'), - )), OrderedDict(( - ('id', 'do_not_commit'), - ('name', 'Block if "DO NOT COMMIT" is found'), - ('entry', 'DO NOT COMMIT'), - ('language', 'pcre'), - ('files', '^(.*)$'), - )))) - )) +def test_post_checkout_hook(cap_out, store, tempdir_factory): + path = git_dir(tempdir_factory) + config = { + 'repo': 'meta', 'hooks': [ + {'id': 'identity', 'stages': ['post-checkout']}, + ], + } + add_config_to_repo(path, config) + + with cwd(path): + _test_run( + cap_out, + store, + path, + {'hook_stage': 'post-checkout'}, + expected_outputs=[b'identity...'], + expected_ret=0, + stage=False, + ) + + +def test_prepare_commit_msg_hook(cap_out, store, prepare_commit_msg_repo): + filename = '.git/COMMIT_EDITMSG' + with open(filename, 'w') as f: + f.write('This is the commit message') + + _test_run( + cap_out, + store, + prepare_commit_msg_repo, + {'hook_stage': 'prepare-commit-msg', 'commit_msg_filename': filename}, + expected_outputs=[b'Add "Signed off by:"', b'Passed'], + expected_ret=0, + stage=False, + ) + + with open(filename) as f: + assert 'Signed off by: ' in f.read() + + +def test_local_hook_passes(cap_out, store, repo_with_passing_hook): + config = { + 'repo': 'local', + 'hooks': [ + { + 'id': 'identity-copy', + 'name': 'identity-copy', + 'entry': '{} -m pre_commit.meta_hooks.identity'.format( + shlex.quote(sys.executable), + ), + 'language': 'system', + 'files': r'\.py$', + }, + { + 'id': 'do_not_commit', + 'name': 'Block if "DO NOT COMMIT" is found', + 'entry': 'DO NOT COMMIT', + 'language': 'pygrep', + }, + ], + } add_config_to_repo(repo_with_passing_hook, config) - with io.open('dummy.py', 'w') as staged_file: + with open('dummy.py', 'w') as staged_file: staged_file.write('"""TODO: something"""\n') cmd_output('git', 'add', 'dummy.py') _test_run( cap_out, + store, repo_with_passing_hook, opts={}, expected_outputs=[b''], @@ -667,27 +788,25 @@ def test_local_hook_passes( ) -def test_local_hook_fails( - cap_out, repo_with_passing_hook, mock_out_store_directory, -): - config = OrderedDict(( - ('repo', 'local'), - ('hooks', [OrderedDict(( - ('id', 'no-todo'), - ('name', 'No TODO'), - ('entry', 'sh -c "! grep -iI todo $@" --'), - ('language', 'system'), - ('files', ''), - ))]) - )) +def test_local_hook_fails(cap_out, store, repo_with_passing_hook): + config = { + 'repo': 'local', + 'hooks': [{ + 'id': 'no-todo', + 'name': 'No TODO', + 'entry': 'sh -c "! grep -iI todo $@" --', + 'language': 'system', + }], + } add_config_to_repo(repo_with_passing_hook, config) - with io.open('dummy.py', 'w') as staged_file: + with open('dummy.py', 'w') as staged_file: staged_file.write('"""TODO: something"""\n') cmd_output('git', 'add', 'dummy.py') _test_run( cap_out, + store, repo_with_passing_hook, opts={}, expected_outputs=[b''], @@ -696,58 +815,57 @@ def test_local_hook_fails( ) -@pytest.yield_fixture +def test_meta_hook_passes(cap_out, store, repo_with_passing_hook): + add_config_to_repo(repo_with_passing_hook, sample_meta_config()) + + _test_run( + cap_out, + store, + repo_with_passing_hook, + opts={}, + expected_outputs=[b'Check for useless excludes'], + expected_ret=0, + stage=False, + ) + + +@pytest.fixture def modified_config_repo(repo_with_passing_hook): with modify_config(repo_with_passing_hook, commit=False) as config: # Some minor modification - config[0]['hooks'][0]['files'] = '' + config['repos'][0]['hooks'][0]['files'] = '' yield repo_with_passing_hook -def test_allow_unstaged_config_option( - cap_out, modified_config_repo, mock_out_store_directory, -): - args = _get_opts(allow_unstaged_config=True) - ret, printed = _do_run(cap_out, modified_config_repo, args) - expected = ( - b'You have an unstaged config file and have specified the ' - b'--allow-unstaged-config option.' - ) - assert expected in printed - assert ret == 0 +def test_error_with_unstaged_config(cap_out, store, modified_config_repo): + args = run_opts() + ret, printed = _do_run(cap_out, store, modified_config_repo, args) + assert b'Your pre-commit configuration is unstaged.' in printed + assert ret == 1 -def test_no_allow_unstaged_config_option( - cap_out, modified_config_repo, mock_out_store_directory, -): - args = _get_opts(allow_unstaged_config=False) - ret, printed = _do_run(cap_out, modified_config_repo, args) - assert b'Your .pre-commit-config.yaml is unstaged.' in printed +def test_commit_msg_missing_filename(cap_out, store, repo_with_passing_hook): + args = run_opts(hook_stage='commit-msg') + ret, printed = _do_run(cap_out, store, repo_with_passing_hook, args) assert ret == 1 + assert printed == ( + b'[ERROR] `--commit-msg-filename` is required for ' + b'`--hook-stage commit-msg`\n' + ) @pytest.mark.parametrize( - 'opts', - ( - {'allow_unstaged_config': False, 'no_stash': True}, - {'all_files': True}, - {'files': [C.CONFIG_FILE]}, - ), + 'opts', (run_opts(all_files=True), run_opts(files=[C.CONFIG_FILE])), ) -def test_unstaged_message_suppressed( - cap_out, modified_config_repo, mock_out_store_directory, opts, +def test_no_unstaged_error_with_all_files_or_files( + cap_out, store, modified_config_repo, opts, ): - args = _get_opts(**opts) - ret, printed = _do_run(cap_out, modified_config_repo, args) - assert b'Your .pre-commit-config.yaml is unstaged.' not in printed + ret, printed = _do_run(cap_out, store, modified_config_repo, opts) + assert b'Your pre-commit configuration is unstaged.' not in printed -def test_files_running_subdir( - repo_with_passing_hook, mock_out_store_directory, tempdir_factory, -): +def test_files_running_subdir(repo_with_passing_hook, tempdir_factory): with cwd(repo_with_passing_hook): - install(Runner(repo_with_passing_hook, C.CONFIG_FILE)) - os.mkdir('subdir') open('subdir/foo.py', 'w').close() cmd_output('git', 'add', 'subdir/foo.py') @@ -760,7 +878,7 @@ def test_files_running_subdir( '--files', 'foo.py', tempdir_factory=tempdir_factory, ) - assert 'subdir/foo.py'.replace('/', os.sep) in stdout + assert 'subdir/foo.py' in stdout @pytest.mark.parametrize( @@ -770,20 +888,125 @@ def test_files_running_subdir( (False, [], b''), (True, ['some', 'args'], b'some args foo.py'), (False, ['some', 'args'], b'some args'), - ) + ), ) def test_pass_filenames( - cap_out, repo_with_passing_hook, mock_out_store_directory, - pass_filenames, - hook_args, - expected_out, + cap_out, store, repo_with_passing_hook, + pass_filenames, hook_args, expected_out, ): with modify_config() as config: - config[0]['hooks'][0]['pass_filenames'] = pass_filenames - config[0]['hooks'][0]['args'] = hook_args + config['repos'][0]['hooks'][0]['pass_filenames'] = pass_filenames + config['repos'][0]['hooks'][0]['args'] = hook_args stage_a_file() ret, printed = _do_run( - cap_out, repo_with_passing_hook, _get_opts(verbose=True), + cap_out, store, repo_with_passing_hook, run_opts(verbose=True), ) assert expected_out + b'\nHello World' in printed assert (b'foo.py' in printed) == pass_filenames + + +def test_fail_fast(cap_out, store, repo_with_failing_hook): + with modify_config() as config: + # More than one hook + config['fail_fast'] = True + config['repos'][0]['hooks'] *= 2 + stage_a_file() + + ret, printed = _do_run(cap_out, store, repo_with_failing_hook, run_opts()) + # it should have only run one hook + assert printed.count(b'Failing hook') == 1 + + +def test_classifier_removes_dne(): + classifier = Classifier(('this_file_does_not_exist',)) + assert classifier.filenames == [] + + +def test_classifier_normalizes_filenames_on_windows_to_forward_slashes(tmpdir): + with tmpdir.as_cwd(): + tmpdir.join('a/b/c').ensure() + with mock.patch.object(os, 'altsep', '/'): + with mock.patch.object(os, 'sep', '\\'): + classifier = Classifier((r'a\b\c',)) + assert classifier.filenames == ['a/b/c'] + + +def test_classifier_does_not_normalize_backslashes_non_windows(tmpdir): + with mock.patch.object(os.path, 'lexists', return_value=True): + with mock.patch.object(os, 'altsep', None): + with mock.patch.object(os, 'sep', '/'): + classifier = Classifier((r'a/b\c',)) + assert classifier.filenames == [r'a/b\c'] + + +@pytest.fixture +def some_filenames(): + return ( + '.pre-commit-hooks.yaml', + 'pre_commit/git.py', + 'pre_commit/main.py', + ) + + +def test_include_exclude_base_case(some_filenames): + ret = filter_by_include_exclude(some_filenames, '', '^$') + assert ret == [ + '.pre-commit-hooks.yaml', + 'pre_commit/git.py', + 'pre_commit/main.py', + ] + + +def test_matches_broken_symlink(tmpdir): + with tmpdir.as_cwd(): + os.symlink('does-not-exist', 'link') + ret = filter_by_include_exclude({'link'}, '', '^$') + assert ret == ['link'] + + +def test_include_exclude_total_match(some_filenames): + ret = filter_by_include_exclude(some_filenames, r'^.*\.py$', '^$') + assert ret == ['pre_commit/git.py', 'pre_commit/main.py'] + + +def test_include_exclude_does_search_instead_of_match(some_filenames): + ret = filter_by_include_exclude(some_filenames, r'\.yaml$', '^$') + assert ret == ['.pre-commit-hooks.yaml'] + + +def test_include_exclude_exclude_removes_files(some_filenames): + ret = filter_by_include_exclude(some_filenames, '', r'\.py$') + assert ret == ['.pre-commit-hooks.yaml'] + + +def test_args_hook_only(cap_out, store, repo_with_passing_hook): + config = { + 'repo': 'local', + 'hooks': [ + { + 'id': 'identity-copy', + 'name': 'identity-copy', + 'entry': '{} -m pre_commit.meta_hooks.identity'.format( + shlex.quote(sys.executable), + ), + 'language': 'system', + 'files': r'\.py$', + 'stages': ['commit'], + }, + { + 'id': 'do_not_commit', + 'name': 'Block if "DO NOT COMMIT" is found', + 'entry': 'DO NOT COMMIT', + 'language': 'pygrep', + }, + ], + } + add_config_to_repo(repo_with_passing_hook, config) + stage_a_file() + ret, printed = _do_run( + cap_out, + store, + repo_with_passing_hook, + run_opts(hook='do_not_commit'), + ) + assert b'identity-copy' not in printed diff --git a/tests/commands/sample_config_test.py b/tests/commands/sample_config_test.py index 88a90d914..11c087649 100644 --- a/tests/commands/sample_config_test.py +++ b/tests/commands/sample_config_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - from pre_commit.commands.sample_config import sample_config @@ -9,10 +6,11 @@ def test_sample_config(capsys): assert ret == 0 out, _ = capsys.readouterr() assert out == '''\ -# See http://pre-commit.com for more information -# See http://pre-commit.com/hooks.html for more hooks +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: - repo: https://github.com/pre-commit/pre-commit-hooks - sha: v0.7.1 + rev: v2.4.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer diff --git a/tests/commands/try_repo_test.py b/tests/commands/try_repo_test.py new file mode 100644 index 000000000..d3ec3fda2 --- /dev/null +++ b/tests/commands/try_repo_test.py @@ -0,0 +1,151 @@ +import os.path +import re +import time +from unittest import mock + +from pre_commit import git +from pre_commit.commands.try_repo import try_repo +from pre_commit.util import cmd_output +from testing.auto_namedtuple import auto_namedtuple +from testing.fixtures import git_dir +from testing.fixtures import make_repo +from testing.fixtures import modify_manifest +from testing.util import cwd +from testing.util import git_commit +from testing.util import run_opts + + +def try_repo_opts(repo, ref=None, **kwargs): + return auto_namedtuple(repo=repo, ref=ref, **run_opts(**kwargs)._asdict()) + + +def _get_out(cap_out): + out = re.sub(r'\[INFO\].+\n', '', cap_out.get()) + start, using_config, config, rest = out.split(f'{"=" * 79}\n') + assert using_config == 'Using config:\n' + return start, config, rest + + +def _add_test_file(): + open('test-file', 'a').close() + cmd_output('git', 'add', '.') + + +def _run_try_repo(tempdir_factory, **kwargs): + repo = make_repo(tempdir_factory, 'modified_file_returns_zero_repo') + with cwd(git_dir(tempdir_factory)): + _add_test_file() + assert not try_repo(try_repo_opts(repo, **kwargs)) + + +def test_try_repo_repo_only(cap_out, tempdir_factory): + with mock.patch.object(time, 'time', return_value=0.0): + _run_try_repo(tempdir_factory, verbose=True) + start, config, rest = _get_out(cap_out) + assert start == '' + assert re.match( + '^repos:\n' + '- repo: .+\n' + ' rev: .+\n' + ' hooks:\n' + ' - id: bash_hook\n' + ' - id: bash_hook2\n' + ' - id: bash_hook3\n$', + config, + ) + assert rest == '''\ +Bash hook............................................(no files to check)Skipped +- hook id: bash_hook +Bash hook................................................................Passed +- hook id: bash_hook2 +- duration: 0s + +test-file + +Bash hook............................................(no files to check)Skipped +- hook id: bash_hook3 +''' + + +def test_try_repo_with_specific_hook(cap_out, tempdir_factory): + _run_try_repo(tempdir_factory, hook='bash_hook', verbose=True) + start, config, rest = _get_out(cap_out) + assert start == '' + assert re.match( + '^repos:\n' + '- repo: .+\n' + ' rev: .+\n' + ' hooks:\n' + ' - id: bash_hook\n$', + config, + ) + assert rest == '''\ +Bash hook............................................(no files to check)Skipped +- hook id: bash_hook +''' + + +def test_try_repo_relative_path(cap_out, tempdir_factory): + repo = make_repo(tempdir_factory, 'modified_file_returns_zero_repo') + with cwd(git_dir(tempdir_factory)): + _add_test_file() + relative_repo = os.path.relpath(repo, '.') + # previously crashed on cloning a relative path + assert not try_repo(try_repo_opts(relative_repo, hook='bash_hook')) + + +def test_try_repo_bare_repo(cap_out, tempdir_factory): + repo = make_repo(tempdir_factory, 'modified_file_returns_zero_repo') + with cwd(git_dir(tempdir_factory)): + _add_test_file() + bare_repo = os.path.join(repo, '.git') + # previously crashed attempting modification changes + assert not try_repo(try_repo_opts(bare_repo, hook='bash_hook')) + + +def test_try_repo_specific_revision(cap_out, tempdir_factory): + repo = make_repo(tempdir_factory, 'script_hooks_repo') + ref = git.head_rev(repo) + git_commit(cwd=repo) + with cwd(git_dir(tempdir_factory)): + _add_test_file() + assert not try_repo(try_repo_opts(repo, ref=ref)) + + _, config, _ = _get_out(cap_out) + assert ref in config + + +def test_try_repo_uncommitted_changes(cap_out, tempdir_factory): + repo = make_repo(tempdir_factory, 'script_hooks_repo') + # make an uncommitted change + with modify_manifest(repo, commit=False) as manifest: + manifest[0]['name'] = 'modified name!' + + with cwd(git_dir(tempdir_factory)): + open('test-fie', 'a').close() + cmd_output('git', 'add', '.') + assert not try_repo(try_repo_opts(repo)) + + start, config, rest = _get_out(cap_out) + assert start == '[WARNING] Creating temporary repo with uncommitted changes...\n' # noqa: E501 + assert re.match( + '^repos:\n' + '- repo: .+shadow-repo\n' + ' rev: .+\n' + ' hooks:\n' + ' - id: bash_hook\n$', + config, + ) + assert rest == 'modified name!...........................................................Passed\n' # noqa: E501 + + +def test_try_repo_staged_changes(tempdir_factory): + repo = make_repo(tempdir_factory, 'modified_file_returns_zero_repo') + + with cwd(repo): + open('staged-file', 'a').close() + open('second-staged-file', 'a').close() + cmd_output('git', 'add', '.') + + with cwd(git_dir(tempdir_factory)): + assert not try_repo(try_repo_opts(repo, hook='bash_hook')) diff --git a/tests/conftest.py b/tests/conftest.py index 140463b15..335d2614f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,35 +1,50 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import functools import io import logging import os.path +from unittest import mock -import mock import pytest -import six -import pre_commit.constants as C from pre_commit import output -from pre_commit.logging_handler import add_logging_handler -from pre_commit.prefixed_command_runner import PrefixedCommandRunner -from pre_commit.runner import Runner +from pre_commit.envcontext import envcontext +from pre_commit.logging_handler import logging_handler from pre_commit.store import Store from pre_commit.util import cmd_output -from pre_commit.util import cwd +from pre_commit.util import make_executable from testing.fixtures import git_dir from testing.fixtures import make_consuming_repo - - -@pytest.yield_fixture +from testing.fixtures import write_config +from testing.util import cwd +from testing.util import git_commit + + +@pytest.fixture(autouse=True) +def no_warnings(recwarn): + yield + warnings = [] + for warning in recwarn: # pragma: no cover + message = str(warning.message) + # ImportWarning: Not importing directory '...' missing __init__(.py) + if not ( + isinstance(warning.message, ImportWarning) and + message.startswith('Not importing directory ') and + ' missing __init__' in message + ): + warnings.append( + f'{warning.filename}:{warning.lineno} {message}', + ) + assert not warnings + + +@pytest.fixture def tempdir_factory(tmpdir): - class TmpdirFactory(object): + class TmpdirFactory: def __init__(self): self.tmpdir_count = 0 def get(self): - path = tmpdir.join(six.text_type(self.tmpdir_count)).strpath + path = tmpdir.join(str(self.tmpdir_count)).strpath self.tmpdir_count += 1 os.mkdir(path) return path @@ -37,40 +52,47 @@ def get(self): yield TmpdirFactory() -@pytest.yield_fixture +@pytest.fixture def in_tmpdir(tempdir_factory): path = tempdir_factory.get() with cwd(path): yield path +@pytest.fixture +def in_git_dir(tmpdir): + repo = tmpdir.join('repo').ensure_dir() + with repo.as_cwd(): + cmd_output('git', 'init') + yield repo + + def _make_conflict(): cmd_output('git', 'checkout', 'origin/master', '-b', 'foo') - with io.open('conflict_file', 'w') as conflict_file: + with open('conflict_file', 'w') as conflict_file: conflict_file.write('herp\nderp\n') cmd_output('git', 'add', 'conflict_file') - with io.open('foo_only_file', 'w') as foo_only_file: + with open('foo_only_file', 'w') as foo_only_file: foo_only_file.write('foo') cmd_output('git', 'add', 'foo_only_file') - cmd_output('git', 'commit', '-m', 'conflict_file') + git_commit(msg=_make_conflict.__name__) cmd_output('git', 'checkout', 'origin/master', '-b', 'bar') - with io.open('conflict_file', 'w') as conflict_file: + with open('conflict_file', 'w') as conflict_file: conflict_file.write('harp\nddrp\n') cmd_output('git', 'add', 'conflict_file') - with io.open('bar_only_file', 'w') as bar_only_file: + with open('bar_only_file', 'w') as bar_only_file: bar_only_file.write('bar') cmd_output('git', 'add', 'bar_only_file') - cmd_output('git', 'commit', '-m', 'conflict_file') + git_commit(msg=_make_conflict.__name__) cmd_output('git', 'merge', 'foo', retcode=None) -@pytest.yield_fixture +@pytest.fixture def in_merge_conflict(tempdir_factory): path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') - with cwd(path): - cmd_output('touch', 'dummy') - cmd_output('git', 'add', 'dummy') - cmd_output('git', 'commit', '-m', 'Add config.') + open(os.path.join(path, 'dummy'), 'a').close() + cmd_output('git', 'add', 'dummy', cwd=path) + git_commit(msg=in_merge_conflict.__name__, cwd=path) conflict_path = tempdir_factory.get() cmd_output('git', 'clone', path, conflict_path) @@ -79,20 +101,86 @@ def in_merge_conflict(tempdir_factory): yield os.path.join(conflict_path) -@pytest.yield_fixture +@pytest.fixture def in_conflicting_submodule(tempdir_factory): git_dir_1 = git_dir(tempdir_factory) git_dir_2 = git_dir(tempdir_factory) - with cwd(git_dir_2): - cmd_output('git', 'commit', '--allow-empty', '-m', 'init!') - with cwd(git_dir_1): - cmd_output('git', 'submodule', 'add', git_dir_2, 'sub') + git_commit(msg=in_conflicting_submodule.__name__, cwd=git_dir_2) + cmd_output('git', 'submodule', 'add', git_dir_2, 'sub', cwd=git_dir_1) with cwd(os.path.join(git_dir_1, 'sub')): _make_conflict() yield -@pytest.yield_fixture(autouse=True, scope='session') +@pytest.fixture +def commit_msg_repo(tempdir_factory): + path = git_dir(tempdir_factory) + config = { + 'repo': 'local', + 'hooks': [{ + 'id': 'must-have-signoff', + 'name': 'Must have "Signed off by:"', + 'entry': 'grep -q "Signed off by:"', + 'language': 'system', + 'stages': ['commit-msg'], + }], + } + write_config(path, config) + with cwd(path): + cmd_output('git', 'add', '.') + git_commit(msg=commit_msg_repo.__name__) + yield path + + +@pytest.fixture +def prepare_commit_msg_repo(tempdir_factory): + path = git_dir(tempdir_factory) + script_name = 'add_sign_off.sh' + config = { + 'repo': 'local', + 'hooks': [{ + 'id': 'add-signoff', + 'name': 'Add "Signed off by:"', + 'entry': f'./{script_name}', + 'language': 'script', + 'stages': ['prepare-commit-msg'], + }], + } + write_config(path, config) + with cwd(path): + with open(script_name, 'w') as script_file: + script_file.write( + '#!/usr/bin/env bash\n' + 'set -eu\n' + 'echo "\nSigned off by: " >> "$1"\n', + ) + make_executable(script_name) + cmd_output('git', 'add', '.') + git_commit(msg=prepare_commit_msg_repo.__name__) + yield path + + +@pytest.fixture +def failing_prepare_commit_msg_repo(tempdir_factory): + path = git_dir(tempdir_factory) + config = { + 'repo': 'local', + 'hooks': [{ + 'id': 'add-signoff', + 'name': 'Add "Signed off by:"', + 'entry': 'bash -c "exit 1"', + 'language': 'system', + 'stages': ['prepare-commit-msg'], + }], + } + write_config(path, config) + with cwd(path): + cmd_output('git', 'add', '.') + git_commit(msg=failing_prepare_commit_msg_repo.__name__) + yield path + + +@pytest.fixture(autouse=True, scope='session') def dont_write_to_home_directory(): """pre_commit.store.Store will by default write to the home directory We'll mock out `Store.get_default_directory` to raise invariantly so we @@ -111,11 +199,12 @@ class YouForgotToExplicitlyChooseAStoreDirectory(AssertionError): @pytest.fixture(autouse=True, scope='session') def configure_logging(): - add_logging_handler(use_color=False) + with logging_handler(use_color=False): + yield -@pytest.yield_fixture -def mock_out_store_directory(tempdir_factory): +@pytest.fixture +def mock_store_dir(tempdir_factory): tmpdir = tempdir_factory.get() with mock.patch.object( Store, @@ -125,34 +214,18 @@ def mock_out_store_directory(tempdir_factory): yield tmpdir -@pytest.yield_fixture +@pytest.fixture def store(tempdir_factory): yield Store(os.path.join(tempdir_factory.get(), '.pre-commit')) -@pytest.yield_fixture -def cmd_runner(tempdir_factory): - yield PrefixedCommandRunner(tempdir_factory.get()) - - -@pytest.yield_fixture -def runner_with_mocked_store(mock_out_store_directory): - yield Runner('/', C.CONFIG_FILE) - - -@pytest.yield_fixture +@pytest.fixture def log_info_mock(): with mock.patch.object(logging.getLogger('pre_commit'), 'info') as mck: yield mck -@pytest.yield_fixture -def log_warning_mock(): - with mock.patch.object(logging.getLogger('pre_commit'), 'warning') as mck: - yield mck - - -class FakeStream(object): +class FakeStream: def __init__(self): self.data = io.BytesIO() @@ -163,35 +236,42 @@ def flush(self): pass -class Fixture(object): +class Fixture: def __init__(self, stream): self._stream = stream def get_bytes(self): """Get the output as-if no encoding occurred""" data = self._stream.data.getvalue() - self._stream = io.BytesIO() - return data + self._stream.data.seek(0) + self._stream.data.truncate() + return data.replace(b'\r\n', b'\n') def get(self): """Get the output assuming it was written as UTF-8 bytes""" - return self.get_bytes().decode('UTF-8') + return self.get_bytes().decode() -@pytest.yield_fixture +@pytest.fixture def cap_out(): stream = FakeStream() write = functools.partial(output.write, stream=stream) - write_line = functools.partial(output.write_line, stream=stream) - with mock.patch.object(output, 'write', write): - with mock.patch.object(output, 'write_line', write_line): - yield Fixture(stream) + write_line_b = functools.partial(output.write_line_b, stream=stream) + with mock.patch.multiple(output, write=write, write_line_b=write_line_b): + yield Fixture(stream) -@pytest.yield_fixture +@pytest.fixture def fake_log_handler(): handler = mock.Mock(level=logging.INFO) logger = logging.getLogger('pre_commit') logger.addHandler(handler) yield handler logger.removeHandler(handler) + + +@pytest.fixture(scope='session', autouse=True) +def set_git_templatedir(tmpdir_factory): + tdir = str(tmpdir_factory.mktemp('git_template_dir')) + with envcontext((('GIT_TEMPLATE_DIR', tdir),)): + yield diff --git a/tests/envcontext_test.py b/tests/envcontext_test.py index c03e94317..f9d4dce69 100644 --- a/tests/envcontext_test.py +++ b/tests/envcontext_test.py @@ -1,9 +1,6 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import os +from unittest import mock -import mock import pytest from pre_commit.envcontext import envcontext @@ -11,12 +8,7 @@ from pre_commit.envcontext import Var -def _test(**kwargs): - before = kwargs.pop('before') - patch = kwargs.pop('patch') - expected = kwargs.pop('expected') - assert not kwargs - +def _test(*, before, patch, expected): env = before.copy() with envcontext(patch, _env=env): assert env == expected @@ -94,16 +86,16 @@ def test_exception_safety(): class MyError(RuntimeError): pass - env = {} + env = {'hello': 'world'} with pytest.raises(MyError): - with envcontext([('foo', 'bar')], _env=env): + with envcontext((('foo', 'bar'),), _env=env): raise MyError() - assert env == {} + assert env == {'hello': 'world'} def test_integration_os_environ(): with mock.patch.dict(os.environ, {'FOO': 'bar'}, clear=True): assert os.environ == {'FOO': 'bar'} - with envcontext([('HERP', 'derp')]): + with envcontext((('HERP', 'derp'),)): assert os.environ == {'FOO': 'bar', 'HERP': 'derp'} assert os.environ == {'FOO': 'bar'} diff --git a/tests/error_handler_test.py b/tests/error_handler_test.py index 1d53c4b7d..833bb8f83 100644 --- a/tests/error_handler_test.py +++ b/tests/error_handler_test.py @@ -1,21 +1,16 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import unicode_literals - -import io import os.path import re import sys +from unittest import mock -import mock import pytest from pre_commit import error_handler -from pre_commit.errors import FatalError +from pre_commit.util import CalledProcessError from testing.util import cmd_output_mocked_pre_commit_home -@pytest.yield_fixture +@pytest.fixture def mocked_log_and_exit(): with mock.patch.object(error_handler, '_log_and_exit') as log_and_exit: yield log_and_exit @@ -28,7 +23,7 @@ def test_error_handler_no_exception(mocked_log_and_exit): def test_error_handler_fatal_error(mocked_log_and_exit): - exc = FatalError('just a test') + exc = error_handler.FatalError('just a test') with error_handler.error_handler(): raise exc @@ -46,7 +41,7 @@ def test_error_handler_fatal_error(mocked_log_and_exit): r' File ".+tests.error_handler_test.py", line \d+, ' r'in test_error_handler_fatal_error\n' r' raise exc\n' - r'(pre_commit\.errors\.)?FatalError: just a test\n', + r'(pre_commit\.error_handler\.)?FatalError: just a test\n', mocked_log_and_exit.call_args[0][2], ) @@ -74,44 +69,102 @@ def test_error_handler_uncaught_error(mocked_log_and_exit): ) -def test_log_and_exit(cap_out, mock_out_store_directory): - with pytest.raises(error_handler.PreCommitSystemExit): +def test_error_handler_keyboardinterrupt(mocked_log_and_exit): + exc = KeyboardInterrupt() + with error_handler.error_handler(): + raise exc + + mocked_log_and_exit.assert_called_once_with( + 'Interrupted (^C)', + exc, + # Tested below + mock.ANY, + ) + assert re.match( + r'Traceback \(most recent call last\):\n' + r' File ".+pre_commit.error_handler.py", line \d+, in error_handler\n' + r' yield\n' + r' File ".+tests.error_handler_test.py", line \d+, ' + r'in test_error_handler_keyboardinterrupt\n' + r' raise exc\n' + r'KeyboardInterrupt\n', + mocked_log_and_exit.call_args[0][2], + ) + + +def test_log_and_exit(cap_out, mock_store_dir): + with pytest.raises(SystemExit): error_handler._log_and_exit( - 'msg', FatalError('hai'), "I'm a stacktrace", + 'msg', error_handler.FatalError('hai'), "I'm a stacktrace", ) printed = cap_out.get() - assert printed == ( - 'msg: FatalError: hai\n' - 'Check the log at ~/.pre-commit/pre-commit.log\n' - ) + log_file = os.path.join(mock_store_dir, 'pre-commit.log') + assert printed == f'msg: FatalError: hai\nCheck the log at {log_file}\n' - log_file = os.path.join(mock_out_store_directory, 'pre-commit.log') assert os.path.exists(log_file) - contents = io.open(log_file).read() - assert contents == ( - 'msg: FatalError: hai\n' - "I'm a stacktrace\n" - ) + with open(log_file) as f: + logged = f.read() + expected = ( + r'^### version information\n' + r'\n' + r'```\n' + r'pre-commit version: \d+\.\d+\.\d+\n' + r'sys.version:\n' + r'( .*\n)*' + r'sys.executable: .*\n' + r'os.name: .*\n' + r'sys.platform: .*\n' + r'```\n' + r'\n' + r'### error information\n' + r'\n' + r'```\n' + r'msg: FatalError: hai\n' + r'```\n' + r'\n' + r'```\n' + r"I'm a stacktrace\n" + r'```\n' + ) + assert re.match(expected, logged) -def test_error_handler_non_ascii_exception(mock_out_store_directory): - with pytest.raises(error_handler.PreCommitSystemExit): +def test_error_handler_non_ascii_exception(mock_store_dir): + with pytest.raises(SystemExit): with error_handler.error_handler(): raise ValueError('β˜ƒ') +def test_error_handler_non_utf8_exception(mock_store_dir): + with pytest.raises(SystemExit): + with error_handler.error_handler(): + raise CalledProcessError(1, ('exe',), 0, b'error: \xa0\xe1', b'') + + +def test_error_handler_non_stringable_exception(mock_store_dir): + class C(Exception): + def __str__(self): + raise RuntimeError('not today!') + + with pytest.raises(SystemExit): + with error_handler.error_handler(): + raise C() + + def test_error_handler_no_tty(tempdir_factory): - output = cmd_output_mocked_pre_commit_home( - sys.executable, '-c', - 'from __future__ import unicode_literals\n' + pre_commit_home = tempdir_factory.get() + ret, out, _ = cmd_output_mocked_pre_commit_home( + sys.executable, + '-c', 'from pre_commit.error_handler import error_handler\n' 'with error_handler():\n' ' raise ValueError("\\u2603")\n', retcode=1, tempdir_factory=tempdir_factory, + pre_commit_home=pre_commit_home, ) - assert output[1].replace('\r', '') == ( - 'An unexpected error has occurred: ValueError: β˜ƒ\n' - 'Check the log at ~/.pre-commit/pre-commit.log\n' - ) + log_file = os.path.join(pre_commit_home, 'pre-commit.log') + out_lines = out.splitlines() + assert out_lines[-2] == 'An unexpected error has occurred: ValueError: β˜ƒ' + assert out_lines[-1] == f'Check the log at {log_file}' diff --git a/tests/git_test.py b/tests/git_test.py index ffe1c1aad..e73a6f240 100644 --- a/tests/git_test.py +++ b/tests/git_test.py @@ -1,52 +1,33 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import os.path import pytest from pre_commit import git -from pre_commit.errors import FatalError from pre_commit.util import cmd_output -from pre_commit.util import cwd -from testing.fixtures import git_dir - - -def test_get_root_at_root(tempdir_factory): - path = git_dir(tempdir_factory) - with cwd(path): - assert os.path.normcase(git.get_root()) == os.path.normcase(path) +from testing.util import git_commit -def test_get_root_deeper(tempdir_factory): - path = git_dir(tempdir_factory) +def test_get_root_at_root(in_git_dir): + expected = os.path.normcase(in_git_dir.strpath) + assert os.path.normcase(git.get_root()) == expected - foo_path = os.path.join(path, 'foo') - os.mkdir(foo_path) - with cwd(foo_path): - assert os.path.normcase(git.get_root()) == os.path.normcase(path) +def test_get_root_deeper(in_git_dir): + expected = os.path.normcase(in_git_dir.strpath) + with in_git_dir.join('foo').ensure_dir().as_cwd(): + assert os.path.normcase(git.get_root()) == expected -def test_get_root_not_git_dir(tempdir_factory): - with cwd(tempdir_factory.get()): - with pytest.raises(FatalError): - git.get_root() +def test_get_staged_files_deleted(in_git_dir): + in_git_dir.join('test').ensure() + cmd_output('git', 'add', 'test') + git_commit() + cmd_output('git', 'rm', '--cached', 'test') + assert git.get_staged_files() == [] -def test_get_staged_files_deleted(tempdir_factory): - path = git_dir(tempdir_factory) - with cwd(path): - open('test', 'a').close() - cmd_output('git', 'add', 'test') - cmd_output('git', 'commit', '-m', 'foo', '--allow-empty') - cmd_output('git', 'rm', '--cached', 'test') - assert git.get_staged_files() == [] - -def test_is_not_in_merge_conflict(tempdir_factory): - path = git_dir(tempdir_factory) - with cwd(path): - assert git.is_in_merge_conflict() is False +def test_is_not_in_merge_conflict(in_git_dir): + assert git.is_in_merge_conflict() is False def test_is_in_merge_conflict(in_merge_conflict): @@ -64,50 +45,6 @@ def test_cherry_pick_conflict(in_merge_conflict): assert git.is_in_merge_conflict() is False -@pytest.fixture -def get_files_matching_func(): - def get_filenames(): - return ( - '.pre-commit-hooks.yaml', - 'pre_commit/main.py', - 'pre_commit/git.py', - 'im_a_file_that_doesnt_exist.py', - 'testing/test_symlink', - ) - - return git.get_files_matching(get_filenames) - - -def test_get_files_matching_base(get_files_matching_func): - ret = get_files_matching_func('', '^$') - assert ret == { - '.pre-commit-hooks.yaml', - 'pre_commit/main.py', - 'pre_commit/git.py', - 'testing/test_symlink' - } - - -def test_get_files_matching_total_match(get_files_matching_func): - ret = get_files_matching_func('^.*\\.py$', '^$') - assert ret == {'pre_commit/main.py', 'pre_commit/git.py'} - - -def test_does_search_instead_of_match(get_files_matching_func): - ret = get_files_matching_func('\\.yaml$', '^$') - assert ret == {'.pre-commit-hooks.yaml'} - - -def test_does_not_include_deleted_fileS(get_files_matching_func): - ret = get_files_matching_func('exist.py', '^$') - assert ret == set() - - -def test_exclude_removes_files(get_files_matching_func): - ret = get_files_matching_func('', '\\.py$') - assert ret == {'.pre-commit-hooks.yaml', 'testing/test_symlink'} - - def resolve_conflict(): with open('conflict_file', 'w') as conflicted_file: conflicted_file.write('herp\nderp\n') @@ -130,8 +67,7 @@ def test_get_conflicted_files_in_submodule(in_conflicting_submodule): def test_get_conflicted_files_unstaged_files(in_merge_conflict): - # If they for whatever reason did pre-commit run --no-stash during a - # conflict + """This case no longer occurs, but it is a useful test nonetheless""" resolve_conflict() # Make unstaged file. @@ -156,3 +92,97 @@ def test_get_conflicted_files_unstaged_files(in_merge_conflict): def test_parse_merge_msg_for_conflicts(input, expected_output): ret = git.parse_merge_msg_for_conflicts(input) assert ret == expected_output + + +def test_get_changed_files(in_git_dir): + git_commit() + in_git_dir.join('a.txt').ensure() + in_git_dir.join('b.txt').ensure() + cmd_output('git', 'add', '.') + git_commit() + files = git.get_changed_files('HEAD^', 'HEAD') + assert files == ['a.txt', 'b.txt'] + + # files changed in source but not in origin should not be returned + files = git.get_changed_files('HEAD', 'HEAD^') + assert files == [] + + +@pytest.mark.parametrize( + ('s', 'expected'), + ( + ('foo\0bar\0', ['foo', 'bar']), + ('foo\0', ['foo']), + ('', []), + ('foo', ['foo']), + ), +) +def test_zsplit(s, expected): + assert git.zsplit(s) == expected + + +@pytest.fixture +def non_ascii_repo(in_git_dir): + git_commit() + in_git_dir.join('ΠΈΠ½Ρ‚Π΅Ρ€Π²ΡŒΡŽ').ensure() + cmd_output('git', 'add', '.') + git_commit() + yield in_git_dir + + +def test_all_files_non_ascii(non_ascii_repo): + ret = git.get_all_files() + assert ret == ['ΠΈΠ½Ρ‚Π΅Ρ€Π²ΡŒΡŽ'] + + +def test_staged_files_non_ascii(non_ascii_repo): + non_ascii_repo.join('ΠΈΠ½Ρ‚Π΅Ρ€Π²ΡŒΡŽ').write('hi') + cmd_output('git', 'add', '.') + assert git.get_staged_files() == ['ΠΈΠ½Ρ‚Π΅Ρ€Π²ΡŒΡŽ'] + + +def test_changed_files_non_ascii(non_ascii_repo): + ret = git.get_changed_files('HEAD^', 'HEAD') + assert ret == ['ΠΈΠ½Ρ‚Π΅Ρ€Π²ΡŒΡŽ'] + + +def test_get_conflicted_files_non_ascii(in_merge_conflict): + open('ΠΈΠ½Ρ‚Π΅Ρ€Π²ΡŒΡŽ', 'a').close() + cmd_output('git', 'add', '.') + ret = git.get_conflicted_files() + assert ret == {'conflict_file', 'ΠΈΠ½Ρ‚Π΅Ρ€Π²ΡŒΡŽ'} + + +def test_intent_to_add(in_git_dir): + in_git_dir.join('a').ensure() + cmd_output('git', 'add', '--intent-to-add', 'a') + + assert git.intent_to_add_files() == ['a'] + + +def test_status_output_with_rename(in_git_dir): + in_git_dir.join('a').write('1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n') + cmd_output('git', 'add', 'a') + git_commit() + cmd_output('git', 'mv', 'a', 'b') + in_git_dir.join('c').ensure() + cmd_output('git', 'add', '--intent-to-add', 'c') + + assert git.intent_to_add_files() == ['c'] + + +def test_no_git_env(): + env = { + 'http_proxy': 'http://myproxy:80', + 'GIT_EXEC_PATH': '/some/git/exec/path', + 'GIT_SSH': '/usr/bin/ssh', + 'GIT_SSH_COMMAND': 'ssh -o', + 'GIT_DIR': '/none/shall/pass', + } + no_git_env = git.no_git_env(env) + assert no_git_env == { + 'http_proxy': 'http://myproxy:80', + 'GIT_EXEC_PATH': '/some/git/exec/path', + 'GIT_SSH': '/usr/bin/ssh', + 'GIT_SSH_COMMAND': 'ssh -o', + } diff --git a/tests/languages/all_test.py b/tests/languages/all_test.py deleted file mode 100644 index 73b89cb5d..000000000 --- a/tests/languages/all_test.py +++ /dev/null @@ -1,35 +0,0 @@ -from __future__ import unicode_literals - -import inspect - -import pytest - -from pre_commit.languages.all import all_languages -from pre_commit.languages.all import languages - - -@pytest.mark.parametrize('language', all_languages) -def test_install_environment_argspec(language): - expected_argspec = inspect.ArgSpec( - args=['repo_cmd_runner', 'version', 'additional_dependencies'], - varargs=None, - keywords=None, - defaults=('default', ()), - ) - argspec = inspect.getargspec(languages[language].install_environment) - assert argspec == expected_argspec - - -@pytest.mark.parametrize('language', all_languages) -def test_ENVIRONMENT_DIR(language): - assert hasattr(languages[language], 'ENVIRONMENT_DIR') - - -@pytest.mark.parametrize('language', all_languages) -def test_run_hook_argpsec(language): - expected_argspec = inspect.ArgSpec( - args=['repo_cmd_runner', 'hook', 'file_args'], - varargs=None, keywords=None, defaults=None, - ) - argspec = inspect.getargspec(languages[language].run_hook) - assert argspec == expected_argspec diff --git a/tests/languages/docker_test.py b/tests/languages/docker_test.py index 6ca2ed5cc..171a3f732 100644 --- a/tests/languages/docker_test.py +++ b/tests/languages/docker_test.py @@ -1,7 +1,4 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - -import mock +from unittest import mock from pre_commit.languages import docker from pre_commit.util import CalledProcessError @@ -9,7 +6,18 @@ def test_docker_is_running_process_error(): with mock.patch( - 'pre_commit.languages.docker.cmd_output', - side_effect=CalledProcessError(*(None,) * 4) + 'pre_commit.languages.docker.cmd_output_b', + side_effect=CalledProcessError(1, (), 0, b'', None), ): assert docker.docker_is_running() is False + + +def test_docker_fallback_user(): + def invalid_attribute(): + raise AttributeError + with mock.patch.multiple( + 'os', create=True, + getuid=invalid_attribute, + getgid=invalid_attribute, + ): + assert docker.get_docker_user() == '1000:1000' diff --git a/tests/languages/golang_test.py b/tests/languages/golang_test.py index e0c9ab429..9a64ed195 100644 --- a/tests/languages/golang_test.py +++ b/tests/languages/golang_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import pytest from pre_commit.languages.golang import guess_go_dir @@ -10,6 +7,7 @@ ('url', 'expected'), ( ('/im/a/path/on/disk', 'unknown_src_dir'), + ('file:///im/a/path/on/disk', 'unknown_src_dir'), ('git@github.com:golang/lint', 'github.com/golang/lint'), ('git://github.com/golang/lint', 'github.com/golang/lint'), ('http://github.com/golang/lint', 'github.com/golang/lint'), diff --git a/tests/languages/helpers_test.py b/tests/languages/helpers_test.py new file mode 100644 index 000000000..c52e947be --- /dev/null +++ b/tests/languages/helpers_test.py @@ -0,0 +1,82 @@ +import multiprocessing +import os +import sys +from unittest import mock + +import pytest + +import pre_commit.constants as C +from pre_commit.languages import helpers +from pre_commit.prefix import Prefix +from pre_commit.util import CalledProcessError +from testing.auto_namedtuple import auto_namedtuple + + +def test_basic_get_default_version(): + assert helpers.basic_get_default_version() == C.DEFAULT + + +def test_basic_healthy(): + assert helpers.basic_healthy(Prefix('.'), 'default') is True + + +def test_failed_setup_command_does_not_unicode_error(): + script = ( + 'import sys\n' + "getattr(sys.stderr, 'buffer', sys.stderr).write(b'\\x81\\xfe')\n" + 'exit(1)\n' + ) + + # an assertion that this does not raise `UnicodeError` + with pytest.raises(CalledProcessError): + helpers.run_setup_cmd(Prefix('.'), (sys.executable, '-c', script)) + + +def test_assert_no_additional_deps(): + with pytest.raises(AssertionError) as excinfo: + helpers.assert_no_additional_deps('lang', ['hmmm']) + msg, = excinfo.value.args + assert msg == ( + 'For now, pre-commit does not support additional_dependencies for lang' + ) + + +SERIAL_FALSE = auto_namedtuple(require_serial=False) +SERIAL_TRUE = auto_namedtuple(require_serial=True) + + +def test_target_concurrency_normal(): + with mock.patch.object(multiprocessing, 'cpu_count', return_value=123): + with mock.patch.dict(os.environ, {}, clear=True): + assert helpers.target_concurrency(SERIAL_FALSE) == 123 + + +def test_target_concurrency_cpu_count_require_serial_true(): + with mock.patch.dict(os.environ, {}, clear=True): + assert helpers.target_concurrency(SERIAL_TRUE) == 1 + + +def test_target_concurrency_testing_env_var(): + with mock.patch.dict( + os.environ, {'PRE_COMMIT_NO_CONCURRENCY': '1'}, clear=True, + ): + assert helpers.target_concurrency(SERIAL_FALSE) == 1 + + +def test_target_concurrency_on_travis(): + with mock.patch.dict(os.environ, {'TRAVIS': '1'}, clear=True): + assert helpers.target_concurrency(SERIAL_FALSE) == 2 + + +def test_target_concurrency_cpu_count_not_implemented(): + with mock.patch.object( + multiprocessing, 'cpu_count', side_effect=NotImplementedError, + ): + with mock.patch.dict(os.environ, {}, clear=True): + assert helpers.target_concurrency(SERIAL_FALSE) == 1 + + +def test_shuffled_is_deterministic(): + seq = [str(i) for i in range(10)] + expected = ['3', '7', '8', '2', '4', '6', '5', '1', '0', '9'] + assert helpers._shuffled(seq) == expected diff --git a/tests/languages/pygrep_test.py b/tests/languages/pygrep_test.py new file mode 100644 index 000000000..cabea22ec --- /dev/null +++ b/tests/languages/pygrep_test.py @@ -0,0 +1,65 @@ +import pytest + +from pre_commit.languages import pygrep + + +@pytest.fixture +def some_files(tmpdir): + tmpdir.join('f1').write_binary(b'foo\nbar\n') + tmpdir.join('f2').write_binary(b'[INFO] hi\n') + tmpdir.join('f3').write_binary(b"with'quotes\n") + with tmpdir.as_cwd(): + yield + + +@pytest.mark.usefixtures('some_files') +@pytest.mark.parametrize( + ('pattern', 'expected_retcode', 'expected_out'), + ( + ('baz', 0, ''), + ('foo', 1, 'f1:1:foo\n'), + ('bar', 1, 'f1:2:bar\n'), + (r'(?i)\[info\]', 1, 'f2:1:[INFO] hi\n'), + ("h'q", 1, "f3:1:with'quotes\n"), + ), +) +def test_main(some_files, cap_out, pattern, expected_retcode, expected_out): + ret = pygrep.main((pattern, 'f1', 'f2', 'f3')) + out = cap_out.get() + assert ret == expected_retcode + assert out == expected_out + + +def test_ignore_case(some_files, cap_out): + ret = pygrep.main(('--ignore-case', 'info', 'f1', 'f2', 'f3')) + out = cap_out.get() + assert ret == 1 + assert out == 'f2:1:[INFO] hi\n' + + +def test_multiline(some_files, cap_out): + ret = pygrep.main(('--multiline', r'foo\nbar', 'f1', 'f2', 'f3')) + out = cap_out.get() + assert ret == 1 + assert out == 'f1:1:foo\nbar\n' + + +def test_multiline_line_number(some_files, cap_out): + ret = pygrep.main(('--multiline', r'ar', 'f1', 'f2', 'f3')) + out = cap_out.get() + assert ret == 1 + assert out == 'f1:2:bar\n' + + +def test_multiline_dotall_flag_is_enabled(some_files, cap_out): + ret = pygrep.main(('--multiline', r'o.*bar', 'f1', 'f2', 'f3')) + out = cap_out.get() + assert ret == 1 + assert out == 'f1:1:foo\nbar\n' + + +def test_multiline_multiline_flag_is_enabled(some_files, cap_out): + ret = pygrep.main(('--multiline', r'foo$.*bar', 'f1', 'f2', 'f3')) + out = cap_out.get() + assert ret == 1 + assert out == 'f1:1:foo\nbar\n' diff --git a/tests/languages/python_test.py b/tests/languages/python_test.py index 78211cb9a..34c6c7fc5 100644 --- a/tests/languages/python_test.py +++ b/tests/languages/python_test.py @@ -1,18 +1,75 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import os.path +import sys +from unittest import mock + +import pytest +import pre_commit.constants as C from pre_commit.languages import python +from pre_commit.prefix import Prefix def test_norm_version_expanduser(): home = os.path.expanduser('~') - if os.name == 'nt': # pragma: no cover (nt) + if os.name == 'nt': # pragma: nt cover path = r'~\python343' - expected_path = r'{}\python343'.format(home) - else: # pragma: no cover (non-nt) + expected_path = fr'{home}\python343' + else: # pragma: nt no cover path = '~/.pyenv/versions/3.4.3/bin/python' - expected_path = home + '/.pyenv/versions/3.4.3/bin/python' + expected_path = f'{home}/.pyenv/versions/3.4.3/bin/python' result = python.norm_version(path) assert result == expected_path + + +@pytest.mark.parametrize('v', ('python3.6', 'python3', 'python')) +def test_sys_executable_matches(v): + with mock.patch.object(sys, 'version_info', (3, 6, 7)): + assert python._sys_executable_matches(v) + + +@pytest.mark.parametrize('v', ('notpython', 'python3.x')) +def test_sys_executable_matches_does_not_match(v): + with mock.patch.object(sys, 'version_info', (3, 6, 7)): + assert not python._sys_executable_matches(v) + + +@pytest.mark.parametrize( + ('exe', 'realpath', 'expected'), ( + ('/usr/bin/python3', '/usr/bin/python3.7', 'python3'), + ('/usr/bin/python', '/usr/bin/python3.7', 'python3.7'), + ('/usr/bin/python', '/usr/bin/python', None), + ('/usr/bin/python3.6m', '/usr/bin/python3.6m', 'python3.6m'), + ('v/bin/python', 'v/bin/pypy', 'pypy'), + ), +) +def test_find_by_sys_executable(exe, realpath, expected): + with mock.patch.object(sys, 'executable', exe): + with mock.patch.object(os.path, 'realpath', return_value=realpath): + with mock.patch.object(python, 'find_executable', lambda x: x): + assert python._find_by_sys_executable() == expected + + +def test_healthy_types_py_in_cwd(tmpdir): + with tmpdir.as_cwd(): + prefix = tmpdir.join('prefix').ensure_dir() + prefix.join('setup.py').write('import setuptools; setuptools.setup()') + prefix = Prefix(str(prefix)) + python.install_environment(prefix, C.DEFAULT, ()) + + # even if a `types.py` file exists, should still be healthy + tmpdir.join('types.py').ensure() + assert python.healthy(prefix, C.DEFAULT) is True + + +def test_healthy_python_goes_missing(tmpdir): + with tmpdir.as_cwd(): + prefix = tmpdir.join('prefix').ensure_dir() + prefix.join('setup.py').write('import setuptools; setuptools.setup()') + prefix = Prefix(str(prefix)) + python.install_environment(prefix, C.DEFAULT, ()) + + exe_name = 'python' if sys.platform != 'win32' else 'python.exe' + py_exe = prefix.path(python.bin_dir('py_env-default'), exe_name) + os.remove(py_exe) + + assert python.healthy(prefix, C.DEFAULT) is False diff --git a/tests/languages/ruby_test.py b/tests/languages/ruby_test.py index 1eddea1d5..36a029d17 100644 --- a/tests/languages/ruby_test.py +++ b/tests/languages/ruby_test.py @@ -1,39 +1,28 @@ -from __future__ import unicode_literals - import os.path -from pre_commit.languages.ruby import _install_rbenv +from pre_commit.languages import ruby +from pre_commit.prefix import Prefix +from pre_commit.util import cmd_output from testing.util import xfailif_windows_no_ruby @xfailif_windows_no_ruby -def test_install_rbenv(cmd_runner): - _install_rbenv(cmd_runner) +def test_install_rbenv(tempdir_factory): + prefix = Prefix(tempdir_factory.get()) + ruby._install_rbenv(prefix) # Should have created rbenv directory - assert os.path.exists(cmd_runner.path('rbenv-default')) - # We should have created our `activate` script - activate_path = cmd_runner.path('rbenv-default', 'bin', 'activate') - assert os.path.exists(activate_path) + assert os.path.exists(prefix.path('rbenv-default')) # Should be able to activate using our script and access rbenv - cmd_runner.run( - [ - 'bash', - '-c', - ". '{prefix}rbenv-default/bin/activate' && rbenv --help", - ], - ) + with ruby.in_env(prefix, 'default'): + cmd_output('rbenv', '--help') @xfailif_windows_no_ruby -def test_install_rbenv_with_version(cmd_runner): - _install_rbenv(cmd_runner, version='1.9.3p547') +def test_install_rbenv_with_version(tempdir_factory): + prefix = Prefix(tempdir_factory.get()) + ruby._install_rbenv(prefix, version='1.9.3p547') # Should be able to activate and use rbenv install - cmd_runner.run( - [ - 'bash', - '-c', - ". '{prefix}rbenv-1.9.3p547/bin/activate' && rbenv install --help", - ], - ) + with ruby.in_env(prefix, '1.9.3p547'): + cmd_output('rbenv', 'install', '--help') diff --git a/tests/logging_handler_test.py b/tests/logging_handler_test.py index 0e72541a2..fe68593b9 100644 --- a/tests/logging_handler_test.py +++ b/tests/logging_handler_test.py @@ -1,27 +1,21 @@ -from __future__ import unicode_literals +import logging from pre_commit import color from pre_commit.logging_handler import LoggingHandler -class FakeLogRecord(object): - def __init__(self, message, levelname, levelno): - self.message = message - self.levelname = levelname - self.levelno = levelno - - def getMessage(self): - return self.message +def _log_record(message, level): + return logging.LogRecord('name', level, '', 1, message, {}, None) def test_logging_handler_color(cap_out): handler = LoggingHandler(True) - handler.emit(FakeLogRecord('hi', 'WARNING', 30)) + handler.emit(_log_record('hi', logging.WARNING)) ret = cap_out.get() - assert ret == color.YELLOW + '[WARNING]' + color.NORMAL + ' hi\n' + assert ret == f'{color.YELLOW}[WARNING]{color.NORMAL} hi\n' def test_logging_handler_no_color(cap_out): handler = LoggingHandler(False) - handler.emit(FakeLogRecord('hi', 'WARNING', 30)) + handler.emit(_log_record('hi', logging.WARNING)) assert cap_out.get() == '[WARNING] hi\n' diff --git a/tests/main_test.py b/tests/main_test.py index 0425b8d27..c4724768c 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -1,25 +1,93 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import argparse +import os.path +from unittest import mock -import mock import pytest +import pre_commit.constants as C from pre_commit import main -from pre_commit.error_handler import PreCommitSystemExit -from pre_commit.util import cwd +from pre_commit.error_handler import FatalError from testing.auto_namedtuple import auto_namedtuple +@pytest.mark.parametrize( + ('argv', 'expected'), + ( + ((), ['f']), + (('--f', 'x'), ['x']), + (('--f', 'x', '--f', 'y'), ['x', 'y']), + ), +) +def test_append_replace_default(argv, expected): + parser = argparse.ArgumentParser() + parser.add_argument('--f', action=main.AppendReplaceDefault, default=['f']) + assert parser.parse_args(argv).f == expected + + +def _args(**kwargs): + kwargs.setdefault('command', 'help') + kwargs.setdefault('config', C.CONFIG_FILE) + return argparse.Namespace(**kwargs) + + +def test_adjust_args_and_chdir_not_in_git_dir(in_tmpdir): + with pytest.raises(FatalError): + main._adjust_args_and_chdir(_args()) + + +def test_adjust_args_and_chdir_in_dot_git_dir(in_git_dir): + with in_git_dir.join('.git').as_cwd(), pytest.raises(FatalError): + main._adjust_args_and_chdir(_args()) + + +def test_adjust_args_and_chdir_noop(in_git_dir): + args = _args(command='run', files=['f1', 'f2']) + main._adjust_args_and_chdir(args) + assert os.getcwd() == in_git_dir + assert args.config == C.CONFIG_FILE + assert args.files == ['f1', 'f2'] + + +def test_adjust_args_and_chdir_relative_things(in_git_dir): + in_git_dir.join('foo/cfg.yaml').ensure() + in_git_dir.join('foo').chdir() + + args = _args(command='run', files=['f1', 'f2'], config='cfg.yaml') + main._adjust_args_and_chdir(args) + assert os.getcwd() == in_git_dir + assert args.config == os.path.join('foo', 'cfg.yaml') + assert args.files == [os.path.join('foo', 'f1'), os.path.join('foo', 'f2')] + + +def test_adjust_args_and_chdir_non_relative_config(in_git_dir): + in_git_dir.join('foo').ensure_dir().chdir() + + args = _args() + main._adjust_args_and_chdir(args) + assert os.getcwd() == in_git_dir + assert args.config == C.CONFIG_FILE + + +def test_adjust_args_try_repo_repo_relative(in_git_dir): + in_git_dir.join('foo').ensure_dir().chdir() + + args = _args(command='try-repo', repo='../foo', files=[]) + assert args.repo is not None + assert os.path.exists(args.repo) + main._adjust_args_and_chdir(args) + assert os.getcwd() == in_git_dir + assert os.path.exists(args.repo) + assert args.repo == 'foo' + + FNS = ( - 'autoupdate', 'clean', 'install', 'install_hooks', 'run', 'sample_config', - 'uninstall', + 'autoupdate', 'clean', 'gc', 'hook_impl', 'install', 'install_hooks', + 'migrate_config', 'run', 'sample_config', 'uninstall', ) CMDS = tuple(fn.replace('_', '-') for fn in FNS) -@pytest.yield_fixture +@pytest.fixture def mock_commands(): mcks = {fn: mock.patch.object(main, fn).start() for fn in FNS} ret = auto_namedtuple(**mcks) @@ -28,19 +96,7 @@ def mock_commands(): mck.stop() -class CalledExit(Exception): - pass - - -@pytest.yield_fixture -def argparse_exit_mock(): - with mock.patch.object( - argparse.ArgumentParser, 'exit', side_effect=CalledExit, - ) as exit_mock: - yield exit_mock - - -@pytest.yield_fixture +@pytest.fixture def argparse_parse_args_spy(): parse_args_mock = mock.Mock() @@ -62,15 +118,13 @@ def assert_only_one_mock_called(mock_objs): assert total_call_count == 1 -def test_overall_help(mock_commands, argparse_exit_mock): - with pytest.raises(CalledExit): +def test_overall_help(mock_commands): + with pytest.raises(SystemExit): main.main(['--help']) -def test_help_command( - mock_commands, argparse_exit_mock, argparse_parse_args_spy, -): - with pytest.raises(CalledExit): +def test_help_command(mock_commands, argparse_parse_args_spy): + with pytest.raises(SystemExit): main.main(['help']) argparse_parse_args_spy.assert_has_calls([ @@ -79,10 +133,8 @@ def test_help_command( ]) -def test_help_other_command( - mock_commands, argparse_exit_mock, argparse_parse_args_spy, -): - with pytest.raises(CalledExit): +def test_help_other_command(mock_commands, argparse_parse_args_spy): + with pytest.raises(SystemExit): main.main(['help', 'run']) argparse_parse_args_spy.assert_has_calls([ @@ -92,23 +144,31 @@ def test_help_other_command( @pytest.mark.parametrize('command', CMDS) -def test_install_command(command, mock_commands): +def test_all_cmds(command, mock_commands, mock_store_dir): main.main((command,)) assert getattr(mock_commands, command.replace('-', '_')).call_count == 1 assert_only_one_mock_called(mock_commands) +def test_try_repo(mock_store_dir): + with mock.patch.object(main, 'try_repo') as patch: + main.main(('try-repo', '.')) + assert patch.call_count == 1 + + +def test_init_templatedir(mock_store_dir): + with mock.patch.object(main, 'init_templatedir') as patch: + main.main(('init-templatedir', 'tdir')) + assert patch.call_count == 1 + + def test_help_cmd_in_empty_directory( + in_tmpdir, mock_commands, - tempdir_factory, - argparse_exit_mock, argparse_parse_args_spy, ): - path = tempdir_factory.get() - - with cwd(path): - with pytest.raises(CalledExit): - main.main(['help', 'run']) + with pytest.raises(SystemExit): + main.main(['help', 'run']) argparse_parse_args_spy.assert_has_calls([ mock.call(['help', 'run']), @@ -116,19 +176,14 @@ def test_help_cmd_in_empty_directory( ]) -def test_expected_fatal_error_no_git_repo( - tempdir_factory, cap_out, mock_out_store_directory, -): - with cwd(tempdir_factory.get()): - with pytest.raises(PreCommitSystemExit): - main.main([]) - assert cap_out.get() == ( +def test_expected_fatal_error_no_git_repo(in_tmpdir, cap_out, mock_store_dir): + with pytest.raises(SystemExit): + main.main([]) + log_file = os.path.join(mock_store_dir, 'pre-commit.log') + cap_out_lines = cap_out.get().splitlines() + assert ( + cap_out_lines[-2] == 'An error has occurred: FatalError: git failed. ' - 'Is it installed, and are you in a Git repository directory?\n' - 'Check the log at ~/.pre-commit/pre-commit.log\n' + 'Is it installed, and are you in a Git repository directory?' ) - - -def test_warning_on_tags_only(mock_commands, cap_out): - main.main(('autoupdate', '--tags-only')) - assert '--tags-only is the default' in cap_out.get() + assert cap_out_lines[-1] == f'Check the log at {log_file}' diff --git a/tests/make_archives_test.py b/tests/make_archives_test.py index f3636b535..6ae2f8e74 100644 --- a/tests/make_archives_test.py +++ b/tests/make_archives_test.py @@ -1,64 +1,46 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - -import os.path import tarfile -import mock -import pytest - +from pre_commit import git from pre_commit import make_archives from pre_commit.util import cmd_output -from pre_commit.util import cwd -from testing.fixtures import git_dir -from testing.util import get_head_sha -from testing.util import skipif_slowtests_false +from testing.util import git_commit -def test_make_archive(tempdir_factory): - output_dir = tempdir_factory.get() - git_path = git_dir(tempdir_factory) +def test_make_archive(in_git_dir, tmpdir): + output_dir = tmpdir.join('output').ensure_dir() # Add a files to the git directory - with cwd(git_path): - cmd_output('touch', 'foo') - cmd_output('git', 'add', '.') - cmd_output('git', 'commit', '-m', 'foo') - # We'll use this sha - head_sha = get_head_sha('.') - # And check that this file doesn't exist - cmd_output('touch', 'bar') - cmd_output('git', 'add', '.') - cmd_output('git', 'commit', '-m', 'bar') + in_git_dir.join('foo').ensure() + cmd_output('git', 'add', '.') + git_commit() + # We'll use this rev + head_rev = git.head_rev('.') + # And check that this file doesn't exist + in_git_dir.join('bar').ensure() + cmd_output('git', 'add', '.') + git_commit() # Do the thing archive_path = make_archives.make_archive( - 'foo', git_path, head_sha, output_dir, + 'foo', in_git_dir.strpath, head_rev, output_dir.strpath, ) - assert archive_path == os.path.join(output_dir, 'foo.tar.gz') - assert os.path.exists(archive_path) + expected = output_dir.join('foo.tar.gz') + assert archive_path == expected.strpath + assert expected.exists() - extract_dir = tempdir_factory.get() - - # Extract the tar + extract_dir = tmpdir.join('extract').ensure_dir() with tarfile.open(archive_path) as tf: - tf.extractall(extract_dir) + tf.extractall(extract_dir.strpath) # Verify the contents of the tar - assert os.path.exists(os.path.join(extract_dir, 'foo')) - assert os.path.exists(os.path.join(extract_dir, 'foo', 'foo')) - assert not os.path.exists(os.path.join(extract_dir, 'foo', '.git')) - assert not os.path.exists(os.path.join(extract_dir, 'foo', 'bar')) - + assert extract_dir.join('foo').isdir() + assert extract_dir.join('foo/foo').exists() + assert not extract_dir.join('foo/.git').exists() + assert not extract_dir.join('foo/bar').exists() -@skipif_slowtests_false -@pytest.mark.integration -def test_main(tempdir_factory): - path = tempdir_factory.get() - # Don't actually want to make these in the current repo - with mock.patch.object(make_archives, 'RESOURCES_DIR', path): - make_archives.main() +def test_main(tmpdir): + make_archives.main(('--dest', tmpdir.strpath)) for archive, _, _ in make_archives.REPOS: - assert os.path.exists(os.path.join(path, archive + '.tar.gz')) + assert tmpdir.join(f'{archive}.tar.gz').exists() diff --git a/tests/manifest_test.py b/tests/manifest_test.py deleted file mode 100644 index 7db886c50..000000000 --- a/tests/manifest_test.py +++ /dev/null @@ -1,78 +0,0 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - -import pytest - -from pre_commit.manifest import Manifest -from testing.fixtures import make_repo -from testing.util import get_head_sha - - -@pytest.yield_fixture -def manifest(store, tempdir_factory): - path = make_repo(tempdir_factory, 'script_hooks_repo') - head_sha = get_head_sha(path) - repo_path = store.clone(path, head_sha) - yield Manifest(repo_path, path) - - -def test_manifest_contents(manifest): - # Should just retrieve the manifest contents - assert manifest.manifest_contents == [{ - 'always_run': False, - 'additional_dependencies': [], - 'args': [], - 'description': '', - 'entry': 'bin/hook.sh', - 'exclude': '^$', - 'files': '', - 'id': 'bash_hook', - 'language': 'script', - 'language_version': 'default', - 'log_file': '', - 'minimum_pre_commit_version': '0', - 'name': 'Bash hook', - 'pass_filenames': True, - 'stages': [], - 'types': ['file'], - 'exclude_types': [], - }] - - -def test_hooks(manifest): - assert manifest.hooks['bash_hook'] == { - 'always_run': False, - 'additional_dependencies': [], - 'args': [], - 'description': '', - 'entry': 'bin/hook.sh', - 'exclude': '^$', - 'files': '', - 'id': 'bash_hook', - 'language': 'script', - 'language_version': 'default', - 'log_file': '', - 'minimum_pre_commit_version': '0', - 'name': 'Bash hook', - 'pass_filenames': True, - 'stages': [], - 'types': ['file'], - 'exclude_types': [], - } - - -def test_legacy_manifest_warn(store, tempdir_factory, log_warning_mock): - path = make_repo(tempdir_factory, 'legacy_hooks_yaml_repo') - head_sha = get_head_sha(path) - repo_path = store.clone(path, head_sha) - Manifest(repo_path, path).manifest_contents - - # Should have printed a warning - assert log_warning_mock.call_args_list[0][0][0] == ( - '{} uses legacy hooks.yaml to provide hooks.\n' - 'In newer versions, this file is called .pre-commit-hooks.yaml\n' - 'This will work in this version of pre-commit but will be removed at ' - 'a later time.\n' - 'If `pre-commit autoupdate` does not silence this warning consider ' - 'making an issue / pull request.'.format(path) - ) diff --git a/testing/resources/python_hooks_repo/foo/__init__.py b/tests/meta_hooks/__init__.py similarity index 100% rename from testing/resources/python_hooks_repo/foo/__init__.py rename to tests/meta_hooks/__init__.py diff --git a/tests/meta_hooks/check_hooks_apply_test.py b/tests/meta_hooks/check_hooks_apply_test.py new file mode 100644 index 000000000..06bdd0455 --- /dev/null +++ b/tests/meta_hooks/check_hooks_apply_test.py @@ -0,0 +1,138 @@ +from pre_commit.meta_hooks import check_hooks_apply +from testing.fixtures import add_config_to_repo + + +def test_hook_excludes_everything(capsys, in_git_dir, mock_store_dir): + config = { + 'repos': [ + { + 'repo': 'meta', + 'hooks': [ + { + 'id': 'check-useless-excludes', + 'exclude': '.pre-commit-config.yaml', + }, + ], + }, + ], + } + + add_config_to_repo(in_git_dir.strpath, config) + + assert check_hooks_apply.main(()) == 1 + + out, _ = capsys.readouterr() + assert 'check-useless-excludes does not apply to this repository' in out + + +def test_hook_includes_nothing(capsys, in_git_dir, mock_store_dir): + config = { + 'repos': [ + { + 'repo': 'meta', + 'hooks': [ + { + 'id': 'check-useless-excludes', + 'files': 'foo', + }, + ], + }, + ], + } + + add_config_to_repo(in_git_dir.strpath, config) + + assert check_hooks_apply.main(()) == 1 + + out, _ = capsys.readouterr() + assert 'check-useless-excludes does not apply to this repository' in out + + +def test_hook_types_not_matched(capsys, in_git_dir, mock_store_dir): + config = { + 'repos': [ + { + 'repo': 'meta', + 'hooks': [ + { + 'id': 'check-useless-excludes', + 'types': ['python'], + }, + ], + }, + ], + } + + add_config_to_repo(in_git_dir.strpath, config) + + assert check_hooks_apply.main(()) == 1 + + out, _ = capsys.readouterr() + assert 'check-useless-excludes does not apply to this repository' in out + + +def test_hook_types_excludes_everything(capsys, in_git_dir, mock_store_dir): + config = { + 'repos': [ + { + 'repo': 'meta', + 'hooks': [ + { + 'id': 'check-useless-excludes', + 'exclude_types': ['yaml'], + }, + ], + }, + ], + } + + add_config_to_repo(in_git_dir.strpath, config) + + assert check_hooks_apply.main(()) == 1 + + out, _ = capsys.readouterr() + assert 'check-useless-excludes does not apply to this repository' in out + + +def test_valid_exceptions(capsys, in_git_dir, mock_store_dir): + config = { + 'repos': [ + { + 'repo': 'local', + 'hooks': [ + # applies to a file + { + 'id': 'check-yaml', + 'name': 'check yaml', + 'entry': './check-yaml', + 'language': 'script', + 'files': r'\.yaml$', + }, + # Should not be reported as an error due to language: fail + { + 'id': 'changelogs-rst', + 'name': 'changelogs must be rst', + 'entry': 'changelog filenames must end in .rst', + 'language': 'fail', + 'files': r'changelog/.*(? str: + exe = shutil.which('echo') + assert exe is not None + return exe + + def test_file_doesnt_exist(): assert parse_shebang.parse_filename('herp derp derp') == () def test_simple_case(tmpdir): x = tmpdir.join('f') - x.write_text('#!/usr/bin/env python', encoding='UTF-8') + x.write('#!/usr/bin/env echo') make_executable(x.strpath) - assert parse_shebang.parse_filename(x.strpath) == ('python',) + assert parse_shebang.parse_filename(x.strpath) == ('echo',) def test_find_executable_full_path(): @@ -31,8 +33,7 @@ def test_find_executable_full_path(): def test_find_executable_on_path(): - expected = distutils.spawn.find_executable('echo') - assert parse_shebang.find_executable('echo') == expected + assert parse_shebang.find_executable('echo') == _echo_exe() def test_find_executable_not_found_none(): @@ -42,8 +43,8 @@ def test_find_executable_not_found_none(): def write_executable(shebang, filename='run'): os.mkdir('bin') path = os.path.join('bin', filename) - with io.open(path, 'w') as f: - f.write('#!{}'.format(shebang)) + with open(path, 'w') as f: + f.write(f'#!{shebang}') make_executable(path) return path @@ -66,9 +67,9 @@ def test_find_executable_path_ext(in_tmpdir): """Windows exports PATHEXT as a list of extensions to automatically add to executables when doing PATH searching. """ - exe_path = os.path.abspath(write_executable( - '/usr/bin/env sh', filename='run.myext', - )) + exe_path = os.path.abspath( + write_executable('/usr/bin/env sh', filename='run.myext'), + ) env_path = {'PATH': os.path.dirname(exe_path)} env_path_ext = dict(env_path, PATHEXT=os.pathsep.join(('.exe', '.myext'))) assert parse_shebang.find_executable('run') is None @@ -85,44 +86,67 @@ def test_normexe_does_not_exist(): assert excinfo.value.args == ('Executable `i-dont-exist-lol` not found',) +def test_normexe_does_not_exist_sep(): + with pytest.raises(OSError) as excinfo: + parse_shebang.normexe('./i-dont-exist-lol') + assert excinfo.value.args == ('Executable `./i-dont-exist-lol` not found',) + + +@pytest.mark.xfail(os.name == 'nt', reason='posix only') +def test_normexe_not_executable(tmpdir): # pragma: win32 no cover + tmpdir.join('exe').ensure() + with tmpdir.as_cwd(), pytest.raises(OSError) as excinfo: + parse_shebang.normexe('./exe') + assert excinfo.value.args == ('Executable `./exe` is not executable',) + + +def test_normexe_is_a_directory(tmpdir): + with tmpdir.as_cwd(): + tmpdir.join('exe').ensure_dir() + exe = os.path.join('.', 'exe') + with pytest.raises(OSError) as excinfo: + parse_shebang.normexe(exe) + msg, = excinfo.value.args + assert msg == f'Executable `{exe}` is a directory' + + def test_normexe_already_full_path(): assert parse_shebang.normexe(sys.executable) == sys.executable def test_normexe_gives_full_path(): - expected = distutils.spawn.find_executable('echo') - assert parse_shebang.normexe('echo') == expected - assert os.sep in expected + assert parse_shebang.normexe('echo') == _echo_exe() + assert os.sep in _echo_exe() def test_normalize_cmd_trivial(): - cmd = (distutils.spawn.find_executable('echo'), 'hi') + cmd = (_echo_exe(), 'hi') assert parse_shebang.normalize_cmd(cmd) == cmd def test_normalize_cmd_PATH(): - cmd = ('python', '--version') - expected = (distutils.spawn.find_executable('python'), '--version') + cmd = ('echo', '--version') + expected = (_echo_exe(), '--version') assert parse_shebang.normalize_cmd(cmd) == expected def test_normalize_cmd_shebang(in_tmpdir): - python = distutils.spawn.find_executable('python') - path = write_executable(python.replace(os.sep, '/')) - assert parse_shebang.normalize_cmd((path,)) == (python, path) + echo = _echo_exe().replace(os.sep, '/') + path = write_executable(echo) + assert parse_shebang.normalize_cmd((path,)) == (echo, path) def test_normalize_cmd_PATH_shebang_full_path(in_tmpdir): - python = distutils.spawn.find_executable('python') - path = write_executable(python.replace(os.sep, '/')) + echo = _echo_exe().replace(os.sep, '/') + path = write_executable(echo) with bin_on_path(): ret = parse_shebang.normalize_cmd(('run',)) - assert ret == (python, os.path.abspath(path)) + assert ret == (echo, os.path.abspath(path)) def test_normalize_cmd_PATH_shebang_PATH(in_tmpdir): - python = distutils.spawn.find_executable('python') - path = write_executable('/usr/bin/env python') + echo = _echo_exe() + path = write_executable('/usr/bin/env echo') with bin_on_path(): ret = parse_shebang.normalize_cmd(('run',)) - assert ret == (python, os.path.abspath(path)) + assert ret == (echo, os.path.abspath(path)) diff --git a/tests/prefix_test.py b/tests/prefix_test.py new file mode 100644 index 000000000..6ce8be127 --- /dev/null +++ b/tests/prefix_test.py @@ -0,0 +1,44 @@ +import os.path + +import pytest + +from pre_commit.prefix import Prefix + + +def norm_slash(*args): + return tuple(x.replace('/', os.sep) for x in args) + + +@pytest.mark.parametrize( + ('prefix', 'path_end', 'expected_output'), + ( + norm_slash('foo', '', 'foo'), + norm_slash('foo', 'bar', 'foo/bar'), + norm_slash('foo/bar', '../baz', 'foo/baz'), + norm_slash('./', 'bar', 'bar'), + norm_slash('./', '', '.'), + norm_slash('/tmp/foo', '/tmp/bar', '/tmp/bar'), + ), +) +def test_path(prefix, path_end, expected_output): + instance = Prefix(prefix) + ret = instance.path(path_end) + assert ret == expected_output + + +def test_path_multiple_args(): + instance = Prefix('foo') + ret = instance.path('bar', 'baz') + assert ret == os.path.join('foo', 'bar', 'baz') + + +def test_exists(tmpdir): + assert not Prefix(str(tmpdir)).exists('foo') + tmpdir.ensure('foo') + assert Prefix(str(tmpdir)).exists('foo') + + +def test_star(tmpdir): + for f in ('a.txt', 'b.txt', 'c.py'): + tmpdir.join(f).ensure() + assert set(Prefix(str(tmpdir)).star('.txt')) == {'a.txt', 'b.txt'} diff --git a/tests/prefixed_command_runner_test.py b/tests/prefixed_command_runner_test.py deleted file mode 100644 index 132c2a86f..000000000 --- a/tests/prefixed_command_runner_test.py +++ /dev/null @@ -1,131 +0,0 @@ -from __future__ import unicode_literals - -import os -import subprocess - -import mock -import pytest - -from pre_commit.prefixed_command_runner import PrefixedCommandRunner -from pre_commit.util import CalledProcessError - - -def norm_slash(input_tup): - return tuple(x.replace('/', os.sep) for x in input_tup) - - -def test_CalledProcessError_str(): - error = CalledProcessError( - 1, [str('git'), str('status')], 0, (str('stdout'), str('stderr')), - ) - assert str(error) == ( - "Command: ['git', 'status']\n" - "Return code: 1\n" - "Expected return code: 0\n" - "Output: \n" - " stdout\n" - "Errors: \n" - " stderr\n" - ) - - -def test_CalledProcessError_str_nooutput(): - error = CalledProcessError( - 1, [str('git'), str('status')], 0, (str(''), str('')) - ) - assert str(error) == ( - "Command: ['git', 'status']\n" - "Return code: 1\n" - "Expected return code: 0\n" - "Output: (none)\n" - "Errors: (none)\n" - ) - - -@pytest.fixture -def popen_mock(): - popen = mock.Mock(spec=subprocess.Popen) - popen.return_value.communicate.return_value = (b'stdout', b'stderr') - return popen - - -@pytest.fixture -def makedirs_mock(): - return mock.Mock(spec=os.makedirs) - - -@pytest.mark.parametrize(('input', 'expected_prefix'), ( - norm_slash(('.', './')), - norm_slash(('foo', 'foo/')), - norm_slash(('bar/', 'bar/')), - norm_slash(('foo/bar', 'foo/bar/')), - norm_slash(('foo/bar/', 'foo/bar/')), -)) -def test_init_normalizes_path_endings(input, expected_prefix): - input = input.replace('/', os.sep) - expected_prefix = expected_prefix.replace('/', os.sep) - instance = PrefixedCommandRunner(input) - assert instance.prefix_dir == expected_prefix - - -def test_run_substitutes_prefix(popen_mock, makedirs_mock): - instance = PrefixedCommandRunner( - 'prefix', popen=popen_mock, makedirs=makedirs_mock, - ) - ret = instance.run(['{prefix}bar', 'baz'], retcode=None) - popen_mock.assert_called_once_with( - (str(os.path.join('prefix', 'bar')), str('baz')), - env=None, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - assert ret == (popen_mock.return_value.returncode, 'stdout', 'stderr') - - -PATH_TESTS = ( - norm_slash(('foo', '', 'foo')), - norm_slash(('foo', 'bar', 'foo/bar')), - norm_slash(('foo/bar', '../baz', 'foo/baz')), - norm_slash(('./', 'bar', 'bar')), - norm_slash(('./', '', '.')), - norm_slash(('/tmp/foo', '/tmp/bar', '/tmp/bar')), -) - - -@pytest.mark.parametrize(('prefix', 'path_end', 'expected_output'), PATH_TESTS) -def test_path(prefix, path_end, expected_output): - instance = PrefixedCommandRunner(prefix) - ret = instance.path(path_end) - assert ret == expected_output - - -def test_path_multiple_args(): - instance = PrefixedCommandRunner('foo') - ret = instance.path('bar', 'baz') - assert ret == os.path.join('foo', 'bar', 'baz') - - -def test_create_path_if_not_exists(in_tmpdir): - instance = PrefixedCommandRunner('foo') - assert not os.path.exists('foo') - instance._create_path_if_not_exists() - assert os.path.exists('foo') - - -def test_exists_does_not_exist(in_tmpdir): - assert not PrefixedCommandRunner('.').exists('foo') - - -def test_exists_does_exist(in_tmpdir): - os.mkdir('foo') - assert PrefixedCommandRunner('.').exists('foo') - - -def test_raises_on_error(popen_mock, makedirs_mock): - popen_mock.return_value.returncode = 1 - with pytest.raises(CalledProcessError): - instance = PrefixedCommandRunner( - '.', popen=popen_mock, makedirs=makedirs_mock, - ) - instance.run(['echo']) diff --git a/tests/repository_test.py b/tests/repository_test.py index f91642ee4..df7e7d1bc 100644 --- a/tests/repository_test.py +++ b/tests/repository_test.py @@ -1,41 +1,65 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - -import io import os.path import re import shutil +import sys +from typing import Any +from typing import Dict +from unittest import mock -import mock +import cfgv import pytest import pre_commit.constants as C -from pre_commit import five -from pre_commit import parse_shebang +from pre_commit.clientlib import CONFIG_SCHEMA from pre_commit.clientlib import load_manifest +from pre_commit.envcontext import envcontext +from pre_commit.hook import Hook from pre_commit.languages import golang from pre_commit.languages import helpers from pre_commit.languages import node -from pre_commit.languages import pcre from pre_commit.languages import python from pre_commit.languages import ruby -from pre_commit.repository import Repository +from pre_commit.languages import rust +from pre_commit.languages.all import languages +from pre_commit.prefix import Prefix +from pre_commit.repository import all_hooks +from pre_commit.repository import install_hook_envs from pre_commit.util import cmd_output -from pre_commit.util import cwd -from testing.fixtures import config_with_local_hooks -from testing.fixtures import git_dir +from pre_commit.util import cmd_output_b from testing.fixtures import make_config_from_repo from testing.fixtures import make_repo from testing.fixtures import modify_manifest +from testing.util import cwd from testing.util import get_resource_path from testing.util import skipif_cant_run_docker from testing.util import skipif_cant_run_swift -from testing.util import skipif_slowtests_false -from testing.util import xfailif_no_pcre_support -from testing.util import xfailif_windows_no_node +from testing.util import xfailif_no_venv from testing.util import xfailif_windows_no_ruby +def _norm_out(b): + return b.replace(b'\r\n', b'\n') + + +def _hook_run(hook, filenames, color): + return languages[hook.language].run_hook(hook, filenames, color) + + +def _get_hook_no_install(repo_config, store, hook_id): + config = {'repos': [repo_config]} + config = cfgv.validate(config, CONFIG_SCHEMA) + config = cfgv.apply_defaults(config, CONFIG_SCHEMA) + hooks = all_hooks(config, store) + hook, = [hook for hook in hooks if hook.id == hook_id] + return hook + + +def _get_hook(repo_config, store, hook_id): + hook = _get_hook_no_install(repo_config, store, hook_id) + install_hook_envs([hook], store) + return hook + + def _test_hook_repo( tempdir_factory, store, @@ -45,28 +69,74 @@ def _test_hook_repo( expected, expected_return_code=0, config_kwargs=None, + color=False, ): path = make_repo(tempdir_factory, repo_path) config = make_config_from_repo(path, **(config_kwargs or {})) - repo = Repository.create(config, store) - hook_dict, = [ - hook for repo_hook_id, hook in repo.hooks if repo_hook_id == hook_id - ] - ret = repo.run_hook(hook_dict, args) - assert ret[0] == expected_return_code - assert ret[1].replace(b'\r\n', b'\n') == expected + hook = _get_hook(config, store, hook_id) + ret, out = _hook_run(hook, args, color=color) + assert ret == expected_return_code + assert _norm_out(out) == expected + + +def test_conda_hook(tempdir_factory, store): + _test_hook_repo( + tempdir_factory, store, 'conda_hooks_repo', + 'sys-exec', [os.devnull], + b'conda-default\n', + ) + + +def test_conda_with_additional_dependencies_hook(tempdir_factory, store): + _test_hook_repo( + tempdir_factory, store, 'conda_hooks_repo', + 'additional-deps', [os.devnull], + b'OK\n', + config_kwargs={ + 'hooks': [{ + 'id': 'additional-deps', + 'args': ['-c', 'import mccabe; print("OK")'], + 'additional_dependencies': ['mccabe'], + }], + }, + ) + + +def test_local_conda_additional_dependencies(store): + config = { + 'repo': 'local', + 'hooks': [{ + 'id': 'local-conda', + 'name': 'local-conda', + 'entry': 'python', + 'language': 'conda', + 'args': ['-c', 'import mccabe; print("OK")'], + 'additional_dependencies': ['mccabe'], + }], + } + hook = _get_hook(config, store, 'local-conda') + ret, out = _hook_run(hook, (), color=False) + assert ret == 0 + assert _norm_out(out) == b'OK\n' -@pytest.mark.integration def test_python_hook(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'python_hooks_repo', 'foo', [os.devnull], - b"['" + five.to_bytes(os.devnull) + b"']\nHello World\n" + f'[{os.devnull!r}]\nHello World\n'.encode(), ) -@pytest.mark.integration +def test_python_hook_default_version(tempdir_factory, store): + # make sure that this continues to work for platforms where default + # language detection does not work + with mock.patch.object( + python, 'get_default_version', return_value=C.DEFAULT, + ): + test_python_hook(tempdir_factory, store) + + def test_python_hook_args_with_spaces(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'python_hooks_repo', @@ -78,61 +148,56 @@ def test_python_hook_args_with_spaces(tempdir_factory, store): 'hooks': [{ 'id': 'foo', 'args': ['i have spaces', 'and"\'quotes', '$and !this'], - }] + }], }, ) -@pytest.mark.integration -def test_python_hook_weird_setup_cfg(tempdir_factory, store): - path = git_dir(tempdir_factory) - with cwd(path): - with io.open('setup.cfg', 'w') as setup_cfg: - setup_cfg.write('[install]\ninstall_scripts=/usr/sbin\n') +def test_python_hook_weird_setup_cfg(in_git_dir, tempdir_factory, store): + in_git_dir.join('setup.cfg').write('[install]\ninstall_scripts=/usr/sbin') - _test_hook_repo( - tempdir_factory, store, 'python_hooks_repo', - 'foo', [os.devnull], - b"['" + five.to_bytes(os.devnull) + b"']\nHello World\n" - ) + _test_hook_repo( + tempdir_factory, store, 'python_hooks_repo', + 'foo', [os.devnull], + f'[{os.devnull!r}]\nHello World\n'.encode(), + ) + + +@xfailif_no_venv +def test_python_venv(tempdir_factory, store): # pragma: no cover (no venv) + _test_hook_repo( + tempdir_factory, store, 'python_venv_hooks_repo', + 'foo', [os.devnull], + f'[{os.devnull!r}]\nHello World\n'.encode(), + ) -@pytest.mark.integration def test_switch_language_versions_doesnt_clobber(tempdir_factory, store): # We're using the python3 repo because it prints the python version path = make_repo(tempdir_factory, 'python3_hooks_repo') def run_on_version(version, expected_output): - config = make_config_from_repo( - path, hooks=[{'id': 'python3-hook', 'language_version': version}], - ) - repo = Repository.create(config, store) - hook_dict, = [ - hook - for repo_hook_id, hook in repo.hooks - if repo_hook_id == 'python3-hook' - ] - ret = repo.run_hook(hook_dict, []) - assert ret[0] == 0 - assert ret[1].replace(b'\r\n', b'\n') == expected_output + config = make_config_from_repo(path) + config['hooks'][0]['language_version'] = version + hook = _get_hook(config, store, 'python3-hook') + ret, out = _hook_run(hook, [], color=False) + assert ret == 0 + assert _norm_out(out) == expected_output - run_on_version('python3.4', b'3.4\n[]\nHello World\n') - run_on_version('python3.5', b'3.5\n[]\nHello World\n') + run_on_version('python2', b'2\n[]\nHello World\n') + run_on_version('python3', b'3\n[]\nHello World\n') -@pytest.mark.integration def test_versioned_python_hook(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'python3_hooks_repo', 'python3-hook', [os.devnull], - b"3.5\n['" + five.to_bytes(os.devnull) + b"']\nHello World\n", + f'3\n[{os.devnull!r}]\nHello World\n'.encode(), ) -@skipif_slowtests_false -@skipif_cant_run_docker -@pytest.mark.integration +@skipif_cant_run_docker # pragma: win32 no cover def test_run_a_docker_hook(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'docker_hooks_repo', @@ -141,9 +206,7 @@ def test_run_a_docker_hook(tempdir_factory, store): ) -@skipif_slowtests_false -@skipif_cant_run_docker -@pytest.mark.integration +@skipif_cant_run_docker # pragma: win32 no cover def test_run_a_docker_hook_with_entry_args(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'docker_hooks_repo', @@ -152,63 +215,60 @@ def test_run_a_docker_hook_with_entry_args(tempdir_factory, store): ) -@skipif_slowtests_false -@skipif_cant_run_docker -@pytest.mark.integration +@skipif_cant_run_docker # pragma: win32 no cover def test_run_a_failing_docker_hook(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'docker_hooks_repo', 'docker-hook-failing', - ['Hello World from docker'], b'', - expected_return_code=1 + ['Hello World from docker'], + mock.ANY, # an error message about `bork` not existing + expected_return_code=127, + ) + + +@skipif_cant_run_docker # pragma: win32 no cover +@pytest.mark.parametrize('hook_id', ('echo-entrypoint', 'echo-cmd')) +def test_run_a_docker_image_hook(tempdir_factory, store, hook_id): + _test_hook_repo( + tempdir_factory, store, 'docker_image_hooks_repo', + hook_id, + ['Hello World from docker'], b'Hello World from docker\n', ) -@skipif_slowtests_false -@xfailif_windows_no_node -@pytest.mark.integration def test_run_a_node_hook(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'node_hooks_repo', - 'foo', ['/dev/null'], b'Hello World\n', + 'foo', [os.devnull], b'Hello World\n', ) -@skipif_slowtests_false -@xfailif_windows_no_node -@pytest.mark.integration def test_run_versioned_node_hook(tempdir_factory, store): _test_hook_repo( - tempdir_factory, store, 'node_0_11_8_hooks_repo', - 'node-11-8-hook', ['/dev/null'], b'v0.11.8\nHello World\n', + tempdir_factory, store, 'node_versioned_hooks_repo', + 'versioned-node-hook', [os.devnull], b'v9.3.0\nHello World\n', ) -@skipif_slowtests_false @xfailif_windows_no_ruby -@pytest.mark.integration def test_run_a_ruby_hook(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'ruby_hooks_repo', - 'ruby_hook', ['/dev/null'], b'Hello world from a ruby hook\n', + 'ruby_hook', [os.devnull], b'Hello world from a ruby hook\n', ) -@skipif_slowtests_false @xfailif_windows_no_ruby -@pytest.mark.integration def test_run_versioned_ruby_hook(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'ruby_versioned_hooks_repo', 'ruby_hook', - ['/dev/null'], - b'2.1.5\nHello world from a ruby hook\n', + [os.devnull], + b'2.5.1\nHello world from a ruby hook\n', ) -@skipif_slowtests_false @xfailif_windows_no_ruby -@pytest.mark.integration def test_run_ruby_hook_with_disable_shared_gems( tempdir_factory, store, @@ -219,36 +279,25 @@ def test_run_ruby_hook_with_disable_shared_gems( tmpdir.join('.bundle').mkdir() tmpdir.join('.bundle', 'config').write( 'BUNDLE_DISABLE_SHARED_GEMS: true\n' - 'BUNDLE_PATH: vendor/gem\n' + 'BUNDLE_PATH: vendor/gem\n', ) with cwd(tmpdir.strpath): _test_hook_repo( tempdir_factory, store, 'ruby_versioned_hooks_repo', 'ruby_hook', - ['/dev/null'], - b'2.1.5\nHello world from a ruby hook\n', + [os.devnull], + b'2.5.1\nHello world from a ruby hook\n', ) -@pytest.mark.integration def test_system_hook_with_spaces(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'system_hook_with_spaces_repo', - 'system-hook-with-spaces', ['/dev/null'], b'Hello World\n', - ) - - -@pytest.mark.integration -def test_repo_with_legacy_hooks_yaml(tempdir_factory, store): - _test_hook_repo( - tempdir_factory, store, 'legacy_hooks_yaml_repo', - 'system-hook-with-spaces', ['/dev/null'], b'Hello World\n', - config_kwargs={'legacy': True}, + 'system-hook-with-spaces', [os.devnull], b'Hello World\n', ) -@skipif_cant_run_swift -@pytest.mark.integration +@skipif_cant_run_swift # pragma: win32 no cover def test_swift_hook(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'swift_hooks_repo', @@ -256,7 +305,6 @@ def test_swift_hook(tempdir_factory, store): ) -@pytest.mark.integration def test_golang_hook(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'golang_hooks_repo', @@ -264,36 +312,68 @@ def test_golang_hook(tempdir_factory, store): ) -@pytest.mark.integration -def test_missing_executable(tempdir_factory, store): +def test_golang_hook_still_works_when_gobin_is_set(tempdir_factory, store): + gobin_dir = tempdir_factory.get() + with envcontext((('GOBIN', gobin_dir),)): + test_golang_hook(tempdir_factory, store) + assert os.listdir(gobin_dir) == [] + + +def test_rust_hook(tempdir_factory, store): _test_hook_repo( - tempdir_factory, store, 'not_found_exe', - 'not-found-exe', ['/dev/null'], - b'Executable `i-dont-exist-lol` not found', - expected_return_code=1, + tempdir_factory, store, 'rust_hooks_repo', + 'rust-hook', [], b'hello world\n', ) -@pytest.mark.integration -def test_missing_pcre_support(tempdir_factory, store): - orig_find_executable = parse_shebang.find_executable +@pytest.mark.parametrize('dep', ('cli:shellharden:3.1.0', 'cli:shellharden')) +def test_additional_rust_cli_dependencies_installed( + tempdir_factory, store, dep, +): + path = make_repo(tempdir_factory, 'rust_hooks_repo') + config = make_config_from_repo(path) + # A small rust package with no dependencies. + config['hooks'][0]['additional_dependencies'] = [dep] + hook = _get_hook(config, store, 'rust-hook') + binaries = os.listdir( + hook.prefix.path( + helpers.environment_dir(rust.ENVIRONMENT_DIR, C.DEFAULT), 'bin', + ), + ) + # normalize for windows + binaries = [os.path.splitext(binary)[0] for binary in binaries] + assert 'shellharden' in binaries - def no_grep(exe, **kwargs): - if exe == pcre.GREP: - return None - else: - return orig_find_executable(exe, **kwargs) - with mock.patch.object(parse_shebang, 'find_executable', no_grep): - _test_hook_repo( - tempdir_factory, store, 'pcre_hooks_repo', - 'regex-with-quotes', ['/dev/null'], - 'Executable `{}` not found'.format(pcre.GREP).encode('UTF-8'), - expected_return_code=1, - ) +def test_additional_rust_lib_dependencies_installed( + tempdir_factory, store, +): + path = make_repo(tempdir_factory, 'rust_hooks_repo') + config = make_config_from_repo(path) + # A small rust package with no dependencies. + deps = ['shellharden:3.1.0'] + config['hooks'][0]['additional_dependencies'] = deps + hook = _get_hook(config, store, 'rust-hook') + binaries = os.listdir( + hook.prefix.path( + helpers.environment_dir(rust.ENVIRONMENT_DIR, C.DEFAULT), 'bin', + ), + ) + # normalize for windows + binaries = [os.path.splitext(binary)[0] for binary in binaries] + assert 'rust-hello-world' in binaries + assert 'shellharden' not in binaries + + +def test_missing_executable(tempdir_factory, store): + _test_hook_repo( + tempdir_factory, store, 'not_found_exe', + 'not-found-exe', [os.devnull], + b'Executable `i-dont-exist-lol` not found', + expected_return_code=1, + ) -@pytest.mark.integration def test_run_a_script_hook(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'script_hooks_repo', @@ -301,7 +381,6 @@ def test_run_a_script_hook(tempdir_factory, store): ) -@pytest.mark.integration def test_run_hook_with_spaced_args(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'arg_per_line_hooks_repo', @@ -311,7 +390,6 @@ def test_run_hook_with_spaced_args(tempdir_factory, store): ) -@pytest.mark.integration def test_run_hook_with_curly_braced_arguments(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'arg_per_line_hooks_repo', @@ -322,217 +400,142 @@ def test_run_hook_with_curly_braced_arguments(tempdir_factory, store): 'hooks': [{ 'id': 'arg-per-line', 'args': ['hi {1}', "I'm {a} problem"], - }] + }], }, ) -@xfailif_no_pcre_support -@pytest.mark.integration -def test_pcre_hook_no_match(tempdir_factory, store): - path = git_dir(tempdir_factory) - with cwd(path): - with io.open('herp', 'w') as herp: - herp.write('foo') - - with io.open('derp', 'w') as derp: - derp.write('bar') - - _test_hook_repo( - tempdir_factory, store, 'pcre_hooks_repo', - 'regex-with-quotes', ['herp', 'derp'], b'', - ) +def test_intermixed_stdout_stderr(tempdir_factory, store): + _test_hook_repo( + tempdir_factory, store, 'stdout_stderr_repo', + 'stdout-stderr', + [], + b'0\n1\n2\n3\n4\n5\n', + ) - _test_hook_repo( - tempdir_factory, store, 'pcre_hooks_repo', - 'other-regex', ['herp', 'derp'], b'', - ) +@pytest.mark.xfail(os.name == 'nt', reason='ptys are posix-only') +def test_output_isatty(tempdir_factory, store): + _test_hook_repo( + tempdir_factory, store, 'stdout_stderr_repo', + 'tty-check', + [], + b'stdin: False\nstdout: True\nstderr: True\n', + color=True, + ) -@xfailif_no_pcre_support -@pytest.mark.integration -def test_pcre_hook_matching(tempdir_factory, store): - path = git_dir(tempdir_factory) - with cwd(path): - with io.open('herp', 'w') as herp: - herp.write("\nherpfoo'bard\n") - with io.open('derp', 'w') as derp: - derp.write('[INFO] information yo\n') +def _make_grep_repo(entry, store, args=()): + config = { + 'repo': 'local', + 'hooks': [{ + 'id': 'grep-hook', + 'name': 'grep-hook', + 'language': 'pygrep', + 'entry': entry, + 'args': args, + 'types': ['text'], + }], + } + return _get_hook(config, store, 'grep-hook') - _test_hook_repo( - tempdir_factory, store, 'pcre_hooks_repo', - 'regex-with-quotes', ['herp', 'derp'], b"herp:2:herpfoo'bard\n", - expected_return_code=1, - ) - _test_hook_repo( - tempdir_factory, store, 'pcre_hooks_repo', - 'other-regex', ['herp', 'derp'], b'derp:1:[INFO] information yo\n', - expected_return_code=1, - ) +@pytest.fixture +def greppable_files(tmpdir): + with tmpdir.as_cwd(): + cmd_output_b('git', 'init', '.') + tmpdir.join('f1').write_binary(b"hello'hi\nworld\n") + tmpdir.join('f2').write_binary(b'foo\nbar\nbaz\n') + tmpdir.join('f3').write_binary(b'[WARN] hi\n') + yield tmpdir -@xfailif_no_pcre_support -@pytest.mark.integration -def test_pcre_hook_case_insensitive_option(tempdir_factory, store): - path = git_dir(tempdir_factory) - with cwd(path): - with io.open('herp', 'w') as herp: - herp.write('FoOoOoObar\n') +def test_grep_hook_matching(greppable_files, store): + hook = _make_grep_repo('ello', store) + ret, out = _hook_run(hook, ('f1', 'f2', 'f3'), color=False) + assert ret == 1 + assert _norm_out(out) == b"f1:1:hello'hi\n" - _test_hook_repo( - tempdir_factory, store, 'pcre_hooks_repo', - 'regex-with-grep-args', ['herp'], b'herp:1:FoOoOoObar\n', - expected_return_code=1, - ) +def test_grep_hook_case_insensitive(greppable_files, store): + hook = _make_grep_repo('ELLO', store, args=['-i']) + ret, out = _hook_run(hook, ('f1', 'f2', 'f3'), color=False) + assert ret == 1 + assert _norm_out(out) == b"f1:1:hello'hi\n" -@xfailif_no_pcre_support -@pytest.mark.integration -def test_pcre_many_files(tempdir_factory, store): - # This is intended to simulate lots of passing files and one failing file - # to make sure it still fails. This is not the case when naively using - # a system hook with `grep -H -n '...'` and expected_return_code=1. - path = git_dir(tempdir_factory) - with cwd(path): - with io.open('herp', 'w') as herp: - herp.write('[INFO] info\n') - _test_hook_repo( - tempdir_factory, store, 'pcre_hooks_repo', - 'other-regex', - ['/dev/null'] * 15000 + ['herp'], - b'herp:1:[INFO] info\n', - expected_return_code=1, - ) +@pytest.mark.parametrize('regex', ('nope', "foo'bar", r'^\[INFO\]')) +def test_grep_hook_not_matching(regex, greppable_files, store): + hook = _make_grep_repo(regex, store) + ret, out = _hook_run(hook, ('f1', 'f2', 'f3'), color=False) + assert (ret, out) == (0, b'') def _norm_pwd(path): # Under windows bash's temp and windows temp is different. # This normalizes to the bash /tmp - return cmd_output( - 'bash', '-c', "cd '{}' && pwd".format(path), - encoding=None, + return cmd_output_b( + 'bash', '-c', f"cd '{path}' && pwd", )[1].strip() -@pytest.mark.integration -def test_cwd_of_hook(tempdir_factory, store): +def test_cwd_of_hook(in_git_dir, tempdir_factory, store): # Note: this doubles as a test for `system` hooks - path = git_dir(tempdir_factory) - with cwd(path): - _test_hook_repo( - tempdir_factory, store, 'prints_cwd_repo', - 'prints_cwd', ['-L'], _norm_pwd(path) + b'\n', - ) + _test_hook_repo( + tempdir_factory, store, 'prints_cwd_repo', + 'prints_cwd', ['-L'], _norm_pwd(in_git_dir.strpath) + b'\n', + ) -@pytest.mark.integration def test_lots_of_files(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'script_hooks_repo', - 'bash_hook', ['/dev/null'] * 15000, mock.ANY, + 'bash_hook', [os.devnull] * 15000, mock.ANY, ) -@pytest.mark.integration -def test_venvs(tempdir_factory, store): - path = make_repo(tempdir_factory, 'python_hooks_repo') - config = make_config_from_repo(path) - repo = Repository.create(config, store) - venv, = repo._venvs - assert venv == (mock.ANY, 'python', 'default', []) - - -@pytest.mark.integration -def test_additional_dependencies(tempdir_factory, store): +def test_additional_dependencies_roll_forward(tempdir_factory, store): path = make_repo(tempdir_factory, 'python_hooks_repo') - config = make_config_from_repo(path) - config['hooks'][0]['additional_dependencies'] = ['pep8'] - repo = Repository.create(config, store) - venv, = repo._venvs - assert venv == (mock.ANY, 'python', 'default', ['pep8']) - -@pytest.mark.integration -def test_additional_dependencies_duplicated( - tempdir_factory, store, log_warning_mock, -): - path = make_repo(tempdir_factory, 'ruby_hooks_repo') - config = make_config_from_repo(path) - deps = ['thread_safe', 'tins', 'thread_safe'] - config['hooks'][0]['additional_dependencies'] = deps - repo = Repository.create(config, store) - venv, = repo._venvs - assert venv == (mock.ANY, 'ruby', 'default', ['thread_safe', 'tins']) + config1 = make_config_from_repo(path) + hook1 = _get_hook(config1, store, 'foo') + with python.in_env(hook1.prefix, hook1.language_version): + assert 'mccabe' not in cmd_output('pip', 'freeze', '-l')[1] + # Make another repo with additional dependencies + config2 = make_config_from_repo(path) + config2['hooks'][0]['additional_dependencies'] = ['mccabe'] + hook2 = _get_hook(config2, store, 'foo') + with python.in_env(hook2.prefix, hook2.language_version): + assert 'mccabe' in cmd_output('pip', 'freeze', '-l')[1] -@pytest.mark.integration -def test_additional_python_dependencies_installed(tempdir_factory, store): - path = make_repo(tempdir_factory, 'python_hooks_repo') - config = make_config_from_repo(path) - config['hooks'][0]['additional_dependencies'] = ['mccabe'] - repo = Repository.create(config, store) - repo.require_installed() - with python.in_env(repo._cmd_runner, 'default'): - output = cmd_output('pip', 'freeze', '-l')[1] - assert 'mccabe' in output + # should not have affected original + with python.in_env(hook1.prefix, hook1.language_version): + assert 'mccabe' not in cmd_output('pip', 'freeze', '-l')[1] -@pytest.mark.integration -def test_additional_dependencies_roll_forward(tempdir_factory, store): - path = make_repo(tempdir_factory, 'python_hooks_repo') - config = make_config_from_repo(path) - # Run the repo once without additional_dependencies - repo = Repository.create(config, store) - repo.require_installed() - # Now run it with additional_dependencies - config['hooks'][0]['additional_dependencies'] = ['mccabe'] - repo = Repository.create(config, store) - repo.require_installed() - # We should see our additional dependency installed - with python.in_env(repo._cmd_runner, 'default'): - output = cmd_output('pip', 'freeze', '-l')[1] - assert 'mccabe' in output - - -@skipif_slowtests_false -@xfailif_windows_no_ruby -@pytest.mark.integration -def test_additional_ruby_dependencies_installed( - tempdir_factory, store, -): # pragma: no cover (non-windows) +@xfailif_windows_no_ruby # pragma: win32 no cover +def test_additional_ruby_dependencies_installed(tempdir_factory, store): path = make_repo(tempdir_factory, 'ruby_hooks_repo') config = make_config_from_repo(path) - config['hooks'][0]['additional_dependencies'] = ['thread_safe', 'tins'] - repo = Repository.create(config, store) - repo.require_installed() - with ruby.in_env(repo._cmd_runner, 'default'): + config['hooks'][0]['additional_dependencies'] = ['tins'] + hook = _get_hook(config, store, 'ruby_hook') + with ruby.in_env(hook.prefix, hook.language_version): output = cmd_output('gem', 'list', '--local')[1] - assert 'thread_safe' in output assert 'tins' in output -@skipif_slowtests_false -@xfailif_windows_no_node -@pytest.mark.integration -def test_additional_node_dependencies_installed( - tempdir_factory, store, -): # pragma: no cover (non-windows) +def test_additional_node_dependencies_installed(tempdir_factory, store): path = make_repo(tempdir_factory, 'node_hooks_repo') config = make_config_from_repo(path) # Careful to choose a small package that's not depped by npm config['hooks'][0]['additional_dependencies'] = ['lodash'] - repo = Repository.create(config, store) - repo.require_installed() - with node.in_env(repo._cmd_runner, 'default'): - cmd_output('npm', 'config', 'set', 'global', 'true') - output = cmd_output('npm', 'ls')[1] + hook = _get_hook(config, store, 'foo') + with node.in_env(hook.prefix, hook.language_version): + output = cmd_output('npm', 'ls', '-g')[1] assert 'lodash' in output -@pytest.mark.integration def test_additional_golang_dependencies_installed( tempdir_factory, store, ): @@ -541,30 +544,101 @@ def test_additional_golang_dependencies_installed( # A small go package deps = ['github.com/golang/example/hello'] config['hooks'][0]['additional_dependencies'] = deps - repo = Repository.create(config, store) - repo.require_installed() - binaries = os.listdir(repo._cmd_runner.path( - helpers.environment_dir(golang.ENVIRONMENT_DIR, 'default'), 'bin', - )) + hook = _get_hook(config, store, 'golang-hook') + binaries = os.listdir( + hook.prefix.path( + helpers.environment_dir(golang.ENVIRONMENT_DIR, C.DEFAULT), 'bin', + ), + ) # normalize for windows binaries = [os.path.splitext(binary)[0] for binary in binaries] assert 'hello' in binaries +def test_local_golang_additional_dependencies(store): + config = { + 'repo': 'local', + 'hooks': [{ + 'id': 'hello', + 'name': 'hello', + 'entry': 'hello', + 'language': 'golang', + 'additional_dependencies': ['github.com/golang/example/hello'], + }], + } + hook = _get_hook(config, store, 'hello') + ret, out = _hook_run(hook, (), color=False) + assert ret == 0 + assert _norm_out(out) == b'Hello, Go examples!\n' + + +def test_local_rust_additional_dependencies(store): + config = { + 'repo': 'local', + 'hooks': [{ + 'id': 'hello', + 'name': 'hello', + 'entry': 'hello', + 'language': 'rust', + 'additional_dependencies': ['cli:hello-cli:0.2.2'], + }], + } + hook = _get_hook(config, store, 'hello') + ret, out = _hook_run(hook, (), color=False) + assert ret == 0 + assert _norm_out(out) == b'Hello World!\n' + + +def test_fail_hooks(store): + config = { + 'repo': 'local', + 'hooks': [{ + 'id': 'fail', + 'name': 'fail', + 'language': 'fail', + 'entry': 'make sure to name changelogs as .rst!', + 'files': r'changelog/.*(? too-much: foo, hello' + assert fake_log_handler.handle.call_args[0][0].msg == expected + + def test_reinstall(tempdir_factory, store, log_info_mock): path = make_repo(tempdir_factory, 'python_hooks_repo') config = make_config_from_repo(path) - repo = Repository.create(config, store) - repo.require_installed() + _get_hook(config, store, 'foo') # We print some logging during clone (1) + install (3) assert log_info_mock.call_count == 4 log_info_mock.reset_mock() - # Reinstall with same repo should not trigger another install - repo.require_installed() - assert log_info_mock.call_count == 0 # Reinstall on another run should not trigger another install - repo = Repository.create(config, store) - repo.require_installed() + _get_hook(config, store, 'foo') assert log_info_mock.call_count == 0 @@ -572,8 +646,7 @@ def test_control_c_control_c_on_install(tempdir_factory, store): """Regression test for #186.""" path = make_repo(tempdir_factory, 'python_hooks_repo') config = make_config_from_repo(path) - repo = Repository.create(config, store) - hook = repo.hooks[0][1] + hooks = [_get_hook_no_install(config, store, 'foo')] class MyKeyboardInterrupt(KeyboardInterrupt): pass @@ -588,108 +661,160 @@ class MyKeyboardInterrupt(KeyboardInterrupt): with mock.patch.object( shutil, 'rmtree', side_effect=MyKeyboardInterrupt, ): - repo.run_hook(hook, []) + install_hook_envs(hooks, store) # Should have made an environment, however this environment is broken! - assert os.path.exists(repo._cmd_runner.path('py_env-default')) + hook, = hooks + assert hook.prefix.exists( + helpers.environment_dir(python.ENVIRONMENT_DIR, hook.language_version), + ) # However, it should be perfectly runnable (reinstall after botched # install) - retv, stdout, stderr = repo.run_hook(hook, []) - assert retv == 0 + install_hook_envs(hooks, store) + ret, out = _hook_run(hook, (), color=False) + assert ret == 0 + + +def test_invalidated_virtualenv(tempdir_factory, store): + # A cached virtualenv may become invalidated if the system python upgrades + # This should not cause every hook in that virtualenv to fail. + path = make_repo(tempdir_factory, 'python_hooks_repo') + config = make_config_from_repo(path) + hook = _get_hook(config, store, 'foo') + + # Simulate breaking of the virtualenv + libdir = hook.prefix.path( + helpers.environment_dir(python.ENVIRONMENT_DIR, hook.language_version), + 'lib', hook.language_version, + ) + paths = [ + os.path.join(libdir, p) for p in ('site.py', 'site.pyc', '__pycache__') + ] + cmd_output_b('rm', '-rf', *paths) + + # pre-commit should rebuild the virtualenv and it should be runnable + hook = _get_hook(config, store, 'foo') + ret, out = _hook_run(hook, (), color=False) + assert ret == 0 -@pytest.mark.integration def test_really_long_file_paths(tempdir_factory, store): base_path = tempdir_factory.get() really_long_path = os.path.join(base_path, 'really_long' * 10) - cmd_output('git', 'init', really_long_path) + cmd_output_b('git', 'init', really_long_path) path = make_repo(tempdir_factory, 'python_hooks_repo') config = make_config_from_repo(path) with cwd(really_long_path): - repo = Repository.create(config, store) - repo.require_installed() + _get_hook(config, store, 'foo') -@pytest.mark.integration def test_config_overrides_repo_specifics(tempdir_factory, store): path = make_repo(tempdir_factory, 'script_hooks_repo') config = make_config_from_repo(path) - repo = Repository.create(config, store) - assert repo.hooks[0][1]['files'] == '' + hook = _get_hook(config, store, 'bash_hook') + assert hook.files == '' # Set the file regex to something else config['hooks'][0]['files'] = '\\.sh$' - repo = Repository.create(config, store) - assert repo.hooks[0][1]['files'] == '\\.sh$' + hook = _get_hook(config, store, 'bash_hook') + assert hook.files == '\\.sh$' def _create_repo_with_tags(tempdir_factory, src, tag): path = make_repo(tempdir_factory, src) - with cwd(path): - cmd_output('git', 'tag', tag) + cmd_output_b('git', 'tag', tag, cwd=path) return path -@pytest.mark.integration def test_tags_on_repositories(in_tmpdir, tempdir_factory, store): tag = 'v1.1' - git_dir_1 = _create_repo_with_tags(tempdir_factory, 'prints_cwd_repo', tag) - git_dir_2 = _create_repo_with_tags( - tempdir_factory, 'script_hooks_repo', tag, - ) + git1 = _create_repo_with_tags(tempdir_factory, 'prints_cwd_repo', tag) + git2 = _create_repo_with_tags(tempdir_factory, 'script_hooks_repo', tag) - repo_1 = Repository.create( - make_config_from_repo(git_dir_1, sha=tag), store, - ) - ret = repo_1.run_hook(repo_1.hooks[0][1], ['-L']) - assert ret[0] == 0 - assert ret[1].strip() == _norm_pwd(in_tmpdir) - - repo_2 = Repository.create( - make_config_from_repo(git_dir_2, sha=tag), store, - ) - ret = repo_2.run_hook(repo_2.hooks[0][1], ['bar']) - assert ret[0] == 0 - assert ret[1] == b'bar\nHello World\n' + config1 = make_config_from_repo(git1, rev=tag) + hook1 = _get_hook(config1, store, 'prints_cwd') + ret1, out1 = _hook_run(hook1, ('-L',), color=False) + assert ret1 == 0 + assert out1.strip() == _norm_pwd(in_tmpdir) + config2 = make_config_from_repo(git2, rev=tag) + hook2 = _get_hook(config2, store, 'bash_hook') + ret2, out2 = _hook_run(hook2, ('bar',), color=False) + assert ret2 == 0 + assert out2 == b'bar\nHello World\n' -def test_local_repository(): - config = config_with_local_hooks() - local_repo = Repository.create(config, 'dummy') - with pytest.raises(NotImplementedError): - local_repo.manifest - assert len(local_repo.hooks) == 1 - -def test_local_python_repo(store): +@pytest.fixture +def local_python_config(): # Make a "local" hooks repo that just installs our other hooks repo repo_path = get_resource_path('python_hooks_repo') manifest = load_manifest(os.path.join(repo_path, C.MANIFEST_FILE)) hooks = [ dict(hook, additional_dependencies=[repo_path]) for hook in manifest ] - config = {'repo': 'local', 'hooks': hooks} - repo = Repository.create(config, store) - (_, hook), = repo.hooks - ret = repo.run_hook(hook, ('filename',)) - assert ret[0] == 0 - assert ret[1].replace(b'\r\n', b'\n') == b"['filename']\nHello World\n" + return {'repo': 'local', 'hooks': hooks} + + +@pytest.mark.xfail( # pragma: win32 no cover + sys.platform == 'win32', + reason='microsoft/azure-pipelines-image-generation#989', +) +def test_local_python_repo(store, local_python_config): + hook = _get_hook(local_python_config, store, 'foo') + # language_version should have been adjusted to the interpreter version + assert hook.language_version != C.DEFAULT + ret, out = _hook_run(hook, ('filename',), color=False) + assert ret == 0 + assert _norm_out(out) == b"['filename']\nHello World\n" + + +def test_default_language_version(store, local_python_config): + config: Dict[str, Any] = { + 'default_language_version': {'python': 'fake'}, + 'default_stages': ['commit'], + 'repos': [local_python_config], + } + + # `language_version` was not set, should default + hook, = all_hooks(config, store) + assert hook.language_version == 'fake' + + # `language_version` is set, should not default + config['repos'][0]['hooks'][0]['language_version'] = 'fake2' + hook, = all_hooks(config, store) + assert hook.language_version == 'fake2' + + +def test_default_stages(store, local_python_config): + config: Dict[str, Any] = { + 'default_language_version': {'python': C.DEFAULT}, + 'default_stages': ['commit'], + 'repos': [local_python_config], + } + + # `stages` was not set, should default + hook, = all_hooks(config, store) + assert hook.stages == ['commit'] + + # `stages` is set, should not default + config['repos'][0]['hooks'][0]['stages'] = ['push'] + hook, = all_hooks(config, store) + assert hook.stages == ['push'] def test_hook_id_not_present(tempdir_factory, store, fake_log_handler): path = make_repo(tempdir_factory, 'script_hooks_repo') config = make_config_from_repo(path) config['hooks'][0]['id'] = 'i-dont-exist' - repo = Repository.create(config, store) with pytest.raises(SystemExit): - repo.require_installed() + _get_hook(config, store, 'i-dont-exist') assert fake_log_handler.handle.call_args[0][0].msg == ( - '`i-dont-exist` is not present in repository {}. ' - 'Typo? Perhaps it is introduced in a newer version? ' - 'Often `pre-commit autoupdate` fixes this.'.format(path) + f'`i-dont-exist` is not present in repository file://{path}. ' + f'Typo? Perhaps it is introduced in a newer version? ' + f'Often `pre-commit autoupdate` fixes this.' ) @@ -698,9 +823,8 @@ def test_too_new_version(tempdir_factory, store, fake_log_handler): with modify_manifest(path) as manifest: manifest[0]['minimum_pre_commit_version'] = '999.0.0' config = make_config_from_repo(path) - repo = Repository.create(config, store) with pytest.raises(SystemExit): - repo.require_installed() + _get_hook(config, store, 'bash_hook') msg = fake_log_handler.handle.call_args[0][0].msg assert re.match( r'^The hook `bash_hook` requires pre-commit version 999\.0\.0 but ' @@ -717,4 +841,62 @@ def test_versions_ok(tempdir_factory, store, version): manifest[0]['minimum_pre_commit_version'] = version config = make_config_from_repo(path) # Should succeed - Repository.create(config, store).require_installed() + _get_hook(config, store, 'bash_hook') + + +def test_manifest_hooks(tempdir_factory, store): + path = make_repo(tempdir_factory, 'script_hooks_repo') + config = make_config_from_repo(path) + hook = _get_hook(config, store, 'bash_hook') + + assert hook == Hook( + src=f'file://{path}', + prefix=Prefix(mock.ANY), + additional_dependencies=[], + alias='', + always_run=False, + args=[], + description='', + entry='bin/hook.sh', + exclude='^$', + exclude_types=[], + files='', + id='bash_hook', + language='script', + language_version='default', + log_file='', + minimum_pre_commit_version='0', + name='Bash hook', + pass_filenames=True, + require_serial=False, + stages=( + 'commit', 'merge-commit', 'prepare-commit-msg', 'commit-msg', + 'manual', 'post-checkout', 'push', + ), + types=['file'], + verbose=False, + ) + + +def test_perl_hook(tempdir_factory, store): + _test_hook_repo( + tempdir_factory, store, 'perl_hooks_repo', + 'perl-hook', [], b'Hello from perl-commit Perl!\n', + ) + + +def test_local_perl_additional_dependencies(store): + config = { + 'repo': 'local', + 'hooks': [{ + 'id': 'hello', + 'name': 'hello', + 'entry': 'perltidy --version', + 'language': 'perl', + 'additional_dependencies': ['SHANCOCK/Perl-Tidy-20200110.tar.gz'], + }], + } + hook = _get_hook(config, store, 'hello') + ret, out = _hook_run(hook, (), color=False) + assert ret == 0 + assert _norm_out(out).startswith(b'This is perltidy, v20200110') diff --git a/tests/runner_test.py b/tests/runner_test.py deleted file mode 100644 index a4f8cb7c0..000000000 --- a/tests/runner_test.py +++ /dev/null @@ -1,133 +0,0 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - -import os.path -from collections import OrderedDict - -import pre_commit.constants as C -from pre_commit.runner import Runner -from pre_commit.util import cmd_output -from pre_commit.util import cwd -from testing.fixtures import add_config_to_repo -from testing.fixtures import git_dir -from testing.fixtures import make_consuming_repo - - -def test_init_has_no_side_effects(tmpdir): - current_wd = os.getcwd() - runner = Runner(tmpdir.strpath, C.CONFIG_FILE) - assert runner.git_root == tmpdir.strpath - assert os.getcwd() == current_wd - - -def test_create_sets_correct_directory(tempdir_factory): - path = git_dir(tempdir_factory) - with cwd(path): - runner = Runner.create(C.CONFIG_FILE) - assert os.path.normcase(runner.git_root) == os.path.normcase(path) - assert os.path.normcase(os.getcwd()) == os.path.normcase(path) - - -def test_create_changes_to_git_root(tempdir_factory): - path = git_dir(tempdir_factory) - with cwd(path): - # Change into some directory, create should set to root - foo_path = os.path.join(path, 'foo') - os.mkdir(foo_path) - os.chdir(foo_path) - assert os.getcwd() != path - - runner = Runner.create(C.CONFIG_FILE) - assert os.path.normcase(runner.git_root) == os.path.normcase(path) - assert os.path.normcase(os.getcwd()) == os.path.normcase(path) - - -def test_config_file_path(): - runner = Runner(os.path.join('foo', 'bar'), C.CONFIG_FILE) - expected_path = os.path.join('foo', 'bar', C.CONFIG_FILE) - assert runner.config_file_path == expected_path - - -def test_repositories(tempdir_factory, mock_out_store_directory): - path = make_consuming_repo(tempdir_factory, 'script_hooks_repo') - runner = Runner(path, C.CONFIG_FILE) - assert len(runner.repositories) == 1 - - -def test_local_hooks(tempdir_factory, mock_out_store_directory): - config = OrderedDict(( - ('repo', 'local'), - ('hooks', (OrderedDict(( - ('id', 'arg-per-line'), - ('name', 'Args per line hook'), - ('entry', 'bin/hook.sh'), - ('language', 'script'), - ('files', ''), - ('args', ['hello', 'world']), - )), OrderedDict(( - ('id', 'do_not_commit'), - ('name', 'Block if "DO NOT COMMIT" is found'), - ('entry', 'DO NOT COMMIT'), - ('language', 'pcre'), - ('files', '^(.*)$'), - )))) - )) - git_path = git_dir(tempdir_factory) - add_config_to_repo(git_path, config) - runner = Runner(git_path, C.CONFIG_FILE) - assert len(runner.repositories) == 1 - assert len(runner.repositories[0].hooks) == 2 - - -def test_local_hooks_alt_config(tempdir_factory, mock_out_store_directory): - config = OrderedDict(( - ('repo', 'local'), - ('hooks', (OrderedDict(( - ('id', 'arg-per-line'), - ('name', 'Args per line hook'), - ('entry', 'bin/hook.sh'), - ('language', 'script'), - ('files', ''), - ('args', ['hello', 'world']), - )), OrderedDict(( - ('id', 'ugly-format-json'), - ('name', 'Ugly format json'), - ('entry', 'ugly-format-json'), - ('language', 'python'), - ('files', ''), - )), OrderedDict(( - ('id', 'do_not_commit'), - ('name', 'Block if "DO NOT COMMIT" is found'), - ('entry', 'DO NOT COMMIT'), - ('language', 'pcre'), - ('files', '^(.*)$'), - )))) - )) - git_path = git_dir(tempdir_factory) - alt_config_file = 'alternate_config.yaml' - add_config_to_repo(git_path, config, config_file=alt_config_file) - runner = Runner(git_path, alt_config_file) - assert len(runner.repositories) == 1 - assert len(runner.repositories[0].hooks) == 3 - - -def test_pre_commit_path(in_tmpdir): - path = os.path.join('foo', 'bar') - cmd_output('git', 'init', path) - runner = Runner(path, C.CONFIG_FILE) - expected_path = os.path.join(path, '.git', 'hooks', 'pre-commit') - assert runner.pre_commit_path == expected_path - - -def test_pre_push_path(in_tmpdir): - path = os.path.join('foo', 'bar') - cmd_output('git', 'init', path) - runner = Runner(path, C.CONFIG_FILE) - expected_path = os.path.join(path, '.git', 'hooks', 'pre-push') - assert runner.pre_push_path == expected_path - - -def test_cmd_runner(mock_out_store_directory): - runner = Runner(os.path.join('foo', 'bar'), C.CONFIG_FILE) - ret = runner.cmd_runner - assert ret.prefix_dir == os.path.join(mock_out_store_directory) + os.sep diff --git a/tests/schema_test.py b/tests/schema_test.py deleted file mode 100644 index 914e60977..000000000 --- a/tests/schema_test.py +++ /dev/null @@ -1,391 +0,0 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - -import json - -import mock -import pytest - -from pre_commit.schema import apply_defaults -from pre_commit.schema import Array -from pre_commit.schema import check_and -from pre_commit.schema import check_any -from pre_commit.schema import check_array -from pre_commit.schema import check_bool -from pre_commit.schema import check_regex -from pre_commit.schema import check_type -from pre_commit.schema import Conditional -from pre_commit.schema import load_from_filename -from pre_commit.schema import Map -from pre_commit.schema import MISSING -from pre_commit.schema import Not -from pre_commit.schema import Optional -from pre_commit.schema import OptionalNoDefault -from pre_commit.schema import remove_defaults -from pre_commit.schema import Required -from pre_commit.schema import RequiredRecurse -from pre_commit.schema import validate -from pre_commit.schema import ValidationError - - -def _assert_exception_trace(e, trace): - inner = e - for ctx in trace[:-1]: - assert inner.ctx == ctx - inner = inner.error_msg - assert inner.error_msg == trace[-1] - - -def test_ValidationError_simple_str(): - assert str(ValidationError('error msg')) == ( - '\n' - '=====> error msg' - ) - - -def test_ValidationError_nested(): - error = ValidationError( - ValidationError( - ValidationError('error msg'), - ctx='At line 1', - ), - ctx='In file foo', - ) - assert str(error) == ( - '\n' - '==> In file foo\n' - '==> At line 1\n' - '=====> error msg' - ) - - -def test_check_regex(): - with pytest.raises(ValidationError) as excinfo: - check_regex(str('(')) - assert excinfo.value.error_msg == "'(' is not a valid python regex" - - -def test_check_regex_ok(): - check_regex('^$') - - -def test_check_array_failed_inner_check(): - check = check_array(check_bool) - with pytest.raises(ValidationError) as excinfo: - check([True, False, 5]) - _assert_exception_trace( - excinfo.value, ('At index 2', 'Expected bool got int'), - ) - - -def test_check_array_ok(): - check_array(check_bool)([True, False]) - - -def test_check_and(): - check = check_and(check_type(str), check_regex) - with pytest.raises(ValidationError) as excinfo: - check(True) - assert excinfo.value.error_msg == 'Expected str got bool' - with pytest.raises(ValidationError) as excinfo: - check(str('(')) - assert excinfo.value.error_msg == "'(' is not a valid python regex" - - -def test_check_and_ok(): - check = check_and(check_type(str), check_regex) - check(str('^$')) - - -@pytest.mark.parametrize( - ('val', 'expected'), - (('bar', True), ('foo', False), (MISSING, False)), -) -def test_not(val, expected): - compared = Not('foo') - assert (val == compared) is expected - assert (compared == val) is expected - - -trivial_array_schema = Array(Map('foo', 'id')) - - -def test_validate_top_level_array_not_an_array(): - with pytest.raises(ValidationError) as excinfo: - validate({}, trivial_array_schema) - assert excinfo.value.error_msg == "Expected array but got 'dict'" - - -def test_validate_top_level_array_no_objects(): - with pytest.raises(ValidationError) as excinfo: - validate([], trivial_array_schema) - assert excinfo.value.error_msg == "Expected at least 1 'foo'" - - -@pytest.mark.parametrize('v', (({},), [{}])) -def test_ok_both_types(v): - validate(v, trivial_array_schema) - - -map_required = Map('foo', 'key', Required('key', check_bool)) -map_optional = Map('foo', 'key', Optional('key', check_bool, False)) -map_no_default = Map('foo', 'key', OptionalNoDefault('key', check_bool)) - - -def test_map_wrong_type(): - with pytest.raises(ValidationError) as excinfo: - validate([], map_required) - assert excinfo.value.error_msg == 'Expected a foo map but got a list' - - -def test_required_missing_key(): - with pytest.raises(ValidationError) as excinfo: - validate({}, map_required) - _assert_exception_trace( - excinfo.value, ('At foo(key=MISSING)', 'Missing required key: key'), - ) - - -@pytest.mark.parametrize( - 'schema', (map_required, map_optional, map_no_default), -) -def test_map_value_wrong_type(schema): - with pytest.raises(ValidationError) as excinfo: - validate({'key': 5}, schema) - _assert_exception_trace( - excinfo.value, - ('At foo(key=5)', 'At key: key', 'Expected bool got int'), - ) - - -@pytest.mark.parametrize( - 'schema', (map_required, map_optional, map_no_default), -) -def test_map_value_correct_type(schema): - validate({'key': True}, schema) - - -@pytest.mark.parametrize('schema', (map_optional, map_no_default)) -def test_optional_key_missing(schema): - validate({}, schema) - - -map_conditional = Map( - 'foo', 'key', - Conditional( - 'key2', check_bool, condition_key='key', condition_value=True, - ), -) -map_conditional_not = Map( - 'foo', 'key', - Conditional( - 'key2', check_bool, condition_key='key', condition_value=Not(False), - ), -) -map_conditional_absent = Map( - 'foo', 'key', - Conditional( - 'key2', check_bool, - condition_key='key', condition_value=True, ensure_absent=True, - ), -) -map_conditional_absent_not = Map( - 'foo', 'key', - Conditional( - 'key2', check_bool, - condition_key='key', condition_value=Not(True), ensure_absent=True, - ), -) - - -@pytest.mark.parametrize('schema', (map_conditional, map_conditional_not)) -@pytest.mark.parametrize( - 'v', - ( - # Conditional check passes, key2 is checked and passes - {'key': True, 'key2': True}, - # Conditional check fails, key2 is not checked - {'key': False, 'key2': 'ohai'}, - ), -) -def test_ok_conditional_schemas(v, schema): - validate(v, schema) - - -@pytest.mark.parametrize('schema', (map_conditional, map_conditional_not)) -def test_not_ok_conditional_schemas(schema): - with pytest.raises(ValidationError) as excinfo: - validate({'key': True, 'key2': 5}, schema) - _assert_exception_trace( - excinfo.value, - ('At foo(key=True)', 'At key: key2', 'Expected bool got int'), - ) - - -def test_ensure_absent_conditional(): - with pytest.raises(ValidationError) as excinfo: - validate({'key': False, 'key2': True}, map_conditional_absent) - _assert_exception_trace( - excinfo.value, - ( - 'At foo(key=False)', - 'Expected key2 to be absent when key is not True, ' - 'found key2: True', - ), - ) - - -def test_ensure_absent_conditional_not(): - with pytest.raises(ValidationError) as excinfo: - validate({'key': True, 'key2': True}, map_conditional_absent_not) - _assert_exception_trace( - excinfo.value, - ( - 'At foo(key=True)', - 'Expected key2 to be absent when key is True, ' - 'found key2: True', - ), - ) - - -def test_no_error_conditional_absent(): - validate({}, map_conditional_absent) - validate({}, map_conditional_absent_not) - validate({'key2': True}, map_conditional_absent) - validate({'key2': True}, map_conditional_absent_not) - - -def test_apply_defaults_copies_object(): - val = {} - ret = apply_defaults(val, map_optional) - assert ret is not val - - -def test_apply_defaults_sets_default(): - ret = apply_defaults({}, map_optional) - assert ret == {'key': False} - - -def test_apply_defaults_does_not_change_non_default(): - ret = apply_defaults({'key': True}, map_optional) - assert ret == {'key': True} - - -def test_apply_defaults_does_nothing_on_non_optional(): - ret = apply_defaults({}, map_required) - assert ret == {} - - -def test_apply_defaults_map_in_list(): - ret = apply_defaults([{}], Array(map_optional)) - assert ret == [{'key': False}] - - -def test_remove_defaults_copies_object(): - val = {'key': False} - ret = remove_defaults(val, map_optional) - assert ret is not val - - -def test_remove_defaults_removes_defaults(): - ret = remove_defaults({'key': False}, map_optional) - assert ret == {} - - -def test_remove_defaults_nothing_to_remove(): - ret = remove_defaults({}, map_optional) - assert ret == {} - - -def test_remove_defaults_does_not_change_non_default(): - ret = remove_defaults({'key': True}, map_optional) - assert ret == {'key': True} - - -def test_remove_defaults_map_in_list(): - ret = remove_defaults([{'key': False}], Array(map_optional)) - assert ret == [{}] - - -def test_remove_defaults_does_nothing_on_non_optional(): - ret = remove_defaults({'key': True}, map_required) - assert ret == {'key': True} - - -nested_schema_required = Map( - 'Repository', 'repo', - Required('repo', check_any), - RequiredRecurse('hooks', Array(map_required)), -) -nested_schema_optional = Map( - 'Repository', 'repo', - Required('repo', check_any), - RequiredRecurse('hooks', Array(map_optional)), -) - - -def test_validate_failure_nested(): - with pytest.raises(ValidationError) as excinfo: - validate({'repo': 1, 'hooks': [{}]}, nested_schema_required) - _assert_exception_trace( - excinfo.value, - ( - 'At Repository(repo=1)', 'At key: hooks', 'At foo(key=MISSING)', - 'Missing required key: key', - ), - ) - - -def test_apply_defaults_nested(): - val = {'repo': 'repo1', 'hooks': [{}]} - ret = apply_defaults(val, nested_schema_optional) - assert ret == {'repo': 'repo1', 'hooks': [{'key': False}]} - - -def test_remove_defaults_nested(): - val = {'repo': 'repo1', 'hooks': [{'key': False}]} - ret = remove_defaults(val, nested_schema_optional) - assert ret == {'repo': 'repo1', 'hooks': [{}]} - - -class Error(Exception): - pass - - -def test_load_from_filename_file_does_not_exist(): - with pytest.raises(Error) as excinfo: - load_from_filename('does_not_exist', map_required, json.loads, Error) - assert excinfo.value.args[0].error_msg == 'does_not_exist does not exist' - - -def test_load_from_filename_fails_load_strategy(tmpdir): - f = tmpdir.join('foo.notjson') - f.write('totes not json') - with pytest.raises(Error) as excinfo: - load_from_filename(f.strpath, map_required, json.loads, Error) - _assert_exception_trace( - excinfo.value.args[0], - # ANY is json's error message - ('File {}'.format(f.strpath), mock.ANY) - ) - - -def test_load_from_filename_validation_error(tmpdir): - f = tmpdir.join('foo.json') - f.write('{}') - with pytest.raises(Error) as excinfo: - load_from_filename(f.strpath, map_required, json.loads, Error) - _assert_exception_trace( - excinfo.value.args[0], - ( - 'File {}'.format(f.strpath), 'At foo(key=MISSING)', - 'Missing required key: key', - ), - ) - - -def test_load_from_filename_applies_defaults(tmpdir): - f = tmpdir.join('foo.json') - f.write('{}') - ret = load_from_filename(f.strpath, map_optional, json.loads, Error) - assert ret == {'key': False} diff --git a/tests/staged_files_only_test.py b/tests/staged_files_only_test.py index 5099c2d34..ddb957435 100644 --- a/tests/staged_files_only_test.py +++ b/tests/staged_files_only_test.py @@ -1,40 +1,39 @@ -# -*- coding: UTF-8 -*- -from __future__ import absolute_import -from __future__ import unicode_literals - -import io -import logging +import itertools import os.path import shutil -import mock import pytest +from pre_commit import git from pre_commit.staged_files_only import staged_files_only from pre_commit.util import cmd_output -from pre_commit.util import cwd from testing.auto_namedtuple import auto_namedtuple from testing.fixtures import git_dir +from testing.util import cwd from testing.util import get_resource_path +from testing.util import git_commit FOO_CONTENTS = '\n'.join(('1', '2', '3', '4', '5', '6', '7', '8', '')) +@pytest.fixture +def patch_dir(tempdir_factory): + return tempdir_factory.get() + + def get_short_git_status(): git_status = cmd_output('git', 'status', '-s')[1] - return dict(reversed(line.split()) for line in git_status.splitlines()) + line_parts = [line.split() for line in git_status.splitlines()] + return {v: k for k, v in line_parts} -@pytest.yield_fixture -def foo_staged(tempdir_factory): - path = git_dir(tempdir_factory) - with cwd(path): - with io.open('foo', 'w') as foo_file: - foo_file.write(FOO_CONTENTS) - cmd_output('git', 'add', 'foo') - foo_filename = os.path.join(path, 'foo') - yield auto_namedtuple(path=path, foo_filename=foo_filename) +@pytest.fixture +def foo_staged(in_git_dir): + foo = in_git_dir.join('foo') + foo.write(FOO_CONTENTS) + cmd_output('git', 'add', 'foo') + yield auto_namedtuple(path=in_git_dir.strpath, foo_filename=foo.strpath) def _test_foo_state( @@ -44,7 +43,8 @@ def _test_foo_state( encoding='UTF-8', ): assert os.path.exists(path.foo_filename) - assert io.open(path.foo_filename, encoding=encoding).read() == foo_contents + with open(path.foo_filename, encoding=encoding) as f: + assert f.read() == foo_contents actual_status = get_short_git_status()['foo'] assert status == actual_status @@ -53,65 +53,74 @@ def test_foo_staged(foo_staged): _test_foo_state(foo_staged) -def test_foo_nothing_unstaged(foo_staged, cmd_runner): - with staged_files_only(cmd_runner): +def test_foo_nothing_unstaged(foo_staged, patch_dir): + with staged_files_only(patch_dir): _test_foo_state(foo_staged) _test_foo_state(foo_staged) -def test_foo_something_unstaged(foo_staged, cmd_runner): - with io.open(foo_staged.foo_filename, 'w') as foo_file: +def test_foo_something_unstaged(foo_staged, patch_dir): + with open(foo_staged.foo_filename, 'w') as foo_file: foo_file.write('herp\nderp\n') _test_foo_state(foo_staged, 'herp\nderp\n', 'AM') - with staged_files_only(cmd_runner): + with staged_files_only(patch_dir): _test_foo_state(foo_staged) _test_foo_state(foo_staged, 'herp\nderp\n', 'AM') -def test_something_unstaged_ext_diff_tool(foo_staged, cmd_runner, tmpdir): +def test_does_not_crash_patch_dir_does_not_exist(foo_staged, patch_dir): + with open(foo_staged.foo_filename, 'w') as foo_file: + foo_file.write('hello\nworld\n') + + shutil.rmtree(patch_dir) + with staged_files_only(patch_dir): + pass + + +def test_something_unstaged_ext_diff_tool(foo_staged, patch_dir, tmpdir): diff_tool = tmpdir.join('diff-tool.sh') diff_tool.write('#!/usr/bin/env bash\necho "$@"\n') cmd_output('git', 'config', 'diff.external', diff_tool.strpath) - test_foo_something_unstaged(foo_staged, cmd_runner) + test_foo_something_unstaged(foo_staged, patch_dir) -def test_foo_something_unstaged_diff_color_always(foo_staged, cmd_runner): +def test_foo_something_unstaged_diff_color_always(foo_staged, patch_dir): cmd_output('git', 'config', '--local', 'color.diff', 'always') - test_foo_something_unstaged(foo_staged, cmd_runner) + test_foo_something_unstaged(foo_staged, patch_dir) -def test_foo_both_modify_non_conflicting(foo_staged, cmd_runner): - with io.open(foo_staged.foo_filename, 'w') as foo_file: - foo_file.write(FOO_CONTENTS + '9\n') +def test_foo_both_modify_non_conflicting(foo_staged, patch_dir): + with open(foo_staged.foo_filename, 'w') as foo_file: + foo_file.write(f'{FOO_CONTENTS}9\n') - _test_foo_state(foo_staged, FOO_CONTENTS + '9\n', 'AM') + _test_foo_state(foo_staged, f'{FOO_CONTENTS}9\n', 'AM') - with staged_files_only(cmd_runner): + with staged_files_only(patch_dir): _test_foo_state(foo_staged) # Modify the file as part of the "pre-commit" - with io.open(foo_staged.foo_filename, 'w') as foo_file: + with open(foo_staged.foo_filename, 'w') as foo_file: foo_file.write(FOO_CONTENTS.replace('1', 'a')) _test_foo_state(foo_staged, FOO_CONTENTS.replace('1', 'a'), 'AM') - _test_foo_state(foo_staged, FOO_CONTENTS.replace('1', 'a') + '9\n', 'AM') + _test_foo_state(foo_staged, f'{FOO_CONTENTS.replace("1", "a")}9\n', 'AM') -def test_foo_both_modify_conflicting(foo_staged, cmd_runner): - with io.open(foo_staged.foo_filename, 'w') as foo_file: +def test_foo_both_modify_conflicting(foo_staged, patch_dir): + with open(foo_staged.foo_filename, 'w') as foo_file: foo_file.write(FOO_CONTENTS.replace('1', 'a')) _test_foo_state(foo_staged, FOO_CONTENTS.replace('1', 'a'), 'AM') - with staged_files_only(cmd_runner): + with staged_files_only(patch_dir): _test_foo_state(foo_staged) # Modify in the same place as the stashed diff - with io.open(foo_staged.foo_filename, 'w') as foo_file: + with open(foo_staged.foo_filename, 'w') as foo_file: foo_file.write(FOO_CONTENTS.replace('1', 'b')) _test_foo_state(foo_staged, FOO_CONTENTS.replace('1', 'b'), 'AM') @@ -119,22 +128,19 @@ def test_foo_both_modify_conflicting(foo_staged, cmd_runner): _test_foo_state(foo_staged, FOO_CONTENTS.replace('1', 'a'), 'AM') -@pytest.yield_fixture -def img_staged(tempdir_factory): - path = git_dir(tempdir_factory) - with cwd(path): - img_filename = os.path.join(path, 'img.jpg') - shutil.copy(get_resource_path('img1.jpg'), img_filename) - cmd_output('git', 'add', 'img.jpg') - yield auto_namedtuple(path=path, img_filename=img_filename) +@pytest.fixture +def img_staged(in_git_dir): + img = in_git_dir.join('img.jpg') + shutil.copy(get_resource_path('img1.jpg'), img.strpath) + cmd_output('git', 'add', 'img.jpg') + yield auto_namedtuple(path=in_git_dir.strpath, img_filename=img.strpath) def _test_img_state(path, expected_file='img1.jpg', status='A'): assert os.path.exists(path.img_filename) - assert ( - io.open(path.img_filename, 'rb').read() == - io.open(get_resource_path(expected_file), 'rb').read() - ) + with open(path.img_filename, 'rb') as f1: + with open(get_resource_path(expected_file), 'rb') as f2: + assert f1.read() == f2.read() actual_status = get_short_git_status()['img.jpg'] assert status == actual_status @@ -143,30 +149,30 @@ def test_img_staged(img_staged): _test_img_state(img_staged) -def test_img_nothing_unstaged(img_staged, cmd_runner): - with staged_files_only(cmd_runner): +def test_img_nothing_unstaged(img_staged, patch_dir): + with staged_files_only(patch_dir): _test_img_state(img_staged) _test_img_state(img_staged) -def test_img_something_unstaged(img_staged, cmd_runner): +def test_img_something_unstaged(img_staged, patch_dir): shutil.copy(get_resource_path('img2.jpg'), img_staged.img_filename) _test_img_state(img_staged, 'img2.jpg', 'AM') - with staged_files_only(cmd_runner): + with staged_files_only(patch_dir): _test_img_state(img_staged) _test_img_state(img_staged, 'img2.jpg', 'AM') -def test_img_conflict(img_staged, cmd_runner): +def test_img_conflict(img_staged, patch_dir): """Admittedly, this shouldn't happen, but just in case.""" shutil.copy(get_resource_path('img2.jpg'), img_staged.img_filename) _test_img_state(img_staged, 'img2.jpg', 'AM') - with staged_files_only(cmd_runner): + with staged_files_only(patch_dir): _test_img_state(img_staged) shutil.copy(get_resource_path('img3.jpg'), img_staged.img_filename) _test_img_state(img_staged, 'img3.jpg', 'AM') @@ -174,30 +180,29 @@ def test_img_conflict(img_staged, cmd_runner): _test_img_state(img_staged, 'img2.jpg', 'AM') -@pytest.yield_fixture +@pytest.fixture def submodule_with_commits(tempdir_factory): path = git_dir(tempdir_factory) with cwd(path): - cmd_output('git', 'commit', '--allow-empty', '-m', 'foo') - sha1 = cmd_output('git', 'rev-parse', 'HEAD')[1].strip() - cmd_output('git', 'commit', '--allow-empty', '-m', 'bar') - sha2 = cmd_output('git', 'rev-parse', 'HEAD')[1].strip() - yield auto_namedtuple(path=path, sha1=sha1, sha2=sha2) + git_commit() + rev1 = cmd_output('git', 'rev-parse', 'HEAD')[1].strip() + git_commit() + rev2 = cmd_output('git', 'rev-parse', 'HEAD')[1].strip() + yield auto_namedtuple(path=path, rev1=rev1, rev2=rev2) -def checkout_submodule(sha): - with cwd('sub'): - cmd_output('git', 'checkout', sha) +def checkout_submodule(rev): + cmd_output('git', 'checkout', rev, cwd='sub') -@pytest.yield_fixture +@pytest.fixture def sub_staged(submodule_with_commits, tempdir_factory): path = git_dir(tempdir_factory) with cwd(path): cmd_output( 'git', 'submodule', 'add', submodule_with_commits.path, 'sub', ) - checkout_submodule(submodule_with_commits.sha1) + checkout_submodule(submodule_with_commits.rev1) cmd_output('git', 'add', 'sub') yield auto_namedtuple( path=path, @@ -206,11 +211,11 @@ def sub_staged(submodule_with_commits, tempdir_factory): ) -def _test_sub_state(path, sha='sha1', status='A'): +def _test_sub_state(path, rev='rev1', status='A'): assert os.path.exists(path.sub_path) with cwd(path.sub_path): - actual_sha = cmd_output('git', 'rev-parse', 'HEAD')[1].strip() - assert actual_sha == getattr(path.submodule, sha) + actual_rev = cmd_output('git', 'rev-parse', 'HEAD')[1].strip() + assert actual_rev == getattr(path.submodule, rev) actual_status = get_short_git_status()['sub'] assert actual_status == status @@ -219,77 +224,48 @@ def test_sub_staged(sub_staged): _test_sub_state(sub_staged) -def test_sub_nothing_unstaged(sub_staged, cmd_runner): - with staged_files_only(cmd_runner): +def test_sub_nothing_unstaged(sub_staged, patch_dir): + with staged_files_only(patch_dir): _test_sub_state(sub_staged) _test_sub_state(sub_staged) -def test_sub_something_unstaged(sub_staged, cmd_runner): - checkout_submodule(sub_staged.submodule.sha2) +def test_sub_something_unstaged(sub_staged, patch_dir): + checkout_submodule(sub_staged.submodule.rev2) - _test_sub_state(sub_staged, 'sha2', 'AM') + _test_sub_state(sub_staged, 'rev2', 'AM') - with staged_files_only(cmd_runner): + with staged_files_only(patch_dir): # This is different from others, we don't want to touch subs - _test_sub_state(sub_staged, 'sha2', 'AM') - - _test_sub_state(sub_staged, 'sha2', 'AM') - - -@pytest.yield_fixture -def fake_logging_handler(): - class FakeHandler(logging.Handler): - def __init__(self): - logging.Handler.__init__(self) - self.logs = [] - - def emit(self, record): - self.logs.append(record) # pragma: no cover (only hit in failure) + _test_sub_state(sub_staged, 'rev2', 'AM') - pre_commit_logger = logging.getLogger('pre_commit') - original_level = pre_commit_logger.getEffectiveLevel() - handler = FakeHandler() - pre_commit_logger.addHandler(handler) - pre_commit_logger.setLevel(logging.WARNING) - yield handler - pre_commit_logger.setLevel(original_level) - pre_commit_logger.removeHandler(handler) + _test_sub_state(sub_staged, 'rev2', 'AM') -def test_diff_returns_1_no_diff_though(fake_logging_handler, foo_staged): - cmd_runner = mock.Mock() - cmd_runner.run.return_value = (1, '', '') - cmd_runner.path.return_value = '.pre-commit-files_patch' - with staged_files_only(cmd_runner): - pass - assert not fake_logging_handler.logs - - -def test_stage_utf8_changes(foo_staged, cmd_runner): +def test_stage_utf8_changes(foo_staged, patch_dir): contents = '\u2603' - with io.open('foo', 'w', encoding='UTF-8') as foo_file: + with open('foo', 'w', encoding='UTF-8') as foo_file: foo_file.write(contents) _test_foo_state(foo_staged, contents, 'AM') - with staged_files_only(cmd_runner): + with staged_files_only(patch_dir): _test_foo_state(foo_staged) _test_foo_state(foo_staged, contents, 'AM') -def test_stage_non_utf8_changes(foo_staged, cmd_runner): +def test_stage_non_utf8_changes(foo_staged, patch_dir): contents = 'ΓΊ' # Produce a latin-1 diff - with io.open('foo', 'w', encoding='latin-1') as foo_file: + with open('foo', 'w', encoding='latin-1') as foo_file: foo_file.write(contents) _test_foo_state(foo_staged, contents, 'AM', encoding='latin-1') - with staged_files_only(cmd_runner): + with staged_files_only(patch_dir): _test_foo_state(foo_staged) _test_foo_state(foo_staged, contents, 'AM', encoding='latin-1') -def test_non_utf8_conflicting_diff(foo_staged, cmd_runner): +def test_non_utf8_conflicting_diff(foo_staged, patch_dir): """Regression test for #397""" # The trailing whitespace is important here, this triggers git to produce # an error message which looks like: @@ -302,13 +278,72 @@ def test_non_utf8_conflicting_diff(foo_staged, cmd_runner): # Previously, the error message (though discarded immediately) was being # decoded with the UTF-8 codec (causing a crash) contents = 'ΓΊ \n' - with io.open('foo', 'w', encoding='latin-1') as foo_file: + with open('foo', 'w', encoding='latin-1') as foo_file: foo_file.write(contents) _test_foo_state(foo_staged, contents, 'AM', encoding='latin-1') - with staged_files_only(cmd_runner): + with staged_files_only(patch_dir): _test_foo_state(foo_staged) # Create a conflicting diff that will need to be rolled back - with io.open('foo', 'w') as foo_file: + with open('foo', 'w') as foo_file: foo_file.write('') _test_foo_state(foo_staged, contents, 'AM', encoding='latin-1') + + +def _write(b): + with open('foo', 'wb') as f: + f.write(b) + + +def assert_no_diff(): + tree = cmd_output('git', 'write-tree')[1].strip() + cmd_output('git', 'diff-index', tree, '--exit-code') + + +bool_product = tuple(itertools.product((True, False), repeat=2)) + + +@pytest.mark.parametrize(('crlf_before', 'crlf_after'), bool_product) +@pytest.mark.parametrize('autocrlf', ('true', 'false', 'input')) +def test_crlf(in_git_dir, patch_dir, crlf_before, crlf_after, autocrlf): + cmd_output('git', 'config', '--local', 'core.autocrlf', autocrlf) + + before, after = b'1\n2\n', b'3\n4\n\n' + before = before.replace(b'\n', b'\r\n') if crlf_before else before + after = after.replace(b'\n', b'\r\n') if crlf_after else after + + _write(before) + cmd_output('git', 'add', 'foo') + _write(after) + with staged_files_only(patch_dir): + assert_no_diff() + + +def test_whitespace_errors(in_git_dir, patch_dir): + cmd_output('git', 'config', '--local', 'apply.whitespace', 'error') + test_crlf(in_git_dir, patch_dir, True, True, 'true') + + +def test_autocrlf_committed_crlf(in_git_dir, patch_dir): + """Regression test for #570""" + cmd_output('git', 'config', '--local', 'core.autocrlf', 'false') + _write(b'1\r\n2\r\n') + cmd_output('git', 'add', 'foo') + git_commit() + + cmd_output('git', 'config', '--local', 'core.autocrlf', 'true') + _write(b'1\r\n2\r\n\r\n\r\n\r\n') + + with staged_files_only(patch_dir): + assert_no_diff() + + +def test_intent_to_add(in_git_dir, patch_dir): + """Regression test for #881""" + _write(b'hello\nworld\n') + cmd_output('git', 'add', '--intent-to-add', 'foo') + + assert git.intent_to_add_files() == ['foo'] + with staged_files_only(patch_dir): + assert_no_diff() + assert git.intent_to_add_files() == ['foo'] diff --git a/tests/store_test.py b/tests/store_test.py index 268579655..586661619 100644 --- a/tests/store_test.py +++ b/tests/store_test.py @@ -1,21 +1,17 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - -import io import os.path import sqlite3 +from unittest import mock -import mock import pytest -import six +from pre_commit import git from pre_commit.store import _get_default_directory from pre_commit.store import Store +from pre_commit.util import CalledProcessError from pre_commit.util import cmd_output -from pre_commit.util import cwd -from pre_commit.util import rmtree from testing.fixtures import git_dir -from testing.util import get_head_sha +from testing.util import cwd +from testing.util import git_commit def test_our_session_fixture_works(): @@ -29,24 +25,30 @@ def test_our_session_fixture_works(): def test_get_default_directory_defaults_to_home(): # Not we use the module level one which is not mocked ret = _get_default_directory() - assert ret == os.path.join(os.path.expanduser('~'), '.pre-commit') + assert ret == os.path.join(os.path.expanduser('~/.cache'), 'pre-commit') + + +def test_adheres_to_xdg_specification(): + with mock.patch.dict( + os.environ, {'XDG_CACHE_HOME': '/tmp/fakehome'}, + ): + ret = _get_default_directory() + assert ret == os.path.join('/tmp/fakehome', 'pre-commit') def test_uses_environment_variable_when_present(): with mock.patch.dict( - os.environ, {'PRE_COMMIT_HOME': '/tmp/pre_commit_home'} + os.environ, {'PRE_COMMIT_HOME': '/tmp/pre_commit_home'}, ): ret = _get_default_directory() assert ret == '/tmp/pre_commit_home' -def test_store_require_created(store): - assert not os.path.exists(store.directory) - store.require_created() +def test_store_init(store): # Should create the store directory assert os.path.exists(store.directory) # Should create a README file indicating what the directory is about - with io.open(os.path.join(store.directory, 'README'), 'r') as readme_file: + with open(os.path.join(store.directory, 'README')) as readme_file: readme_contents = readme_file.read() for text_line in ( 'This directory is maintained by the pre-commit project.', @@ -55,41 +57,17 @@ def test_store_require_created(store): assert text_line in readme_contents -def test_store_require_created_does_not_create_twice(store): - assert not os.path.exists(store.directory) - store.require_created() - # We intentionally delete the directory here so we can figure out if it - # calls it again. - rmtree(store.directory) - assert not os.path.exists(store.directory) - # Call require_created, this should not trigger a call to create - store.require_created() - assert not os.path.exists(store.directory) - - -def test_does_not_recreate_if_directory_already_exists(store): - assert not os.path.exists(store.directory) - # We manually create the directory. - # Note: we're intentionally leaving out the README file. This is so we can - # know that `Store` didn't call create - os.mkdir(store.directory) - io.open(store.db_path, 'a+').close() - # Call require_created, this should not call create - store.require_created() - assert not os.path.exists(os.path.join(store.directory, 'README')) - - def test_clone(store, tempdir_factory, log_info_mock): path = git_dir(tempdir_factory) with cwd(path): - cmd_output('git', 'commit', '--allow-empty', '-m', 'foo') - sha = get_head_sha(path) - cmd_output('git', 'commit', '--allow-empty', '-m', 'bar') + git_commit() + rev = git.head_rev(path) + git_commit() - ret = store.clone(path, sha) + ret = store.clone(path, rev) # Should have printed some stuff assert log_info_mock.call_args_list[0][0][0].startswith( - 'Initializing environment for ' + 'Initializing environment for ', ) # Should return a directory inside of the store @@ -98,55 +76,141 @@ def test_clone(store, tempdir_factory, log_info_mock): # Directory should start with `repo` _, dirname = os.path.split(ret) assert dirname.startswith('repo') - # Should be checked out to the sha we specified - assert get_head_sha(ret) == sha + # Should be checked out to the rev we specified + assert git.head_rev(ret) == rev # Assert there's an entry in the sqlite db for this - with sqlite3.connect(store.db_path) as db: - path, = db.execute( - 'SELECT path from repos WHERE repo = ? and ref = ?', - [path, sha], - ).fetchone() - assert path == ret + assert store.select_all_repos() == [(path, rev, ret)] def test_clone_cleans_up_on_checkout_failure(store): - try: + with pytest.raises(Exception) as excinfo: # This raises an exception because you can't clone something that # doesn't exist! - store.clone('/i_dont_exist_lol', 'fake_sha') - except Exception as e: - assert '/i_dont_exist_lol' in six.text_type(e) + store.clone('/i_dont_exist_lol', 'fake_rev') + assert '/i_dont_exist_lol' in str(excinfo.value) - things_starting_with_repo = [ - thing for thing in os.listdir(store.directory) - if thing.startswith('repo') + repo_dirs = [ + d for d in os.listdir(store.directory) if d.startswith('repo') ] - assert things_starting_with_repo == [] - - -def test_has_cmd_runner_at_directory(store): - ret = store.cmd_runner - assert ret.prefix_dir == store.directory + os.sep + assert repo_dirs == [] def test_clone_when_repo_already_exists(store): # Create an entry in the sqlite db that makes it look like the repo has # been cloned. - store.require_created() - with sqlite3.connect(store.db_path) as db: db.execute( 'INSERT INTO repos (repo, ref, path) ' - 'VALUES ("fake_repo", "fake_ref", "fake_path")' + 'VALUES ("fake_repo", "fake_ref", "fake_path")', ) assert store.clone('fake_repo', 'fake_ref') == 'fake_path' -def test_require_created_when_directory_exists_but_not_db(store): +def test_clone_shallow_failure_fallback_to_complete( + store, tempdir_factory, + log_info_mock, +): + path = git_dir(tempdir_factory) + with cwd(path): + git_commit() + rev = git.head_rev(path) + git_commit() + + # Force shallow clone failure + def fake_shallow_clone(self, *args, **kwargs): + raise CalledProcessError(1, (), 0, b'', None) + store._shallow_clone = fake_shallow_clone + + ret = store.clone(path, rev) + + # Should have printed some stuff + assert log_info_mock.call_args_list[0][0][0].startswith( + 'Initializing environment for ', + ) + + # Should return a directory inside of the store + assert os.path.exists(ret) + assert ret.startswith(store.directory) + # Directory should start with `repo` + _, dirname = os.path.split(ret) + assert dirname.startswith('repo') + # Should be checked out to the rev we specified + assert git.head_rev(ret) == rev + + # Assert there's an entry in the sqlite db for this + assert store.select_all_repos() == [(path, rev, ret)] + + +def test_clone_tag_not_on_mainline(store, tempdir_factory): + path = git_dir(tempdir_factory) + with cwd(path): + git_commit() + cmd_output('git', 'checkout', 'master', '-b', 'branch') + git_commit() + cmd_output('git', 'tag', 'v1') + cmd_output('git', 'checkout', 'master') + cmd_output('git', 'branch', '-D', 'branch') + + # previously crashed on unreachable refs + store.clone(path, 'v1') + + +def test_create_when_directory_exists_but_not_db(store): # In versions <= 0.3.5, there was no sqlite db causing a need for # backward compatibility - os.makedirs(store.directory) - store.require_created() + os.remove(store.db_path) + store = Store(store.directory) assert os.path.exists(store.db_path) + + +def test_create_when_store_already_exists(store): + # an assertion that this is idempotent and does not crash + Store(store.directory) + + +def test_db_repo_name(store): + assert store.db_repo_name('repo', ()) == 'repo' + assert store.db_repo_name('repo', ('b', 'a', 'c')) == 'repo:a,b,c' + + +def test_local_resources_reflects_reality(): + on_disk = { + res[len('empty_template_'):] + for res in os.listdir('pre_commit/resources') + if res.startswith('empty_template_') + } + assert on_disk == set(Store.LOCAL_RESOURCES) + + +def test_mark_config_as_used(store, tmpdir): + with tmpdir.as_cwd(): + f = tmpdir.join('f').ensure() + store.mark_config_used('f') + assert store.select_all_configs() == [f.strpath] + + +def test_mark_config_as_used_idempotent(store, tmpdir): + test_mark_config_as_used(store, tmpdir) + test_mark_config_as_used(store, tmpdir) + + +def test_mark_config_as_used_does_not_exist(store): + store.mark_config_used('f') + assert store.select_all_configs() == [] + + +def _simulate_pre_1_14_0(store): + with store.connect() as db: + db.executescript('DROP TABLE configs') + + +def test_select_all_configs_roll_forward(store): + _simulate_pre_1_14_0(store) + assert store.select_all_configs() == [] + + +def test_mark_config_as_used_roll_forward(store, tmpdir): + _simulate_pre_1_14_0(store) + test_mark_config_as_used(store, tmpdir) diff --git a/tests/util_test.py b/tests/util_test.py index ba2b4a821..01afbd4bf 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -1,46 +1,42 @@ -from __future__ import unicode_literals - import os.path -import random +import stat +import subprocess import pytest +from pre_commit.util import CalledProcessError from pre_commit.util import clean_path_on_failure from pre_commit.util import cmd_output -from pre_commit.util import cwd -from pre_commit.util import memoize_by_cwd +from pre_commit.util import cmd_output_b +from pre_commit.util import cmd_output_p +from pre_commit.util import make_executable +from pre_commit.util import parse_version +from pre_commit.util import rmtree from pre_commit.util import tmpdir -@pytest.fixture -def memoized_by_cwd(): - @memoize_by_cwd - def func(arg): - return arg + str(random.getrandbits(64)) - - return func - - -def test_memoized_by_cwd_returns_same_twice_in_a_row(memoized_by_cwd): - ret = memoized_by_cwd('baz') - ret2 = memoized_by_cwd('baz') - assert ret is ret2 - +def test_CalledProcessError_str(): + error = CalledProcessError(1, ('exe',), 0, b'output', b'errors') + assert str(error) == ( + "command: ('exe',)\n" + 'return code: 1\n' + 'expected return code: 0\n' + 'stdout:\n' + ' output\n' + 'stderr:\n' + ' errors' + ) -def test_memoized_by_cwd_returns_different_for_different_args(memoized_by_cwd): - ret = memoized_by_cwd('baz') - ret2 = memoized_by_cwd('bar') - assert ret.startswith('baz') - assert ret2.startswith('bar') - assert ret != ret2 - -def test_memoized_by_cwd_changes_with_different_cwd(memoized_by_cwd): - ret = memoized_by_cwd('baz') - with cwd('.git'): - ret2 = memoized_by_cwd('baz') - - assert ret != ret2 +def test_CalledProcessError_str_nooutput(): + error = CalledProcessError(1, ('exe',), 0, b'', b'') + assert str(error) == ( + "command: ('exe',)\n" + 'return code: 1\n' + 'expected return code: 0\n' + 'stdout: (none)\n' + 'stderr: (none)' + ) def test_clean_on_failure_noop(in_tmpdir): @@ -85,6 +81,42 @@ def test_tmpdir(): def test_cmd_output_exe_not_found(): - ret, out, _ = cmd_output('i-dont-exist', retcode=None) + ret, out, _ = cmd_output('dne', retcode=None) + assert ret == 1 + assert out == 'Executable `dne` not found' + + +@pytest.mark.parametrize('fn', (cmd_output_b, cmd_output_p)) +def test_cmd_output_exe_not_found_bytes(fn): + ret, out, _ = fn('dne', retcode=None, stderr=subprocess.STDOUT) + assert ret == 1 + assert out == b'Executable `dne` not found' + + +@pytest.mark.parametrize('fn', (cmd_output_b, cmd_output_p)) +def test_cmd_output_no_shebang(tmpdir, fn): + f = tmpdir.join('f').ensure() + make_executable(f) + + # previously this raised `OSError` -- the output is platform specific + ret, out, _ = fn(str(f), retcode=None, stderr=subprocess.STDOUT) assert ret == 1 - assert out == 'Executable `i-dont-exist` not found' + assert isinstance(out, bytes) + assert out.endswith(b'\n') + + +def test_parse_version(): + assert parse_version('0.0') == parse_version('0.0') + assert parse_version('0.1') > parse_version('0.0') + assert parse_version('2.1') >= parse_version('2') + + +def test_rmtree_read_only_directories(tmpdir): + """Simulates the go module tree. See #1042""" + tmpdir.join('x/y/z').ensure_dir().join('a').ensure() + mode = os.stat(str(tmpdir.join('x'))).st_mode + mode_no_w = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) + tmpdir.join('x/y/z').chmod(mode_no_w) + tmpdir.join('x/y/z').chmod(mode_no_w) + tmpdir.join('x/y/z').chmod(mode_no_w) + rmtree(str(tmpdir.join('x'))) diff --git a/tests/xargs_test.py b/tests/xargs_test.py index 529eb197c..1fc920725 100644 --- a/tests/xargs_test.py +++ b/tests/xargs_test.py @@ -1,17 +1,50 @@ -from __future__ import absolute_import -from __future__ import unicode_literals +import concurrent.futures +import os +import sys +import time +from typing import Tuple +from unittest import mock import pytest +from pre_commit import parse_shebang from pre_commit import xargs +@pytest.mark.parametrize( + ('env', 'expected'), + ( + ({}, 0), + ({b'x': b'1'}, 12), + ({b'x': b'12'}, 13), + ({b'x': b'1', b'y': b'2'}, 24), + ), +) +def test_environ_size(env, expected): + # normalize integer sizing + assert xargs._environ_size(_env=env) == expected + + +@pytest.fixture +def win32_mock(): + with mock.patch.object(sys, 'getfilesystemencoding', return_value='utf-8'): + with mock.patch.object(sys, 'platform', 'win32'): + yield + + +@pytest.fixture +def linux_mock(): + with mock.patch.object(sys, 'getfilesystemencoding', return_value='utf-8'): + with mock.patch.object(sys, 'platform', 'linux'): + yield + + def test_partition_trivial(): - assert xargs.partition(('cmd',), ()) == (('cmd',),) + assert xargs.partition(('cmd',), (), 1) == (('cmd',),) def test_partition_simple(): - assert xargs.partition(('cmd',), ('foo',)) == (('cmd', 'foo'),) + assert xargs.partition(('cmd',), ('foo',), 1) == (('cmd', 'foo'),) def test_partition_limits(): @@ -25,7 +58,8 @@ def test_partition_limits(): '.' * 5, '.' * 6, ), - _max_length=20, + 1, + _max_length=21, ) assert ret == ( ('ninechars', '.' * 5, '.' * 4), @@ -35,43 +69,129 @@ def test_partition_limits(): ) +def test_partition_limit_win32(win32_mock): + cmd = ('ninechars',) + # counted as half because of utf-16 encode + varargs = ('πŸ˜‘' * 5,) + ret = xargs.partition(cmd, varargs, 1, _max_length=21) + assert ret == (cmd + varargs,) + + +def test_partition_limit_linux(linux_mock): + cmd = ('ninechars',) + varargs = ('πŸ˜‘' * 5,) + ret = xargs.partition(cmd, varargs, 1, _max_length=31) + assert ret == (cmd + varargs,) + + +def test_argument_too_long_with_large_unicode(linux_mock): + cmd = ('ninechars',) + varargs = ('πŸ˜‘' * 10,) # 4 bytes * 10 + with pytest.raises(xargs.ArgumentTooLongError): + xargs.partition(cmd, varargs, 1, _max_length=20) + + +def test_partition_target_concurrency(): + ret = xargs.partition( + ('foo',), ('A',) * 22, + 4, + _max_length=50, + ) + assert ret == ( + ('foo',) + ('A',) * 6, + ('foo',) + ('A',) * 6, + ('foo',) + ('A',) * 6, + ('foo',) + ('A',) * 4, + ) + + +def test_partition_target_concurrency_wont_make_tiny_partitions(): + ret = xargs.partition( + ('foo',), ('A',) * 10, + 4, + _max_length=50, + ) + assert ret == ( + ('foo',) + ('A',) * 4, + ('foo',) + ('A',) * 4, + ('foo',) + ('A',) * 2, + ) + + def test_argument_too_long(): with pytest.raises(xargs.ArgumentTooLongError): - xargs.partition(('a' * 5,), ('a' * 5,), _max_length=10) + xargs.partition(('a' * 5,), ('a' * 5,), 1, _max_length=10) def test_xargs_smoke(): - ret, out, err = xargs.xargs(('echo',), ('hello', 'world')) + ret, out = xargs.xargs(('echo',), ('hello', 'world')) assert ret == 0 - assert out == b'hello world\n' - assert err == b'' + assert out.replace(b'\r\n', b'\n') == b'hello world\n' -exit_cmd = ('bash', '-c', 'exit $1', '--') +exit_cmd = parse_shebang.normalize_cmd(('bash', '-c', 'exit $1', '--')) # Abuse max_length to control the exit code -max_length = len(' '.join(exit_cmd)) + 2 +max_length = len(' '.join(exit_cmd)) + 3 -def test_xargs_negate(): - ret, _, _ = xargs.xargs( - exit_cmd, ('1',), negate=True, _max_length=max_length, - ) +def test_xargs_retcode_normal(): + ret, _ = xargs.xargs(exit_cmd, ('0',), _max_length=max_length) assert ret == 0 - ret, _, _ = xargs.xargs( - exit_cmd, ('1', '0'), negate=True, _max_length=max_length, - ) + ret, _ = xargs.xargs(exit_cmd, ('0', '1'), _max_length=max_length) assert ret == 1 + # takes the maximum return code + ret, _ = xargs.xargs(exit_cmd, ('0', '5', '1'), _max_length=max_length) + assert ret == 5 -def test_xargs_negate_command_not_found(): - ret, _, _ = xargs.xargs(('cmd-not-found',), ('1',), negate=True) - assert ret != 0 +def test_xargs_concurrency(): + bash_cmd = parse_shebang.normalize_cmd(('bash', '-c')) + print_pid = ('sleep 0.5 && echo $$',) -def test_xargs_retcode_normal(): - ret, _, _ = xargs.xargs(exit_cmd, ('0',), _max_length=max_length) + start = time.time() + ret, stdout = xargs.xargs( + bash_cmd, print_pid * 5, + target_concurrency=5, + _max_length=len(' '.join(bash_cmd + print_pid)) + 1, + ) + elapsed = time.time() - start assert ret == 0 + pids = stdout.splitlines() + assert len(pids) == 5 + # It would take 0.5*5=2.5 seconds ot run all of these in serial, so if it + # takes less, they must have run concurrently. + assert elapsed < 2.5 - ret, _, _ = xargs.xargs(exit_cmd, ('0', '1'), _max_length=max_length) - assert ret == 1 + +def test_thread_mapper_concurrency_uses_threadpoolexecutor_map(): + with xargs._thread_mapper(10) as thread_map: + _self = thread_map.__self__ # type: ignore + assert isinstance(_self, concurrent.futures.ThreadPoolExecutor) + + +def test_thread_mapper_concurrency_uses_regular_map(): + with xargs._thread_mapper(1) as thread_map: + assert thread_map is map + + +def test_xargs_propagate_kwargs_to_cmd(): + env = {'PRE_COMMIT_TEST_VAR': 'Pre commit is awesome'} + cmd: Tuple[str, ...] = ('bash', '-c', 'echo $PRE_COMMIT_TEST_VAR', '--') + cmd = parse_shebang.normalize_cmd(cmd) + + ret, stdout = xargs.xargs(cmd, ('1',), env=env) + assert ret == 0 + assert b'Pre commit is awesome' in stdout + + +@pytest.mark.xfail(os.name == 'nt', reason='posix only') +def test_xargs_color_true_makes_tty(): + retcode, out = xargs.xargs( + (sys.executable, '-c', 'import sys; print(sys.stdout.isatty())'), + ('1',), + color=True, + ) + assert retcode == 0 + assert out == b'True\n' diff --git a/tox.ini b/tox.ini index 872b4c359..d9f9420c9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,24 +1,22 @@ [tox] -project = pre_commit -# These should match the travis env list -envlist = py27,py35,py36,pypy +envlist = py36,py37,py38,pypy3,pre-commit [testenv] deps = -rrequirements-dev.txt -passenv = GOROOT HOME HOMEPATH PROGRAMDATA TERM +passenv = HOME LOCALAPPDATA RUSTUP_HOME commands = coverage erase coverage run -m pytest {posargs:tests} - # TODO: change to 100 - coverage report --fail-under 99 - pre-commit run --all-files + coverage report + pre-commit install -[testenv:venv] -envdir = venv-{[tox]project} -commands = +[testenv:pre-commit] +skip_install = true +deps = pre-commit +commands = pre-commit run --all-files --show-diff-on-failure [pep8] -ignore = E265,E309,E501 +ignore = E265,E501,W504 [pytest] env = @@ -27,3 +25,4 @@ env = GIT_AUTHOR_EMAIL=test@example.com GIT_COMMITTER_EMAIL=test@example.com VIRTUALENV_NO_DOWNLOAD=1 + PRE_COMMIT_NO_CONCURRENCY=1