Skip to content

Commit 82369fd

Browse files
asottileasottile
committed
Add utility for parsing shebangs and resolving PATH
1 parent a932315 commit 82369fd

File tree

6 files changed

+267
-13
lines changed

6 files changed

+267
-13
lines changed

‎pre_commit/commands/install_uninstall.py‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@
55
import logging
66
import os
77
import os.path
8-
import stat
98
import sys
109

1110
from pre_commit.logging_handler import LoggingHandler
11+
from pre_commit.util import make_executable
1212
from pre_commit.util import mkdirp
1313
from pre_commit.util import resource_filename
1414

@@ -42,14 +42,6 @@ def is_previous_pre_commit(filename):
4242
return any(hash in contents for hash in PREVIOUS_IDENTIFYING_HASHES)
4343

4444

45-
def make_executable(filename):
46-
original_mode = os.stat(filename).st_mode
47-
os.chmod(
48-
filename,
49-
original_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH,
50-
)
51-
52-
5345
def install(runner, overwrite=False, hooks=False, hook_type='pre-commit'):
5446
"""Install the pre-commit hooks."""
5547
hook_path = runner.get_hook_path(hook_type)

‎pre_commit/parse_shebang.py‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
from __future__ import absolute_import
2+
from __future__ import unicode_literals
3+
4+
import io
5+
import os.path
6+
import shlex
7+
import string
8+
9+
from pre_commit import five
10+
11+
12+
printable = frozenset(string.printable)
13+
14+
15+
def parse_bytesio(bytesio):
16+
"""Parse the shebang from a file opened for reading binary."""
17+
if bytesio.read(2) != b'#!':
18+
return ()
19+
first_line = bytesio.readline()
20+
try:
21+
first_line = first_line.decode('US-ASCII')
22+
except UnicodeDecodeError:
23+
return ()
24+
25+
# Require only printable ascii
26+
for c in first_line:
27+
if c not in printable:
28+
return ()
29+
30+
# shlex.split is horribly broken in py26 on text strings
31+
cmd = tuple(shlex.split(five.n(first_line)))
32+
if cmd[0] == '/usr/bin/env':
33+
cmd = cmd[1:]
34+
return cmd
35+
36+
37+
def parse_filename(filename):
38+
"""Parse the shebang given a filename."""
39+
if not os.path.exists(filename) or not os.access(filename, os.X_OK):
40+
return ()
41+
42+
with io.open(filename, 'rb') as f:
43+
return parse_bytesio(f)
44+
45+
46+
def find_executable(exe, _environ=None):
47+
exe = os.path.normpath(exe)
48+
if os.sep in exe:
49+
return exe
50+
51+
environ = _environ if _environ is not None else os.environ
52+
53+
if 'PATHEXT' in environ:
54+
possible_exe_names = (exe,) + tuple(
55+
exe + ext.lower() for ext in environ['PATHEXT'].split(os.pathsep)
56+
)
57+
else:
58+
possible_exe_names = (exe,)
59+
60+
for path in environ.get('PATH', '').split(os.pathsep):
61+
for possible_exe_name in possible_exe_names:
62+
joined = os.path.join(path, possible_exe_name)
63+
if os.path.isfile(joined) and os.access(joined, os.X_OK):
64+
return joined
65+
else:
66+
return None
67+
68+
69+
def normexe(orig_exe):
70+
if os.sep not in orig_exe:
71+
exe = find_executable(orig_exe)
72+
if exe is None:
73+
raise OSError('Executable {0} not found'.format(orig_exe))
74+
return exe
75+
else:
76+
return orig_exe
77+
78+
79+
def normalize_cmd(cmd):
80+
"""Fixes for the following issues on windows
81+
- http://bugs.python.org/issue8557
82+
- windows does not parse shebangs
83+
84+
This function also makes deep-path shebangs work just fine
85+
"""
86+
# Use PATH to determine the executable
87+
exe = normexe(cmd[0])
88+
89+
# Figure out the shebang from the resulting command
90+
cmd = parse_filename(exe) + (exe,) + cmd[1:]
91+
92+
# This could have given us back another bare executable
93+
exe = normexe(cmd[0])
94+
95+
return (exe,) + cmd[1:]

‎pre_commit/util.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import pkg_resources
1515

1616
from pre_commit import five
17+
from pre_commit import parse_shebang
1718

1819

1920
@contextlib.contextmanager
@@ -110,6 +111,14 @@ def resource_filename(filename):
110111
)
111112

112113

114+
def make_executable(filename):
115+
original_mode = os.stat(filename).st_mode
116+
os.chmod(
117+
filename,
118+
original_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH,
119+
)
120+
121+
113122
class CalledProcessError(RuntimeError):
114123
def __init__(self, returncode, cmd, expected_returncode, output=None):
115124
super(CalledProcessError, self).__init__(
@@ -166,12 +175,14 @@ def cmd_output(*cmd, **kwargs):
166175
}
167176

168177
# py2/py3 on windows are more strict about the types here
169-
cmd = [five.n(arg) for arg in cmd]
178+
cmd = tuple(five.n(arg) for arg in cmd)
170179
kwargs['env'] = dict(
171180
(five.n(key), five.n(value))
172181
for key, value in kwargs.pop('env', {}).items()
173182
) or None
174183

184+
cmd = parse_shebang.normalize_cmd(cmd)
185+
175186
popen_kwargs.update(kwargs)
176187
proc = __popen(cmd, **popen_kwargs)
177188
stdout, stderr = proc.communicate()

‎tests/commands/install_uninstall_test.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,12 @@
1515
from pre_commit.commands.install_uninstall import install
1616
from pre_commit.commands.install_uninstall import is_our_pre_commit
1717
from pre_commit.commands.install_uninstall import is_previous_pre_commit
18-
from pre_commit.commands.install_uninstall import make_executable
1918
from pre_commit.commands.install_uninstall import PREVIOUS_IDENTIFYING_HASHES
2019
from pre_commit.commands.install_uninstall import uninstall
2120
from pre_commit.runner import Runner
2221
from pre_commit.util import cmd_output
2322
from pre_commit.util import cwd
23+
from pre_commit.util import make_executable
2424
from pre_commit.util import mkdirp
2525
from pre_commit.util import resource_filename
2626
from testing.fixtures import git_dir
@@ -473,6 +473,8 @@ def test_installed_from_venv(tempdir_factory):
473473
'TERM': os.environ.get('TERM', ''),
474474
# Windows needs this to import `random`
475475
'SYSTEMROOT': os.environ.get('SYSTEMROOT', ''),
476+
# Windows needs this to resolve executables
477+
'PATHEXT': os.environ.get('PATHEXT', ''),
476478
},
477479
)
478480
assert ret == 0

‎tests/parse_shebang_test.py‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
from __future__ import absolute_import
2+
from __future__ import unicode_literals
3+
4+
import contextlib
5+
import distutils.spawn
6+
import io
7+
import os
8+
import sys
9+
10+
import pytest
11+
12+
from pre_commit import parse_shebang
13+
from pre_commit.envcontext import envcontext
14+
from pre_commit.envcontext import Var
15+
from pre_commit.util import make_executable
16+
17+
18+
@pytest.mark.parametrize(
19+
('s', 'expected'),
20+
(
21+
(b'', ()),
22+
(b'#!/usr/bin/python', ('/usr/bin/python',)),
23+
(b'#!/usr/bin/env python', ('python',)),
24+
(b'#! /usr/bin/python', ('/usr/bin/python',)),
25+
(b'#!/usr/bin/foo python', ('/usr/bin/foo', 'python')),
26+
(b'\xf9\x93\x01\x42\xcd', ()),
27+
(b'#!\xf9\x93\x01\x42\xcd', ()),
28+
(b'#!\x00\x00\x00\x00', ()),
29+
),
30+
)
31+
def test_parse_bytesio(s, expected):
32+
assert parse_shebang.parse_bytesio(io.BytesIO(s)) == expected
33+
34+
35+
def test_file_doesnt_exist():
36+
assert parse_shebang.parse_filename('herp derp derp') == ()
37+
38+
39+
@pytest.mark.xfail(
40+
sys.platform == 'win32', reason='Windows says everything is X_OK',
41+
)
42+
def test_file_not_executable(tmpdir):
43+
x = tmpdir.join('f')
44+
x.write_text('#!/usr/bin/env python', encoding='UTF-8')
45+
assert parse_shebang.parse_filename(x.strpath) == ()
46+
47+
48+
def test_simple_case(tmpdir):
49+
x = tmpdir.join('f')
50+
x.write_text('#!/usr/bin/env python', encoding='UTF-8')
51+
make_executable(x.strpath)
52+
assert parse_shebang.parse_filename(x.strpath) == ('python',)
53+
54+
55+
def test_find_executable_full_path():
56+
assert parse_shebang.find_executable(sys.executable) == sys.executable
57+
58+
59+
def test_find_executable_on_path():
60+
expected = distutils.spawn.find_executable('echo')
61+
assert parse_shebang.find_executable('echo') == expected
62+
63+
64+
def test_find_executable_not_found_none():
65+
assert parse_shebang.find_executable('not-a-real-executable') is None
66+
67+
68+
def write_executable(shebang, filename='run'):
69+
os.mkdir('bin')
70+
path = os.path.join('bin', filename)
71+
with io.open(path, 'w') as f:
72+
f.write('#!{0}'.format(shebang))
73+
make_executable(path)
74+
return path
75+
76+
77+
@contextlib.contextmanager
78+
def bin_on_path():
79+
bindir = os.path.join(os.getcwd(), 'bin')
80+
with envcontext((('PATH', (bindir, os.pathsep, Var('PATH'))),)):
81+
yield
82+
83+
84+
def test_find_executable_path_added(in_tmpdir):
85+
path = os.path.abspath(write_executable('/usr/bin/env sh'))
86+
assert parse_shebang.find_executable('run') is None
87+
with bin_on_path():
88+
assert parse_shebang.find_executable('run') == path
89+
90+
91+
def test_find_executable_path_ext(in_tmpdir):
92+
"""Windows exports PATHEXT as a list of extensions to automatically add
93+
to executables when doing PATH searching.
94+
"""
95+
exe_path = os.path.abspath(write_executable(
96+
'/usr/bin/env sh', filename='run.myext',
97+
))
98+
env_path = {'PATH': os.path.dirname(exe_path)}
99+
env_path_ext = dict(env_path, PATHEXT=os.pathsep.join(('.exe', '.myext')))
100+
assert parse_shebang.find_executable('run') is None
101+
assert parse_shebang.find_executable('run', _environ=env_path) is None
102+
ret = parse_shebang.find_executable('run.myext', _environ=env_path)
103+
assert ret == exe_path
104+
ret = parse_shebang.find_executable('run', _environ=env_path_ext)
105+
assert ret == exe_path
106+
107+
108+
def test_normexe_does_not_exist():
109+
with pytest.raises(OSError) as excinfo:
110+
parse_shebang.normexe('i-dont-exist-lol')
111+
assert excinfo.value.args == ('Executable i-dont-exist-lol not found',)
112+
113+
114+
def test_normexe_already_full_path():
115+
assert parse_shebang.normexe(sys.executable) == sys.executable
116+
117+
118+
def test_normexe_gives_full_path():
119+
expected = distutils.spawn.find_executable('echo')
120+
assert parse_shebang.normexe('echo') == expected
121+
assert os.sep in expected
122+
123+
124+
def test_normalize_cmd_trivial():
125+
cmd = (distutils.spawn.find_executable('echo'), 'hi')
126+
assert parse_shebang.normalize_cmd(cmd) == cmd
127+
128+
129+
def test_normalize_cmd_PATH():
130+
cmd = ('python', '--version')
131+
expected = (distutils.spawn.find_executable('python'), '--version')
132+
assert parse_shebang.normalize_cmd(cmd) == expected
133+
134+
135+
def test_normalize_cmd_shebang(in_tmpdir):
136+
python = distutils.spawn.find_executable('python')
137+
path = write_executable(python.replace(os.sep, '/'))
138+
assert parse_shebang.normalize_cmd((path,)) == (python, path)
139+
140+
141+
def test_normalize_cmd_PATH_shebang_full_path(in_tmpdir):
142+
python = distutils.spawn.find_executable('python')
143+
path = write_executable(python.replace(os.sep, '/'))
144+
with bin_on_path():
145+
ret = parse_shebang.normalize_cmd(('run',))
146+
assert ret == (python, os.path.abspath(path))
147+
148+
149+
def test_normalize_cmd_PATH_shebang_PATH(in_tmpdir):
150+
python = distutils.spawn.find_executable('python')
151+
path = write_executable('/usr/bin/env python')
152+
with bin_on_path():
153+
ret = parse_shebang.normalize_cmd(('run',))
154+
assert ret == (python, os.path.abspath(path))

‎tests/prefixed_command_runner_test.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ def test_run_substitutes_prefix(popen_mock, makedirs_mock):
7878
)
7979
ret = instance.run(['{prefix}bar', 'baz'], retcode=None)
8080
popen_mock.assert_called_once_with(
81-
[five.n(os.path.join('prefix', 'bar')), five.n('baz')],
81+
(five.n(os.path.join('prefix', 'bar')), five.n('baz')),
8282
env=None,
8383
stdin=subprocess.PIPE,
8484
stdout=subprocess.PIPE,
@@ -132,4 +132,4 @@ def test_raises_on_error(popen_mock, makedirs_mock):
132132
instance = PrefixedCommandRunner(
133133
'.', popen=popen_mock, makedirs=makedirs_mock,
134134
)
135-
instance.run(['foo'])
135+
instance.run(['echo'])

0 commit comments

Comments
 (0)