This repository was archived by the owner on Sep 28, 2023. It is now read-only.
forked from googleapis/google-cloud-python
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathrun_pylint.py
More file actions
211 lines (174 loc) · 7.19 KB
/
run_pylint.py
File metadata and controls
211 lines (174 loc) · 7.19 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Custom script to run PyLint on gcloud codebase.
This runs pylint as a script via subprocess in two different
subprocesses. The first lints the production/library code
using the default rc file (PRODUCTION_RC). The second lints the
demo/test code using an rc file (TEST_RC) which allows more style
violations (hence it has a reduced number of style checks).
"""
import ConfigParser
import copy
import os
import subprocess
import sys
IGNORED_DIRECTORIES = [
'_gcloud_vendor/',
]
IGNORED_FILES = [
'gcloud/datastore/_datastore_v1_pb2.py',
'docs/conf.py',
'setup.py',
]
PRODUCTION_RC = 'pylintrc_default'
TEST_RC = 'pylintrc_reduced'
TEST_DISABLED_MESSAGES = [
'attribute-defined-outside-init',
'exec-used',
'import-error',
'invalid-name',
'missing-docstring',
'no-init',
'no-self-use',
'superfluous-parens',
'too-few-public-methods',
'too-many-locals',
'too-many-public-methods',
'unbalanced-tuple-unpacking',
]
TEST_RC_ADDITIONS = {
'MESSAGES CONTROL': {
'disable': ', '.join(TEST_DISABLED_MESSAGES),
},
}
def read_config(filename):
"""Reads pylintrc config onto native ConfigParser object."""
config = ConfigParser.ConfigParser()
with open(filename, 'r') as file_obj:
config.readfp(file_obj)
return config
def make_test_rc(base_rc_filename, additions_dict, target_filename):
"""Combines a base rc and test additions into single file."""
main_cfg = read_config(base_rc_filename)
# Create fresh config for test, which must extend production.
test_cfg = ConfigParser.ConfigParser()
test_cfg._sections = copy.deepcopy(main_cfg._sections)
for section, opts in additions_dict.items():
curr_section = test_cfg._sections.setdefault(
section, test_cfg._dict())
for opt, opt_val in opts.items():
curr_val = curr_section.get(opt)
if curr_val is None:
raise KeyError('Expected to be adding to existing option.')
curr_section[opt] = '%s, %s' % (curr_val, opt_val)
with open(target_filename, 'w') as file_obj:
test_cfg.write(file_obj)
def valid_filename(filename):
"""Checks if a file is a Python file and is not ignored."""
for directory in IGNORED_DIRECTORIES:
if filename.startswith(directory):
return False
return (filename.endswith('.py') and
filename not in IGNORED_FILES)
def is_production_filename(filename):
"""Checks if the file contains production code.
:rtype: boolean
:returns: Boolean indicating production status.
"""
return not ('demo' in filename or 'test' in filename
or filename.startswith('regression'))
def get_files_for_linting():
"""Gets a list of files in the repository.
By default, returns all files via `git ls-files`. However, in some cases
uses a specific commit or branch (a so-called diff base) to compare
against for changed files.
To speed up linting on Travis pull requests against master, we manually
set the diff base to origin/master. We don't do this on non-pull requests
since origin/master will be equivalent to the currently checked out code.
One could potentially use ${TRAVIS_COMMIT_RANGE} to find a diff base but
this value is not dependable.
To allow faster local `tox` runs, the environment variables
`GCLOUD_REMOTE_FOR_LINT` and `GCLOUD_BRANCH_FOR_LINT` can be set to specify
a remote branch to diff against.
"""
diff_base = None
if (os.getenv('TRAVIS_BRANCH') == 'master' and
os.getenv('TRAVIS_PULL_REQUEST') != 'false'):
# In the case of a pull request into master, we want to
# diff against HEAD in master.
diff_base = 'origin/master'
elif os.getenv('TRAVIS') is None:
# Only allow specified remote and branch in local dev.
remote = os.getenv('GCLOUD_REMOTE_FOR_LINT')
branch = os.getenv('GCLOUD_BRANCH_FOR_LINT')
if remote is not None and branch is not None:
diff_base = '%s/%s' % (remote, branch)
if diff_base is None:
print 'Diff base not specified, listing all files in repository.'
result = subprocess.check_output(['git', 'ls-files'])
else:
result = subprocess.check_output(['git', 'diff', '--name-only',
diff_base])
print 'Using files changed relative to %s:' % (diff_base,)
print '-' * 60
print result.rstrip('\n') # Don't print trailing newlines.
print '-' * 60
return result.rstrip('\n').split('\n')
def get_python_files():
"""Gets a list of all Python files in the repository that need linting.
Relies on get_files_for_linting() to determine which files should be
considered.
NOTE: This requires `git` to be installed and requires that this
is run within the `git` repository.
:rtype: `tuple`
:returns: A tuple containing two lists. The first list contains
all production files and the next all test/demo files.
"""
all_files = get_files_for_linting()
library_files = []
non_library_files = []
for filename in all_files:
if valid_filename(filename):
if is_production_filename(filename):
library_files.append(filename)
else:
non_library_files.append(filename)
return library_files, non_library_files
def lint_fileset(filenames, rcfile, description):
"""Lints a group of files using a given rcfile."""
# Only lint filenames that exist. For example, 'git diff --name-only'
# could spit out deleted / renamed files. Another alternative could
# be to use 'git diff --name-status' and filter out files with a
# status of 'D'.
filenames = [filename for filename in filenames
if os.path.exists(filename)]
if filenames:
rc_flag = '--rcfile=%s' % (rcfile,)
pylint_shell_command = ['pylint', rc_flag] + filenames
status_code = subprocess.call(pylint_shell_command)
if status_code != 0:
error_message = ('Pylint failed on %s with '
'status %d.' % (description, status_code))
print >> sys.stderr, error_message
sys.exit(status_code)
else:
print 'Skipping %s, no files to lint.' % (description,)
def main():
"""Script entry point. Lints both sets of files."""
make_test_rc(PRODUCTION_RC, TEST_RC_ADDITIONS, TEST_RC)
library_files, non_library_files = get_python_files()
lint_fileset(library_files, PRODUCTION_RC, 'library code')
lint_fileset(non_library_files, TEST_RC, 'test and demo code')
if __name__ == '__main__':
main()