Skip to content

Commit 6f2275b

Browse files
author
Doug Greiman
committed
Move some utility functions into their own file
1 parent 36d0969 commit 6f2275b

File tree

4 files changed

+221
-166
lines changed

4 files changed

+221
-166
lines changed

‎scripts/local_cloudbuild.py‎

Lines changed: 12 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@
4242

4343
import yaml
4444

45+
import validation_utils
46+
4547

4648
# Exclude non-printable control characters (including newlines)
4749
PRINTABLE_REGEX = re.compile(r"""^[^\x00-\x1f]*$""")
@@ -59,10 +61,6 @@
5961
)
6062
""")
6163

62-
# For easier development, we allow redefining builtins like
63-
# --substitutions=PROJECT_ID=foo even though gcloud doesn't.
64-
KEY_VALUE_REGEX = re.compile(r'^([A-Z_][A-Z0-9_]*)=(.*)$')
65-
6664
# Default builtin substitutions
6765
DEFAULT_SUBSTITUTIONS = {
6866
'BRANCH_NAME': '',
@@ -149,51 +147,6 @@ def sub(match):
149147
quoted_s = shlex.quote(substituted_s)
150148
return quoted_s
151149

152-
def get_field_value(container, field_name, field_type):
153-
"""Fetch a field from a container with typechecking and default values.
154-
155-
The field value is coerced to the desired type. If the field is
156-
not present, a instance of `field_type` is constructed with no
157-
arguments and used as the default value.
158-
159-
Args:
160-
container (dict): Object decoded from yaml
161-
field_name (str): Field that should be present in `container`
162-
field_type (type): Expected type for field value
163-
164-
Returns:
165-
Any: Fetched or default value of field
166-
167-
Raises:
168-
ValueError: if field value cannot be converted to the desired type
169-
"""
170-
try:
171-
value = container[field_name]
172-
except (IndexError, KeyError):
173-
return field_type()
174-
175-
msg = 'Expected "{}" field to be of type "{}", but found type "{}"'
176-
if not isinstance(value, field_type):
177-
# list('some string') is a successful type cast as far as Python
178-
# is concerned, but doesn't exactly produce the results we want.
179-
# We have a whitelist of conversions we will attempt.
180-
whitelist = (
181-
(float, str),
182-
(int, str),
183-
(str, float),
184-
(str, int),
185-
(int, float),
186-
)
187-
if (type(value), field_type) not in whitelist:
188-
raise ValueError(msg.format(field_name, field_type, type(value)))
189-
190-
try:
191-
value = field_type(value)
192-
except ValueError as e:
193-
e.message = msg.format(field_name, field_type, type(value))
194-
raise
195-
return value
196-
197150

198151
def get_cloudbuild(raw_config, args):
199152
"""Read and validate a cloudbuild recipe
@@ -210,7 +163,7 @@ def get_cloudbuild(raw_config, args):
210163
'Expected {} contents to be of type "dict", but found type "{}"'.
211164
format(args.config, type(raw_config)))
212165

213-
raw_steps = get_field_value(raw_config, 'steps', list)
166+
raw_steps = validation_utils.get_field_value(raw_config, 'steps', list)
214167
if not raw_steps:
215168
raise ValueError('No steps defined in {}'.format(args.config))
216169

@@ -236,14 +189,14 @@ def get_step(raw_step):
236189
raise ValueError(
237190
'Expected step to be of type "dict", but found type "{}"'.
238191
format(type(raw_step)))
239-
raw_args = get_field_value(raw_step, 'args', list)
240-
args = [get_field_value(raw_args, index, str)
192+
raw_args = validation_utils.get_field_value(raw_step, 'args', list)
193+
args = [validation_utils.get_field_value(raw_args, index, str)
241194
for index in range(len(raw_args))]
242-
dir_ = get_field_value(raw_step, 'dir', str)
243-
raw_env = get_field_value(raw_step, 'env', list)
244-
env = [get_field_value(raw_env, index, str)
195+
dir_ = validation_utils.get_field_value(raw_step, 'dir', str)
196+
raw_env = validation_utils.get_field_value(raw_step, 'env', list)
197+
env = [validation_utils.get_field_value(raw_env, index, str)
245198
for index in range(len(raw_env))]
246-
name = get_field_value(raw_step, 'name', str)
199+
name = validation_utils.get_field_value(raw_step, 'name', str)
247200
return Step(
248201
args=args,
249202
dir_=dir_,
@@ -373,46 +326,21 @@ def local_cloudbuild(args):
373326
subprocess.check_call(args)
374327

375328

376-
def validate_arg_regex(flag_value, flag_regex):
377-
"""Check a named command line flag against a regular expression"""
378-
if not re.match(flag_regex, flag_value):
379-
raise argparse.ArgumentTypeError(
380-
'Value "{}" does not match pattern "{}"'.format(
381-
flag_value, flag_regex.pattern))
382-
return flag_value
383-
384-
385-
def validate_arg_dict(flag_value):
386-
"""Parse a command line flag as a key=val,... dict"""
387-
if not flag_value:
388-
return {}
389-
entries = flag_value.split(',')
390-
pairs = []
391-
for entry in entries:
392-
match = re.match(KEY_VALUE_REGEX, entry)
393-
if not match:
394-
raise argparse.ArgumentTypeError(
395-
'Value "{}" should be a list like _KEY1=value1,_KEY2=value2"'.format(
396-
flag_value))
397-
pairs.append((match.group(1), match.group(2)))
398-
return dict(pairs)
399-
400-
401329
def parse_args(argv):
402330
"""Parse and validate command line flags"""
403331
parser = argparse.ArgumentParser(
404332
description='Process cloudbuild.yaml locally to build Docker images')
405333
parser.add_argument(
406334
'--config',
407335
type=functools.partial(
408-
validate_arg_regex, flag_regex=PRINTABLE_REGEX),
336+
validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX),
409337
default='cloudbuild.yaml',
410338
help='Path to cloudbuild.yaml file'
411339
)
412340
parser.add_argument(
413341
'--output_script',
414342
type=functools.partial(
415-
validate_arg_regex, flag_regex=PRINTABLE_REGEX),
343+
validation_utils.validate_arg_regex, flag_regex=PRINTABLE_REGEX),
416344
help='Filename to write shell script to',
417345
)
418346
parser.add_argument(
@@ -423,7 +351,7 @@ def parse_args(argv):
423351
)
424352
parser.add_argument(
425353
'--substitutions',
426-
type=validate_arg_dict,
354+
type=validation_utils.validate_arg_dict,
427355
default={},
428356
help='Parameters to be substituted in the build specification',
429357
)

‎scripts/local_cloudbuild_test.py‎

Lines changed: 1 addition & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -34,91 +34,10 @@
3434
STAGING_DIR_REGEX = re.compile(
3535
b'(?m)Copying source to staging directory (.+)$')
3636

37-
class ValidationUtilsTest(unittest.TestCase):
38-
39-
def test_get_field_value(self):
40-
valid_cases = (
41-
# Normal case, field present and correct type
42-
({ 'present': 1 }, 'present', int, 1),
43-
({ 'present': '1' }, 'present', str, '1'),
44-
({ 'present': [1] }, 'present', list, [1]),
45-
({ 'present': {1: 2} }, 'present', dict, {1: 2}),
46-
# Missing field replaced by default
47-
({}, 'missing', str, ''),
48-
# Valid conversions
49-
({ 'str_to_int': '1' }, 'str_to_int', int, 1),
50-
({ 'int_to_str': 1 }, 'int_to_str', str, '1'),
51-
)
52-
for valid_case in valid_cases:
53-
with self.subTest(valid_case=valid_case):
54-
container, field_name, field_type, expected = valid_case
55-
self.assertEqual(
56-
local_cloudbuild.get_field_value(
57-
container, field_name, field_type),
58-
expected)
59-
60-
invalid_cases = (
61-
# Type conversion failures
62-
({ 'bad_list_to_dict': [1] }, 'bad_list_to_dict', dict),
63-
({ 'bad_list_to_str': [1] }, 'bad_list_to_str', str),
64-
({ 'bad_dict_to_list': {1: 2} }, 'bad_dict_to_list', list),
65-
({ 'bad_str_to_int': 'not_an_int' }, 'bad_str_to_int', int),
66-
({ 'bad_str_to_list': 'abc' }, 'bad_str_to_list', list),
67-
)
68-
for invalid_case in invalid_cases:
69-
with self.subTest(invalid_case=invalid_case):
70-
container, field_name, field_type = invalid_case
71-
with self.assertRaises(ValueError):
72-
local_cloudbuild.get_field_value(
73-
container, field_name, field_type)
74-
75-
def test_validate_arg_regex(self):
76-
self.assertEqual(
77-
local_cloudbuild.validate_arg_regex('abc', re.compile('a[b]c')),
78-
'abc')
79-
with self.assertRaises(argparse.ArgumentTypeError):
80-
local_cloudbuild.validate_arg_regex('abc', re.compile('a[d]c'))
81-
82-
83-
def test_validate_arg_dict(self):
84-
valid_cases = (
85-
# Normal case, field present and correct type
86-
('', {}),
87-
('_A=1', {'_A':'1'}),
88-
('_A=1,_B=2', {'_A':'1', '_B':'2'}),
89-
# Repeated key is ok
90-
('_A=1,_A=2', {'_A':'2'}),
91-
# Extra = is ok
92-
('_A=x=y=z,_B=2', {'_A':'x=y=z', '_B':'2'}),
93-
# No value is ok
94-
('_A=', {'_A':''}),
95-
)
96-
for valid_case in valid_cases:
97-
with self.subTest(valid_case=valid_case):
98-
s, expected = valid_case
99-
self.assertEqual(
100-
local_cloudbuild.validate_arg_dict(s),
101-
expected)
102-
103-
invalid_cases = (
104-
# No key
105-
',_A',
106-
'_A,',
107-
# Invalid variable name
108-
'_Aa=1',
109-
'_aA=1',
110-
'0A=1',
111-
)
112-
for invalid_case in invalid_cases:
113-
with self.subTest(invalid_case=invalid_case):
114-
with self.assertRaises(argparse.ArgumentTypeError):
115-
local_cloudbuild.validate_arg_dict(invalid_case)
116-
117-
11837
class LocalCloudbuildTest(unittest.TestCase):
11938

12039
def setUp(self):
121-
self.testdata_dir = 'testdata'
40+
self.testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') # Sigh
12241
assert os.path.isdir(self.testdata_dir), 'Could not run test: testdata directory not found'
12342

12443
def test_sub_and_quote(self):

‎scripts/validation_utils.py‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
#!/usr/bin/env python3
2+
3+
# Copyright 2017 Google Inc. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
"""Utilities for schema and command line validation"""
18+
19+
import argparse
20+
import re
21+
22+
23+
# For easier development, we allow redefining builtins like
24+
# --substitutions=PROJECT_ID=foo even though gcloud doesn't.
25+
KEY_VALUE_REGEX = re.compile(r'^([A-Z_][A-Z0-9_]*)=(.*)$')
26+
27+
28+
def get_field_value(container, field_name, field_type):
29+
"""Fetch a field from a container with typechecking and default values.
30+
31+
The field value is coerced to the desired type. If the field is
32+
not present, a instance of `field_type` is constructed with no
33+
arguments and used as the default value.
34+
35+
Args:
36+
container (dict): Object decoded from yaml
37+
field_name (str): Field that should be present in `container`
38+
field_type (type): Expected type for field value
39+
40+
Returns:
41+
Any: Fetched or default value of field
42+
43+
Raises:
44+
ValueError: if field value cannot be converted to the desired type
45+
"""
46+
try:
47+
value = container[field_name]
48+
if value is None:
49+
return field_type()
50+
except (IndexError, KeyError):
51+
return field_type()
52+
53+
msg = 'Expected "{}" field to be of type "{}", but found type "{}"'
54+
if not isinstance(value, field_type):
55+
# list('some string') is a successful type cast as far as Python
56+
# is concerned, but doesn't exactly produce the results we want.
57+
# We have a whitelist of conversions we will attempt.
58+
whitelist = (
59+
(float, str),
60+
(int, str),
61+
(str, float),
62+
(str, int),
63+
(int, float),
64+
)
65+
if (type(value), field_type) not in whitelist:
66+
raise ValueError(msg.format(field_name, field_type, type(value)))
67+
68+
try:
69+
value = field_type(value)
70+
except ValueError as e:
71+
e.message = msg.format(field_name, field_type, type(value))
72+
raise
73+
return value
74+
75+
76+
def validate_arg_regex(flag_value, flag_regex):
77+
"""Check a named command line flag against a regular expression"""
78+
if not re.match(flag_regex, flag_value):
79+
raise argparse.ArgumentTypeError(
80+
'Value "{}" does not match pattern "{}"'.format(
81+
flag_value, flag_regex.pattern))
82+
return flag_value
83+
84+
85+
def validate_arg_dict(flag_value):
86+
"""Parse a command line flag as a key=val,... dict"""
87+
if not flag_value:
88+
return {}
89+
entries = flag_value.split(',')
90+
pairs = []
91+
for entry in entries:
92+
match = re.match(KEY_VALUE_REGEX, entry)
93+
if not match:
94+
raise argparse.ArgumentTypeError(
95+
'Value "{}" should be a list like _KEY1=value1,_KEY2=value2"'.format(
96+
flag_value))
97+
pairs.append((match.group(1), match.group(2)))
98+
return dict(pairs)

0 commit comments

Comments
 (0)