From fd4a0f28cbf7034a58a528e7fab91bfd78569cc7 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 6 Feb 2019 15:54:35 -0500 Subject: [PATCH 1/6] Add explicit unit tests for '_scalar_field_to_json' helper. --- bigquery/tests/unit/test__helpers.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/bigquery/tests/unit/test__helpers.py b/bigquery/tests/unit/test__helpers.py index c2c4f9f7f787..bc26dd3b4970 100644 --- a/bigquery/tests/unit/test__helpers.py +++ b/bigquery/tests/unit/test__helpers.py @@ -776,6 +776,31 @@ def test_w_datetime(self): self.assertEqual(self._call_fut(when), "12:13:41") +class Test_scalar_field_to_json(unittest.TestCase): + def _call_fut(self, field, value): + from google.cloud.bigquery._helpers import _scalar_field_to_json + + return _scalar_field_to_json(field, value) + + @staticmethod + def _make_field(field_type): + from google.cloud.bigquery.schema import SchemaField + + return SchemaField(name='testing', field_type=field_type) + + def test_w_unknown_field_type(self): + field = self._make_field('UNKNOWN') + original = object() + converted = self._call_fut(field, original) + self.assertIs(converted, original) + + def test_w_known_field_type(self): + field = self._make_field('INT64') + original = 42 + converted = self._call_fut(field, original) + self.assertEqual(converted, str(original)) + + class Test_snake_to_camel_case(unittest.TestCase): def _call_fut(self, value): from google.cloud.bigquery._helpers import _snake_to_camel_case From 41b4369605d998e460b21044457f23917ec00547 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 6 Feb 2019 16:06:34 -0500 Subject: [PATCH 2/6] Add explicit unit tests for '_repeated_field_to_json' helper. --- bigquery/tests/unit/test__helpers.py | 37 ++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/bigquery/tests/unit/test__helpers.py b/bigquery/tests/unit/test__helpers.py index bc26dd3b4970..7cdf3318e406 100644 --- a/bigquery/tests/unit/test__helpers.py +++ b/bigquery/tests/unit/test__helpers.py @@ -776,31 +776,52 @@ def test_w_datetime(self): self.assertEqual(self._call_fut(when), "12:13:41") +def _make_field(field_type, mode='NULLABLE'): + from google.cloud.bigquery.schema import SchemaField + + return SchemaField(name='testing', field_type=field_type, mode=mode) + + class Test_scalar_field_to_json(unittest.TestCase): def _call_fut(self, field, value): from google.cloud.bigquery._helpers import _scalar_field_to_json return _scalar_field_to_json(field, value) - @staticmethod - def _make_field(field_type): - from google.cloud.bigquery.schema import SchemaField - - return SchemaField(name='testing', field_type=field_type) - def test_w_unknown_field_type(self): - field = self._make_field('UNKNOWN') + field = _make_field('UNKNOWN') original = object() converted = self._call_fut(field, original) self.assertIs(converted, original) def test_w_known_field_type(self): - field = self._make_field('INT64') + field = _make_field('INT64') original = 42 converted = self._call_fut(field, original) self.assertEqual(converted, str(original)) +class Test_repeated_field_to_json(unittest.TestCase): + def _call_fut(self, field, value): + from google.cloud.bigquery._helpers import _repeated_field_to_json + + return _repeated_field_to_json(field, value) + + def test_w_empty(self): + field = _make_field('INT64', mode='REPEATED') + original = [] + converted = self._call_fut(field, original) + self.assertEqual(converted, original) + self.assertEqual(field.mode, 'REPEATED') + + def test_w_non_empty(self): + field = _make_field('INT64', mode='REPEATED') + original = [42] + converted = self._call_fut(field, original) + self.assertEqual(converted, [str(value) for value in original]) + self.assertEqual(field.mode, 'REPEATED') + + class Test_snake_to_camel_case(unittest.TestCase): def _call_fut(self, value): from google.cloud.bigquery._helpers import _snake_to_camel_case From 6e3eb86f6e8d15d433ec4bbee023143e64a93bb3 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 6 Feb 2019 16:13:03 -0500 Subject: [PATCH 3/6] Add explicit unit tests for current '_record_field_to_json' helper behavior. --- bigquery/tests/unit/test__helpers.py | 35 ++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/bigquery/tests/unit/test__helpers.py b/bigquery/tests/unit/test__helpers.py index 7cdf3318e406..fba69d03d299 100644 --- a/bigquery/tests/unit/test__helpers.py +++ b/bigquery/tests/unit/test__helpers.py @@ -776,10 +776,10 @@ def test_w_datetime(self): self.assertEqual(self._call_fut(when), "12:13:41") -def _make_field(field_type, mode='NULLABLE'): +def _make_field(field_type, mode='NULLABLE', name='testing'): from google.cloud.bigquery.schema import SchemaField - return SchemaField(name='testing', field_type=field_type, mode=mode) + return SchemaField(name=name, field_type=field_type, mode=mode) class Test_scalar_field_to_json(unittest.TestCase): @@ -822,6 +822,37 @@ def test_w_non_empty(self): self.assertEqual(field.mode, 'REPEATED') +class Test_record_field_to_json(unittest.TestCase): + def _call_fut(self, field, value): + from google.cloud.bigquery._helpers import _record_field_to_json + + return _record_field_to_json(field, value) + + def test_w_empty(self): + fields = [] + original = [] + converted = self._call_fut(fields, original) + self.assertEqual(converted, {}) + + def test_w_non_empty_list(self): + fields = [ + _make_field('INT64', name='one', mode='NULLABLE'), + _make_field('STRING', name='two', mode='NULLABLE'), + ] + original = [42, 'two'] + converted = self._call_fut(fields, original) + self.assertEqual(converted, {'one': '42', 'two': 'two'}) + + def test_w_non_empty_dict(self): + fields = [ + _make_field('INT64', name='one', mode='NULLABLE'), + _make_field('STRING', name='two', mode='NULLABLE'), + ] + original = {'one': 42, 'two': 'two'} + converted = self._call_fut(fields, original) + self.assertEqual(converted, {'one': '42', 'two': 'two'}) + + class Test_snake_to_camel_case(unittest.TestCase): def _call_fut(self, value): from google.cloud.bigquery._helpers import _snake_to_camel_case From 88efb51330971b208bccfde2d8ed36e1a0e1bc2d Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 6 Feb 2019 16:22:53 -0500 Subject: [PATCH 4/6] Allow missing values in record-type fields. Closes #7294. --- bigquery/google/cloud/bigquery/_helpers.py | 5 ++++- bigquery/tests/unit/test__helpers.py | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index 10753cfc998b..72e1fa276dd7 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -398,7 +398,10 @@ def _record_field_to_json(fields, row_value): for subindex, subfield in enumerate(fields): subname = subfield.name - subvalue = row_value[subname] if isdict else row_value[subindex] + if isdict: + subvalue = row_value.get(subname) + else: + subvalue = row_value[subindex] record[subname] = _field_to_json(subfield, subvalue) return record diff --git a/bigquery/tests/unit/test__helpers.py b/bigquery/tests/unit/test__helpers.py index fba69d03d299..0e8574dd0170 100644 --- a/bigquery/tests/unit/test__helpers.py +++ b/bigquery/tests/unit/test__helpers.py @@ -852,6 +852,15 @@ def test_w_non_empty_dict(self): converted = self._call_fut(fields, original) self.assertEqual(converted, {'one': '42', 'two': 'two'}) + def test_w_missing_nullable(self): + fields = [ + _make_field('INT64', name='one', mode='NULLABLE'), + _make_field('STRING', name='two', mode='NULLABLE'), + ] + original = {'one': 42} + converted = self._call_fut(fields, original) + self.assertEqual(converted, {'one': '42', 'two': None}) + class Test_snake_to_camel_case(unittest.TestCase): def _call_fut(self, value): From 950cca40dc3574b1a84f5b8085e684da6cb5af25 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 6 Feb 2019 16:23:30 -0500 Subject: [PATCH 5/6] Add explicit unit tests for '_field_to_json' helper. --- bigquery/tests/unit/test__helpers.py | 39 ++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/bigquery/tests/unit/test__helpers.py b/bigquery/tests/unit/test__helpers.py index 0e8574dd0170..cf77eda7671d 100644 --- a/bigquery/tests/unit/test__helpers.py +++ b/bigquery/tests/unit/test__helpers.py @@ -776,10 +776,10 @@ def test_w_datetime(self): self.assertEqual(self._call_fut(when), "12:13:41") -def _make_field(field_type, mode='NULLABLE', name='testing'): +def _make_field(field_type, mode='NULLABLE', name='testing', fields=()): from google.cloud.bigquery.schema import SchemaField - return SchemaField(name=name, field_type=field_type, mode=mode) + return SchemaField(name=name, field_type=field_type, mode=mode, fields=fields) class Test_scalar_field_to_json(unittest.TestCase): @@ -862,6 +862,41 @@ def test_w_missing_nullable(self): self.assertEqual(converted, {'one': '42', 'two': None}) +class Test_field_to_json(unittest.TestCase): + def _call_fut(self, field, value): + from google.cloud.bigquery._helpers import _field_to_json + + return _field_to_json(field, value) + + def test_w_none(self): + field = _make_field('INT64') + original = None + converted = self._call_fut(field, original) + self.assertIsNone(converted) + + def test_w_repeated(self): + field = _make_field('INT64', mode='REPEATED') + original = [42, 17] + converted = self._call_fut(field, original) + self.assertEqual(converted, [str(value) for value in original]) + + def test_w_record(self): + subfields = [ + _make_field('INT64', name='one'), + _make_field('STRING', name='two'), + ] + field = _make_field('RECORD', fields=subfields) + original = {'one': 42, 'two': 'two'} + converted = self._call_fut(field, original) + self.assertEqual(converted, {'one': '42', 'two': 'two'}) + + def test_w_scalar(self): + field = _make_field('INT64') + original = 42 + converted = self._call_fut(field, original) + self.assertEqual(converted, str(original)) + + class Test_snake_to_camel_case(unittest.TestCase): def _call_fut(self, value): from google.cloud.bigquery._helpers import _snake_to_camel_case From 45244de53cdcbeaeb75a592f030d4f1a82038332 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 6 Feb 2019 16:25:19 -0500 Subject: [PATCH 6/6] Blacken. --- bigquery/tests/unit/test__helpers.py | 54 ++++++++++++++-------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/bigquery/tests/unit/test__helpers.py b/bigquery/tests/unit/test__helpers.py index cf77eda7671d..3884695d83af 100644 --- a/bigquery/tests/unit/test__helpers.py +++ b/bigquery/tests/unit/test__helpers.py @@ -776,7 +776,7 @@ def test_w_datetime(self): self.assertEqual(self._call_fut(when), "12:13:41") -def _make_field(field_type, mode='NULLABLE', name='testing', fields=()): +def _make_field(field_type, mode="NULLABLE", name="testing", fields=()): from google.cloud.bigquery.schema import SchemaField return SchemaField(name=name, field_type=field_type, mode=mode, fields=fields) @@ -789,13 +789,13 @@ def _call_fut(self, field, value): return _scalar_field_to_json(field, value) def test_w_unknown_field_type(self): - field = _make_field('UNKNOWN') + field = _make_field("UNKNOWN") original = object() converted = self._call_fut(field, original) self.assertIs(converted, original) def test_w_known_field_type(self): - field = _make_field('INT64') + field = _make_field("INT64") original = 42 converted = self._call_fut(field, original) self.assertEqual(converted, str(original)) @@ -808,18 +808,18 @@ def _call_fut(self, field, value): return _repeated_field_to_json(field, value) def test_w_empty(self): - field = _make_field('INT64', mode='REPEATED') + field = _make_field("INT64", mode="REPEATED") original = [] converted = self._call_fut(field, original) self.assertEqual(converted, original) - self.assertEqual(field.mode, 'REPEATED') + self.assertEqual(field.mode, "REPEATED") def test_w_non_empty(self): - field = _make_field('INT64', mode='REPEATED') + field = _make_field("INT64", mode="REPEATED") original = [42] converted = self._call_fut(field, original) self.assertEqual(converted, [str(value) for value in original]) - self.assertEqual(field.mode, 'REPEATED') + self.assertEqual(field.mode, "REPEATED") class Test_record_field_to_json(unittest.TestCase): @@ -836,30 +836,30 @@ def test_w_empty(self): def test_w_non_empty_list(self): fields = [ - _make_field('INT64', name='one', mode='NULLABLE'), - _make_field('STRING', name='two', mode='NULLABLE'), + _make_field("INT64", name="one", mode="NULLABLE"), + _make_field("STRING", name="two", mode="NULLABLE"), ] - original = [42, 'two'] + original = [42, "two"] converted = self._call_fut(fields, original) - self.assertEqual(converted, {'one': '42', 'two': 'two'}) + self.assertEqual(converted, {"one": "42", "two": "two"}) def test_w_non_empty_dict(self): fields = [ - _make_field('INT64', name='one', mode='NULLABLE'), - _make_field('STRING', name='two', mode='NULLABLE'), + _make_field("INT64", name="one", mode="NULLABLE"), + _make_field("STRING", name="two", mode="NULLABLE"), ] - original = {'one': 42, 'two': 'two'} + original = {"one": 42, "two": "two"} converted = self._call_fut(fields, original) - self.assertEqual(converted, {'one': '42', 'two': 'two'}) + self.assertEqual(converted, {"one": "42", "two": "two"}) def test_w_missing_nullable(self): fields = [ - _make_field('INT64', name='one', mode='NULLABLE'), - _make_field('STRING', name='two', mode='NULLABLE'), + _make_field("INT64", name="one", mode="NULLABLE"), + _make_field("STRING", name="two", mode="NULLABLE"), ] - original = {'one': 42} + original = {"one": 42} converted = self._call_fut(fields, original) - self.assertEqual(converted, {'one': '42', 'two': None}) + self.assertEqual(converted, {"one": "42", "two": None}) class Test_field_to_json(unittest.TestCase): @@ -869,29 +869,29 @@ def _call_fut(self, field, value): return _field_to_json(field, value) def test_w_none(self): - field = _make_field('INT64') + field = _make_field("INT64") original = None converted = self._call_fut(field, original) self.assertIsNone(converted) def test_w_repeated(self): - field = _make_field('INT64', mode='REPEATED') + field = _make_field("INT64", mode="REPEATED") original = [42, 17] converted = self._call_fut(field, original) self.assertEqual(converted, [str(value) for value in original]) def test_w_record(self): subfields = [ - _make_field('INT64', name='one'), - _make_field('STRING', name='two'), + _make_field("INT64", name="one"), + _make_field("STRING", name="two"), ] - field = _make_field('RECORD', fields=subfields) - original = {'one': 42, 'two': 'two'} + field = _make_field("RECORD", fields=subfields) + original = {"one": 42, "two": "two"} converted = self._call_fut(field, original) - self.assertEqual(converted, {'one': '42', 'two': 'two'}) + self.assertEqual(converted, {"one": "42", "two": "two"}) def test_w_scalar(self): - field = _make_field('INT64') + field = _make_field("INT64") original = 42 converted = self._call_fut(field, original) self.assertEqual(converted, str(original))