Skip to content

Commit 102f0f8

Browse files
alixhamitheacodes
alixhami
authored and
theacodes
committed
BigQuery: Adds schema update options and snippets (googleapis#5415)
1 parent 5a79d5d commit 102f0f8

5 files changed

Lines changed: 351 additions & 2 deletions

File tree

‎bigquery/google/cloud/bigquery/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
from google.cloud.bigquery.job import QueryJob
5050
from google.cloud.bigquery.job import QueryJobConfig
5151
from google.cloud.bigquery.job import QueryPriority
52+
from google.cloud.bigquery.job import SchemaUpdateOption
5253
from google.cloud.bigquery.job import SourceFormat
5354
from google.cloud.bigquery.job import UnknownJob
5455
from google.cloud.bigquery.job import WriteDisposition
@@ -113,6 +114,7 @@
113114
'DestinationFormat',
114115
'Encoding',
115116
'QueryPriority',
117+
'SchemaUpdateOption',
116118
'SourceFormat',
117119
'WriteDisposition'
118120
]

‎bigquery/google/cloud/bigquery/job.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,18 @@ class WriteDisposition(object):
206206
returned in the job result."""
207207

208208

209+
class SchemaUpdateOption(object):
210+
"""Specifies an update to the destination table schema as a side effect of
211+
a load job.
212+
"""
213+
214+
ALLOW_FIELD_ADDITION = 'ALLOW_FIELD_ADDITION'
215+
"""Allow adding a nullable field to the schema."""
216+
217+
ALLOW_FIELD_RELAXATION = 'ALLOW_FIELD_RELAXATION'
218+
"""Allow relaxing a required field in the original schema to nullable."""
219+
220+
209221
class _JobReference(object):
210222
"""A reference to a job.
211223
@@ -1004,6 +1016,18 @@ def time_partitioning(self, value):
10041016
api_repr = value.to_api_repr()
10051017
self._set_sub_prop('timePartitioning', api_repr)
10061018

1019+
@property
1020+
def schema_update_options(self):
1021+
"""List[google.cloud.bigquery.job.SchemaUpdateOption]: Specifies
1022+
updates to the destination table schema to allow as a side effect of
1023+
the load job.
1024+
"""
1025+
return self._get_sub_prop('schemaUpdateOptions')
1026+
1027+
@schema_update_options.setter
1028+
def schema_update_options(self, values):
1029+
self._set_sub_prop('schemaUpdateOptions', values)
1030+
10071031

10081032
class LoadJob(_AsyncJob):
10091033
"""Asynchronous job for loading data into a table.
@@ -1158,6 +1182,13 @@ def time_partitioning(self):
11581182
"""
11591183
return self._configuration.time_partitioning
11601184

1185+
@property
1186+
def schema_update_options(self):
1187+
"""See
1188+
:attr:`google.cloud.bigquery.job.LoadJobConfig.schema_update_options`.
1189+
"""
1190+
return self._configuration.schema_update_options
1191+
11611192
@property
11621193
def input_file_bytes(self):
11631194
"""Count of bytes loaded from source files.
@@ -1971,6 +2002,18 @@ def time_partitioning(self, value):
19712002
api_repr = value.to_api_repr()
19722003
self._set_sub_prop('timePartitioning', api_repr)
19732004

2005+
@property
2006+
def schema_update_options(self):
2007+
"""List[google.cloud.bigquery.job.SchemaUpdateOption]: Specifies
2008+
updates to the destination table schema to allow as a side effect of
2009+
the query job.
2010+
"""
2011+
return self._get_sub_prop('schemaUpdateOptions')
2012+
2013+
@schema_update_options.setter
2014+
def schema_update_options(self, values):
2015+
self._set_sub_prop('schemaUpdateOptions', values)
2016+
19742017
def to_api_repr(self):
19752018
"""Build an API representation of the query job config.
19762019
@@ -2149,6 +2192,13 @@ def time_partitioning(self):
21492192
"""
21502193
return self._configuration.time_partitioning
21512194

2195+
@property
2196+
def schema_update_options(self):
2197+
"""See
2198+
:attr:`google.cloud.bigquery.job.QueryJobConfig.schema_update_options`.
2199+
"""
2200+
return self._configuration.schema_update_options
2201+
21522202
def _build_resource(self):
21532203
"""Generate a resource for :meth:`begin`."""
21542204
configuration = self._configuration.to_api_repr()

‎bigquery/tests/unit/test_job.py‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -380,6 +380,11 @@ def _verifyEnumConfigProperties(self, job, config):
380380
config['writeDisposition'])
381381
else:
382382
self.assertIsNone(job.write_disposition)
383+
if 'schemaUpdateOptions' in config:
384+
self.assertEqual(
385+
job.schema_update_options, config['schemaUpdateOptions'])
386+
else:
387+
self.assertIsNone(job.schema_update_options)
383388

384389
def _verifyResourceProperties(self, job, resource):
385390
self._verifyReadonlyResourceProperties(job, resource)
@@ -467,6 +472,7 @@ def test_ctor(self):
467472
self.assertIsNone(job.write_disposition)
468473
self.assertIsNone(job.destination_encryption_configuration)
469474
self.assertIsNone(job.time_partitioning)
475+
self.assertIsNone(job.schema_update_options)
470476

471477
def test_ctor_w_config(self):
472478
from google.cloud.bigquery.schema import SchemaField
@@ -780,6 +786,7 @@ def test_begin_w_autodetect(self):
780786

781787
def test_begin_w_alternate_client(self):
782788
from google.cloud.bigquery.job import CreateDisposition
789+
from google.cloud.bigquery.job import SchemaUpdateOption
783790
from google.cloud.bigquery.job import WriteDisposition
784791
from google.cloud.bigquery.schema import SchemaField
785792

@@ -817,7 +824,10 @@ def test_begin_w_alternate_client(self):
817824
'mode': 'REQUIRED',
818825
'description': None,
819826
},
820-
]}
827+
]},
828+
'schemaUpdateOptions': [
829+
SchemaUpdateOption.ALLOW_FIELD_ADDITION,
830+
],
821831
}
822832
RESOURCE['configuration']['load'] = LOAD_CONFIGURATION
823833
conn1 = _make_connection()
@@ -842,6 +852,9 @@ def test_begin_w_alternate_client(self):
842852
config.skip_leading_rows = 1
843853
config.source_format = 'CSV'
844854
config.write_disposition = WriteDisposition.WRITE_TRUNCATE
855+
config.schema_update_options = [
856+
SchemaUpdateOption.ALLOW_FIELD_ADDITION,
857+
]
845858

846859
job._begin(client=client2)
847860

@@ -2127,6 +2140,11 @@ def _verifyResourceProperties(self, job, resource):
21272140
'kmsKeyName'])
21282141
else:
21292142
self.assertIsNone(job.destination_encryption_configuration)
2143+
if 'schemaUpdateOptions' in query_config:
2144+
self.assertEqual(
2145+
job.schema_update_options, query_config['schemaUpdateOptions'])
2146+
else:
2147+
self.assertIsNone(job.schema_update_options)
21302148

21312149
def test_ctor_defaults(self):
21322150
client = _make_client(project=self.PROJECT)
@@ -2157,6 +2175,7 @@ def test_ctor_defaults(self):
21572175
self.assertIsNone(job.table_definitions)
21582176
self.assertIsNone(job.destination_encryption_configuration)
21592177
self.assertIsNone(job.time_partitioning)
2178+
self.assertIsNone(job.schema_update_options)
21602179

21612180
def test_ctor_w_udf_resources(self):
21622181
from google.cloud.bigquery.job import QueryJobConfig
@@ -2248,6 +2267,7 @@ def test_from_api_repr_with_encryption(self):
22482267

22492268
def test_from_api_repr_w_properties(self):
22502269
from google.cloud.bigquery.job import CreateDisposition
2270+
from google.cloud.bigquery.job import SchemaUpdateOption
22512271
from google.cloud.bigquery.job import WriteDisposition
22522272

22532273
client = _make_client(project=self.PROJECT)
@@ -2260,6 +2280,9 @@ def test_from_api_repr_w_properties(self):
22602280
'datasetId': self.DS_ID,
22612281
'tableId': self.DESTINATION_TABLE,
22622282
}
2283+
query_config['schemaUpdateOptions'] = [
2284+
SchemaUpdateOption.ALLOW_FIELD_ADDITION,
2285+
]
22632286
klass = self._get_target_class()
22642287
job = klass.from_api_repr(RESOURCE, client=client)
22652288
self.assertIs(job._client, client)
@@ -2841,6 +2864,7 @@ def test_begin_w_alternate_client(self):
28412864
from google.cloud.bigquery.job import CreateDisposition
28422865
from google.cloud.bigquery.job import QueryJobConfig
28432866
from google.cloud.bigquery.job import QueryPriority
2867+
from google.cloud.bigquery.job import SchemaUpdateOption
28442868
from google.cloud.bigquery.job import WriteDisposition
28452869

28462870
PATH = '/projects/%s/jobs' % (self.PROJECT,)
@@ -2866,7 +2890,10 @@ def test_begin_w_alternate_client(self):
28662890
'useLegacySql': True,
28672891
'writeDisposition': WriteDisposition.WRITE_TRUNCATE,
28682892
'maximumBillingTier': 4,
2869-
'maximumBytesBilled': '123456'
2893+
'maximumBytesBilled': '123456',
2894+
'schemaUpdateOptions': [
2895+
SchemaUpdateOption.ALLOW_FIELD_RELAXATION,
2896+
]
28702897
}
28712898
RESOURCE['configuration']['query'] = QUERY_CONFIGURATION
28722899
RESOURCE['configuration']['dryRun'] = True
@@ -2890,6 +2917,9 @@ def test_begin_w_alternate_client(self):
28902917
config.use_query_cache = True
28912918
config.write_disposition = WriteDisposition.WRITE_TRUNCATE
28922919
config.maximum_bytes_billed = 123456
2920+
config.schema_update_options = [
2921+
SchemaUpdateOption.ALLOW_FIELD_RELAXATION,
2922+
]
28932923
job = self._make_one(
28942924
self.JOB_ID, self.QUERY, client1, job_config=config)
28952925

‎docs/bigquery/reference.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ Job-Related Types
6262
job.QueryPriority
6363
job.SourceFormat
6464
job.WriteDisposition
65+
job.SchemaUpdateOption
6566

6667

6768
Dataset

0 commit comments

Comments
 (0)