From 245cc03c3b5f79c27146153a635496af03d4b5e7 Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Mon, 3 Dec 2018 11:42:27 -0800 Subject: [PATCH 1/2] add avro load samples --- bigquery/docs/snippets.py | 42 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/bigquery/docs/snippets.py b/bigquery/docs/snippets.py index a60f587d03cf..e507e9689f33 100644 --- a/bigquery/docs/snippets.py +++ b/bigquery/docs/snippets.py @@ -1313,6 +1313,39 @@ def test_load_table_from_file(client, to_delete): assert row2 in rows +def test_load_table_from_uri_avro(client, to_delete, capsys): + dataset_id = 'load_table_from_uri_avro_{}'.format(_millis()) + dataset = bigquery.Dataset(client.dataset(dataset_id)) + client.create_dataset(dataset) + to_delete.append(dataset) + + # [START bigquery_load_table_gcs_avro] + # from google.cloud import bigquery + # client = bigquery.Client() + # dataset_id = 'my_dataset' + + dataset_ref = client.dataset(dataset_id) + job_config = bigquery.LoadJobConfig() + job_config.source_format = bigquery.SourceFormat.AVRO + uri = 'gs://cloud-samples-data/bigquery/us-states/us-states.avro' + + load_job = client.load_table_from_uri( + uri, + dataset_ref.table('us_states'), + job_config=job_config) # API request + print('Starting job {}'.format(load_job.job_id)) + + load_job.result() # Waits for table load to complete. + print('Job finished.') + + destination_table = client.get_table(dataset_ref.table('us_states')) + print('Loaded {} rows.'.format(destination_table.num_rows)) + # [END bigquery_load_table_gcs_avro] + + out, _ = capsys.readouterr() + assert 'Loaded 50 rows.' in out + + def test_load_table_from_uri_csv(client, to_delete, capsys): dataset_id = "load_table_from_uri_csv_{}".format(_millis()) dataset = bigquery.Dataset(client.dataset(dataset_id)) @@ -1590,6 +1623,7 @@ def test_load_table_from_uri_truncate(client, to_delete, capsys): client.load_table_from_file(body, table_ref, job_config=job_config).result() # Shared code + # [START bigquery_load_table_gcs_avro_truncate] # [START bigquery_load_table_gcs_csv_truncate] # [START bigquery_load_table_gcs_json_truncate] # [START bigquery_load_table_gcs_parquet_truncate] @@ -1603,12 +1637,18 @@ def test_load_table_from_uri_truncate(client, to_delete, capsys): job_config = bigquery.LoadJobConfig() job_config.write_disposition = bigquery.WriteDisposition.WRITE_TRUNCATE + # [END bigquery_load_table_gcs_avro_truncate] # [END bigquery_load_table_gcs_csv_truncate] # [END bigquery_load_table_gcs_json_truncate] # [END bigquery_load_table_gcs_parquet_truncate] # [END bigquery_load_table_gcs_orc_truncate] # Format-specific code + # [START bigquery_load_table_gcs_avro_truncate] + job_config.source_format = bigquery.SourceFormat.AVRO + uri = "gs://cloud-samples-data/bigquery/us-states/us-states.avro" + # [END bigquery_load_table_gcs_avro_truncate] + # [START bigquery_load_table_gcs_csv_truncate] job_config.skip_leading_rows = 1 # The source format defaults to CSV, so the line below is optional. @@ -1634,6 +1674,7 @@ def test_load_table_from_uri_truncate(client, to_delete, capsys): # [END bigquery_load_table_gcs_orc_truncate] # Shared code + # [START bigquery_load_table_gcs_avro_truncate] # [START bigquery_load_table_gcs_csv_truncate] # [START bigquery_load_table_gcs_json_truncate] # [START bigquery_load_table_gcs_parquet_truncate] @@ -1648,6 +1689,7 @@ def test_load_table_from_uri_truncate(client, to_delete, capsys): destination_table = client.get_table(table_ref) print("Loaded {} rows.".format(destination_table.num_rows)) + # [END bigquery_load_table_gcs_avro_truncate] # [END bigquery_load_table_gcs_csv_truncate] # [END bigquery_load_table_gcs_json_truncate] # [END bigquery_load_table_gcs_parquet_truncate] From 1664ca87f77e8d0627cb63bd213be9e02b69dd22 Mon Sep 17 00:00:00 2001 From: Alix Hamilton Date: Mon, 3 Dec 2018 12:09:20 -0800 Subject: [PATCH 2/2] move assert out of sample --- bigquery/docs/snippets.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bigquery/docs/snippets.py b/bigquery/docs/snippets.py index e507e9689f33..90762f98b37e 100644 --- a/bigquery/docs/snippets.py +++ b/bigquery/docs/snippets.py @@ -1621,6 +1621,8 @@ def test_load_table_from_uri_truncate(client, to_delete, capsys): table_ref = dataset.table("us_states") body = six.BytesIO(b"Washington,WA") client.load_table_from_file(body, table_ref, job_config=job_config).result() + previous_rows = client.get_table(table_ref).num_rows + assert previous_rows > 0 # Shared code # [START bigquery_load_table_gcs_avro_truncate] @@ -1632,9 +1634,6 @@ def test_load_table_from_uri_truncate(client, to_delete, capsys): # client = bigquery.Client() # table_ref = client.dataset('my_dataset').table('existing_table') - previous_rows = client.get_table(table_ref).num_rows - assert previous_rows > 0 - job_config = bigquery.LoadJobConfig() job_config.write_disposition = bigquery.WriteDisposition.WRITE_TRUNCATE # [END bigquery_load_table_gcs_avro_truncate]