Skip to content

Commit 01f6826

Browse files
emar-karemar-kar
authored
refactor(bigquery): update code samples to use strings for table and dataset IDs (#9136)
1 parent 65f02a7 commit 01f6826

42 files changed

Lines changed: 392 additions & 198 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎bigquery/docs/snippets.py‎

Lines changed: 0 additions & 170 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,9 @@
1313
# limitations under the License.
1414

1515
"""Testable usage examples for Google BigQuery API wrapper
16-
1716
Each example function takes a ``client`` argument (which must be an instance
1817
of :class:`google.cloud.bigquery.client.Client`) and uses it to perform a task
1918
with the API.
20-
2119
To facilitate running the examples as system tests, each example is also passed
2220
a ``to_delete`` list; the function adds to the list any objects created which
2321
need to be deleted during teardown.
@@ -303,47 +301,6 @@ def test_load_and_query_partitioned_table(client, to_delete):
303301
assert len(rows) == 29
304302

305303

306-
# [START bigquery_table_exists]
307-
def table_exists(client, table_reference):
308-
"""Return if a table exists.
309-
310-
Args:
311-
client (google.cloud.bigquery.client.Client):
312-
A client to connect to the BigQuery API.
313-
table_reference (google.cloud.bigquery.table.TableReference):
314-
A reference to the table to look for.
315-
316-
Returns:
317-
bool: ``True`` if the table exists, ``False`` otherwise.
318-
"""
319-
from google.cloud.exceptions import NotFound
320-
321-
try:
322-
client.get_table(table_reference)
323-
return True
324-
except NotFound:
325-
return False
326-
327-
328-
# [END bigquery_table_exists]
329-
330-
331-
def test_table_exists(client, to_delete):
332-
"""Determine if a table exists."""
333-
DATASET_ID = "get_table_dataset_{}".format(_millis())
334-
TABLE_ID = "get_table_table_{}".format(_millis())
335-
dataset = bigquery.Dataset(client.dataset(DATASET_ID))
336-
dataset = client.create_dataset(dataset)
337-
to_delete.append(dataset)
338-
339-
table_ref = dataset.table(TABLE_ID)
340-
table = bigquery.Table(table_ref, schema=SCHEMA)
341-
table = client.create_table(table)
342-
343-
assert table_exists(client, table_ref)
344-
assert not table_exists(client, dataset.table("i_dont_exist"))
345-
346-
347304
@pytest.mark.skip(
348305
reason=(
349306
"update_table() is flaky "
@@ -698,36 +655,6 @@ def test_manage_views(client, to_delete):
698655
# [END bigquery_grant_view_access]
699656

700657

701-
def test_table_insert_rows(client, to_delete):
702-
"""Insert / fetch table data."""
703-
dataset_id = "table_insert_rows_dataset_{}".format(_millis())
704-
table_id = "table_insert_rows_table_{}".format(_millis())
705-
dataset = bigquery.Dataset(client.dataset(dataset_id))
706-
dataset = client.create_dataset(dataset)
707-
dataset.location = "US"
708-
to_delete.append(dataset)
709-
710-
table = bigquery.Table(dataset.table(table_id), schema=SCHEMA)
711-
table = client.create_table(table)
712-
713-
# [START bigquery_table_insert_rows]
714-
# TODO(developer): Uncomment the lines below and replace with your values.
715-
# from google.cloud import bigquery
716-
# client = bigquery.Client()
717-
# dataset_id = 'my_dataset' # replace with your dataset ID
718-
# For this sample, the table must already exist and have a defined schema
719-
# table_id = 'my_table' # replace with your table ID
720-
# table_ref = client.dataset(dataset_id).table(table_id)
721-
# table = client.get_table(table_ref) # API request
722-
723-
rows_to_insert = [(u"Phred Phlyntstone", 32), (u"Wylma Phlyntstone", 29)]
724-
725-
errors = client.insert_rows(table, rows_to_insert) # API request
726-
727-
assert errors == []
728-
# [END bigquery_table_insert_rows]
729-
730-
731658
def test_load_table_from_file(client, to_delete):
732659
"""Upload table data from a CSV file."""
733660
dataset_id = "load_table_from_file_dataset_{}".format(_millis())
@@ -993,12 +920,10 @@ def test_load_table_from_uri_orc(client, to_delete, capsys):
993920

994921
def test_load_table_from_uri_autodetect(client, to_delete, capsys):
995922
"""Load table from a GCS URI using various formats and auto-detected schema
996-
997923
Each file format has its own tested load from URI sample. Because most of
998924
the code is common for autodetect, append, and truncate, this sample
999925
includes snippets for all supported formats but only calls a single load
1000926
job.
1001-
1002927
This code snippet is made up of shared code, then format-specific code,
1003928
followed by more shared code. Note that only the last format in the
1004929
format-specific code section will be tested in this test.
@@ -1058,12 +983,10 @@ def test_load_table_from_uri_autodetect(client, to_delete, capsys):
1058983

1059984
def test_load_table_from_uri_truncate(client, to_delete, capsys):
1060985
"""Replaces table data with data from a GCS URI using various formats
1061-
1062986
Each file format has its own tested load from URI sample. Because most of
1063987
the code is common for autodetect, append, and truncate, this sample
1064988
includes snippets for all supported formats but only calls a single load
1065989
job.
1066-
1067990
This code snippet is made up of shared code, then format-specific code,
1068991
followed by more shared code. Note that only the last format in the
1069992
format-specific code section will be tested in this test.
@@ -1303,38 +1226,6 @@ def test_load_table_relax_column(client, to_delete):
13031226
assert table.num_rows > 0
13041227

13051228

1306-
def test_copy_table(client, to_delete):
1307-
dataset_id = "copy_table_dataset_{}".format(_millis())
1308-
dest_dataset = bigquery.Dataset(client.dataset(dataset_id))
1309-
dest_dataset.location = "US"
1310-
dest_dataset = client.create_dataset(dest_dataset)
1311-
to_delete.append(dest_dataset)
1312-
1313-
# [START bigquery_copy_table]
1314-
# from google.cloud import bigquery
1315-
# client = bigquery.Client()
1316-
1317-
source_dataset = client.dataset("samples", project="bigquery-public-data")
1318-
source_table_ref = source_dataset.table("shakespeare")
1319-
1320-
# dataset_id = 'my_dataset'
1321-
dest_table_ref = client.dataset(dataset_id).table("destination_table")
1322-
1323-
job = client.copy_table(
1324-
source_table_ref,
1325-
dest_table_ref,
1326-
# Location must match that of the source and destination tables.
1327-
location="US",
1328-
) # API request
1329-
1330-
job.result() # Waits for job to complete.
1331-
1332-
assert job.state == "DONE"
1333-
dest_table = client.get_table(dest_table_ref) # API request
1334-
assert dest_table.num_rows > 0
1335-
# [END bigquery_copy_table]
1336-
1337-
13381229
def test_copy_table_multiple_source(client, to_delete):
13391230
dest_dataset_id = "dest_dataset_{}".format(_millis())
13401231
dest_dataset = bigquery.Dataset(client.dataset(dest_dataset_id))
@@ -1601,31 +1492,6 @@ def test_undelete_table(client, to_delete):
16011492
# [END bigquery_undelete_table]
16021493

16031494

1604-
def test_client_query(client):
1605-
"""Run a simple query."""
1606-
1607-
# [START bigquery_query]
1608-
# from google.cloud import bigquery
1609-
# client = bigquery.Client()
1610-
1611-
query = (
1612-
"SELECT name FROM `bigquery-public-data.usa_names.usa_1910_2013` "
1613-
'WHERE state = "TX" '
1614-
"LIMIT 100"
1615-
)
1616-
query_job = client.query(
1617-
query,
1618-
# Location must match that of the dataset(s) referenced in the query.
1619-
location="US",
1620-
) # API request - starts the query
1621-
1622-
for row in query_job: # API request - fetches results
1623-
# Row values can be accessed by field name or index
1624-
assert row[0] == row.name == row["name"]
1625-
print(row)
1626-
# [END bigquery_query]
1627-
1628-
16291495
def test_client_query_legacy_sql(client):
16301496
"""Run a query with Legacy SQL explicitly set"""
16311497
# [START bigquery_query_legacy]
@@ -2360,42 +2226,6 @@ def test_ddl_create_view(client, to_delete, capsys):
23602226
assert len(df) == 0
23612227

23622228

2363-
def test_client_list_jobs(client):
2364-
"""List jobs for a project."""
2365-
2366-
# [START bigquery_list_jobs]
2367-
# TODO(developer): Uncomment the lines below and replace with your values.
2368-
# from google.cloud import bigquery
2369-
# project = 'my_project' # replace with your project ID
2370-
# client = bigquery.Client(project=project)
2371-
import datetime
2372-
2373-
# List the 10 most recent jobs in reverse chronological order.
2374-
# Omit the max_results parameter to list jobs from the past 6 months.
2375-
print("Last 10 jobs:")
2376-
for job in client.list_jobs(max_results=10): # API request(s)
2377-
print(job.job_id)
2378-
2379-
# The following are examples of additional optional parameters:
2380-
2381-
# Use min_creation_time and/or max_creation_time to specify a time window.
2382-
print("Jobs from the last ten minutes:")
2383-
ten_mins_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
2384-
for job in client.list_jobs(min_creation_time=ten_mins_ago):
2385-
print(job.job_id)
2386-
2387-
# Use all_users to include jobs run by all users in the project.
2388-
print("Last 10 jobs run by all users:")
2389-
for job in client.list_jobs(max_results=10, all_users=True):
2390-
print("{} run by user: {}".format(job.job_id, job.user_email))
2391-
2392-
# Use state_filter to filter by job state.
2393-
print("Jobs currently running:")
2394-
for job in client.list_jobs(state_filter="RUNNING"):
2395-
print(job.job_id)
2396-
# [END bigquery_list_jobs]
2397-
2398-
23992229
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
24002230
def test_query_results_as_dataframe(client):
24012231
# [START bigquery_query_results_dataframe]

‎bigquery/docs/usage/jobs.rst‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,20 @@
11
Managing Jobs
22
~~~~~~~~~~~~~
33

4-
List jobs for a project
5-
^^^^^^^^^^^^^^^^^^^^^^^
6-
74
Jobs describe actions performed on data in BigQuery tables:
85

96
- Load data into a table
107
- Run a query against data in one or more tables
118
- Extract data from a table
129
- Copy a table
1310

14-
.. literalinclude:: ../snippets.py
11+
Listing jobs
12+
^^^^^^^^^^^^
13+
14+
List jobs for a project with the
15+
:func:`~google.cloud.bigquery.client.Client.list_jobs` method:
16+
17+
.. literalinclude:: ../samples/client_list_jobs.py
1518
:language: python
1619
:dedent: 4
1720
:start-after: [START bigquery_list_jobs]

‎bigquery/docs/usage/queries.rst‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,10 @@ Running Queries
44
Querying data
55
^^^^^^^^^^^^^
66

7-
Run a query and wait for it to finish:
7+
Run a query and wait for it to finish with the
8+
:func:`~google.cloud.bigquery.client.Client.query` method:
89

9-
.. literalinclude:: ../snippets.py
10+
.. literalinclude:: ../samples/client_query.py
1011
:language: python
1112
:dedent: 4
1213
:start-after: [START bigquery_query]

‎bigquery/docs/usage/tables.rst‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,15 @@ Get a table resource with the
2828
:start-after: [START bigquery_get_table]
2929
:end-before: [END bigquery_get_table]
3030

31+
Determine if a table exists with the
32+
:func:`~google.cloud.bigquery.client.Client.get_table` method:
33+
34+
.. literalinclude:: ../samples/table_exists.py
35+
:language: python
36+
:dedent: 4
37+
:start-after: [START bigquery_table_exists]
38+
:end-before: [END bigquery_table_exists]
39+
3140
Browse data rows in a table with the
3241
:func:`~google.cloud.bigquery.client.Client.list_rows` method:
3342

@@ -107,7 +116,7 @@ Update a property in a table's metadata with the
107116
Insert rows into a table's data with the
108117
:func:`~google.cloud.bigquery.client.Client.insert_rows` method:
109118

110-
.. literalinclude:: ../snippets.py
119+
.. literalinclude:: ../samples/table_insert_rows.py
111120
:language: python
112121
:dedent: 4
113122
:start-after: [START bigquery_table_insert_rows]
@@ -128,7 +137,7 @@ Copying a Table
128137
Copy a table with the
129138
:func:`~google.cloud.bigquery.client.Client.copy_table` method:
130139

131-
.. literalinclude:: ../snippets.py
140+
.. literalinclude:: ../samples/copy_table.py
132141
:language: python
133142
:dedent: 4
134143
:start-after: [START bigquery_copy_table]

‎bigquery/samples/add_empty_column.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ def add_empty_column(client, table_id):
2828
table = client.get_table(table_id) # Make an API request.
2929

3030
original_schema = table.schema
31-
new_schema = original_schema[:] # creates a copy of the schema
31+
new_schema = original_schema[:] # Creates a copy of the schema.
3232
new_schema.append(bigquery.SchemaField("phone", "STRING"))
3333

3434
table.schema = new_schema

‎bigquery/samples/browse_table_data.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ def browse_table_data(client, table_id):
3939

4040
# Specify selected fields to limit the results to certain columns.
4141
table = client.get_table(table_id) # Make an API request.
42-
fields = table.schema[:2] # first two columns
42+
fields = table.schema[:2] # First two columns.
4343
rows_iter = client.list_rows(table_id, selected_fields=fields, max_results=10)
4444
rows = list(rows_iter)
4545
print("Selected {} columns from table {}.".format(len(rows_iter.schema), table_id))

‎bigquery/samples/client_list_jobs.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# Copyright 2019 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
def client_list_jobs(client):
17+
18+
# [START bigquery_list_jobs]
19+
# TODO(developer): Import the client library.
20+
# from google.cloud import bigquery
21+
22+
import datetime
23+
24+
# TODO(developer): Construct a BigQuery client object.
25+
# client = bigquery.Client()
26+
27+
# List the 10 most recent jobs in reverse chronological order.
28+
# Omit the max_results parameter to list jobs from the past 6 months.
29+
print("Last 10 jobs:")
30+
for job in client.list_jobs(max_results=10): # API request(s)
31+
print("{}".format(job.job_id))
32+
33+
# The following are examples of additional optional parameters:
34+
35+
# Use min_creation_time and/or max_creation_time to specify a time window.
36+
print("Jobs from the last ten minutes:")
37+
ten_mins_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
38+
for job in client.list_jobs(min_creation_time=ten_mins_ago):
39+
print("{}".format(job.job_id))
40+
41+
# Use all_users to include jobs run by all users in the project.
42+
print("Last 10 jobs run by all users:")
43+
for job in client.list_jobs(max_results=10, all_users=True):
44+
print("{} run by user: {}".format(job.job_id, job.user_email))
45+
46+
# Use state_filter to filter by job state.
47+
print("Last 10 jobs done:")
48+
for job in client.list_jobs(max_results=10, state_filter="DONE"):
49+
print("{}".format(job.job_id))
50+
# [END bigquery_list_jobs]

0 commit comments

Comments
 (0)