Skip to content

Commit 2823215

Browse files
authored
BigQuery: address dataset leaks, conflicts in systests (#6099)
Closes #6098. Closes #6097.
1 parent 48452b7 commit 2823215

3 files changed

Lines changed: 38 additions & 15 deletions

File tree

‎bigquery/docs/snippets.py‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1212,8 +1212,8 @@ def test_table_insert_rows(client, to_delete):
12121212

12131213
def test_load_table_from_file(client, to_delete):
12141214
"""Upload table data from a CSV file."""
1215-
dataset_id = 'table_upload_from_file_dataset_{}'.format(_millis())
1216-
table_id = 'table_upload_from_file_table_{}'.format(_millis())
1215+
dataset_id = 'load_table_from_file_dataset_{}'.format(_millis())
1216+
table_id = 'load_table_from_file_table_{}'.format(_millis())
12171217
dataset = bigquery.Dataset(client.dataset(dataset_id))
12181218
dataset.location = 'US'
12191219
client.create_dataset(dataset)
@@ -1261,7 +1261,7 @@ def test_load_table_from_file(client, to_delete):
12611261

12621262

12631263
def test_load_table_from_uri_csv(client, to_delete, capsys):
1264-
dataset_id = 'load_table_dataset_{}'.format(_millis())
1264+
dataset_id = 'load_table_from_uri_csv_{}'.format(_millis())
12651265
dataset = bigquery.Dataset(client.dataset(dataset_id))
12661266
client.create_dataset(dataset)
12671267
to_delete.append(dataset)
@@ -1300,7 +1300,7 @@ def test_load_table_from_uri_csv(client, to_delete, capsys):
13001300

13011301

13021302
def test_load_table_from_uri_json(client, to_delete, capsys):
1303-
dataset_id = 'load_table_dataset_{}'.format(_millis())
1303+
dataset_id = 'load_table_from_uri_json_{}'.format(_millis())
13041304
dataset = bigquery.Dataset(client.dataset(dataset_id))
13051305
dataset.location = 'US'
13061306
client.create_dataset(dataset)
@@ -1381,7 +1381,7 @@ def test_load_table_from_uri_cmek(client, to_delete):
13811381

13821382

13831383
def test_load_table_from_uri_parquet(client, to_delete, capsys):
1384-
dataset_id = 'load_table_dataset_{}'.format(_millis())
1384+
dataset_id = 'load_table_from_uri_parquet_{}'.format(_millis())
13851385
dataset = bigquery.Dataset(client.dataset(dataset_id))
13861386
client.create_dataset(dataset)
13871387
to_delete.append(dataset)
@@ -1414,7 +1414,7 @@ def test_load_table_from_uri_parquet(client, to_delete, capsys):
14141414

14151415

14161416
def test_load_table_from_uri_orc(client, to_delete, capsys):
1417-
dataset_id = 'load_table_dataset_{}'.format(_millis())
1417+
dataset_id = 'load_table_from_uri_orc_{}'.format(_millis())
14181418
dataset = bigquery.Dataset(client.dataset(dataset_id))
14191419
client.create_dataset(dataset)
14201420
to_delete.append(dataset)
@@ -1458,7 +1458,7 @@ def test_load_table_from_uri_autodetect(client, to_delete, capsys):
14581458
followed by more shared code. Note that only the last format in the
14591459
format-specific code section will be tested in this test.
14601460
"""
1461-
dataset_id = 'load_table_dataset_{}'.format(_millis())
1461+
dataset_id = 'load_table_from_uri_auto_{}'.format(_millis())
14621462
dataset = bigquery.Dataset(client.dataset(dataset_id))
14631463
client.create_dataset(dataset)
14641464
to_delete.append(dataset)
@@ -1524,7 +1524,7 @@ def test_load_table_from_uri_truncate(client, to_delete, capsys):
15241524
followed by more shared code. Note that only the last format in the
15251525
format-specific code section will be tested in this test.
15261526
"""
1527-
dataset_id = 'load_table_dataset_{}'.format(_millis())
1527+
dataset_id = 'load_table_from_uri_trunc_{}'.format(_millis())
15281528
dataset = bigquery.Dataset(client.dataset(dataset_id))
15291529
client.create_dataset(dataset)
15301530
to_delete.append(dataset)
@@ -2972,7 +2972,7 @@ def test_list_rows_as_dataframe(client):
29722972
@pytest.mark.skipif(pandas is None, reason='Requires `pandas`')
29732973
@pytest.mark.skipif(pyarrow is None, reason='Requires `pyarrow`')
29742974
def test_load_table_from_dataframe(client, to_delete):
2975-
dataset_id = 'load_table_dataframe_dataset_{}'.format(_millis())
2975+
dataset_id = 'load_table_from_dataframe_{}'.format(_millis())
29762976
dataset = bigquery.Dataset(client.dataset(dataset_id))
29772977
client.create_dataset(dataset)
29782978
to_delete.append(dataset)

‎bigquery/tests/scrub_datasets.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
import re
2+
import sys
3+
4+
from google.api_core.exceptions import NotFound
5+
from google.cloud.bigquery import Client
6+
7+
8+
def main(prefixes):
9+
client = Client()
10+
11+
pattern = re.compile(
12+
'|'.join('^{}.*$'.format(prefix) for prefix in prefixes))
13+
14+
ds_items = list(client.list_datasets())
15+
for dataset in ds_items:
16+
ds_id = dataset.dataset_id
17+
if pattern.match(ds_id):
18+
print("Deleting dataset: {}".format(ds_id))
19+
try:
20+
client.delete_dataset(dataset.reference, delete_contents=True)
21+
except NotFound:
22+
print(" NOT FOUND")
23+
24+
25+
if __name__ == '__main__':
26+
main(sys.argv[1:])

‎bigquery/tests/system.py‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -295,10 +295,7 @@ def test_delete_dataset_delete_contents_true(self):
295295

296296
def test_delete_dataset_delete_contents_false(self):
297297
from google.api_core import exceptions
298-
dataset_id = _make_dataset_id('delete_table_false')
299-
dataset = retry_403(Config.CLIENT.create_dataset)(
300-
Dataset(Config.CLIENT.dataset(dataset_id)))
301-
298+
dataset = self.temp_dataset(_make_dataset_id('delete_table_false'))
302299
table_id = 'test_table'
303300
table_arg = Table(dataset.table(table_id), schema=SCHEMA)
304301

@@ -1191,7 +1188,7 @@ def _load_table_for_dml(self, rows, dataset_id, table_id):
11911188
self._fetch_single_page(table)
11921189

11931190
def test_query_w_dml(self):
1194-
dataset_name = _make_dataset_id('dml_tests')
1191+
dataset_name = _make_dataset_id('dml_query')
11951192
table_name = 'test_table'
11961193
self._load_table_for_dml([('Hello World',)], dataset_name, table_name)
11971194
query_template = """UPDATE {}.{}
@@ -1207,7 +1204,7 @@ def test_query_w_dml(self):
12071204
self.assertEqual(query_job.num_dml_affected_rows, 1)
12081205

12091206
def test_dbapi_w_dml(self):
1210-
dataset_name = _make_dataset_id('dml_tests')
1207+
dataset_name = _make_dataset_id('dml_dbapi')
12111208
table_name = 'test_table'
12121209
self._load_table_for_dml([('Hello World',)], dataset_name, table_name)
12131210
query_template = """UPDATE {}.{}

0 commit comments

Comments
 (0)