From 980173dd9b5d5684949fd045c7126691abb42dff Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 27 Aug 2019 12:23:12 +0300 Subject: [PATCH 1/7] add prefix support --- bigquery/google/cloud/bigquery/_helpers.py | 22 ++++++++++++++++++---- bigquery/tests/unit/test_table.py | 12 ++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index bb3998732a5a..bb41f756cec6 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -18,6 +18,7 @@ import copy import datetime import decimal +import re from google.cloud._helpers import UTC from google.cloud._helpers import _date_from_iso8601_date @@ -29,6 +30,12 @@ _RFC3339_MICROS_NO_ZULU = "%Y-%m-%dT%H:%M:%S.%f" _TIMEONLY_WO_MICROS = "%H:%M:%S" _TIMEONLY_W_MICROS = "%H:%M:%S.%f" +_PROJECT_PREFIX_PATTERN = re.compile( + r""" + (?P\S+\:[^.]+)\.(?P[^.]+)\.(?P[^.]+)$ +""", + re.VERBOSE, +) def _not_null(value, field): @@ -590,20 +597,27 @@ def _parse_3_part_id(full_id, default_project=None, property_name="table_id"): output_project_id = default_project output_dataset_id = None output_resource_id = None - parts = full_id.split(".") + with_prefix = _PROJECT_PREFIX_PATTERN.match(full_id) + if with_prefix is None: + parts = full_id.split(".") + else: + project_id = with_prefix.group("project_id") + dataset_id = with_prefix.group("dataset_id") + table_id = with_prefix.group("table_id") + parts = [project_id, dataset_id, table_id] if len(parts) != 2 and len(parts) != 3: raise ValueError( "{property_name} must be a fully-qualified ID in " - 'standard SQL format. e.g. "project.dataset.{property_name}", ' + 'standard SQL format, e.g., "project.dataset.{property_name}", ' "got {}".format(full_id, property_name=property_name) ) if len(parts) == 2 and not default_project: raise ValueError( "When default_project is not set, {property_name} must be a " - "fully-qualified ID in standard SQL format. " - 'e.g. "project.dataset_id.{property_name}", got {}'.format( + "fully-qualified ID in standard SQL format, " + 'e.g., "project.dataset_id.{property_name}", got {}'.format( full_id, property_name=property_name ) ) diff --git a/bigquery/tests/unit/test_table.py b/bigquery/tests/unit/test_table.py index 8ba7fee892e5..2f72dd3b542e 100644 --- a/bigquery/tests/unit/test_table.py +++ b/bigquery/tests/unit/test_table.py @@ -215,11 +215,23 @@ def test_from_string(self): self.assertEqual(got.dataset_id, "string_dataset") self.assertEqual(got.table_id, "string_table") + def test_from_string_w_prefix(self): + cls = self._get_target_class() + got = cls.from_string("google.com:string-project.string_dataset.string_table") + self.assertEqual(got.project, "google.com:string-project") + self.assertEqual(got.dataset_id, "string_dataset") + self.assertEqual(got.table_id, "string_table") + def test_from_string_legacy_string(self): cls = self._get_target_class() with self.assertRaises(ValueError): cls.from_string("string-project:string_dataset.string_table") + def test_from_string_w_incorrect_prefix(self): + cls = self._get_target_class() + with self.assertRaises(ValueError): + cls.from_string("google.com.string-project.string_dataset.string_table") + def test_from_string_not_fully_qualified(self): cls = self._get_target_class() with self.assertRaises(ValueError): From 525939b7d522d83caee1ac9ee403f1128d8ce762 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 27 Aug 2019 18:20:06 +0300 Subject: [PATCH 2/7] Update _helpers.py --- bigquery/google/cloud/bigquery/_helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index bb41f756cec6..596a14709bfa 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -32,7 +32,7 @@ _TIMEONLY_W_MICROS = "%H:%M:%S.%f" _PROJECT_PREFIX_PATTERN = re.compile( r""" - (?P\S+\:[^.]+)\.(?P[^.]+)\.(?P[^.]+)$ + (?P\S+\:[^.]+)\.(?P[^.]+)\.(?P[^.]+)$ """, re.VERBOSE, ) @@ -603,8 +603,8 @@ def _parse_3_part_id(full_id, default_project=None, property_name="table_id"): else: project_id = with_prefix.group("project_id") dataset_id = with_prefix.group("dataset_id") - table_id = with_prefix.group("table_id") - parts = [project_id, dataset_id, table_id] + custom_id = with_prefix.group("custom_id") + parts = [project_id, dataset_id, custom_id] if len(parts) != 2 and len(parts) != 3: raise ValueError( From d7f30bb9859464582d47038ebc436275546b47f6 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 30 Aug 2019 16:12:15 +0300 Subject: [PATCH 3/7] consolidate the regex --- bigquery/google/cloud/bigquery/_helpers.py | 16 +++++++++++----- bigquery/google/cloud/bigquery/dataset.py | 16 +--------------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index 596a14709bfa..57bed1a5295c 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -32,7 +32,7 @@ _TIMEONLY_W_MICROS = "%H:%M:%S.%f" _PROJECT_PREFIX_PATTERN = re.compile( r""" - (?P\S+\:[^.]+)\.(?P[^.]+)\.(?P[^.]+)$ + (?P\S+\:[^.]+)\.(?P[^.]+)(?:$|\.(?P[^.]+)$) """, re.VERBOSE, ) @@ -593,10 +593,7 @@ def _str_or_none(value): return str(value) -def _parse_3_part_id(full_id, default_project=None, property_name="table_id"): - output_project_id = default_project - output_dataset_id = None - output_resource_id = None +def _parse_id(full_id): with_prefix = _PROJECT_PREFIX_PATTERN.match(full_id) if with_prefix is None: parts = full_id.split(".") @@ -605,6 +602,15 @@ def _parse_3_part_id(full_id, default_project=None, property_name="table_id"): dataset_id = with_prefix.group("dataset_id") custom_id = with_prefix.group("custom_id") parts = [project_id, dataset_id, custom_id] + parts = [part for part in parts if part] + return parts + + +def _parse_3_part_id(full_id, default_project=None, property_name="table_id"): + output_project_id = default_project + output_dataset_id = None + output_resource_id = None + parts = _parse_id(full_id) if len(parts) != 2 and len(parts) != 3: raise ValueError( diff --git a/bigquery/google/cloud/bigquery/dataset.py b/bigquery/google/cloud/bigquery/dataset.py index 494c219d4f67..eaa3d0f2adb5 100644 --- a/bigquery/google/cloud/bigquery/dataset.py +++ b/bigquery/google/cloud/bigquery/dataset.py @@ -27,14 +27,6 @@ from google.cloud.bigquery.table import TableReference -_PROJECT_PREFIX_PATTERN = re.compile( - r""" - (?P\S+\:[^.]+)\.(?P[^.]+)$ -""", - re.VERBOSE, -) - - def _get_table_reference(self, table_id): """Constructs a TableReference. @@ -299,13 +291,7 @@ def from_string(cls, dataset_id, default_project=None): """ output_dataset_id = dataset_id output_project_id = default_project - with_prefix = _PROJECT_PREFIX_PATTERN.match(dataset_id) - if with_prefix is None: - parts = dataset_id.split(".") - else: - project_id = with_prefix.group("project_id") - dataset_id = with_prefix.group("dataset_id") - parts = [project_id, dataset_id] + parts = _helpers._parse_id(dataset_id) if len(parts) == 1 and not default_project: raise ValueError( From 63ea561ae8080cffc1876638185fc4fa59d50c59 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 3 Sep 2019 11:13:33 +0300 Subject: [PATCH 4/7] update split_id method * '_parse_id' method renamed to '_split_id' * switched to 're.groups' implementation instead of partly grouping --- bigquery/google/cloud/bigquery/_helpers.py | 9 +++------ bigquery/google/cloud/bigquery/dataset.py | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index 57bed1a5295c..01beabaf8c6a 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -593,15 +593,12 @@ def _str_or_none(value): return str(value) -def _parse_id(full_id): +def _split_id(full_id): with_prefix = _PROJECT_PREFIX_PATTERN.match(full_id) if with_prefix is None: parts = full_id.split(".") else: - project_id = with_prefix.group("project_id") - dataset_id = with_prefix.group("dataset_id") - custom_id = with_prefix.group("custom_id") - parts = [project_id, dataset_id, custom_id] + parts = with_prefix.groups() parts = [part for part in parts if part] return parts @@ -610,7 +607,7 @@ def _parse_3_part_id(full_id, default_project=None, property_name="table_id"): output_project_id = default_project output_dataset_id = None output_resource_id = None - parts = _parse_id(full_id) + parts = _split_id(full_id) if len(parts) != 2 and len(parts) != 3: raise ValueError( diff --git a/bigquery/google/cloud/bigquery/dataset.py b/bigquery/google/cloud/bigquery/dataset.py index eaa3d0f2adb5..016274bcde99 100644 --- a/bigquery/google/cloud/bigquery/dataset.py +++ b/bigquery/google/cloud/bigquery/dataset.py @@ -291,7 +291,7 @@ def from_string(cls, dataset_id, default_project=None): """ output_dataset_id = dataset_id output_project_id = default_project - parts = _helpers._parse_id(dataset_id) + parts = _helpers._split_id(dataset_id) if len(parts) == 1 and not default_project: raise ValueError( From 4e8182f413c44d8311b072bf232d2002efc82c39 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Tue, 3 Sep 2019 15:02:42 +0300 Subject: [PATCH 5/7] Update dataset.py flake8 fixed --- bigquery/google/cloud/bigquery/dataset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bigquery/google/cloud/bigquery/dataset.py b/bigquery/google/cloud/bigquery/dataset.py index 016274bcde99..67a7353f94e7 100644 --- a/bigquery/google/cloud/bigquery/dataset.py +++ b/bigquery/google/cloud/bigquery/dataset.py @@ -18,7 +18,6 @@ import six import copy -import re import google.cloud._helpers from google.cloud.bigquery import _helpers From 03ac092f337d28a08867debc0575bbc77b405493 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Thu, 5 Sep 2019 17:09:07 +0300 Subject: [PATCH 6/7] Update _helpers.py * added the docstring for the '_split_id' method --- bigquery/google/cloud/bigquery/_helpers.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index 01beabaf8c6a..b7b161ef19ba 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -594,6 +594,14 @@ def _str_or_none(value): def _split_id(full_id): + """Helper: split full_id into composite parts. + + Args: + full_id (str): Fully-qualified ID in standard SQL format. + + Returns: + List[str]: ID's parts separated into components. + """ with_prefix = _PROJECT_PREFIX_PATTERN.match(full_id) if with_prefix is None: parts = full_id.split(".") From 61f3b8194833d8884d39e4dfeb2df2e832a7bcd8 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Thu, 5 Sep 2019 18:00:29 +0300 Subject: [PATCH 7/7] fix lint failure --- bigquery/google/cloud/bigquery/_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index b7b161ef19ba..eb5161c9fe71 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -595,7 +595,7 @@ def _str_or_none(value): def _split_id(full_id): """Helper: split full_id into composite parts. - + Args: full_id (str): Fully-qualified ID in standard SQL format.