Skip to content

Commit 273c267

Browse files
abecerrilsalasleahecolesteffnay
authored
feat: add support for table clones (#1235)
* feat: add support for table clones * feat: clone test * feat: debugging * feat: more debugging * feat: more debugging * feat: even more debugging * feat: debugging test * feat: even more test debugging * feat: check * feat: modify test * feat: deleting print statement * feat: testing * feat: test update * feat: change table name * feat: debugging table name * feat: cleaning up test * feat: degubbing test * feat: add properties check to test * feat: test change * feat: added more properties * Update samples/snippets/requirements.txt Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com> Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com> Co-authored-by: Steffany Brown <30247553+steffnay@users.noreply.github.com>
1 parent 40efd8e commit 273c267

7 files changed

Lines changed: 172 additions & 1 deletion

File tree

‎packages/google-cloud-bigquery/docs/reference.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ Table
9797
table.Row
9898
table.RowIterator
9999
table.SnapshotDefinition
100+
table.CloneDefinition
100101
table.Table
101102
table.TableListItem
102103
table.TableReference

‎packages/google-cloud-bigquery/google/cloud/bigquery/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@
101101
from google.cloud.bigquery.table import RangePartitioning
102102
from google.cloud.bigquery.table import Row
103103
from google.cloud.bigquery.table import SnapshotDefinition
104+
from google.cloud.bigquery.table import CloneDefinition
104105
from google.cloud.bigquery.table import Table
105106
from google.cloud.bigquery.table import TableReference
106107
from google.cloud.bigquery.table import TimePartitioningType
@@ -132,6 +133,7 @@
132133
"RangePartitioning",
133134
"Row",
134135
"SnapshotDefinition",
136+
"CloneDefinition",
135137
"TimePartitioning",
136138
"TimePartitioningType",
137139
# Jobs

‎packages/google-cloud-bigquery/google/cloud/bigquery/job/copy_.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ class OperationType:
4040
SNAPSHOT = "SNAPSHOT"
4141
"""The source table type is TABLE and the destination table type is SNAPSHOT."""
4242

43+
CLONE = "CLONE"
44+
"""The source table type is TABLE and the destination table type is CLONE."""
45+
4346
RESTORE = "RESTORE"
4447
"""The source table type is SNAPSHOT and the destination table type is TABLE."""
4548

‎packages/google-cloud-bigquery/google/cloud/bigquery/table.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,7 @@ class Table(_TableBase):
356356
"time_partitioning": "timePartitioning",
357357
"schema": "schema",
358358
"snapshot_definition": "snapshotDefinition",
359+
"clone_definition": "cloneDefinition",
359360
"streaming_buffer": "streamingBuffer",
360361
"self_link": "selfLink",
361362
"time_partitioning": "timePartitioning",
@@ -929,6 +930,19 @@ def snapshot_definition(self) -> Optional["SnapshotDefinition"]:
929930
snapshot_info = SnapshotDefinition(snapshot_info)
930931
return snapshot_info
931932

933+
@property
934+
def clone_definition(self) -> Optional["CloneDefinition"]:
935+
"""Information about the clone. This value is set via clone creation.
936+
937+
See: https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#Table.FIELDS.clone_definition
938+
"""
939+
clone_info = self._properties.get(
940+
self._PROPERTY_TO_API_FIELD["clone_definition"]
941+
)
942+
if clone_info is not None:
943+
clone_info = CloneDefinition(clone_info)
944+
return clone_info
945+
932946
@classmethod
933947
def from_string(cls, full_table_id: str) -> "Table":
934948
"""Construct a table from fully-qualified table ID.
@@ -1304,6 +1318,29 @@ def __init__(self, resource: Dict[str, Any]):
13041318
)
13051319

13061320

1321+
class CloneDefinition:
1322+
"""Information about base table and clone time of the clone.
1323+
1324+
See https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#clonedefinition
1325+
1326+
Args:
1327+
resource: Clone definition representation returned from the API.
1328+
"""
1329+
1330+
def __init__(self, resource: Dict[str, Any]):
1331+
self.base_table_reference = None
1332+
if "baseTableReference" in resource:
1333+
self.base_table_reference = TableReference.from_api_repr(
1334+
resource["baseTableReference"]
1335+
)
1336+
1337+
self.clone_time = None
1338+
if "cloneTime" in resource:
1339+
self.clone_time = google.cloud._helpers._rfc3339_to_datetime(
1340+
resource["cloneTime"]
1341+
)
1342+
1343+
13071344
class Row(object):
13081345
"""A BigQuery row.
13091346

‎packages/google-cloud-bigquery/samples/magics/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ grpcio==1.46.1
55
ipython===7.31.1; python_version == '3.7'
66
ipython===8.0.1; python_version == '3.8'
77
ipython==8.3.0; python_version >= '3.9'
8-
matplotlib==3.5.2
8+
matplotlib==3.5.1
99
pandas===1.3.5; python_version == '3.7'
1010
pandas==1.4.2; python_version >= '3.8'
1111
pyarrow==8.0.0

‎packages/google-cloud-bigquery/tests/system/test_client.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2190,3 +2190,57 @@ def test_table_snapshots(dataset_id):
21902190
rows_iter = client.list_rows(source_table_path)
21912191
rows = sorted(row.values() for row in rows_iter)
21922192
assert rows == [(1, "one"), (2, "two")]
2193+
2194+
2195+
def test_table_clones(dataset_id):
2196+
from google.cloud.bigquery import CopyJobConfig
2197+
from google.cloud.bigquery import OperationType
2198+
2199+
client = Config.CLIENT
2200+
2201+
table_path_source = f"{client.project}.{dataset_id}.test_table_clone"
2202+
clone_table_path = f"{table_path_source}_clone"
2203+
2204+
# Create the table before loading so that the column order is predictable.
2205+
schema = [
2206+
bigquery.SchemaField("foo", "INTEGER"),
2207+
bigquery.SchemaField("bar", "STRING"),
2208+
]
2209+
source_table = helpers.retry_403(Config.CLIENT.create_table)(
2210+
Table(table_path_source, schema=schema)
2211+
)
2212+
2213+
# Populate the table with initial data.
2214+
rows = [{"foo": 1, "bar": "one"}, {"foo": 2, "bar": "two"}]
2215+
load_job = Config.CLIENT.load_table_from_json(rows, source_table)
2216+
load_job.result()
2217+
2218+
# Now create a clone before modifying the original table data.
2219+
copy_config = CopyJobConfig()
2220+
copy_config.operation_type = OperationType.CLONE
2221+
copy_config.write_disposition = bigquery.WriteDisposition.WRITE_TRUNCATE
2222+
2223+
copy_job = client.copy_table(
2224+
sources=table_path_source,
2225+
destination=clone_table_path,
2226+
job_config=copy_config,
2227+
)
2228+
copy_job.result()
2229+
2230+
# List rows from the source table and compare them to rows from the clone.
2231+
rows_iter = client.list_rows(table_path_source)
2232+
rows = sorted(row.values() for row in rows_iter)
2233+
assert rows == [(1, "one"), (2, "two")]
2234+
2235+
rows_iter = client.list_rows(clone_table_path)
2236+
rows = sorted(row.values() for row in rows_iter)
2237+
assert rows == [(1, "one"), (2, "two")]
2238+
2239+
# Compare properties of the source and clone table.
2240+
source_table_props = client.get_table(table_path_source)
2241+
clone_table_props = client.get_table(clone_table_path)
2242+
2243+
assert source_table_props.schema == clone_table_props.schema
2244+
assert source_table_props.num_bytes == clone_table_props.num_bytes
2245+
assert source_table_props.num_rows == clone_table_props.num_rows
2246+
assert source_table_props.description == clone_table_props.description

‎packages/google-cloud-bigquery/tests/unit/test_table.py‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -841,6 +841,40 @@ def test_snapshot_definition_set(self):
841841
2010, 9, 28, 10, 20, 30, 123000, tzinfo=UTC
842842
)
843843

844+
def test_clone_definition_not_set(self):
845+
dataset = DatasetReference(self.PROJECT, self.DS_ID)
846+
table_ref = dataset.table(self.TABLE_NAME)
847+
table = self._make_one(table_ref)
848+
849+
assert table.clone_definition is None
850+
851+
def test_clone_definition_set(self):
852+
from google.cloud._helpers import UTC
853+
from google.cloud.bigquery.table import CloneDefinition
854+
855+
dataset = DatasetReference(self.PROJECT, self.DS_ID)
856+
table_ref = dataset.table(self.TABLE_NAME)
857+
table = self._make_one(table_ref)
858+
859+
table._properties["cloneDefinition"] = {
860+
"baseTableReference": {
861+
"projectId": "project_x",
862+
"datasetId": "dataset_y",
863+
"tableId": "table_z",
864+
},
865+
"cloneTime": "2010-09-28T10:20:30.123Z",
866+
}
867+
868+
clone = table.clone_definition
869+
870+
assert isinstance(clone, CloneDefinition)
871+
assert clone.base_table_reference.path == (
872+
"/projects/project_x/datasets/dataset_y/tables/table_z"
873+
)
874+
assert clone.clone_time == datetime.datetime(
875+
2010, 9, 28, 10, 20, 30, 123000, tzinfo=UTC
876+
)
877+
844878
def test_description_setter_bad_value(self):
845879
dataset = DatasetReference(self.PROJECT, self.DS_ID)
846880
table_ref = dataset.table(self.TABLE_NAME)
@@ -1789,6 +1823,46 @@ def test_ctor_full_resource(self):
17891823
assert instance.snapshot_time == expected_time
17901824

17911825

1826+
class TestCloneDefinition:
1827+
@staticmethod
1828+
def _get_target_class():
1829+
from google.cloud.bigquery.table import CloneDefinition
1830+
1831+
return CloneDefinition
1832+
1833+
@classmethod
1834+
def _make_one(cls, *args, **kwargs):
1835+
klass = cls._get_target_class()
1836+
return klass(*args, **kwargs)
1837+
1838+
def test_ctor_empty_resource(self):
1839+
instance = self._make_one(resource={})
1840+
assert instance.base_table_reference is None
1841+
assert instance.clone_time is None
1842+
1843+
def test_ctor_full_resource(self):
1844+
from google.cloud._helpers import UTC
1845+
from google.cloud.bigquery.table import TableReference
1846+
1847+
resource = {
1848+
"baseTableReference": {
1849+
"projectId": "my-project",
1850+
"datasetId": "your-dataset",
1851+
"tableId": "our-table",
1852+
},
1853+
"cloneTime": "2005-06-07T19:35:02.123Z",
1854+
}
1855+
instance = self._make_one(resource)
1856+
1857+
expected_table_ref = TableReference.from_string(
1858+
"my-project.your-dataset.our-table"
1859+
)
1860+
assert instance.base_table_reference == expected_table_ref
1861+
1862+
expected_time = datetime.datetime(2005, 6, 7, 19, 35, 2, 123000, tzinfo=UTC)
1863+
assert instance.clone_time == expected_time
1864+
1865+
17921866
class TestRow(unittest.TestCase):
17931867
def test_row(self):
17941868
from google.cloud.bigquery.table import Row

0 commit comments

Comments
 (0)