Skip to content

Commit 0c625c5

Browse files
tseavertseaver
authored
Merge pull request #1930 from tseaver/bigtable-v2-system_tests
Bigtable system tests passing against V2 endpoints
2 parents 0ac2def + 6f4aced commit 0c625c5

File tree

10 files changed

+362
-178
lines changed

10 files changed

+362
-178
lines changed

‎gcloud/bigtable/_generated_v2/operations_grpc_pb2.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,12 @@
1-
1+
from google.longrunning.operations_pb2 import (
2+
CancelOperationRequest,
3+
DeleteOperationRequest,
4+
GetOperationRequest,
5+
ListOperationsRequest,
6+
ListOperationsResponse,
7+
Operation,
8+
google_dot_protobuf_dot_empty__pb2,
9+
)
210
from grpc.beta import implementations as beta_implementations
311
from grpc.beta import interfaces as beta_interfaces
412
from grpc.framework.common import cardinality

‎gcloud/bigtable/client.py‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,9 @@
4343
from gcloud.bigtable._generated_v2 import (
4444
operations_grpc_pb2 as operations_grpc_v2_pb2)
4545

46+
from gcloud.bigtable.cluster import DEFAULT_SERVE_NODES
4647
from gcloud.bigtable.instance import Instance
48+
from gcloud.bigtable.instance import _EXISTING_INSTANCE_LOCATION_ID
4749
from gcloud.client import _ClientFactoryMixin
4850
from gcloud.client import _ClientProjectMixin
4951
from gcloud.credentials import get_credentials
@@ -375,22 +377,33 @@ def __exit__(self, exc_type, exc_val, exc_t):
375377
"""Stops the client as a context manager."""
376378
self.stop()
377379

378-
def instance(self, instance_id, display_name=None):
380+
def instance(self, instance_id, location=_EXISTING_INSTANCE_LOCATION_ID,
381+
display_name=None, serve_nodes=DEFAULT_SERVE_NODES):
379382
"""Factory to create a instance associated with this client.
380383
381384
:type instance_id: str
382385
:param instance_id: The ID of the instance.
383386
387+
:type location: string
388+
:param location: location name, in form
389+
``projects/<project>/locations/<location>``; used to
390+
set up the instance's cluster.
391+
384392
:type display_name: str
385393
:param display_name: (Optional) The display name for the instance in
386394
the Cloud Console UI. (Must be between 4 and 30
387395
characters.) If this value is not set in the
388396
constructor, will fall back to the instance ID.
389397
398+
:type serve_nodes: int
399+
:param serve_nodes: (Optional) The number of nodes in the instance's
400+
cluster; used to set up the instance's cluster.
401+
390402
:rtype: :class:`.Instance`
391403
:returns: an instance owned by this client.
392404
"""
393-
return Instance(instance_id, self, display_name=display_name)
405+
return Instance(instance_id, self, location,
406+
display_name=display_name, serve_nodes=serve_nodes)
394407

395408
def list_instances(self):
396409
"""List instances owned by the project.

‎gcloud/bigtable/column_family.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,7 @@ def create(self):
261261
id=self.column_family_id,
262262
create=column_family,
263263
)
264-
client = self._table._cluster._client
264+
client = self._table._instance._client
265265
# We expect a `.table_v2_pb2.ColumnFamily`. We ignore it since the only
266266
# data it contains are the GC rule and the column family ID already
267267
# stored on this instance.
@@ -286,7 +286,7 @@ def update(self):
286286
request_pb.modifications.add(
287287
id=self.column_family_id,
288288
update=column_family)
289-
client = self._table._cluster._client
289+
client = self._table._instance._client
290290
# We expect a `.table_v2_pb2.ColumnFamily`. We ignore it since the only
291291
# data it contains are the GC rule and the column family ID already
292292
# stored on this instance.
@@ -300,7 +300,7 @@ def delete(self):
300300
request_pb.modifications.add(
301301
id=self.column_family_id,
302302
drop=True)
303-
client = self._table._cluster._client
303+
client = self._table._instance._client
304304
# We expect a `google.protobuf.empty_pb2.Empty`
305305
client._table_stub.ModifyColumnFamilies(request_pb,
306306
client.timeout_seconds)

‎gcloud/bigtable/instance.py‎

Lines changed: 45 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,17 @@
2727
from gcloud.bigtable._generated_v2 import (
2828
bigtable_table_admin_pb2 as table_messages_v2_pb2)
2929
from gcloud.bigtable.cluster import Cluster
30+
from gcloud.bigtable.cluster import DEFAULT_SERVE_NODES
3031
from gcloud.bigtable.table import Table
3132

3233

34+
_EXISTING_INSTANCE_LOCATION_ID = 'see-existing-cluster'
3335
_INSTANCE_NAME_RE = re.compile(r'^projects/(?P<project>[^/]+)/'
3436
r'instances/(?P<instance_id>[a-z][-a-z0-9]*)$')
3537
_OPERATION_NAME_RE = re.compile(r'^operations/projects/([^/]+)/'
36-
r'instances/([a-z][-a-z0-9]*)/operations/'
37-
r'(?P<operation_id>\d+)$')
38+
r'instances/([a-z][-a-z0-9]*)/'
39+
r'locations/(?P<location_id>[a-z][-a-z0-9]*)/'
40+
r'operations/(?P<operation_id>\d+)$')
3841
_TYPE_URL_BASE = 'type.googleapis.com/google.bigtable.'
3942
_ADMIN_TYPE_URL_BASE = _TYPE_URL_BASE + 'admin.v2.'
4043
_INSTANCE_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'CreateInstanceMetadata'
@@ -53,13 +56,19 @@ def _prepare_create_request(instance):
5356
:returns: The CreateInstance request object containing the instance info.
5457
"""
5558
parent_name = ('projects/' + instance._client.project)
56-
return messages_v2_pb2.CreateInstanceRequest(
59+
message = messages_v2_pb2.CreateInstanceRequest(
5760
parent=parent_name,
5861
instance_id=instance.instance_id,
5962
instance=data_v2_pb2.Instance(
6063
display_name=instance.display_name,
6164
),
6265
)
66+
cluster = message.clusters[instance.instance_id]
67+
cluster.name = instance.name + '/clusters/' + instance.instance_id
68+
cluster.location = (
69+
parent_name + '/locations/' + instance._cluster_location_id)
70+
cluster.serve_nodes = instance._cluster_serve_nodes
71+
return message
6372

6473

6574
def _parse_pb_any_to_native(any_val, expected_type=None):
@@ -91,25 +100,24 @@ def _process_operation(operation_pb):
91100
:param operation_pb: The long-running operation response from a
92101
Create/Update/Undelete instance request.
93102
94-
:rtype: tuple
95-
:returns: A pair of an integer and datetime stamp. The integer is the ID
96-
of the operation (``operation_id``) and the timestamp when
97-
the create operation began (``operation_begin``).
103+
:rtype: (int, str, datetime)
104+
:returns: (operation_id, location_id, operation_begin).
98105
:raises: :class:`ValueError <exceptions.ValueError>` if the operation name
99106
doesn't match the :data:`_OPERATION_NAME_RE` regex.
100107
"""
101108
match = _OPERATION_NAME_RE.match(operation_pb.name)
102109
if match is None:
103110
raise ValueError('Operation name was not in the expected '
104-
'format after a instance modification.',
111+
'format after instance creation.',
105112
operation_pb.name)
113+
location_id = match.group('location_id')
106114
operation_id = int(match.group('operation_id'))
107115

108116
request_metadata = _parse_pb_any_to_native(operation_pb.metadata)
109117
operation_begin = _pb_timestamp_to_datetime(
110118
request_metadata.request_time)
111119

112-
return operation_id, operation_begin
120+
return operation_id, location_id, operation_begin
113121

114122

115123
class Operation(object):
@@ -128,14 +136,18 @@ class Operation(object):
128136
:type begin: :class:`datetime.datetime`
129137
:param begin: The time when the operation was started.
130138
139+
:type location_id: str
140+
:param location_id: ID of the location in which the operation is running
141+
131142
:type instance: :class:`Instance`
132143
:param instance: The instance that created the operation.
133144
"""
134145

135-
def __init__(self, op_type, op_id, begin, instance=None):
146+
def __init__(self, op_type, op_id, begin, location_id, instance=None):
136147
self.op_type = op_type
137148
self.op_id = op_id
138149
self.begin = begin
150+
self.location_id = location_id
139151
self._instance = instance
140152
self._complete = False
141153

@@ -145,6 +157,7 @@ def __eq__(self, other):
145157
return (other.op_type == self.op_type and
146158
other.op_id == self.op_id and
147159
other.begin == self.begin and
160+
other.location_id == self.location_id and
148161
other._instance == self._instance and
149162
other._complete == self._complete)
150163

@@ -162,8 +175,9 @@ def finished(self):
162175
if self._complete:
163176
raise ValueError('The operation has completed.')
164177

165-
operation_name = ('operations/' + self._instance.name +
166-
'/operations/%d' % (self.op_id,))
178+
operation_name = (
179+
'operations/%s/locations/%s/operations/%d' %
180+
(self._instance.name, self.location_id, self.op_id))
167181
request_pb = operations_pb2.GetOperationRequest(name=operation_name)
168182
# We expect a `google.longrunning.operations_pb2.Operation`.
169183
operation_pb = self._instance._client._operations_stub.GetOperation(
@@ -199,17 +213,30 @@ class Instance(object):
199213
:param client: The client that owns the instance. Provides
200214
authorization and a project ID.
201215
216+
:type location_id: str
217+
:param location_id: ID of the location in which the instance will be
218+
created. Required for instances which do not yet
219+
exist.
220+
202221
:type display_name: str
203222
:param display_name: (Optional) The display name for the instance in the
204223
Cloud Console UI. (Must be between 4 and 30
205224
characters.) If this value is not set in the
206225
constructor, will fall back to the instance ID.
226+
227+
:type serve_nodes: int
228+
:param serve_nodes: (Optional) The number of nodes in the instance's
229+
cluster; used to set up the instance's cluster.
207230
"""
208231

209232
def __init__(self, instance_id, client,
210-
display_name=None):
233+
location_id=_EXISTING_INSTANCE_LOCATION_ID,
234+
display_name=None,
235+
serve_nodes=DEFAULT_SERVE_NODES):
211236
self.instance_id = instance_id
212237
self.display_name = display_name or instance_id
238+
self._cluster_location_id = location_id
239+
self._cluster_serve_nodes = serve_nodes
213240
self._client = client
214241

215242
def _update_from_pb(self, instance_pb):
@@ -246,8 +273,9 @@ def from_pb(cls, instance_pb, client):
246273
if match.group('project') != client.project:
247274
raise ValueError('Project ID on instance does not match the '
248275
'project ID on the client')
276+
instance_id = match.group('instance_id')
249277

250-
result = cls(match.group('instance_id'), client)
278+
result = cls(instance_id, client, _EXISTING_INSTANCE_LOCATION_ID)
251279
result._update_from_pb(instance_pb)
252280
return result
253281

@@ -262,6 +290,7 @@ def copy(self):
262290
"""
263291
new_client = self._client.copy()
264292
return self.__class__(self.instance_id, new_client,
293+
self._cluster_location_id,
265294
display_name=self.display_name)
266295

267296
@property
@@ -332,8 +361,8 @@ def create(self):
332361
operation_pb = self._client._instance_stub.CreateInstance(
333362
request_pb, self._client.timeout_seconds)
334363

335-
op_id, op_begin = _process_operation(operation_pb)
336-
return Operation('create', op_id, op_begin, instance=self)
364+
op_id, loc_id, op_begin = _process_operation(operation_pb)
365+
return Operation('create', op_id, op_begin, loc_id, instance=self)
337366

338367
def update(self):
339368
"""Update this instance.

‎gcloud/bigtable/row.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -395,7 +395,7 @@ def commit(self):
395395
mutations=mutations_list,
396396
)
397397
# We expect a `google.protobuf.empty_pb2.Empty`
398-
client = self._table._cluster._client
398+
client = self._table._instance._client
399399
client._data_stub.MutateRow(request_pb, client.timeout_seconds)
400400
self.clear()
401401

@@ -512,7 +512,7 @@ def commit(self):
512512
false_mutations=false_mutations,
513513
)
514514
# We expect a `.messages_v2_pb2.CheckAndMutateRowResponse`
515-
client = self._table._cluster._client
515+
client = self._table._instance._client
516516
resp = client._data_stub.CheckAndMutateRow(
517517
request_pb, client.timeout_seconds)
518518
self.clear()
@@ -800,7 +800,7 @@ def commit(self):
800800
rules=self._rule_pb_list,
801801
)
802802
# We expect a `.data_v2_pb2.Row`
803-
client = self._table._cluster._client
803+
client = self._table._instance._client
804804
row_response = client._data_stub.ReadModifyWriteRow(
805805
request_pb, client.timeout_seconds)
806806

‎gcloud/bigtable/test_client.py‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -526,20 +526,47 @@ def test_stop_while_stopped(self):
526526
# Make sure the cluster stub did not change.
527527
self.assertEqual(client._instance_stub_internal, instance_stub)
528528

529-
def test_instance_factory(self):
529+
def test_instance_factory_defaults(self):
530+
from gcloud.bigtable.cluster import DEFAULT_SERVE_NODES
530531
from gcloud.bigtable.instance import Instance
532+
from gcloud.bigtable.instance import _EXISTING_INSTANCE_LOCATION_ID
531533

532534
PROJECT = 'PROJECT'
533535
INSTANCE_ID = 'instance-id'
534536
DISPLAY_NAME = 'display-name'
535-
536537
credentials = _Credentials()
537538
client = self._makeOne(project=PROJECT, credentials=credentials)
538539

539540
instance = client.instance(INSTANCE_ID, display_name=DISPLAY_NAME)
541+
542+
self.assertTrue(isinstance(instance, Instance))
543+
self.assertEqual(instance.instance_id, INSTANCE_ID)
544+
self.assertEqual(instance.display_name, DISPLAY_NAME)
545+
self.assertEqual(instance._cluster_location_id,
546+
_EXISTING_INSTANCE_LOCATION_ID)
547+
self.assertEqual(instance._cluster_serve_nodes, DEFAULT_SERVE_NODES)
548+
self.assertTrue(instance._client is client)
549+
550+
def test_instance_factory_w_explicit_serve_nodes(self):
551+
from gcloud.bigtable.instance import Instance
552+
553+
PROJECT = 'PROJECT'
554+
INSTANCE_ID = 'instance-id'
555+
DISPLAY_NAME = 'display-name'
556+
LOCATION_ID = 'locname'
557+
SERVE_NODES = 5
558+
credentials = _Credentials()
559+
client = self._makeOne(project=PROJECT, credentials=credentials)
560+
561+
instance = client.instance(
562+
INSTANCE_ID, display_name=DISPLAY_NAME,
563+
location=LOCATION_ID, serve_nodes=SERVE_NODES)
564+
540565
self.assertTrue(isinstance(instance, Instance))
541566
self.assertEqual(instance.instance_id, INSTANCE_ID)
542567
self.assertEqual(instance.display_name, DISPLAY_NAME)
568+
self.assertEqual(instance._cluster_location_id, LOCATION_ID)
569+
self.assertEqual(instance._cluster_serve_nodes, SERVE_NODES)
543570
self.assertTrue(instance._client is client)
544571

545572
def test_list_instances(self):
@@ -549,6 +576,7 @@ def test_list_instances(self):
549576
bigtable_instance_admin_pb2 as messages_v2_pb2)
550577
from gcloud.bigtable._testing import _FakeStub
551578

579+
LOCATION = 'projects/' + self.PROJECT + '/locations/locname'
552580
FAILED_LOCATION = 'FAILED'
553581
INSTANCE_ID1 = 'instance-id1'
554582
INSTANCE_ID2 = 'instance-id2'
@@ -593,8 +621,8 @@ def test_list_instances(self):
593621
# Create expected_result.
594622
failed_locations = [FAILED_LOCATION]
595623
instances = [
596-
client.instance(INSTANCE_ID1),
597-
client.instance(INSTANCE_ID2),
624+
client.instance(INSTANCE_ID1, LOCATION),
625+
client.instance(INSTANCE_ID2, LOCATION),
598626
]
599627
expected_result = (instances, failed_locations)
600628

‎gcloud/bigtable/test_column_family.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -650,7 +650,7 @@ def _ColumnFamilyPB(*args, **kw):
650650
return table_v2_pb2.ColumnFamily(*args, **kw)
651651

652652

653-
class _Cluster(object):
653+
class _Instance(object):
654654

655655
def __init__(self, client=None):
656656
self._client = client
@@ -666,4 +666,4 @@ class _Table(object):
666666

667667
def __init__(self, name, client=None):
668668
self.name = name
669-
self._cluster = _Cluster(client)
669+
self._instance = _Instance(client)

0 commit comments

Comments
 (0)