Skip to content

Commit 471199d

Browse files
author
Chris Rossi
authored
feat: customer managed keys (CMEK) (#249)
* feat: customer managed keys (CMEK) Implement customer managed keys (CMEK) feature. WIP. DO NOT MERGE. * Wrap Status. * Wrapper for Status, reorganize to avoid circular imports. * Blacken. * Make system tests in charge of their own key. * Consolidate system tests. Get KMS_KEY_NAME from user's environment. * Fix test. * Lint. * Put system tests where nox is expecting to find them. * Test backup with CMEK. * Differentiate instance and cluster names for cmek test, so tests aren't stepping on each other's toes. Remove bogus backup with cmek test. * rename `encryption.py` to `encryption_info.py` * make sure `kms_key_name` is set to `None` if `encryption_info` is not PB. * Fix typo. Use more realistic looking test strings.
1 parent 70d7b1f commit 471199d

11 files changed

Lines changed: 724 additions & 9 deletions

File tree

‎packages/google-cloud-bigtable/google/cloud/bigtable/backup.py‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
from google.cloud._helpers import _datetime_to_pb_timestamp
2020
from google.cloud.bigtable_admin_v2 import BigtableTableAdminClient
2121
from google.cloud.bigtable_admin_v2.types import table
22+
from google.cloud.bigtable.encryption_info import EncryptionInfo
2223
from google.cloud.bigtable.policy import Policy
2324
from google.cloud.exceptions import NotFound
2425
from google.protobuf import field_mask_pb2
@@ -67,13 +68,20 @@ class Backup(object):
6768
"""
6869

6970
def __init__(
70-
self, backup_id, instance, cluster_id=None, table_id=None, expire_time=None
71+
self,
72+
backup_id,
73+
instance,
74+
cluster_id=None,
75+
table_id=None,
76+
expire_time=None,
77+
encryption_info=None,
7178
):
7279
self.backup_id = backup_id
7380
self._instance = instance
7481
self._cluster = cluster_id
7582
self.table_id = table_id
7683
self._expire_time = expire_time
84+
self._encryption_info = encryption_info
7785

7886
self._parent = None
7987
self._source_table = None
@@ -176,6 +184,15 @@ def expire_time(self):
176184
def expire_time(self, new_expire_time):
177185
self._expire_time = new_expire_time
178186

187+
@property
188+
def encryption_info(self):
189+
"""Encryption info for this Backup.
190+
191+
:rtype: :class:`google.cloud.bigtable.encryption.EncryptionInfo`
192+
:returns: The encryption information for this backup.
193+
"""
194+
return self._encryption_info
195+
179196
@property
180197
def start_time(self):
181198
"""The time this Backup was started.
@@ -255,13 +272,15 @@ def from_pb(cls, backup_pb, instance):
255272
table_id = match.group("table_id") if match else None
256273

257274
expire_time = backup_pb._pb.expire_time
275+
encryption_info = EncryptionInfo._from_pb(backup_pb.encryption_info)
258276

259277
backup = cls(
260278
backup_id,
261279
instance,
262280
cluster_id=cluster_id,
263281
table_id=table_id,
264282
expire_time=expire_time,
283+
encryption_info=encryption_info,
265284
)
266285
backup._start_time = backup_pb._pb.start_time
267286
backup._end_time = backup_pb._pb.end_time

‎packages/google-cloud-bigtable/google/cloud/bigtable/cluster.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,19 @@ class Cluster(object):
6363
Defaults to
6464
:data:`google.cloud.bigtable.enums.StorageType.UNSPECIFIED`.
6565
66+
:type kms_key_name: str
67+
:param kms_key_name: (Optional, Creation Only) The name of the KMS customer managed
68+
encryption key (CMEK) to use for at-rest encryption of data in
69+
this cluster. If omitted, Google's default encryption will be
70+
used. If specified, the requirements for this key are:
71+
72+
1) The Cloud Bigtable service account associated with the
73+
project that contains the cluster must be granted the
74+
``cloudkms.cryptoKeyEncrypterDecrypter`` role on the CMEK.
75+
2) Only regional keys can be used and the region of the CMEK
76+
key must match the region of the cluster.
77+
3) All clusters within an instance must use the same CMEK key.
78+
6679
:type _state: int
6780
:param _state: (`OutputOnly`)
6881
The current state of the cluster.
@@ -81,13 +94,15 @@ def __init__(
8194
location_id=None,
8295
serve_nodes=None,
8396
default_storage_type=None,
97+
kms_key_name=None,
8498
_state=None,
8599
):
86100
self.cluster_id = cluster_id
87101
self._instance = instance
88102
self.location_id = location_id
89103
self.serve_nodes = serve_nodes
90104
self.default_storage_type = default_storage_type
105+
self._kms_key_name = kms_key_name
91106
self._state = _state
92107

93108
@classmethod
@@ -145,6 +160,10 @@ def _update_from_pb(self, cluster_pb):
145160
self.location_id = cluster_pb.location.split("/")[-1]
146161
self.serve_nodes = cluster_pb.serve_nodes
147162
self.default_storage_type = cluster_pb.default_storage_type
163+
if cluster_pb.encryption_config:
164+
self._kms_key_name = cluster_pb.encryption_config.kms_key_name
165+
else:
166+
self._kms_key_name = None
148167
self._state = cluster_pb.state
149168

150169
@property
@@ -187,6 +206,11 @@ def state(self):
187206
"""
188207
return self._state
189208

209+
@property
210+
def kms_key_name(self):
211+
"""str: Customer managed encryption key for the cluster."""
212+
return self._kms_key_name
213+
190214
def __eq__(self, other):
191215
if not isinstance(other, self.__class__):
192216
return NotImplemented
@@ -356,4 +380,8 @@ def _to_pb(self):
356380
serve_nodes=self.serve_nodes,
357381
default_storage_type=self.default_storage_type,
358382
)
383+
if self._kms_key_name:
384+
cluster_pb.encryption_config = instance.Cluster.EncryptionConfig(
385+
kms_key_name=self._kms_key_name,
386+
)
359387
return cluster_pb

‎packages/google-cloud-bigtable/google/cloud/bigtable/encryption_info.py‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Copyright 2021 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Class for encryption info for tables and backups."""
16+
17+
from google.cloud.bigtable.error import Status
18+
19+
20+
class EncryptionInfo:
21+
"""Encryption information for a given resource.
22+
23+
If this resource is protected with customer managed encryption, the in-use Google
24+
Cloud Key Management Service (KMS) key versions will be specified along with their
25+
status.
26+
27+
:type encryption_type: int
28+
:param encryption_type: See :class:`enums.EncryptionInfo.EncryptionType`
29+
30+
:type encryption_status: google.cloud.bigtable.encryption.Status
31+
:param encryption_status: The encryption status.
32+
33+
:type kms_key_version: str
34+
:param kms_key_version: The key version used for encryption.
35+
"""
36+
37+
@classmethod
38+
def _from_pb(cls, info_pb):
39+
return cls(
40+
info_pb.encryption_type,
41+
Status(info_pb.encryption_status),
42+
info_pb.kms_key_version,
43+
)
44+
45+
def __init__(self, encryption_type, encryption_status, kms_key_version):
46+
self.encryption_type = encryption_type
47+
self.encryption_status = encryption_status
48+
self.kms_key_version = kms_key_version
49+
50+
def __eq__(self, other):
51+
if self is other:
52+
return True
53+
54+
if not isinstance(other, type(self)):
55+
return NotImplemented
56+
57+
return (
58+
self.encryption_type == other.encryption_type
59+
and self.encryption_status == other.encryption_status
60+
and self.kms_key_version == other.kms_key_version
61+
)
62+
63+
def __ne__(self, other):
64+
return not self == other

‎packages/google-cloud-bigtable/google/cloud/bigtable/enums.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,7 @@ class View(object):
156156
NAME_ONLY = table.Table.View.NAME_ONLY
157157
SCHEMA_VIEW = table.Table.View.SCHEMA_VIEW
158158
REPLICATION_VIEW = table.Table.View.REPLICATION_VIEW
159+
ENCRYPTION_VIEW = table.Table.View.ENCRYPTION_VIEW
159160
FULL = table.Table.View.FULL
160161

161162
class ReplicationState(object):
@@ -191,3 +192,32 @@ class ReplicationState(object):
191192
table.Table.ClusterState.ReplicationState.UNPLANNED_MAINTENANCE
192193
)
193194
READY = table.Table.ClusterState.ReplicationState.READY
195+
196+
197+
class EncryptionInfo:
198+
class EncryptionType:
199+
"""Possible encryption types for a resource.
200+
201+
Attributes:
202+
ENCRYPTION_TYPE_UNSPECIFIED (int): Encryption type was not specified, though
203+
data at rest remains encrypted.
204+
GOOGLE_DEFAULT_ENCRYPTION (int): The data backing this resource is encrypted
205+
at rest with a key that is fully managed by Google. No key version or
206+
status will be populated. This is the default state.
207+
CUSTOMER_MANAGED_ENCRYPTION (int): The data backing this resource is
208+
encrypted at rest with a key that is managed by the customer. The in-use
209+
version of the key and its status are populated for CMEK-protected
210+
tables. CMEK-protected backups are pinned to the key version that was in
211+
use at the time the backup was taken. This key version is populated but
212+
its status is not tracked and is reported as `UNKNOWN`.
213+
"""
214+
215+
ENCRYPTION_TYPE_UNSPECIFIED = (
216+
table.EncryptionInfo.EncryptionType.ENCRYPTION_TYPE_UNSPECIFIED
217+
)
218+
GOOGLE_DEFAULT_ENCRYPTION = (
219+
table.EncryptionInfo.EncryptionType.GOOGLE_DEFAULT_ENCRYPTION
220+
)
221+
CUSTOMER_MANAGED_ENCRYPTION = (
222+
table.EncryptionInfo.EncryptionType.CUSTOMER_MANAGED_ENCRYPTION
223+
)

‎packages/google-cloud-bigtable/google/cloud/bigtable/error.py‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Copyright 2021 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Class for error status."""
16+
17+
18+
class Status:
19+
"""A status, comprising a code and a message.
20+
21+
See: `Cloud APIs Errors <https://cloud.google.com/apis/design/errors>`_
22+
23+
This is a thin wrapper for ``google.rpc.status_pb2.Status``.
24+
25+
:type status_pb: google.rpc.status_pb2.Status
26+
:param status_pb: The status protocol buffer.
27+
"""
28+
29+
def __init__(self, status_pb):
30+
self.status_pb = status_pb
31+
32+
@property
33+
def code(self):
34+
"""The status code.
35+
36+
Values are defined in ``google.rpc.code_pb2.Code``.
37+
38+
See: `google.rpc.Code
39+
<https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto>`_
40+
41+
:rtype: int
42+
:returns: The status code.
43+
"""
44+
return self.status_pb.code
45+
46+
@property
47+
def message(self):
48+
"""A human readable status message.
49+
50+
:rypte: str
51+
:returns: The status message.
52+
"""
53+
return self.status_pb.message
54+
55+
def __repr__(self):
56+
return repr(self.status_pb)
57+
58+
def __eq__(self, other):
59+
if isinstance(other, type(self)):
60+
return self.status_pb == other.status_pb
61+
return NotImplemented
62+
63+
def __ne__(self, other):
64+
return not self == other

‎packages/google-cloud-bigtable/google/cloud/bigtable/instance.py‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -540,7 +540,12 @@ def test_iam_permissions(self, permissions):
540540
return list(resp.permissions)
541541

542542
def cluster(
543-
self, cluster_id, location_id=None, serve_nodes=None, default_storage_type=None
543+
self,
544+
cluster_id,
545+
location_id=None,
546+
serve_nodes=None,
547+
default_storage_type=None,
548+
kms_key_name=None,
544549
):
545550
"""Factory to create a cluster associated with this instance.
546551
@@ -576,13 +581,30 @@ def cluster(
576581
577582
:rtype: :class:`~google.cloud.bigtable.instance.Cluster`
578583
:returns: a cluster owned by this instance.
584+
585+
:type kms_key_name: str
586+
:param kms_key_name: (Optional, Creation Only) The name of the KMS customer
587+
managed encryption key (CMEK) to use for at-rest encryption
588+
of data in this cluster. If omitted, Google's default
589+
encryption will be used. If specified, the requirements for
590+
this key are:
591+
592+
1) The Cloud Bigtable service account associated with the
593+
project that contains the cluster must be granted the
594+
``cloudkms.cryptoKeyEncrypterDecrypter`` role on the
595+
CMEK.
596+
2) Only regional keys can be used and the region of the
597+
CMEK key must match the region of the cluster.
598+
3) All clusters within an instance must use the same CMEK
599+
key.
579600
"""
580601
return Cluster(
581602
cluster_id,
582603
self,
583604
location_id=location_id,
584605
serve_nodes=serve_nodes,
585606
default_storage_type=default_storage_type,
607+
kms_key_name=kms_key_name,
586608
)
587609

588610
def list_clusters(self):

‎packages/google-cloud-bigtable/google/cloud/bigtable/table.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
from google.cloud.bigtable.column_family import ColumnFamily
2929
from google.cloud.bigtable.batcher import MutationsBatcher
3030
from google.cloud.bigtable.batcher import FLUSH_COUNT, MAX_ROW_BYTES
31+
from google.cloud.bigtable.encryption_info import EncryptionInfo
3132
from google.cloud.bigtable.policy import Policy
3233
from google.cloud.bigtable.row import AppendRow
3334
from google.cloud.bigtable.row import ConditionalRow
@@ -484,6 +485,33 @@ def get_cluster_states(self):
484485
for cluster_id, value_pb in table_pb.cluster_states.items()
485486
}
486487

488+
def get_encryption_info(self):
489+
"""List the encryption info for each cluster owned by this table.
490+
491+
Gets the current encryption info for the table across all of the clusters. The
492+
returned dict will be keyed by cluster id and contain a status for all of the
493+
keys in use.
494+
495+
:rtype: dict
496+
:returns: Dictionary of encryption info for this table. Keys are cluster ids and
497+
values are tuples of :class:`google.cloud.bigtable.encryption.EncryptionInfo` instances.
498+
"""
499+
ENCRYPTION_VIEW = enums.Table.View.ENCRYPTION_VIEW
500+
table_client = self._instance._client.table_admin_client
501+
table_pb = table_client.get_table(
502+
request={"name": self.name, "view": ENCRYPTION_VIEW}
503+
)
504+
505+
return {
506+
cluster_id: tuple(
507+
(
508+
EncryptionInfo._from_pb(info_pb)
509+
for info_pb in value_pb.encryption_info
510+
)
511+
)
512+
for cluster_id, value_pb in table_pb.cluster_states.items()
513+
}
514+
487515
def read_row(self, row_key, filter_=None):
488516
"""Read a single row from this table.
489517

0 commit comments

Comments
 (0)