Skip to content

Commit d9e7a62

Browse files
vnghiavnghia
authored
test(storage): add utils for new gcs emulator (#5057)
1 parent d4e14a7 commit d9e7a62

6 files changed

Lines changed: 473 additions & 0 deletions

File tree

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,5 @@ ci/kokoro/install/ccache-contents/
2727

2828
google/cloud/storage/testbench/__pycache__/
2929
google/cloud/storage/testbench/*.pyc
30+
google/cloud/storage/emulator/**/__pycache__/
31+
google/cloud/storage/emulator/**/*.pyc

‎google/cloud/storage/emulator/requirements.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
git+git://github.com/googleapis/python-storage@gapic-generation
2+
grpcio==1.32.0
3+
flask==1.1.2
4+
grpc-google-iam-v1==0.12.3
5+
pysimdjson==3.0.0

‎google/cloud/storage/emulator/utils/__init__.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Copyright 2020 Google Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from utils import common, error, generation

‎google/cloud/storage/emulator/utils/acl.py‎

Lines changed: 347 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,347 @@
1+
# Copyright 2020 Google LLC.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Utils related to access control"""
16+
17+
import hashlib
18+
import os
19+
20+
import utils
21+
from google.cloud.storage_v1.proto import storage_resources_pb2 as resources_pb2
22+
23+
PROJECT_NUMBER = os.getenv(
24+
"GOOGLE_CLOUD_CPP_STORAGE_EMULATOR_PROJECT_NUMBER", "123456789"
25+
)
26+
OBJECT_OWNER_ENTITY = os.getenv(
27+
"GOOGLE_CLOUD_CPP_STORAGE_EMULATOR_OBJECT_OWNER_ENTITY",
28+
"user-object.owners@example.com",
29+
)
30+
OBJECT_READER_ENTITY = os.getenv(
31+
"GOOGLE_CLOUD_CPP_STORAGE_EMULATOR_OBJECT_READER_ENTITY",
32+
"user-object.viewers@example.com",
33+
)
34+
35+
36+
# === EXTRACT INFORMATION FROM ENTITY === #
37+
38+
39+
def __extract_email(entity):
40+
if entity.startswith("user-"):
41+
return entity[len("user-") :]
42+
elif entity.startswith("group-") and "@" in entity:
43+
return entity[len("group-") :]
44+
return ""
45+
46+
47+
def __extract_domain(entity):
48+
if entity.startswith("domain-"):
49+
return entity[len("domain-") :]
50+
return ""
51+
52+
53+
def __extract_team(entity):
54+
if entity.startswith("project-"):
55+
return entity.split("-")[1]
56+
return ""
57+
58+
59+
# === ENTITY UTILS === #
60+
61+
62+
def get_canonical_entity(entity):
63+
if entity == "allUsers" or entity == "allAuthenticatedUsers":
64+
return entity
65+
if entity.startswith("project-owners-"):
66+
entity = "project-owners-" + PROJECT_NUMBER
67+
if entity.startswith("project-editors-"):
68+
entity = "project-editors-" + PROJECT_NUMBER
69+
if entity.startswith("project-viewers-"):
70+
entity = "project-viewers-" + PROJECT_NUMBER
71+
return entity.lower()
72+
73+
74+
def get_project_entity(team, context):
75+
if team not in ["editors", "owners", "viewers"]:
76+
utils.error.invaild("Team %s for project" % team)
77+
return "project-%s-%s" % (team, PROJECT_NUMBER)
78+
79+
80+
def get_object_entity(role, context):
81+
if role == "OWNER":
82+
return OBJECT_OWNER_ENTITY
83+
elif role == "READER":
84+
return OBJECT_READER_ENTITY
85+
else:
86+
utils.error.invaild("Role %s for object acl" % role, context)
87+
88+
89+
# === CREATE ACL === #
90+
91+
92+
def create_bucket_acl(bucket_name, entity, role, context):
93+
entity = get_canonical_entity(entity)
94+
if role not in ["OWNER", "WRITER", "READER"]:
95+
utils.error.invaild("Role %s for bucket acl" % role, context)
96+
etag = hashlib.md5((bucket_name + entity + role).encode("utf-8")).hexdigest()
97+
acl = resources_pb2.BucketAccessControl(
98+
role=role,
99+
etag=etag,
100+
id=etag,
101+
bucket=bucket_name,
102+
entity=entity,
103+
entity_id=hashlib.md5(entity.encode("utf-8")).hexdigest(),
104+
email=__extract_email(entity),
105+
domain=__extract_domain(entity),
106+
project_team={"project_number": PROJECT_NUMBER, "team": __extract_team(entity)},
107+
)
108+
return acl
109+
110+
111+
def create_default_object_acl(bucket_name, entity, role, context):
112+
entity = get_canonical_entity(entity)
113+
if role not in ["OWNER", "READER"]:
114+
utils.error.invaild("Role %s for object acl" % role, context)
115+
etag = hashlib.md5(
116+
(bucket_name + entity + role + "storage#objectAccessControl").encode("utf-8")
117+
).hexdigest()
118+
acl = resources_pb2.ObjectAccessControl(
119+
role=role,
120+
etag=etag,
121+
bucket=bucket_name,
122+
entity=entity,
123+
entity_id=hashlib.md5(entity.encode("utf-8")).hexdigest(),
124+
email=__extract_email(entity),
125+
domain=__extract_domain(entity),
126+
project_team={"project_number": PROJECT_NUMBER, "team": __extract_team(entity)},
127+
)
128+
return acl
129+
130+
131+
def create_object_acl_from_default_object_acl(
132+
object_name, generation, default_object_acl, context
133+
):
134+
acl = resources_pb2.ObjectAccessControl()
135+
acl.CopyFrom(default_object_acl)
136+
acl.id = hashlib.md5(
137+
(acl.bucket + object_name + generation + acl.entity + acl.role).encode("utf-8")
138+
).hexdigest()
139+
acl.etag = acl.id
140+
acl.object = object_name
141+
acl.generation = generation
142+
return acl
143+
144+
145+
def create_object_acl(bucket_name, object_name, generation, entity, role, context):
146+
entity = get_canonical_entity(entity)
147+
default_object_acl = create_default_object_acl(bucket_name, entity, role, context)
148+
acl = create_object_acl_from_default_object_acl(
149+
object_name, generation, default_object_acl, context
150+
)
151+
return acl
152+
153+
154+
# === EXTRACT INFORMATION FROM REQUEST === #
155+
156+
157+
def extract_predefined_acl(request, is_destination, context):
158+
if context is not None:
159+
extract_field = (
160+
"predefined_acl" if not is_destination else "destination_predefined_acl"
161+
)
162+
return getattr(request, extract_field, None)
163+
else:
164+
extract_field = (
165+
"predefinedAcl" if not is_destination else "destinationPredefinedAcl"
166+
)
167+
return request.args.get(extract_field, "")
168+
169+
170+
def extract_predefined_default_object_acl(request, context):
171+
return (
172+
request.args.get("predefinedDefaultObjectAcl", "")
173+
if context is None
174+
else request.predefined_default_object_acl
175+
)
176+
177+
178+
# === COMPUTE PREDEFINED ACL === #
179+
180+
predefined_bucket_acl_map = {
181+
"authenticated_read": 1,
182+
"private": 2,
183+
"project_private": 3,
184+
"public_read": 4,
185+
"public_read_write": 5,
186+
}
187+
188+
189+
def compute_predefined_bucket_acl(bucket_name, predefined_acl, context):
190+
if context is None:
191+
predefined_acl = utils.common.to_snake_case(predefined_acl)
192+
predefined_acl = predefined_bucket_acl_map.get(predefined_acl)
193+
if predefined_acl is None:
194+
return []
195+
acls = []
196+
if predefined_acl == 1:
197+
acls.append(
198+
create_bucket_acl(
199+
bucket_name, get_project_entity("owners", context), "OWNER", context
200+
)
201+
)
202+
acls.append(
203+
create_bucket_acl(bucket_name, "allAuthenticatedUsers", "READER", context)
204+
)
205+
elif predefined_acl == 2:
206+
acls.append(
207+
create_bucket_acl(
208+
bucket_name, get_project_entity("owners", context), "OWNER", context
209+
)
210+
)
211+
elif predefined_acl == 3:
212+
acls.append(
213+
create_bucket_acl(
214+
bucket_name, get_project_entity("owners", context), "OWNER", context
215+
)
216+
)
217+
acls.append(
218+
create_bucket_acl(
219+
bucket_name, get_project_entity("editors", context), "WRITER", context
220+
)
221+
)
222+
acls.append(
223+
create_bucket_acl(
224+
bucket_name, get_project_entity("viewers", context), "READER", context
225+
)
226+
)
227+
elif predefined_acl == 4:
228+
acls.append(
229+
create_bucket_acl(
230+
bucket_name, get_project_entity("owners", context), "OWNER", context
231+
)
232+
)
233+
acls.append(create_bucket_acl(bucket_name, "allUsers", "READER", context))
234+
elif predefined_acl == 5:
235+
acls.append(
236+
create_bucket_acl(
237+
bucket_name, get_project_entity("owners", context), "OWNER", context
238+
)
239+
)
240+
acls.append(create_bucket_acl(bucket_name, "allUsers", "WRITER", context))
241+
return acls
242+
243+
244+
predefined_object_acl_map = {
245+
"authenticated_read": 1,
246+
"bucket_owner_full_control": 2,
247+
"bucket_owner_read": 3,
248+
"private": 4,
249+
"project_private": 5,
250+
"public_read": 6,
251+
}
252+
253+
254+
def __compute_predefined_object_acl(bucket_name, predefined_acl, acl_factory, context):
255+
if context is None:
256+
predefined_acl = utils.common.to_snake_case(predefined_acl)
257+
predefined_acl = predefined_object_acl_map.get(predefined_acl)
258+
if predefined_acl is None:
259+
return []
260+
acls = []
261+
if predefined_acl == 1:
262+
acls.append(
263+
acl_factory(
264+
bucket_name, get_object_entity("OWNER", context), "OWNER", context
265+
)
266+
)
267+
acls.append(
268+
acl_factory(bucket_name, "allAuthenticatedUsers", "READER", context)
269+
)
270+
elif predefined_acl == 2:
271+
acls.append(
272+
acl_factory(
273+
bucket_name, get_object_entity("OWNER", context), "OWNER", context
274+
)
275+
)
276+
acls.append(
277+
acl_factory(
278+
bucket_name, get_project_entity("owners", context), "OWNER", context
279+
)
280+
)
281+
elif predefined_acl == 3:
282+
acls.append(
283+
acl_factory(
284+
bucket_name, get_object_entity("OWNER", context), "OWNER", context
285+
)
286+
)
287+
acls.append(
288+
acl_factory(
289+
bucket_name, get_project_entity("owners", context), "READER", context
290+
)
291+
)
292+
elif predefined_acl == 4:
293+
acls.append(
294+
acl_factory(
295+
bucket_name, get_object_entity("OWNER", context), "OWNER", context
296+
)
297+
)
298+
elif predefined_acl == 5:
299+
acls.append(
300+
acl_factory(
301+
bucket_name, get_object_entity("OWNER", context), "OWNER", context
302+
)
303+
)
304+
acls.append(
305+
acl_factory(
306+
bucket_name, get_project_entity("owners", context), "OWNER", context
307+
)
308+
)
309+
acls.append(
310+
acl_factory(
311+
bucket_name, get_project_entity("editors", context), "OWNER", context
312+
)
313+
)
314+
acls.append(
315+
acl_factory(
316+
bucket_name, get_project_entity("viewers", context), "READER", context
317+
)
318+
)
319+
elif predefined_acl == 6:
320+
acls.append(
321+
acl_factory(
322+
bucket_name, get_object_entity("OWNER", context), "OWNER", context
323+
)
324+
)
325+
acls.append(acl_factory(bucket_name, "allUsers", "READER", context))
326+
return acls
327+
328+
329+
def compute_predefined_default_object_acl(
330+
bucket_name, predefined_default_object_acl, context
331+
):
332+
return __compute_predefined_object_acl(
333+
bucket_name, predefined_default_object_acl, create_default_object_acl, context
334+
)
335+
336+
337+
def compute_predefined_object_acl(
338+
bucket_name, object_name, generation, predefined_acl, context
339+
):
340+
def object_acl_factory(bucket_name, entity, role, context):
341+
return create_object_acl(
342+
bucket_name, object_name, generation, entity, role, context
343+
)
344+
345+
return __compute_predefined_object_acl(
346+
bucket_name, predefined_acl, object_acl_factory, context
347+
)

0 commit comments

Comments
 (0)