Skip to content

Commit 795bb32

Browse files
jeanbzatseaver
jeanbza
authored and
tseaver
committed
Add support for 'STORAGE_EMULATOR_HOST' env var; add 'benchwrapper' script. (googleapis#9219)
1 parent 7572488 commit 795bb32

File tree

9 files changed

+417
-7
lines changed

9 files changed

+417
-7
lines changed

‎storage/google/cloud/storage/_helpers.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,16 @@
1919

2020
import base64
2121
from hashlib import md5
22+
import os
23+
24+
STORAGE_EMULATOR_ENV_VAR = "STORAGE_EMULATOR_HOST"
25+
"""Environment variable defining host for Storage emulator."""
26+
27+
_DEFAULT_STORAGE_HOST = u"https://www.googleapis.com"
28+
29+
30+
def _get_storage_host():
31+
return os.environ.get(STORAGE_EMULATOR_ENV_VAR, _DEFAULT_STORAGE_HOST)
2232

2333

2434
def _validate_name(name):

‎storage/google/cloud/storage/blob.py‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,27 +47,25 @@
4747
from google.resumable_media.requests import ResumableUpload
4848

4949
from google.cloud import exceptions
50+
from google.cloud._helpers import _bytes_to_unicode
5051
from google.cloud._helpers import _rfc3339_to_datetime
5152
from google.cloud._helpers import _to_bytes
52-
from google.cloud._helpers import _bytes_to_unicode
5353
from google.cloud.exceptions import NotFound
5454
from google.api_core.iam import Policy
55+
from google.cloud.storage._helpers import _get_storage_host
5556
from google.cloud.storage._helpers import _PropertyMixin
5657
from google.cloud.storage._helpers import _scalar_property
5758
from google.cloud.storage._signing import generate_signed_url_v2
5859
from google.cloud.storage._signing import generate_signed_url_v4
5960
from google.cloud.storage.acl import ACL
6061
from google.cloud.storage.acl import ObjectACL
6162

63+
_STORAGE_HOST = _get_storage_host()
6264

6365
_API_ACCESS_ENDPOINT = "https://storage.googleapis.com"
6466
_DEFAULT_CONTENT_TYPE = u"application/octet-stream"
65-
_DOWNLOAD_URL_TEMPLATE = (
66-
u"https://www.googleapis.com/download/storage/v1{path}?alt=media"
67-
)
68-
_BASE_UPLOAD_TEMPLATE = (
69-
u"https://www.googleapis.com/upload/storage/v1{bucket_path}/o?uploadType="
70-
)
67+
_DOWNLOAD_URL_TEMPLATE = _STORAGE_HOST + u"/download/storage/v1{path}?alt=media"
68+
_BASE_UPLOAD_TEMPLATE = _STORAGE_HOST + u"/upload/storage/v1{bucket_path}/o?uploadType="
7169
_MULTIPART_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + u"multipart"
7270
_RESUMABLE_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + u"resumable"
7371
# NOTE: "acl" is also writeable but we defer ACL management to

‎storage/google/cloud/storage/client.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
from google.cloud._helpers import _LocalStack
2323
from google.cloud.client import ClientWithProject
2424
from google.cloud.exceptions import NotFound
25+
from google.cloud.storage._helpers import _get_storage_host
2526
from google.cloud.storage._http import Connection
2627
from google.cloud.storage.batch import Batch
2728
from google.cloud.storage.bucket import Bucket
@@ -94,6 +95,9 @@ def __init__(
9495
)
9596

9697
kw_args = {"client_info": client_info}
98+
99+
kw_args["api_endpoint"] = _get_storage_host()
100+
97101
if client_options:
98102
if type(client_options) == dict:
99103
client_options = google.api_core.client_options.from_dict(

‎storage/test_utils/README.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# storage benchwrapp
2+
3+
main.py is a gRPC wrapper around the storage library for benchmarking purposes.
4+
5+
## Running
6+
7+
```
8+
export STORAGE_EMULATOR_HOST=localhost:8080
9+
pip install grpcio
10+
cd storage
11+
pip install -e . # install google.cloud.storage locally
12+
cd test_utils
13+
python3 benchwrapper.py --port 8081
14+
```
15+
16+
## Re-generating protos
17+
18+
```
19+
pip install grpcio-tools
20+
python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. *.proto
21+
```

‎storage/test_utils/benchwrapper.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import argparse
2+
import sys
3+
import time
4+
import grpc
5+
import os
6+
from concurrent import futures
7+
import storage_pb2_grpc
8+
import storage_pb2
9+
from google.cloud import storage
10+
11+
_ONE_DAY_IN_SECONDS = 60 * 60 * 24
12+
13+
parser = argparse.ArgumentParser()
14+
15+
if os.environ.get('STORAGE_EMULATOR_HOST') == None:
16+
sys.exit('This benchmarking server only works when connected to an emulator. Please set STORAGE_EMULATOR_HOST.')
17+
18+
parser.add_argument('--port', help='The port to run on.')
19+
20+
args = parser.parse_args()
21+
22+
if args.port == None:
23+
sys.exit('Usage: python3 main.py --port 8081')
24+
25+
client = storage.Client()
26+
27+
class StorageBenchWrapperServicer(storage_pb2_grpc.StorageBenchWrapperServicer):
28+
def Write(self, request, context):
29+
# TODO(deklerk): implement this
30+
return storage_pb2.EmptyResponse()
31+
32+
def Read(self, request, context):
33+
bucket = client.bucket(request.bucketName)
34+
blob = storage.Blob(request.objectName, bucket)
35+
blob.download_as_string()
36+
return storage_pb2.EmptyResponse()
37+
38+
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
39+
storage_pb2_grpc.add_StorageBenchWrapperServicer_to_server(StorageBenchWrapperServicer(), server)
40+
41+
print('listening on localhost:'+args.port)
42+
server.add_insecure_port('[::]:'+args.port)
43+
server.start()
44+
try:
45+
while True:
46+
time.sleep(_ONE_DAY_IN_SECONDS)
47+
except KeyboardInterrupt:
48+
server.stop(0)

‎storage/test_utils/storage.proto‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
// Copyright 2019 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
syntax = "proto3";
16+
17+
package storage_bench;
18+
19+
message ObjectRead{
20+
// The bucket string identifier.
21+
string bucketName = 1;
22+
// The object/blob string identifier.
23+
string objectName = 2;
24+
}
25+
26+
message ObjectWrite{
27+
// The bucket string identifier.
28+
string bucketName = 1;
29+
// The object/blob string identifiers.
30+
string objectName = 2;
31+
// The string containing the upload file path.
32+
string destination = 3;
33+
}
34+
35+
message EmptyResponse{
36+
}
37+
38+
service StorageBenchWrapper{
39+
// Performs an upload from a specific object.
40+
rpc Write(ObjectWrite) returns (EmptyResponse) {}
41+
// Read a specific object.
42+
rpc Read(ObjectRead) returns (EmptyResponse){}
43+
}

‎storage/test_utils/storage_pb2.py‎

Lines changed: 195 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)