test(storage): fix S3 SDK tests (GoogleCloudPlatform#6074)

tritone · web-flow · commit e16ff67c3130 · 2021-06-03T18:52:15.000Z
These tests are currently failing because credentials have expired. This change rewrites the tests to create their own HMAC Key rather than relying on keys in a separate project. Inspired by similar approach in GoogleCloudPlatform/golang-samples#1217 Fixes GoogleCloudPlatform#5906 Fixes GoogleCloudPlatform#5907 ## Checklist - [x] I have followed [Sample Guidelines from AUTHORING_GUIDE.MD](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md) - [x] README is updated to include [all relevant information](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md#readme-file) - [x] **Tests** pass: `nox -s py-3.6` (see [Test Environment Setup](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md#test-environment-setup)) - [x] **Lint** pass: `nox -s lint` (see [Test Environment Setup](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md#test-environment-setup)) - [ ] These samples need a new **API enabled** in testing projects to pass (let us know which ones) - [x] These samples need a new/updated **env vars** in testing projects set to pass (let us know which ones) - [x] Please **merge** this PR for me once it is approved. - [ ] This sample adds a new sample directory, and I updated the [CODEOWNERS file](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/.github/CODEOWNERS) with the codeowners for this sample
diff --git a/storage/s3-sdk/list_gcs_buckets_test.py b/storage/s3-sdk/list_gcs_buckets_test.py
diff --git a/storage/s3-sdk/list_gcs_objects_test.py b/storage/s3-sdk/list_gcs_objects_test.py
diff --git a/storage/s3-sdk/noxfile_config.py b/storage/s3-sdk/noxfile_config.py
@@ -0,0 +1,50 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+
+
+# We are reaching maximum number of HMAC keys on the service account.
+# We change the service account based on the value of
+# RUN_TESTS_SESSION. The reason we can not use multiple project is
+# that our new projects are enforced to have
+# 'constraints/iam.disableServiceAccountKeyCreation' policy.
+def get_service_account_email():
+    session = os.environ.get('RUN_TESTS_SESSION')
+    if session == 'py-3.6':
+        return ('py36-storage-test@'
+                'python-docs-samples-tests.iam.gserviceaccount.com')
+    if session == 'py-3.7':
+        return ('py37-storage-test@'
+                'python-docs-samples-tests.iam.gserviceaccount.com')
+    if session == 'py-3.8':
+        return ('py38-storage-test@'
+                'python-docs-samples-tests.iam.gserviceaccount.com')
+    return os.environ['HMAC_KEY_TEST_SERVICE_ACCOUNT']
+
+
+TEST_CONFIG_OVERRIDE = {
+    # A dictionary you want to inject into your test. Don't put any
+    # secrets here. These values will override predefined values.
+    'envs': {
+        'HMAC_KEY_TEST_SERVICE_ACCOUNT': get_service_account_email(),
+        # Some tests can not use multiple projects because of several reasons:
+        # 1. The new projects is enforced to have the
+        # 'constraints/iam.disableServiceAccountKeyCreation' policy.
+        # 2. The new projects buckets need to have universal permission model.
+        # For those tests, we'll use the original project.
+        'MAIN_GOOGLE_CLOUD_PROJECT': 'python-docs-samples-tests'
+    },
+}
diff --git a/storage/s3-sdk/requirements-test.txt b/storage/s3-sdk/requirements-test.txt
@@ -1 +1,3 @@
+backoff==1.10.0
 pytest==6.2.4
+google-cloud-storage==1.38.0
diff --git a/storage/s3-sdk/s3_sdk_test.py b/storage/s3-sdk/s3_sdk_test.py
@@ -0,0 +1,93 @@
+# Copyright 2021 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import uuid
+
+import backoff
+from botocore.exceptions import ClientError
+from google.cloud import storage
+import pytest
+
+import list_gcs_buckets
+import list_gcs_objects
+
+
+PROJECT_ID = os.environ["MAIN_GOOGLE_CLOUD_PROJECT"]
+SERVICE_ACCOUNT_EMAIL = os.environ["HMAC_KEY_TEST_SERVICE_ACCOUNT"]
+STORAGE_CLIENT = storage.Client(project=PROJECT_ID)
+
+
+@pytest.fixture(scope="module")
+def hmac_fixture():
+    """
+    Creates an HMAC Key and secret to supply to the S3 SDK tests. The key
+    will be deleted after the test session.
+    """
+    hmac_key, secret = STORAGE_CLIENT.create_hmac_key(
+        service_account_email=SERVICE_ACCOUNT_EMAIL, project_id=PROJECT_ID
+    )
+    yield hmac_key, secret
+    hmac_key.state = "INACTIVE"
+    hmac_key.update()
+    hmac_key.delete()
+
+
+@pytest.fixture(scope="module")
+def test_bucket():
+    """Yields a bucket that is deleted after the test completes."""
+    bucket = None
+    while bucket is None or bucket.exists():
+        bucket_name = "storage-snippets-test-{}".format(uuid.uuid4())
+        bucket = storage.Client().bucket(bucket_name)
+    bucket.create()
+    yield bucket
+    bucket.delete(force=True)
+
+
+@pytest.fixture(scope="module")
+def test_blob(test_bucket):
+    """Yields a blob that is deleted after the test completes."""
+    bucket = test_bucket
+    blob = bucket.blob("storage_snippets_test_sigil-{}".format(uuid.uuid4()))
+    blob.upload_from_string("Hello, is it me you're looking for?")
+    yield blob
+
+
+def test_list_buckets(capsys, hmac_fixture, test_bucket):
+    # Retry request because the created key may not be fully propagated for up
+    # to 15s.
+    @backoff.on_exception(backoff.constant, ClientError, interval=1, max_time=15)
+    def list_buckets():
+        list_gcs_buckets.list_gcs_buckets(
+            google_access_key_id=hmac_fixture[0].access_id, google_access_key_secret=hmac_fixture[1]
+        )
+        out, _ = capsys.readouterr()
+        assert "Buckets:" in out
+        assert test_bucket.name in out
+
+
+def test_list_blobs(capsys, hmac_fixture, test_bucket, test_blob):
+    # Retry request because the created key may not be fully propagated for up
+    # to 15s.
+    @backoff.on_exception(backoff.constant, ClientError, interval=1, max_time=15)
+    def list_objects():
+        list_gcs_objects.list_gcs_objects(
+            google_access_key_id=hmac_fixture[0].access_id,
+            google_access_key_secret=hmac_fixture[1],
+            bucket_name=test_bucket.name,
+        )
+        out, _ = capsys.readouterr()
+        assert "Objects:" in out
+        assert test_blob.name in out
diff --git a/testing/test-env.tmpl.sh b/testing/test-env.tmpl.sh
@@ -50,11 +50,6 @@ export SLACK_TEST_SIGNATURE=
 export SLACK_SECRET=
 export FUNCTIONS_TOPIC=
 
-# HMAC SA Credentials for S3 SDK samples
-export GOOGLE_CLOUD_PROJECT_S3_SDK=
-export STORAGE_HMAC_ACCESS_KEY_ID=
-export STORAGE_HMAC_ACCESS_SECRET_KEY=
-
 # Service account for HMAC samples
 export HMAC_KEY_TEST_SERVICE_ACCOUNT=
 

Original file line number	Diff line number	Diff line change
`@@ -1 +1,3 @@`
	`1`	`+backoff==1.10.0`
`1`	`2`	`pytest==6.2.4`
	`3`	`+google-cloud-storage==1.38.0`