From ac4316819324c2be3fb43a084317900cf9e62a1e Mon Sep 17 00:00:00 2001 From: Miguel Trejo Date: Mon, 11 Apr 2022 22:24:38 -0500 Subject: [PATCH 1/2] feat: DynamoDBOnlineStoreCreator Signed-off-by: Miguel Trejo --- .../feature_repos/repo_configuration.py | 9 ++++++- .../universal/online_store/dynamodb.py | 26 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 sdk/python/tests/integration/feature_repos/universal/online_store/dynamodb.py diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 6557394cae6..f340ba68f87 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -18,6 +18,7 @@ from feast.data_source import DataSource from feast.errors import FeastModuleImportError from feast.repo_config import RegistryConfig, RepoConfig +from sdk.python.tests.integration.feature_repos.universal.online_store.dynamodb import DynamoDBOnlineStoreCreator from tests.integration.feature_repos.integration_test_repo_config import ( IntegrationTestRepoConfig, ) @@ -47,6 +48,9 @@ from tests.integration.feature_repos.universal.online_store.datastore import ( DatastoreOnlineStoreCreator, ) +from tests.integration.feature_repos.universal.online_store.dynamodb import ( + DynamoDBOnlineStoreCreator +) from tests.integration.feature_repos.universal.online_store.redis import ( RedisOnlineStoreCreator, ) @@ -130,7 +134,10 @@ if os.getenv("FEAST_LOCAL_ONLINE_CONTAINER", "False").lower() == "true": replacements = {"datastore": DatastoreOnlineStoreCreator} - replacement_dicts = [(REDIS_CONFIG, RedisOnlineStoreCreator)] + replacement_dicts = [ + (REDIS_CONFIG, RedisOnlineStoreCreator), + (DYNAMO_CONFIG, DynamoDBOnlineStoreCreator) + ] for c in FULL_REPO_CONFIGS: if isinstance(c.online_store, dict): for _replacement in replacement_dicts: diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/dynamodb.py b/sdk/python/tests/integration/feature_repos/universal/online_store/dynamodb.py new file mode 100644 index 00000000000..4dce9e5e2a1 --- /dev/null +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/dynamodb.py @@ -0,0 +1,26 @@ +from typing import Dict + +from testcontainers.core.container import DockerContainer +from testcontainers.core.waiting_utils import wait_for_logs + +from tests.integration.feature_repos.universal.online_store_creator import ( + OnlineStoreCreator, +) + +class DynamoDBOnlineStoreCreator(OnlineStoreCreator): + def __init__(self, project_name: str): + super().__init__(project_name) + self.container = DockerContainer("amazon/dynamodb-local").with_exposed_ports("8000") + + def create_online_store(self) -> Dict[str, str]: + self.container.start() + log_string_to_wait_for = "Initializing DynamoDB Local with the following configuration:" + wait_for_logs( + container=self.container, predicate=log_string_to_wait_for, timeout=5 + ) + exposed_port = self.container.get_exposed_port("8000") + return {"type": "dynamodb", "connection_string": f"localhost:{exposed_port}"} + + def teardown(self): + self.container.stop() + From 41a0cfce09ac26e8e548a0be5787e5486f406e51 Mon Sep 17 00:00:00 2001 From: Miguel Trejo Date: Tue, 12 Apr 2022 21:40:06 -0500 Subject: [PATCH 2/2] fix: DynamoDBOnlineStoreCreator return endpoint_url and aws region Signed-off-by: Miguel Trejo --- CONTRIBUTING.md | 1 + .../feature_repos/repo_configuration.py | 5 ++--- .../universal/online_store/dynamodb.py | 16 ++++++++++++---- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2a4cb74634c..1def64cf186 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -171,6 +171,7 @@ These tests create new temporary tables / datasets locally only, and they are cl The services with containerized replacements currently implemented are: - Datastore +- DynamoDB - Redis You can run `make test-python-integration-container` to run tests against the containerized versions of dependencies. diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index f340ba68f87..bb63131bd8c 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -18,7 +18,6 @@ from feast.data_source import DataSource from feast.errors import FeastModuleImportError from feast.repo_config import RegistryConfig, RepoConfig -from sdk.python.tests.integration.feature_repos.universal.online_store.dynamodb import DynamoDBOnlineStoreCreator from tests.integration.feature_repos.integration_test_repo_config import ( IntegrationTestRepoConfig, ) @@ -49,7 +48,7 @@ DatastoreOnlineStoreCreator, ) from tests.integration.feature_repos.universal.online_store.dynamodb import ( - DynamoDBOnlineStoreCreator + DynamoDBOnlineStoreCreator, ) from tests.integration.feature_repos.universal.online_store.redis import ( RedisOnlineStoreCreator, @@ -136,7 +135,7 @@ replacements = {"datastore": DatastoreOnlineStoreCreator} replacement_dicts = [ (REDIS_CONFIG, RedisOnlineStoreCreator), - (DYNAMO_CONFIG, DynamoDBOnlineStoreCreator) + (DYNAMO_CONFIG, DynamoDBOnlineStoreCreator), ] for c in FULL_REPO_CONFIGS: if isinstance(c.online_store, dict): diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/dynamodb.py b/sdk/python/tests/integration/feature_repos/universal/online_store/dynamodb.py index 4dce9e5e2a1..e4d8e0c3d02 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/dynamodb.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/dynamodb.py @@ -7,20 +7,28 @@ OnlineStoreCreator, ) + class DynamoDBOnlineStoreCreator(OnlineStoreCreator): def __init__(self, project_name: str): super().__init__(project_name) - self.container = DockerContainer("amazon/dynamodb-local").with_exposed_ports("8000") + self.container = DockerContainer( + "amazon/dynamodb-local:latest" + ).with_exposed_ports("8000") def create_online_store(self) -> Dict[str, str]: self.container.start() - log_string_to_wait_for = "Initializing DynamoDB Local with the following configuration:" + log_string_to_wait_for = ( + "Initializing DynamoDB Local with the following configuration:" + ) wait_for_logs( container=self.container, predicate=log_string_to_wait_for, timeout=5 ) exposed_port = self.container.get_exposed_port("8000") - return {"type": "dynamodb", "connection_string": f"localhost:{exposed_port}"} + return { + "type": "dynamodb", + "endpoint_url": f"http://localhost:{exposed_port}", + "region": "us-west-2", + } def teardown(self): self.container.stop() -