feat: Add local tests for s3 registry using minio (#4029)

tokoko · web-flow · commit d82d1ecb534a · 2024-03-30T12:14:10.000-07:00
diff --git a/Makefile b/Makefile
@@ -67,7 +67,7 @@ test-python-unit:
 	python -m pytest -n 8 --color=yes sdk/python/tests
 
 test-python-integration:
-	python -m pytest -n 8 --integration --color=yes --durations=5 --timeout=1200 --timeout_method=thread sdk/python/tests
+	python -m pytest -n 8 --integration -k "not minio_registry" --color=yes --durations=5 --timeout=1200 --timeout_method=thread sdk/python/tests
 
 test-python-integration-local:
 	@(docker info > /dev/null 2>&1 && \
diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py
@@ -14,9 +14,11 @@
 import os
 import time
 from datetime import timedelta
+from unittest import mock
 
 import pytest
 from pytest_lazyfixture import lazy_fixture
+from testcontainers.core.container import DockerContainer
 
 from feast import FileSource
 from feast.data_format import ParquetFormat
@@ -60,12 +62,56 @@ def s3_registry() -> Registry:
     return Registry("project", registry_config, None)
 
 
+@pytest.fixture
+def minio_registry() -> Registry:
+    minio_user = "minio99"
+    minio_password = "minio123"
+    bucket_name = "test-bucket"
+
+    container: DockerContainer = (
+        DockerContainer("quay.io/minio/minio")
+        .with_exposed_ports(9000, 9001)
+        .with_env("MINIO_ROOT_USER", minio_user)
+        .with_env("MINIO_ROOT_PASSWORD", minio_password)
+        .with_command('server /data --console-address ":9001"')
+        .with_exposed_ports()
+    )
+
+    container.start()
+
+    exposed_port = container.get_exposed_port("9000")
+    container_host = container.get_container_host_ip()
+
+    container.exec(f"mkdir /data/{bucket_name}")
+
+    registry_config = RegistryConfig(
+        path=f"s3://{bucket_name}/registry.db", cache_ttl_seconds=600
+    )
+
+    mock_environ = {
+        "FEAST_S3_ENDPOINT_URL": f"http://{container_host}:{exposed_port}",
+        "AWS_ACCESS_KEY_ID": minio_user,
+        "AWS_SECRET_ACCESS_KEY": minio_password,
+        "AWS_SESSION_TOKEN": "",
+    }
+
+    with mock.patch.dict(os.environ, mock_environ):
+        yield Registry("project", registry_config, None)
+
+    container.stop()
+
+
 @pytest.mark.integration
 @pytest.mark.parametrize(
     "test_registry",
-    [lazy_fixture("gcs_registry"), lazy_fixture("s3_registry")],
+    [
+        lazy_fixture("gcs_registry"),
+        lazy_fixture("s3_registry"),
+        lazy_fixture("minio_registry"),
+    ],
 )
 def test_apply_entity_integration(test_registry):
+
     entity = Entity(
         name="driver_car_id",
         description="Car driver id",
@@ -106,7 +152,11 @@ def test_apply_entity_integration(test_registry):
 @pytest.mark.integration
 @pytest.mark.parametrize(
     "test_registry",
-    [lazy_fixture("gcs_registry"), lazy_fixture("s3_registry")],
+    [
+        lazy_fixture("gcs_registry"),
+        lazy_fixture("s3_registry"),
+        lazy_fixture("minio_registry"),
+    ],
 )
 def test_apply_feature_view_integration(test_registry):
     # Create Feature Views
@@ -183,7 +233,11 @@ def test_apply_feature_view_integration(test_registry):
 @pytest.mark.integration
 @pytest.mark.parametrize(
     "test_registry",
-    [lazy_fixture("gcs_registry"), lazy_fixture("s3_registry")],
+    [
+        lazy_fixture("gcs_registry"),
+        lazy_fixture("s3_registry"),
+        lazy_fixture("minio_registry"),
+    ],
 )
 def test_apply_data_source_integration(test_registry: Registry):
     validate_registry_data_source_apply(test_registry)
