Add redis online provider - integration tests

Signed-off-by: qooba <dev@qooba.net>
feast-dev · feast-ci-bot · Jun 9, 2021 · Apr 25, 2021 · Apr 25, 2021 · Apr 26, 2021
commit e434e59a848b450875e2cda8b3198961968f36aa
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -16,6 +16,16 @@ jobs:
     env:
       OS: ${{ matrix.os }}
       PYTHON: ${{ matrix.python-version }}
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
       - uses: actions/checkout@v2
       - name: Setup Python
@@ -43,6 +53,9 @@ jobs:
         run: make install-python-ci-dependencies
       - name: Test python
         run: FEAST_TELEMETRY=False pytest --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration
+        env:
+          REDIS_TYPE: REDIS
+          REDIS_CONNECTION_STRING: localhost:6379,db=0
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1
         with:
@@ -51,4 +64,4 @@ jobs:
           flags: integrationtests
           env_vars: OS,PYTHON
           fail_ci_if_error: true
-          verbose: true
+          verbose: true
@@ -0,0 +1,55 @@
+import random
+import string
+import tempfile
+from pathlib import Path
+from textwrap import dedent
+
+import pytest
+
+from feast.feature_store import FeatureStore
+from tests.cli_utils import CliRunner
+from tests.online_read_write_test import basic_rw_test
+
+
+@pytest.mark.integration
+def test_basic() -> None:
+    project_id = "".join(
+        random.choice(string.ascii_lowercase + string.digits) for _ in range(10)
+    )
+    runner = CliRunner()
+    with tempfile.TemporaryDirectory() as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name:
+
+        repo_path = Path(repo_dir_name)
+        data_path = Path(data_dir_name)
+
+        repo_config = repo_path / "feature_store.yaml"
+
+        repo_config.write_text(
+            dedent(
+                f"""
+        project: {project_id}
+        registry: {data_path / "registry.db"}
+        provider: redis
+        """
+            )
+        )
+
+        repo_example = repo_path / "example.py"
+        repo_example.write_text(
+            (Path(__file__).parent / "example_feature_repo_1.py").read_text()
+        )
+
+        result = runner.run(["apply"], cwd=repo_path)
+        assert result.returncode == 0
+
+        # Doing another apply should be a no op, and should not cause errors
+        result = runner.run(["apply"], cwd=repo_path)
+        assert result.returncode == 0
+
+        basic_rw_test(
+            FeatureStore(repo_path=str(repo_path), config=None),
+            view_name="driver_locations",
+        )
+
+        result = runner.run(["teardown"], cwd=repo_path)
+        assert result.returncode == 0
@@ -146,6 +146,39 @@ def prep_local_fs_and_fv() -> Iterator[Tuple[FeatureStore, FeatureView]]:
             yield fs, fv
 
 
+@contextlib.contextmanager
+def prep_redis_fs_and_fv() -> Iterator[Tuple[FeatureStore, FeatureView]]:
+    with tempfile.NamedTemporaryFile(suffix=".parquet") as f:
+        df = create_dataset()
+        f.close()
+        df.to_parquet(f.name)
+        file_source = FileSource(
+            file_format=ParquetFormat(),
+            file_url=f"file://{f.name}",
+            event_timestamp_column="ts",
+            created_timestamp_column="created_ts",
+            date_partition_column="",
+            field_mapping={"ts_1": "ts", "id": "driver_id"},
+        )
+        fv = get_feature_view(file_source)
+        e = Entity(
+            name="driver",
+            description="id for driver",
+            join_key="driver_id",
+            value_type=ValueType.INT32,
+        )
+        with tempfile.TemporaryDirectory() as repo_dir_name, tempfile.TemporaryDirectory():
+            config = RepoConfig(
+                registry=str(Path(repo_dir_name) / "registry.db"),
+                project=f"test_bq_correctness_{str(uuid.uuid4()).replace('-', '')}",
+                provider="redis",
+            )
+            fs = FeatureStore(config=config)
+            fs.apply([fv, e])
+
+            yield fs, fv
+
+
 # Checks that both offline & online store values are as expected
 def check_offline_and_online_features(
     fs: FeatureStore,
@@ -221,6 +254,12 @@ def test_bq_offline_online_store_consistency(bq_source_type: str):
         run_offline_online_store_consistency_test(fs, fv)
 
 
+@pytest.mark.integration
+def test_redis_offline_online_store_consistency():
+    with prep_redis_fs_and_fv() as (fs, fv):
+        run_offline_online_store_consistency_test(fs, fv)
+
+
 def test_local_offline_online_store_consistency():
     with prep_local_fs_and_fv() as (fs, fv):
         run_offline_online_store_consistency_test(fs, fv)