Cleanup e2e and docker-compose tests (feast-dev#1035)

terryyylim · web-flow · commit 5dd68a864c68 · 2020-10-08T15:12:08.000+08:00
Signed-off-by: Terence &lt;terencelimxp@gmail.com&gt;
diff --git a/core/src/main/java/feast/core/validators/Matchers.java b/core/src/main/java/feast/core/validators/Matchers.java
@@ -23,7 +23,7 @@
 public class Matchers {
 
   private static Pattern BIGQUERY_TABLE_REF_REGEX =
-      Pattern.compile("[a-zA-Z0-9-]+[:]+[a-zA-Z0-9]+[.]+[a-zA-Z0-9_]*");
+      Pattern.compile("[a-zA-Z0-9-]+[:]+[a-zA-Z0-9_]+[.]+[a-zA-Z0-9_]*");
   private static Pattern UPPER_SNAKE_CASE_REGEX = Pattern.compile("^[A-Z0-9]+(_[A-Z0-9]+)*$");
   private static Pattern LOWER_SNAKE_CASE_REGEX = Pattern.compile("^[a-z0-9]+(_[a-z0-9]+)*$");
   private static Pattern VALID_CHARACTERS_REGEX = Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_]*$");
diff --git a/infra/scripts/test-docker-compose.sh b/infra/scripts/test-docker-compose.sh
@@ -63,4 +63,4 @@ export FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .N
 ${PROJECT_ROOT_DIR}/infra/scripts/wait-for-it.sh ${FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS}:6566 --timeout=120
 
 # Run e2e tests for Redis
-docker exec feast_jupyter_1 bash -c 'cd /feast/tests/e2e && pytest *.py --core_url core:6565 --serving_url=online_serving:6566 --kafka_brokers=kafka:9092'
+docker exec feast_jupyter_1 bash -c 'cd /feast/tests/e2e && pytest *.py -m "not bq" --core_url core:6565 --serving_url=online_serving:6566 --kafka_brokers=kafka:9092'
diff --git a/sdk/python/feast/feature.py b/sdk/python/feast/feature.py
@@ -46,6 +46,9 @@ def __eq__(self, other):
             return False
         return True
 
+    def __lt__(self, other):
+        return self.name < other.name
+
     @property
     def name(self):
         """
diff --git a/sdk/python/feast/feature_table.py b/sdk/python/feast/feature_table.py
@@ -80,9 +80,9 @@ def __eq__(self, other):
         ):
             return False
 
-        if self.entities != other.entities:
+        if sorted(self.entities) != sorted(other.entities):
             return False
-        if self.features != other.features:
+        if sorted(self.features) != sorted(other.features):
             return False
         if self.batch_source != other.batch_source:
             return False
diff --git a/tests/e2e/test-register.py b/tests/e2e/test-register.py
@@ -1,5 +1,4 @@
 import os
-import time
 import uuid
 from datetime import datetime
 
@@ -16,6 +15,7 @@
 from feast.feature import Feature
 from feast.feature_table import FeatureTable
 from feast.value_type import ValueType
+from feast.wait import wait_retry_backoff
 
 DIR_PATH = os.path.dirname(os.path.realpath(__file__))
 PROJECT_NAME = "basic_" + uuid.uuid4().hex.upper()[0:6]
@@ -236,6 +236,7 @@ def test_get_list_alltypes(
     assert actual_list_feature_table == alltypes_featuretable
 
 
+@pytest.mark.bq
 def test_ingest(
     client: Client,
     customer_entity: Entity,
@@ -255,12 +256,31 @@ def test_ingest(
     client.apply_feature_table(bq_featuretable)
     client.ingest(bq_featuretable, bq_dataset, timeout=120)
 
-    # Give time to allow data to propagate to BQ table
-    time.sleep(15)
-
+    from google.api_core.exceptions import NotFound
     from google.cloud import bigquery
 
     bq_client = bigquery.Client(project=gcp_project)
+
+    # Poll BQ for table until the table has been created
+    def try_get_table():
+        table_exist = False
+        table_resp = None
+        try:
+            table_resp = bq_client.get_table(bq_table_id)
+
+            if table_resp and table_resp.table_id == bq_table_id.split(".")[-1]:
+                table_exist = True
+        except NotFound:
+            pass
+
+        return table_resp, table_exist
+
+    wait_retry_backoff(
+        retry_fn=try_get_table,
+        timeout_secs=30,
+        timeout_msg="Timed out trying to get bigquery table",
+    )
+
     query_string = f"SELECT * FROM `{bq_table_id}`"
 
     job = bq_client.query(query_string)
