test: Adding more tests to DQM to display behavior (#3295)

franciscojavierarceo · web-flow · commit af4bac377b91 · 2022-10-18T00:35:06.000-07:00
* adding additional unit test to show profilers

Signed-off-by: Francisco Javier Arceo &lt;franciscojavierarceo@users.noreply.github.com&gt;

* linted

Signed-off-by: Francisco Javier Arceo &lt;franciscojavierarceo@users.noreply.github.com&gt;

* updated test case

Signed-off-by: Francisco Javier Arceo &lt;franciscojavierarceo@users.noreply.github.com&gt;

Signed-off-by: Francisco Javier Arceo &lt;franciscojavierarceo@users.noreply.github.com&gt;
diff --git a/sdk/python/tests/integration/e2e/test_validation.py b/sdk/python/tests/integration/e2e/test_validation.py
@@ -104,6 +104,13 @@ def test_historical_retrieval_fails_on_validation(environment, universal_data_so
         features=_features,
     )
 
+    ds = store.get_saved_dataset("my_other_dataset")
+    profiler_expectation_suite = ds.get_profile(
+        profiler=profiler_with_unrealistic_expectations
+    )
+
+    assert len(profiler_expectation_suite.expectation_suite["expectations"]) == 3
+
     with pytest.raises(ValidationFailed) as exc_info:
         job.to_df(
             validation_reference=store.get_saved_dataset(
@@ -360,6 +367,7 @@ def profiler_with_feature_metadata(dataset: PandasDataset) -> ExpectationSuite:
 
 @ge_profiler
 def profiler_with_unrealistic_expectations(dataset: PandasDataset) -> ExpectationSuite:
+    # note: there are 4 expectations here and only 3 are returned from the profiler
     # need to create dataframe with corrupted data first
     df = pd.DataFrame()
     df["current_balance"] = [-100]
@@ -371,5 +379,7 @@ def profiler_with_unrealistic_expectations(dataset: PandasDataset) -> Expectatio
 
     # this should pass
     other_ds.expect_column_min_to_be_between("avg_passenger_count", 0, 1000)
+    # this should fail
+    other_ds.expect_column_to_exist("missing random column")
 
     return other_ds.get_expectation_suite()
