fix for handling missing feature data (#1163)

anirudhagar13 · Anirudh Agarwal · web-flow · commit 7d91781871c5 · 2023-05-09T16:35:24.000-07:00
Co-authored-by: Anirudh Agarwal &lt;aniagarwal@linkedin.biz&gt;
diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/AclCheckUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/AclCheckUtils.scala
@@ -68,12 +68,12 @@ private[offline] object AclCheckUtils {
     } yield featureAnchorWithSource.source.path
 
     val shouldSkipFeature = FeathrUtils.getFeathrJobParam(ss.sparkContext.getConf, FeathrUtils.SKIP_MISSING_FEATURE).toBoolean
-    val shouldAddDefaultCol = FeathrUtils.getFeathrJobParam(ss.sparkContext.getConf, FeathrUtils.SKIP_MISSING_FEATURE).toBoolean
+    val shouldAddDefaultCol = FeathrUtils.getFeathrJobParam(ss.sparkContext.getConf, FeathrUtils.ADD_DEFAULT_COL_FOR_MISSING_DATA).toBoolean
     val invalidPaths = AclCheckUtils.checkReadAuthorization(conf, allRequiredPaths.distinct)
     if (invalidPaths.isEmpty) {
       (Success(()), invalidPaths.map(_._2))
     } else {
-      if (!shouldSkipFeature || !shouldAddDefaultCol) {
+      if (!shouldSkipFeature && !shouldAddDefaultCol) {
         (Failure(
           new RuntimeException(
             "Can not verify read authorization on the following paths. This can be due to" +
diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/SlidingWindowAggIntegTest.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/SlidingWindowAggIntegTest.scala
@@ -820,6 +820,7 @@ class SlidingWindowAggIntegTest extends FeathrIntegTest {
     setFeathrJobParam(SKIP_MISSING_FEATURE, "false")
   }
 
+  @Test
   def testSWAWithMissingFeatureDataFlag(): Unit = {
     setFeathrJobParam(FeathrUtils.ADD_DEFAULT_COL_FOR_MISSING_DATA, "true")
     val joinConfigAsString =
diff --git a/gradle.properties b/gradle.properties
@@ -1,3 +1,3 @@
-version=1.0.3-rc3
+version=1.0.4-rc1
 SONATYPE_AUTOMATIC_RELEASE=true
 POM_ARTIFACT_ID=feathr_2.12

Original file line number	Diff line number	Diff line change
`@@ -820,6 +820,7 @@ class SlidingWindowAggIntegTest extends FeathrIntegTest {`
`820`	`820`	`setFeathrJobParam(SKIP_MISSING_FEATURE, "false")`
`821`	`821`	`}`
`822`	`822`
	`823`	`+ @Test`
`823`	`824`	`def testSWAWithMissingFeatureDataFlag(): Unit = {`
`824`	`825`	`setFeathrJobParam(FeathrUtils.ADD_DEFAULT_COL_FOR_MISSING_DATA, "true")`
`825`	`826`	`val joinConfigAsString =`