diff --git a/build.gradle b/build.gradle index 19b9d62a7..1db1198c5 100644 --- a/build.gradle +++ b/build.gradle @@ -125,7 +125,7 @@ project.ext.spec = [ ], 'scala' : [ 'scala_library' : "org.scala-lang:scala-library:$ver.scala", - 'scalatest' : "org.scalatest:scalatest_$ver.scala_rt:3.0.0", + 'scalatest' : "org.scalatest:scalatest_$ver.scala_rt:3.2.9", ], 'avro' : "org.apache.avro:avro:1.10.2", "avroUtil": "com.linkedin.avroutil1:helper-all:0.2.100", @@ -153,7 +153,7 @@ project.ext.spec = [ "antlr": "org.antlr:antlr4:4.8", "antlrRuntime": "org.antlr:antlr4-runtime:4.8", "jsqlparser": "com.github.jsqlparser:jsqlparser:3.1", - + "scalaTestPlus": "org.scalatestplus:mockito-3-4_2.12:3.3.0.0-SNAP3", ] ] diff --git a/feathr-impl/build.gradle b/feathr-impl/build.gradle index 055fcd4c0..bd7db600d 100644 --- a/feathr-impl/build.gradle +++ b/feathr-impl/build.gradle @@ -81,6 +81,7 @@ dependencies { testImplementation spec.product.scala.scalatest testImplementation spec.product.testing testImplementation spec.product.jdiagnostics + testImplementation spec.product.scalaTestPlus } // Since there are cross-calls from Scala to Java, we use joint compiler diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/BatchDataLoader.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/BatchDataLoader.scala index e3714217b..b6bd7c1ee 100644 --- a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/BatchDataLoader.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/source/dataloader/BatchDataLoader.scala @@ -4,8 +4,6 @@ import com.linkedin.feathr.common.exception.{ErrorLabel, FeathrInputDataExceptio import com.linkedin.feathr.offline.config.location.DataLocation import com.linkedin.feathr.offline.generation.SparkIOUtils import com.linkedin.feathr.offline.job.DataSourceUtils.getSchemaFromAvroDataFile -import com.linkedin.feathr.offline.source.dataloader.DataLoaderHandler -import com.linkedin.feathr.offline.util.DelimiterUtils.checkDelimiterOption import org.apache.avro.Schema import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.hadoop.mapred.JobConf @@ -16,7 +14,9 @@ import org.apache.spark.sql.{DataFrame, SparkSession} * @param ss the spark session * @param path input data path */ -private[offline] class BatchDataLoader(ss: SparkSession, location: DataLocation, dataLoaderHandlers: List[DataLoaderHandler]) extends DataLoader { +private[offline] class BatchDataLoader(val ss: SparkSession, + val location: DataLocation, + val dataLoaderHandlers: List[DataLoaderHandler]) extends DataLoader { /** * get the schema of the source. It's only used in the deprecated DataSource.getDataSetAndSchema @@ -48,16 +48,16 @@ private[offline] class BatchDataLoader(ss: SparkSession, location: DataLocation, * @return an dataframe */ override def loadDataFrame(): DataFrame = { - loadDataFrame(Map(), new JobConf(ss.sparkContext.hadoopConfiguration)) + loadDataFrameWithRetry(Map(), new JobConf(ss.sparkContext.hadoopConfiguration), MAX_DATA_LOAD_RETRY) } - /** * load the source data as dataframe. * @param dataIOParameters extra parameters * @param jobConf Hadoop JobConf to be passed + * @param retry number of times to retry when data loading fails * @return an dataframe */ - def loadDataFrame(dataIOParameters: Map[String, String], jobConf: JobConf): DataFrame = { + def loadDataFrameWithRetry(dataIOParameters: Map[String, String], jobConf: JobConf, retry: Int): DataFrame = { val sparkConf = ss.sparkContext.getConf val inputSplitSize = sparkConf.get("spark.feathr.input.split.size", "") val dataIOParametersWithSplitSize = Map(SparkIOUtils.SPLIT_SIZE -> inputSplitSize) ++ dataIOParameters @@ -65,9 +65,6 @@ private[offline] class BatchDataLoader(ss: SparkSession, location: DataLocation, log.info(s"Loading ${location} as DataFrame, using parameters ${dataIOParametersWithSplitSize}") - // Get csvDelimiterOption set with spark.feathr.inputFormat.csvOptions.sep and check if it is set properly (Only for CSV and TSV) - val csvDelimiterOption = checkDelimiterOption(ss.sqlContext.getConf("spark.feathr.inputFormat.csvOptions.sep", ",")) - try { import scala.util.control.Breaks._ @@ -87,12 +84,20 @@ private[offline] class BatchDataLoader(ss: SparkSession, location: DataLocation, } df } catch { - case feathrException: FeathrInputDataException => - println(feathrException.toString) - throw feathrException // Throwing exception to avoid dataLoaderHandler hook exception from being swallowed. - case e: Throwable => //TODO: Analyze all thrown exceptions, instead of swalling them all, and reading as a csv - println(e.toString) - ss.read.format("csv").option("header", "true").option("delimiter", csvDelimiterOption).load(dataPath) + case _: Throwable => + // If data loading from source failed, retry it automatically, as it might due to data source still being written into. + log.info(s"Loading ${location} failed, retrying for ${retry}-th time..") + if (retry > 0) { + Thread.sleep(DATA_LOAD_WAIT_IN_MS) + loadDataFrameWithRetry(dataIOParameters, jobConf, retry - 1) + } else { + // Throwing exception to avoid dataLoaderHandler hook exception from being swallowed. + throw new FeathrInputDataException(ErrorLabel.FEATHR_USER_ERROR, s"Failed to load ${dataPath} after ${MAX_DATA_LOAD_RETRY} retries.") + } } } + // Retry 2 times if data source loading fails + val MAX_DATA_LOAD_RETRY = 2 + // Wait for 10 minutes and retry if data source loading fails + val DATA_LOAD_WAIT_IN_MS = 10*60*1000 } \ No newline at end of file diff --git a/feathr-impl/src/test/java/com/linkedin/feathr/common/AutoTensorizableTypesTest.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/AutoTensorizableTypesTest.java index 70c33872c..3da83c34c 100644 --- a/feathr-impl/src/test/java/com/linkedin/feathr/common/AutoTensorizableTypesTest.java +++ b/feathr-impl/src/test/java/com/linkedin/feathr/common/AutoTensorizableTypesTest.java @@ -6,7 +6,6 @@ import com.linkedin.feathr.common.types.PrimitiveType; import java.util.Optional; -import org.scalatest.testng.TestNGSuite; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -14,7 +13,7 @@ import static org.testng.Assert.*; -public class AutoTensorizableTypesTest extends TestNGSuite { +public class AutoTensorizableTypesTest { private static final TensorType NTV_EQUIVALENT_TENSOR_TYPE = new TensorType(PrimitiveType.FLOAT, singletonList( PrimitiveDimensionType.STRING)); diff --git a/feathr-impl/src/test/java/com/linkedin/feathr/common/FeatureTypeConfigTest.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/FeatureTypeConfigTest.java index 0c5eb303c..f05faeb1f 100644 --- a/feathr-impl/src/test/java/com/linkedin/feathr/common/FeatureTypeConfigTest.java +++ b/feathr-impl/src/test/java/com/linkedin/feathr/common/FeatureTypeConfigTest.java @@ -1,7 +1,6 @@ package com.linkedin.feathr.common; import com.linkedin.feathr.common.tensor.TensorType; -import org.scalatest.testng.TestNGSuite; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -11,7 +10,7 @@ * Unit tests for {@link FeatureTypeConfig} * */ -public class FeatureTypeConfigTest extends TestNGSuite { +public class FeatureTypeConfigTest { @DataProvider public Object[][] FeatureTypeConstructorWithAutoTZTestCases() { diff --git a/feathr-impl/src/test/java/com/linkedin/feathr/common/TestFeatureDependencyGraph.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/TestFeatureDependencyGraph.java index d6214e443..37c456a27 100644 --- a/feathr-impl/src/test/java/com/linkedin/feathr/common/TestFeatureDependencyGraph.java +++ b/feathr-impl/src/test/java/com/linkedin/feathr/common/TestFeatureDependencyGraph.java @@ -12,7 +12,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import org.scalatest.testng.TestNGSuite; import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; @@ -21,7 +20,7 @@ import static org.testng.Assert.*; -public class TestFeatureDependencyGraph extends TestNGSuite { +public class TestFeatureDependencyGraph { /* Description of test scenario: diff --git a/feathr-impl/src/test/java/com/linkedin/feathr/common/TestFeatureValue.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/TestFeatureValue.java index b5b8ea0c3..a5e13071e 100644 --- a/feathr-impl/src/test/java/com/linkedin/feathr/common/TestFeatureValue.java +++ b/feathr-impl/src/test/java/com/linkedin/feathr/common/TestFeatureValue.java @@ -19,7 +19,6 @@ import nl.jqno.equalsverifier.EqualsVerifier; import nl.jqno.equalsverifier.Warning; -import org.scalatest.testng.TestNGSuite; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -27,7 +26,7 @@ import static java.util.Collections.*; -public class TestFeatureValue extends TestNGSuite { +public class TestFeatureValue { private static final float DEFAULT_VALUE = FeatureValue.DEFAULT_VALUE; private static final String EMPTY_TERM = FeatureValue.EMPTY_TERM; diff --git a/feathr-impl/src/test/java/com/linkedin/feathr/common/types/TestFeatureTypes.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/types/TestFeatureTypes.java index 8407d4c86..f7be28be0 100644 --- a/feathr-impl/src/test/java/com/linkedin/feathr/common/types/TestFeatureTypes.java +++ b/feathr-impl/src/test/java/com/linkedin/feathr/common/types/TestFeatureTypes.java @@ -3,13 +3,12 @@ import java.util.Collections; import com.linkedin.feathr.common.tensor.TensorType; -import org.scalatest.testng.TestNGSuite; import org.testng.annotations.Test; import static org.testng.Assert.*; -public class TestFeatureTypes extends TestNGSuite { +public class TestFeatureTypes { @Test public void checkBasicTypes() { assertEquals(BooleanFeatureType.INSTANCE.getBasicType(), FeatureType.BasicType.BOOLEAN); diff --git a/feathr-impl/src/test/java/com/linkedin/feathr/common/types/TestQuinceFeatureTypeMapper.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/types/TestQuinceFeatureTypeMapper.java index c87aeaa56..f802a6bbb 100644 --- a/feathr-impl/src/test/java/com/linkedin/feathr/common/types/TestQuinceFeatureTypeMapper.java +++ b/feathr-impl/src/test/java/com/linkedin/feathr/common/types/TestQuinceFeatureTypeMapper.java @@ -6,7 +6,6 @@ import com.linkedin.feathr.common.tensor.TensorCategory; import com.linkedin.feathr.common.tensor.TensorType; import com.linkedin.feathr.common.value.QuinceFeatureTypeMapper; -import org.scalatest.testng.TestNGSuite; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -14,7 +13,7 @@ import static org.testng.Assert.*; -public class TestQuinceFeatureTypeMapper extends TestNGSuite { +public class TestQuinceFeatureTypeMapper { @DataProvider public Object[][] quinceTypeMapperCases() { return new Object[][] { diff --git a/feathr-impl/src/test/java/com/linkedin/feathr/common/util/MvelUDFExpressionTests.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/util/MvelUDFExpressionTests.java index 5b56de6f8..8ef18f292 100644 --- a/feathr-impl/src/test/java/com/linkedin/feathr/common/util/MvelUDFExpressionTests.java +++ b/feathr-impl/src/test/java/com/linkedin/feathr/common/util/MvelUDFExpressionTests.java @@ -7,7 +7,6 @@ import org.mvel2.MVEL; import org.mvel2.ParserConfiguration; import org.mvel2.ParserContext; -import org.scalatest.testng.TestNGSuite; import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; @@ -17,7 +16,7 @@ /** * Unit tests for {@link MvelContextUDFs} using expressions */ -public class MvelUDFExpressionTests extends TestNGSuite { +public class MvelUDFExpressionTests { private static final ParserConfiguration PARSER_CONFIG = new ParserConfiguration(); private ParserContext parserContext; diff --git a/feathr-impl/src/test/java/com/linkedin/feathr/common/util/TestMvelContextUDFs.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/util/TestMvelContextUDFs.java index dcdf4674c..d7f2017b8 100644 --- a/feathr-impl/src/test/java/com/linkedin/feathr/common/util/TestMvelContextUDFs.java +++ b/feathr-impl/src/test/java/com/linkedin/feathr/common/util/TestMvelContextUDFs.java @@ -11,7 +11,6 @@ import java.util.List; import java.util.Map; -import org.scalatest.testng.TestNGSuite; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -23,7 +22,7 @@ /** * Unit tests for {@link MvelContextUDFs} */ -public class TestMvelContextUDFs extends TestNGSuite { +public class TestMvelContextUDFs { @Test public void testGetDataType() { Assert.assertEquals(get_data_type("A"), "java.lang.String"); diff --git a/feathr-impl/src/test/java/com/linkedin/feathr/common/value/TestFeatureValueOldAPICompatibility.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/value/TestFeatureValueOldAPICompatibility.java index ed7eb0651..0752886fc 100644 --- a/feathr-impl/src/test/java/com/linkedin/feathr/common/value/TestFeatureValueOldAPICompatibility.java +++ b/feathr-impl/src/test/java/com/linkedin/feathr/common/value/TestFeatureValueOldAPICompatibility.java @@ -14,14 +14,13 @@ import java.util.Collections; import java.util.Map; -import org.scalatest.testng.TestNGSuite; import org.testng.annotations.Test; import static java.util.Collections.*; import static org.testng.Assert.*; -public class TestFeatureValueOldAPICompatibility extends TestNGSuite { +public class TestFeatureValueOldAPICompatibility { @Test public void basicEqualityChecks() { { diff --git a/feathr-impl/src/test/java/com/linkedin/feathr/common/value/TestFeatureValues.java b/feathr-impl/src/test/java/com/linkedin/feathr/common/value/TestFeatureValues.java index 991313207..13befd584 100644 --- a/feathr-impl/src/test/java/com/linkedin/feathr/common/value/TestFeatureValues.java +++ b/feathr-impl/src/test/java/com/linkedin/feathr/common/value/TestFeatureValues.java @@ -15,7 +15,6 @@ import java.util.Arrays; import java.util.Map; -import org.scalatest.testng.TestNGSuite; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -23,7 +22,7 @@ import static org.testng.Assert.*; -public class TestFeatureValues extends TestNGSuite { +public class TestFeatureValues { private static final Representable[] TENSOR_STRING_FLOAT_TYPE = new Representable[]{Primitive.STRING, Primitive.FLOAT}; private static final Representable[] TENSOR_INT_FLOAT_TYPE = new Representable[]{Primitive.INT, Primitive.FLOAT}; diff --git a/feathr-impl/src/test/java/com/linkedin/feathr/offline/TestMvelContext.java b/feathr-impl/src/test/java/com/linkedin/feathr/offline/TestMvelContext.java index b7559710c..1bd54f32b 100644 --- a/feathr-impl/src/test/java/com/linkedin/feathr/offline/TestMvelContext.java +++ b/feathr-impl/src/test/java/com/linkedin/feathr/offline/TestMvelContext.java @@ -1,6 +1,5 @@ package com.linkedin.feathr.offline; -import org.scalatest.testng.TestNGSuite; import org.testng.annotations.Test; import java.util.HashMap; @@ -10,7 +9,7 @@ import static org.testng.Assert.assertEquals; -public class TestMvelContext extends TestNGSuite { +public class TestMvelContext { @Test public void testCosineSimilarity() { // Test basic cosine similarity calculation diff --git a/feathr-impl/src/test/java/com/linkedin/feathr/offline/TestMvelExpression.java b/feathr-impl/src/test/java/com/linkedin/feathr/offline/TestMvelExpression.java index e8bf97810..8752c4716 100644 --- a/feathr-impl/src/test/java/com/linkedin/feathr/offline/TestMvelExpression.java +++ b/feathr-impl/src/test/java/com/linkedin/feathr/offline/TestMvelExpression.java @@ -12,7 +12,6 @@ import org.mvel2.integration.VariableResolverFactory; import org.mvel2.integration.impl.MapVariableResolverFactory; import org.mvel2.optimizers.OptimizerFactory; -import org.scalatest.testng.TestNGSuite; import org.testng.Assert; import org.testng.annotations.Test; @@ -22,7 +21,7 @@ /** * Test MVEL expression evaluator */ -public class TestMvelExpression extends TestNGSuite { +public class TestMvelExpression { @Test(description = "test mvel expression foo.bar on a map field of GenericRecord, where foo is the map field " + "and bar is the target key in the map") public void testMVELExpressionOnMap() { diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/FeathrIntegTest.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/FeathrIntegTest.scala index 13bf5578e..6cedc5498 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/FeathrIntegTest.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/FeathrIntegTest.scala @@ -21,7 +21,7 @@ abstract class FeathrIntegTest extends TestFeathr { val generatedDataFolder = "src/integTest/generated" val mockDataFolder = generatedDataFolder + "/mockData" - val trainingData = "src/test/resources/obs/" + val trainingData = "src/test/resources/obs/obs.csv" val featureData = mockDataFolder + "/a_feature_data" /** diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/SlidingWindowAggIntegTest.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/SlidingWindowAggIntegTest.scala index 965052432..c686fe319 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/SlidingWindowAggIntegTest.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/SlidingWindowAggIntegTest.scala @@ -1000,7 +1000,6 @@ class SlidingWindowAggIntegTest extends FeathrIntegTest { } - /** @Test def testSWACountDistinct(): Unit = { val featureDefAsString = @@ -1080,5 +1079,5 @@ class SlidingWindowAggIntegTest extends FeathrIntegTest { val dfs = runLocalFeatureJoinForTest(featureJoinAsString, featureDefAsString, "featuresWithFilterObs.avro.json").data validateRows(dfs.select(keyField, features: _*).collect().sortBy(row => row.getAs[Int](keyField)), expectedRows) - }*/ + } } diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathr.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathr.scala index f052663e3..e9fb8840e 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathr.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathr.scala @@ -10,14 +10,13 @@ import com.linkedin.feathr.offline.util.FeathrTestUtils import org.apache.avro.generic.GenericRecord import org.apache.hadoop.conf.Configuration import org.apache.spark.sql.SparkSession -import org.scalatest.testng.TestNGSuite import org.testng.annotations.{AfterClass, BeforeClass} import scala.collection.convert.wrapAll._ import scala.reflect.ClassTag // abstract class for all feathr tests -abstract class TestFeathr extends TestNGSuite { +abstract class TestFeathr { protected var ss: SparkSession = _ protected var conf: Configuration = _ protected var feathr: FeathrClient = _ diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathrUtils.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathrUtils.scala index 837aa8078..3b2859e56 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathrUtils.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/TestFeathrUtils.scala @@ -1,11 +1,11 @@ package com.linkedin.feathr.offline import com.linkedin.feathr.offline.util.FeathrUtils -import org.scalatest.testng.TestNGSuite + import org.testng.Assert._ import org.testng.annotations.Test -class TestFeathrUtils extends TestNGSuite { +class TestFeathrUtils { private val TEST_FEATHR_VERSION = "1.2.3" /** diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/client/TestDataFrameColName.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/client/TestDataFrameColName.scala index 540cae74a..bf66747b3 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/client/TestDataFrameColName.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/client/TestDataFrameColName.scala @@ -6,7 +6,7 @@ import com.linkedin.feathr.offline.anchored.feature.{FeatureAnchor, FeatureAncho import org.apache.spark.sql.Row import org.apache.spark.sql.types.{DoubleType, StringType, StructField, StructType} import org.mockito.Mockito.when -import org.scalatest.mockito.MockitoSugar.mock +import org.scalatestplus.mockito.MockitoSugar.mock import org.testng.Assert.assertEquals import org.testng.annotations.Test diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestDataSourceLoader.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestDataSourceLoader.scala index 627f2af73..d44741ccb 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestDataSourceLoader.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestDataSourceLoader.scala @@ -7,12 +7,12 @@ import com.jasonclawson.jackson.dataformat.hocon.HoconFactory import com.linkedin.feathr.common.FeathrJacksonScalaModule import com.linkedin.feathr.offline.config.location.{Jdbc, LocationUtils, Snowflake} import com.linkedin.feathr.offline.source.{DataSource, SourceFormatType} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite import scala.collection.mutable -class TestDataSourceLoader extends FunSuite { +class TestDataSourceLoader extends AnyFunSuite { /// Base line test to ensure backward compatibility test("DataSourceLoader.deserialize BaseLine") { val configDoc = diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestFeatureGroupsGenerator.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestFeatureGroupsGenerator.scala index 9c3c7f13b..867274d39 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestFeatureGroupsGenerator.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestFeatureGroupsGenerator.scala @@ -1,13 +1,13 @@ package com.linkedin.feathr.offline.config import com.linkedin.feathr.common.exception.FeathrConfigException -import org.scalatest.testng.TestNGSuite + import org.testng.Assert import org.testng.annotations.Test import scala.io.Source -class TestFeatureGroupsGenerator extends TestNGSuite{ +class TestFeatureGroupsGenerator{ private val _feathrConfigLoader = FeathrConfigLoader() diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestFeatureJoinConfig.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestFeatureJoinConfig.scala index bb17fc44c..d10c830b5 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestFeatureJoinConfig.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/TestFeatureJoinConfig.scala @@ -2,14 +2,14 @@ package com.linkedin.feathr.offline.config import com.linkedin.feathr.common.{DateParam, JoiningFeatureParams} import com.linkedin.feathr.offline.anchored.WindowTimeUnit -import org.scalatest.testng.TestNGSuite + import org.testng.Assert import org.testng.annotations.Test /** * Test suite to test parsing of join configs */ -class TestFeatureJoinConfig extends TestNGSuite{ +class TestFeatureJoinConfig{ /** * Settings config with observationDataTimeSettings and joinTimeSettings. diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/location/TestDesLocation.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/location/TestDesLocation.scala index 7b1f6fed0..95432f636 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/location/TestDesLocation.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/config/location/TestDesLocation.scala @@ -1,21 +1,19 @@ package com.linkedin.feathr.offline.config.location -import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} import com.fasterxml.jackson.databind.module.SimpleModule +import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} import com.fasterxml.jackson.module.caseclass.mapper.CaseClassObjectMapper import com.jasonclawson.jackson.dataformat.hocon.HoconFactory import com.linkedin.feathr.common.FeathrJacksonScalaModule import com.linkedin.feathr.offline.config.DataSourceLoader import com.linkedin.feathr.offline.config.location.LocationUtils.envSubstitute -import com.linkedin.feathr.offline.config.location.{DataLocation, Jdbc, SimplePath} import com.linkedin.feathr.offline.generation.SparkIOUtils import com.linkedin.feathr.offline.source.DataSource -import org.apache.spark.{SparkConf, SparkContext} -import org.apache.spark.sql.SparkSession import org.apache.hadoop.mapred.JobConf -import org.scalatest.FunSuite +import org.apache.spark.sql.SparkSession +import org.scalatest.funsuite.AnyFunSuite -class TestDesLocation extends FunSuite { +class TestDesLocation extends AnyFunSuite { val jackson = (new ObjectMapper(new HoconFactory) with CaseClassObjectMapper) .registerModule(FeathrJacksonScalaModule) // DefaultScalaModule causes a fail on holdem .configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true) diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/TestSequentialJoinAsDerivation.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/TestSequentialJoinAsDerivation.scala index 33e4ac822..0d230b9a7 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/TestSequentialJoinAsDerivation.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/derived/TestSequentialJoinAsDerivation.scala @@ -18,7 +18,7 @@ import org.apache.spark.sql.functions.{when => _, _} import org.apache.spark.sql.types._ import org.apache.spark.sql.{AnalysisException, DataFrame, Row, SparkSession} import org.mockito.Mockito._ -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert._ import org.testng.annotations.{DataProvider, Test} import org.apache.spark.sql.internal.SQLConf diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestFeatureGenFeatureGrouper.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestFeatureGenFeatureGrouper.scala index 99bfaa746..e630b20f5 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestFeatureGenFeatureGrouper.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestFeatureGenFeatureGrouper.scala @@ -6,7 +6,7 @@ import com.linkedin.feathr.offline.{AssertFeatureUtils, FeatureDataFrame, JoinKe import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema import org.apache.spark.sql.functions.col import org.apache.spark.sql.types._ -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert._ import org.testng.annotations.Test diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestFeatureGenKeyTagAnalyzer.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestFeatureGenKeyTagAnalyzer.scala index 2ee8dfd4d..90a3ba675 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestFeatureGenKeyTagAnalyzer.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestFeatureGenKeyTagAnalyzer.scala @@ -8,7 +8,7 @@ import com.linkedin.feathr.offline.derived.DerivedFeature import com.linkedin.feathr.offline.job.FeatureGenSpec import com.linkedin.feathr.offline.logical.FeatureGroups import org.mockito.Mockito._ -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert._ import org.testng.annotations.Test diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestIncrementalAggSnapshotLoader.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestIncrementalAggSnapshotLoader.scala index 520b6173a..12ed1e615 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestIncrementalAggSnapshotLoader.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestIncrementalAggSnapshotLoader.scala @@ -8,7 +8,7 @@ import com.typesafe.config.Config import org.apache.hadoop.fs.FileSystem import org.apache.spark.sql.DataFrame import org.mockito.Mockito.{verify, when} -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert.{assertEquals, assertTrue} import org.testng.annotations.Test diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestPostGenPruner.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestPostGenPruner.scala index 7b1f68f46..96456d126 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestPostGenPruner.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestPostGenPruner.scala @@ -8,7 +8,7 @@ import com.linkedin.feathr.offline.derived.DerivedFeature import com.linkedin.feathr.offline.job.FeatureGenSpec import com.linkedin.feathr.offline.logical.{FeatureGroups, MultiStageJoinPlan} import org.mockito.Mockito.when -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert.assertEquals import org.testng.annotations.Test diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestPushToRedisOutputProcessor.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestPushToRedisOutputProcessor.scala index b331a827e..1d5677a07 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestPushToRedisOutputProcessor.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestPushToRedisOutputProcessor.scala @@ -7,7 +7,7 @@ import com.linkedin.feathr.offline.{AssertFeatureUtils, TestFeathr} import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.encoders.RowEncoder import org.apache.spark.sql.types.{ArrayType, BooleanType, FloatType, IntegerType, StringType, StructField, StructType} -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.annotations.Test import java.util.Base64 diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestStageEvaluator.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestStageEvaluator.scala index 65e80bb14..f6cb7072f 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestStageEvaluator.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/generation/TestStageEvaluator.scala @@ -7,7 +7,7 @@ import com.linkedin.feathr.offline.evaluator.{BaseDataFrameMetadata, DerivedFeat import com.linkedin.feathr.offline.logical.{FeatureGroups, MultiStageJoinPlan} import com.linkedin.feathr.offline.{FeatureDataFrame, TestFeathr} import org.mockito.Mockito._ -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert._ import org.testng.annotations.Test diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureJoinJob.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureJoinJob.scala index 9c187c675..91a550df2 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureJoinJob.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureJoinJob.scala @@ -7,13 +7,13 @@ import org.apache.hadoop.conf.Configuration import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkConf import org.apache.spark.sql.{Row, SparkSession} -import org.scalatest.testng.TestNGSuite + import org.testng.Assert._ import org.testng.annotations.{AfterClass, BeforeClass, Test} import scala.collection.mutable -class TestFeatureJoinJob extends TestNGSuite{ +class TestFeatureJoinJob{ private val generatedDataFolder = "src/test/generated" var ss: SparkSession = _ diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureJoinJobUtils.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureJoinJobUtils.scala index c07e36f8f..19986e451 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureJoinJobUtils.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureJoinJobUtils.scala @@ -1,10 +1,10 @@ package com.linkedin.feathr.offline.job import com.linkedin.feathr.offline.config.FeatureJoinConfig -import org.scalatest.testng.TestNGSuite + import org.testng.annotations.Test -class TestJoinJobUtils extends TestNGSuite{ +class TestJoinJobUtils{ @Test( expectedExceptions = Array(classOf[RuntimeException]), diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureTransformation.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureTransformation.scala index 776c2a910..2a37a8ff6 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureTransformation.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/TestFeatureTransformation.scala @@ -15,7 +15,7 @@ import com.linkedin.feathr.sparkcommon.{SimpleAnchorExtractorSpark, SourceKeyExt import org.apache.spark.sql.Row import org.apache.spark.sql.types._ import org.mockito.Mockito._ -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert.assertEquals import org.testng.annotations.{DataProvider, Test} diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenConfigOverrider.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenConfigOverrider.scala index cee3d22e7..f815dead8 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenConfigOverrider.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenConfigOverrider.scala @@ -2,14 +2,14 @@ package com.linkedin.feathr.offline.job.featureGen import com.linkedin.feathr.offline.config.FeathrConfigLoader import com.linkedin.feathr.offline.job.{FeatureGenConfigOverrider, FeatureGenJobContext} -import org.scalatest.testng.TestNGSuite + import org.testng.Assert.{assertEquals, assertTrue} import org.testng.annotations.Test /** * Test suite to test the [[FeatureGenConfigOverrider]] class */ -class TestFeatureGenConfigOverrider extends TestNGSuite{ +class TestFeatureGenConfigOverrider{ private val feathrConfigLoader = FeathrConfigLoader() @Test(description = "test feature definition override parsing for feature-gen job") diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenJobParser.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenJobParser.scala index 01568fab0..7758271b1 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenJobParser.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenJobParser.scala @@ -1,14 +1,14 @@ package com.linkedin.feathr.offline.job.featureGen import com.linkedin.feathr.offline.job.FeatureGenJobContext -import org.scalatest.testng.TestNGSuite + import org.testng.Assert.assertEquals import org.testng.annotations.Test /** * Test the [[FeatureGenJobContext]] class' parser */ -class TestFeatureGenJobParser extends TestNGSuite{ +class TestFeatureGenJobParser{ @Test(description = "test params-override parameter parsing for feature-gen job") def featureGenJobParamsTest(): Unit = { diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenSpecParser.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenSpecParser.scala index 7d1a911b4..5af33b3d0 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenSpecParser.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/job/featureGen/TestFeatureGenSpecParser.scala @@ -1,11 +1,11 @@ package com.linkedin.feathr.offline.job.featureGen import com.linkedin.feathr.offline.job.{FeatureGenJobContext, FeatureGenSpec} -import org.scalatest.testng.TestNGSuite + import org.testng.Assert.assertEquals import org.testng.annotations.Test -class TestFeatureGenSpecParser extends TestNGSuite{ +class TestFeatureGenSpecParser{ @Test(description = "test feature gen spec parser") def featureGenSpecParserTest(): Unit = { diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestJoinConditionBuilder.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestJoinConditionBuilder.scala index 7c646ed6f..3bfc9a67a 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestJoinConditionBuilder.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestJoinConditionBuilder.scala @@ -5,7 +5,7 @@ import com.linkedin.feathr.offline.TestFeathr import org.apache.spark.sql.functions.{expr, lit} import org.apache.spark.sql.{DataFrame, Row} import org.apache.spark.sql.types.{DataTypes, FloatType, StringType, StructField, StructType} -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert._ import org.testng.annotations.Test diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestJoinKeyColumnsAppender.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestJoinKeyColumnsAppender.scala index 8a627761f..5cf9ac6f6 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestJoinKeyColumnsAppender.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestJoinKeyColumnsAppender.scala @@ -4,7 +4,7 @@ import com.linkedin.feathr.common.exception.FeathrFeatureJoinException import com.linkedin.feathr.offline.TestFeathr import org.apache.spark.sql.DataFrame import org.mockito.Mockito.{verify, when} -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.annotations.Test class TestJoinKeyColumnsAppender extends TestFeathr with MockitoSugar { diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestSparkJoin.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestSparkJoin.scala index a61a22496..8263c998c 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestSparkJoin.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestSparkJoin.scala @@ -7,7 +7,7 @@ import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema import org.apache.spark.sql.functions.col import org.apache.spark.sql.types._ import org.mockito.Mockito.verifyNoInteractions -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert.assertEquals import org.testng.annotations.Test diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestSparkSaltedJoin.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestSparkSaltedJoin.scala index 4145c818b..9dc51519e 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestSparkSaltedJoin.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/algorithms/TestSparkSaltedJoin.scala @@ -9,7 +9,7 @@ import org.apache.commons.math3.distribution.ZipfDistribution import org.apache.spark.sql.DataFrame import org.apache.spark.sql.functions._ import org.mockito.Mockito.when -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert.{assertEquals, assertTrue} import org.testng.annotations._ diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/workflow/TestAnchoredFeatureJoinStep.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/workflow/TestAnchoredFeatureJoinStep.scala index f9ec6d50e..011149495 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/workflow/TestAnchoredFeatureJoinStep.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/workflow/TestAnchoredFeatureJoinStep.scala @@ -16,7 +16,7 @@ import org.testng.annotations.Test import org.testng.Assert._ import org.mockito.Mockito._ import org.mockito.ArgumentMatchers._ -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar /** * Unit test class for [[AnchoredFeatureJoinStep]]. diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/workflow/TestDerivedFeatureJoinStep.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/workflow/TestDerivedFeatureJoinStep.scala index 7e703eacc..828de7cd7 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/workflow/TestDerivedFeatureJoinStep.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/join/workflow/TestDerivedFeatureJoinStep.scala @@ -10,7 +10,7 @@ import org.apache.spark.sql.types.{StringType, StructField, StructType} import org.apache.spark.sql.{DataFrame, Row} import org.mockito.ArgumentMatchers._ import org.mockito.Mockito._ -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert._ import org.testng.annotations.Test diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/logical/TestMultiStageJoinPlan.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/logical/TestMultiStageJoinPlan.scala index 2009ea5ba..61d6aa83f 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/logical/TestMultiStageJoinPlan.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/logical/TestMultiStageJoinPlan.scala @@ -6,7 +6,7 @@ import com.linkedin.feathr.offline.anchored.feature.FeatureAnchorWithSource import com.linkedin.feathr.offline.config.{FeatureGroupsGenerator, FeatureJoinConfig} import com.linkedin.feathr.offline.derived.DerivedFeature import com.linkedin.feathr.offline.{ErasedEntityTaggedFeature, TestFeathr} -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert.{assertEquals, assertTrue} import org.testng.annotations.Test diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestBatchDataLoader.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestBatchDataLoader.scala index 83e46c1d0..11345cb9b 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestBatchDataLoader.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestBatchDataLoader.scala @@ -1,8 +1,14 @@ package com.linkedin.feathr.offline.source.dataloader +import com.linkedin.feathr.common.exception.FeathrInputDataException import com.linkedin.feathr.offline.TestFeathr import com.linkedin.feathr.offline.config.location.SimplePath +import org.apache.hadoop.mapred.JobConf +import org.apache.log4j.Logger import org.apache.spark.sql.Row +import org.mockito.ArgumentMatchers.any +import org.mockito.Mockito.{times, verify, when} +import org.scalatestplus.mockito.MockitoSugar.mock import org.testng.Assert.assertEquals import org.testng.annotations.Test @@ -10,7 +16,6 @@ import org.testng.annotations.Test * unit tests for [[BatchDataLoader]] */ class TestBatchDataLoader extends TestFeathr { - def escape(raw: String): String = { import scala.reflect.runtime.universe._ Literal(Constant(raw)).toString @@ -56,4 +61,19 @@ class TestBatchDataLoader extends TestFeathr { sqlContext.setConf("spark.feathr.inputFormat.csvOptions.sep", "") } + @Test(description = "Verify loadDataFrame with retry works", expectedExceptions = Array(classOf[FeathrInputDataException])) + def testRetry(): Unit = { + val path = SimplePath("anchor1-source") + val loader = mock[BatchDataLoader] + val log = mock[Logger] + when(loader.loadDataFrame()).thenCallRealMethod() + when(loader.loadDataFrameWithRetry(any(classOf[Map[String, String]]), any(classOf[JobConf]), any(classOf[Int]))).thenCallRealMethod() + when(loader.ss).thenReturn(ss) + when(loader.location).thenReturn(path) + when(loader.dataLoaderHandlers).thenReturn(List()) + when(loader.log).thenReturn(log) + when(loader.MAX_DATA_LOAD_RETRY).thenReturn(2) + loader.loadDataFrame() + verify(loader, times(3)).loadDataFrameWithRetry(any(classOf[Map[String, String]]), any(classOf[JobConf]), any(classOf[Int])) + } } \ No newline at end of file diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestCaseInsensitiveGenericRecordWrapper.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestCaseInsensitiveGenericRecordWrapper.scala index 7234869cc..f8d7d836f 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestCaseInsensitiveGenericRecordWrapper.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/dataloader/TestCaseInsensitiveGenericRecordWrapper.scala @@ -3,7 +3,7 @@ package com.linkedin.feathr.offline.source.dataloader import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper import org.apache.avro.generic.{GenericData, GenericRecord} import org.apache.avro.{AvroRuntimeException, Schema} -import org.scalatest.testng.TestNGSuite +import org.scalatest.Assertions.assertThrows import org.testng.Assert.{assertEquals, assertFalse, assertTrue} import org.testng.annotations.Test @@ -12,7 +12,7 @@ import scala.collection.JavaConverters._ /** * unit test for CaseInsensitiveGenericRecordWrapper */ -class TestCaseInsensitiveGenericRecordWrapper extends TestNGSuite{ +class TestCaseInsensitiveGenericRecordWrapper{ val record = createRecord() val childRecord = record.get("child").asInstanceOf[GenericRecord] diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestPathChecker.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestPathChecker.scala index dedec4d50..45b6b4fe4 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestPathChecker.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestPathChecker.scala @@ -9,7 +9,7 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.DataFrame import org.apache.hadoop.mapred.JobConf import org.mockito.Mockito.when -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert.{assertEquals, assertTrue} import org.testng.annotations.Test diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestTimeBasedHdfsPathAnalyzer.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestTimeBasedHdfsPathAnalyzer.scala index ca380eb0c..1bf1faae1 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestTimeBasedHdfsPathAnalyzer.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestTimeBasedHdfsPathAnalyzer.scala @@ -3,7 +3,7 @@ package com.linkedin.feathr.offline.source.pathutil import com.linkedin.feathr.common.DateTimeResolution import com.linkedin.feathr.offline.TestFeathr import org.mockito.Mockito._ -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert.assertEquals import org.testng.annotations.Test diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestTimeBasedHdfsPathGenerator.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestTimeBasedHdfsPathGenerator.scala index 682ffc34a..90dc9a01c 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestTimeBasedHdfsPathGenerator.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/source/pathutil/TestTimeBasedHdfsPathGenerator.scala @@ -3,7 +3,7 @@ package com.linkedin.feathr.offline.source.pathutil import com.linkedin.feathr.common.DateTimeResolution import com.linkedin.feathr.offline.{TestFeathr, TestUtils} import org.mockito.Mockito._ -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert.assertEquals import org.testng.annotations.Test diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/swa/TestSlidingWindowFeatureUtils.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/swa/TestSlidingWindowFeatureUtils.scala index ec1537408..c07f15720 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/swa/TestSlidingWindowFeatureUtils.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/swa/TestSlidingWindowFeatureUtils.scala @@ -13,7 +13,7 @@ import org.apache.spark.sql.types.{StringType, StructField, StructType} import org.apache.spark.sql.{DataFrame, Row} import org.testng.Assert.{assertEquals, assertFalse, assertTrue} import org.testng.annotations._ -import org.scalatest.mockito.MockitoSugar.mock +import org.scalatestplus.mockito.MockitoSugar.mock import org.mockito.Mockito.when import scala.collection.JavaConverters._ diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestAnchorToDataSourceMapper.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestAnchorToDataSourceMapper.scala index 1f0e41887..69aedda38 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestAnchorToDataSourceMapper.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestAnchorToDataSourceMapper.scala @@ -8,7 +8,7 @@ import com.linkedin.feathr.offline.{TestFeathr, TestUtils} import org.apache.avro.Schema import org.apache.avro.generic.{GenericRecord, GenericRecordBuilder} import org.apache.spark.sql.Row -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert.assertEquals import org.testng.annotations.{AfterClass, Test} diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestDataFrameExt.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestDataFrameExt.scala index d23a91bd7..80f29bb6f 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestDataFrameExt.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestDataFrameExt.scala @@ -1,7 +1,7 @@ package com.linkedin.feathr.offline.transformation import com.linkedin.feathr.offline.TestFeathr -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.annotations.Test import com.linkedin.feathr.offline.transformation.DataFrameExt._ import com.linkedin.feathr.offline.AssertFeatureUtils._ diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestDefaultValueToColumnConverter.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestDefaultValueToColumnConverter.scala index 8f64db6e9..c6bd65bfe 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestDefaultValueToColumnConverter.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestDefaultValueToColumnConverter.scala @@ -5,7 +5,7 @@ import com.linkedin.feathr.common.FeatureValue import com.linkedin.feathr.common.exception.{FeathrException, FeathrFeatureTransformationException} import com.linkedin.feathr.offline.TestFeathr import org.mockito.Mockito.{verify, when} -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.annotations.{DataProvider, Test} import java.{lang => jl, util => ju} diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestFDSConversionUtils.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestFDSConversionUtils.scala index 0367feb91..eadcd7f99 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestFDSConversionUtils.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/transformation/TestFDSConversionUtils.scala @@ -2,7 +2,7 @@ package com.linkedin.feathr.offline.transformation import com.linkedin.feathr.common.exception.FeathrException import com.linkedin.feathr.offline.TestFeathr -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert.assertEquals import org.testng.annotations.{DataProvider, Test} diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestDataFrameSplitterMerger.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestDataFrameSplitterMerger.scala index 38f3dad75..4f9391974 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestDataFrameSplitterMerger.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestDataFrameSplitterMerger.scala @@ -8,9 +8,9 @@ import com.linkedin.feathr.offline.util.DataFrameSplitterMerger._ import org.apache.spark.sql.{DataFrame, Row} import org.apache.spark.sql.types.{StringType, StructField, StructType} import com.linkedin.feathr.offline.util.TestSplitMergeDataProvider._ -import org.scalatest.testng.TestNGSuite -class TestDataFrameSplitterMerger extends TestNGSuite { + +class TestDataFrameSplitterMerger { /** * Test partitioning when NO nulls exist in column. diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestFDSConversionUtil.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestFDSConversionUtil.scala index f3b75024e..4f5d50af0 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestFDSConversionUtil.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestFDSConversionUtil.scala @@ -8,7 +8,7 @@ import com.linkedin.feathr.offline.transformation.FDSConversionUtils import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.expressions.{GenericRow, GenericRowWithSchema} import org.apache.spark.sql.types._ -import org.scalatest.testng.TestNGSuite + import org.testng.Assert.{assertEquals, assertTrue} import org.testng.annotations.{DataProvider, Test} @@ -16,7 +16,7 @@ import java.util import java.util.Collections import scala.collection.mutable -class TestFDSConversionUtil extends TestNGSuite { +class TestFDSConversionUtil { val EMPTY_STRING = "" val EXPECTED_DENSE_VECTOR_FDS_DATA_TYPE = ArrayType(FloatType, false) diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestPartitionLimiter.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestPartitionLimiter.scala index e54686a08..f28749fa7 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestPartitionLimiter.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/TestPartitionLimiter.scala @@ -4,7 +4,7 @@ import com.linkedin.feathr.offline.TestFeathr import org.apache.spark.SparkContext import org.apache.spark.sql.{DataFrame, SparkSession} import org.mockito.Mockito.when -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.testng.Assert.assertEquals import org.testng.annotations.{BeforeClass, DataProvider, Test} diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestDateTimeInterval.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestDateTimeInterval.scala index 18cdbc165..d5ca243f6 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestDateTimeInterval.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestDateTimeInterval.scala @@ -3,14 +3,14 @@ package com.linkedin.feathr.offline.util.datetime import com.linkedin.feathr.common.DateTimeResolution import com.linkedin.feathr.offline.TestUtils.{createDailyInterval, createHourlyInterval} import com.linkedin.feathr.offline.util.datetime.OfflineDateTimeUtils.createTimeFromString -import org.scalatest.testng.TestNGSuite + import org.testng.annotations.Test import org.testng.Assert.{assertEquals, assertFalse, assertTrue} /** * unit tests for [[DateTimeInterval]] */ -class TestDateTimeInterval extends TestNGSuite { +class TestDateTimeInterval { @Test(description = "test creation of DateTimeInterval") def testCreation(): Unit = { diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestDateTimePeriod.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestDateTimePeriod.scala index 50303b878..31611381b 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestDateTimePeriod.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestDateTimePeriod.scala @@ -2,14 +2,14 @@ package com.linkedin.feathr.offline.util.datetime import com.linkedin.feathr.common.DateTimeResolution import com.linkedin.feathr.common.exception.FeathrConfigException -import org.scalatest.testng.TestNGSuite + import org.testng.Assert.assertEquals import org.testng.annotations.Test /** * A unit test class for DateTimePeriod. */ -class TestDateTimePeriod extends TestNGSuite { +class TestDateTimePeriod { /** * Unit test for parsing days. diff --git a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestOfflineDateTimeUtils.scala b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestOfflineDateTimeUtils.scala index 65579729b..715235ad0 100644 --- a/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestOfflineDateTimeUtils.scala +++ b/feathr-impl/src/test/scala/com/linkedin/feathr/offline/util/datetime/TestOfflineDateTimeUtils.scala @@ -3,7 +3,7 @@ package com.linkedin.feathr.offline.util.datetime import com.linkedin.feathr.common.exception.FeathrConfigException import com.linkedin.feathr.common.{DateParam, DateTimeResolution} import com.linkedin.feathr.offline.TestUtils.{createDailyInterval, createHourlyInterval} -import org.scalatest.testng.TestNGSuite +import org.scalatest.Assertions.assertThrows import org.testng.Assert.assertEquals import org.testng.annotations.Test @@ -11,7 +11,7 @@ import java.time.format.DateTimeFormatter import java.time.temporal.ChronoUnit import java.time.{Duration, LocalDate, ZoneId, ZonedDateTime} -class TestOfflineDateTimeUtils extends TestNGSuite { +class TestOfflineDateTimeUtils { @Test(description = "test dateRange for creating datepartition filter expression") def testDateRange(): Unit = {