From c41846e642bb8c666d6fe6042500334d49ff58ba Mon Sep 17 00:00:00 2001 From: Jinghui Mo Date: Fri, 20 May 2022 11:44:34 -0700 Subject: [PATCH] Fix millisecond timestamp handling --- .../offline/swa/SlidingWindowFeatureUtils.scala | 12 ++++++++---- .../offline/swa/TestSlidingWindowFeatureUtils.scala | 11 +++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/main/scala/com/linkedin/feathr/offline/swa/SlidingWindowFeatureUtils.scala b/src/main/scala/com/linkedin/feathr/offline/swa/SlidingWindowFeatureUtils.scala index f0a4dbedc..88d23784e 100644 --- a/src/main/scala/com/linkedin/feathr/offline/swa/SlidingWindowFeatureUtils.scala +++ b/src/main/scala/com/linkedin/feathr/offline/swa/SlidingWindowFeatureUtils.scala @@ -35,7 +35,8 @@ private[offline] object SlidingWindowFeatureUtils { private val EPOCH_MILLIS = "epoch_millis" private val MILLIS_IN_SECOND = 1000 - val TIMESTAMP_WITH_TIMEZONE_FORMAT = "yyyy-MM-dd HH:mm:ssZ" // Z is timezone pattern letter + val TIMESTAMP_WITHOUT_TIMEZONE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS" + val TIMESTAMP_WITH_TIMEZONE_FORMAT = TIMESTAMP_WITHOUT_TIMEZONE_FORMAT + "Z" // Z is timezone pattern letter val UTC_TIMEZONE_OFFSET = "-0000" // PDT/PST val DEFAULT_TIME_DELAY = "Default-time-delay" val TIMESTAMP_PARTITION_COLUMN = "__feathr_timestamp_column_from_partition" @@ -218,9 +219,12 @@ private[offline] object SlidingWindowFeatureUtils { // specified timezone to epoch s"""to_unix_timestamp( | concat( - | to_utc_timestamp( - | to_timestamp($timeStampCol, "$timeStampFormat"), - | "$parsedTimeZone" + | date_format( + | to_utc_timestamp( + | to_timestamp($timeStampCol, "$timeStampFormat"), + | "$parsedTimeZone" + | ), + | "$TIMESTAMP_WITHOUT_TIMEZONE_FORMAT" | ), | "$UTC_TIMEZONE_OFFSET" | ), diff --git a/src/test/scala/com/linkedin/feathr/offline/swa/TestSlidingWindowFeatureUtils.scala b/src/test/scala/com/linkedin/feathr/offline/swa/TestSlidingWindowFeatureUtils.scala index 57bb8c69d..ec1537408 100644 --- a/src/test/scala/com/linkedin/feathr/offline/swa/TestSlidingWindowFeatureUtils.scala +++ b/src/test/scala/com/linkedin/feathr/offline/swa/TestSlidingWindowFeatureUtils.scala @@ -77,6 +77,17 @@ class TestSlidingWindowFeatureUtils extends TestFeathr { assertEquals(epoch.first().getLong(0), 1556866800) } + @Test(enabled = true) + def testTimestampWithMilliSecondsTime(): Unit = { + val timeStampFormat = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'" + val expr = SlidingWindowFeatureUtils.constructTimeStampExpr("\"2019-05-03T15:00:00.123Z\"", timeStampFormat, Some("America/Los_Angeles")) + + val epoch = ss.sql(s"""select $expr + """.stripMargin) + + assertEquals(epoch.first().getLong(0), 1556920800) + } + /** * Test constructing timestamp with daylight saving at: * Sunnyvale (USA - California) Friday, May 3, 2019 0:00:00 GMT-07:00 DST