preserve fractional seconds in datetimes

Paul Santa Clara · Paul Santa Clara · commit ed7f1fc3fee0 · 2017-08-08T17:16:23.000-04:00
diff --git a/singer/utils.py b/singer/utils.py
@@ -7,12 +7,15 @@
 
 from singer.catalog import Catalog
 
-DATETIME_FMT = "%Y-%m-%dT%H:%M:%SZ"
+DATETIME_PARSE = "%Y-%m-%dT%H:%M:%SZ"
+DATETIME_FMT = "%Y-%m-%dT%H:%M:%S.%fZ"
 
 
 def strptime(dtime):
-    return datetime.datetime.strptime(dtime, DATETIME_FMT)
-
+    try:
+        return datetime.datetime.strptime(dtime, DATETIME_FMT)
+    except Exception:
+        return datetime.datetime.strptime(dtime, DATETIME_PARSE)
 
 def strftime(dtime):
     return dtime.strftime(DATETIME_FMT)
diff --git a/tests/test_transform.py b/tests/test_transform.py
@@ -32,11 +32,12 @@ def test_null_transform(self):
     def test_datetime_transform(self):
         schema = {"type": "string", "format": "date-time"}
         string_datetime = "2017-01-01T00:00:00Z"
-        self.assertEqual(string_datetime, transform(string_datetime, schema, NO_INTEGER_DATETIME_PARSING))
-        self.assertEqual('1970-01-02T00:00:00Z', transform(86400, schema, UNIX_SECONDS_INTEGER_DATETIME_PARSING))
-        self.assertEqual(string_datetime, transform(string_datetime, schema, UNIX_SECONDS_INTEGER_DATETIME_PARSING))
-        self.assertEqual('1970-01-01T00:01:26Z', transform(86400, schema, UNIX_MILLISECONDS_INTEGER_DATETIME_PARSING))
-        self.assertEqual(string_datetime, transform(string_datetime, schema, UNIX_MILLISECONDS_INTEGER_DATETIME_PARSING))
+        transformed_string_datetime = "2017-01-01T00:00:00.000000Z"
+        self.assertEqual(transformed_string_datetime, transform(string_datetime, schema, NO_INTEGER_DATETIME_PARSING))
+        self.assertEqual('1970-01-02T00:00:00.000000Z', transform(86400, schema, UNIX_SECONDS_INTEGER_DATETIME_PARSING))
+        self.assertEqual(transformed_string_datetime, transform(string_datetime, schema, UNIX_SECONDS_INTEGER_DATETIME_PARSING))
+        self.assertEqual('1970-01-01T00:01:26.400000Z', transform(86400, schema, UNIX_MILLISECONDS_INTEGER_DATETIME_PARSING))
+        self.assertEqual(transformed_string_datetime, transform(string_datetime, schema, UNIX_MILLISECONDS_INTEGER_DATETIME_PARSING))
 
         trans = Transformer(NO_INTEGER_DATETIME_PARSING)
         self.assertIsNone(trans._transform_datetime('cat'))
@@ -45,10 +46,16 @@ def test_datetime_transform(self):
         trans.integer_datetime_fmt = UNIX_SECONDS_INTEGER_DATETIME_PARSING
         self.assertIsNone(trans._transform_datetime('cat'))
 
+    def test_datetime_fractional_seconds_transform(self):
+        schema = {"type": "string", "format": "date-time"}
+        string_datetime = "2017-01-01T00:00:00.123000Z"
+        self.assertEqual(string_datetime, transform(string_datetime, schema, NO_INTEGER_DATETIME_PARSING))
+
     def test_anyof_datetime(self):
         schema = {'anyOf': [{'type': 'null'}, {'format': 'date-time', 'type': 'string'}]}
         string_datetime = '2016-03-10T18:47:20Z'
-        self.assertEqual(string_datetime, transform(string_datetime, schema))
+        transformed_string_datetime = '2016-03-10T18:47:20.000000Z'
+        self.assertEqual(transformed_string_datetime, transform(string_datetime, schema))
         self.assertIsNone(transform(None, schema))
 
     def test_error_path(self):
