Skip to content

Commit c674bea

Browse files
MattDelacMattDelac
authored
Add Unix Timestamp value type (#1520)
* Add Unix Timestamp value type Signed-off-by: Matt Delacour <matt.delacour@shopify.com> * Update docs Signed-off-by: Matt Delacour <matt.delacour@shopify.com> * Handle Scalar type Signed-off-by: Matt Delacour <matt.delacour@shopify.com>
1 parent 75dfa63 commit c674bea

File tree

11 files changed

+997
-1060
lines changed

11 files changed

+997
-1060
lines changed

‎docs/specs/offline_store_format.md‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,16 @@ Here's how Feast types map to Pandas types for Feast APIs that take in or return
3636
| BYTES | `bytes` |
3737
| STRING | `str` , `category`|
3838
| INT32 | `int32`, `uint32` |
39-
| INT64 | `int64`, `uint64`, `datetime64[ns]`, `datetime64[ns, tz]` |
39+
| INT64 | `int64`, `uint64` |
40+
| UNIX_TIMESTAMP | `datetime64[ns]`, `datetime64[ns, tz]` |
4041
| DOUBLE | `float64` |
4142
| FLOAT | `float32` |
4243
| BOOL | `bool`|
4344
| BYTES\_LIST | `list[bytes]` |
4445
| STRING\_LIST | `list[str]`|
4546
| INT32\_LIST | `list[int]`|
4647
| INT64\_LIST | `list[int]`|
48+
| UNIX_TIMESTAMP\_LIST | `list[unix_timestamp]`|
4749
| DOUBLE\_LIST | `list[float]`|
4850
| FLOAT\_LIST | `list[float]`|
4951
| BOOL\_LIST | `list[bool]`|
@@ -52,7 +54,7 @@ Note that this mapping is non-injective, that is more than one Pandas type may c
5254

5355
Feast array types are mapped to a pandas column with object dtype, that contains a Python array of corresponding type.
5456

55-
Another thing to note is Feast doesn't support timestamp type for entity and feature columns. Values of datetime type in pandas dataframe are converted to int64 if they are found in entity and feature columns.
57+
Another thing to note is Feast doesn't support timestamp type for entity and feature columns. Values of datetime type in pandas dataframe are converted to int64 if they are found in entity and feature columns. In order to easily differentiate int64 to timestamp features, there is a UNIX_TIMESTAMP type that is an int64 under the hood.
5658

5759
#### BigQuery types
5860
Here's how Feast types map to BigQuery types when using BigQuery for offline storage when reading data from BigQuery to the online store:
@@ -64,13 +66,15 @@ Here's how Feast types map to BigQuery types when using BigQuery for offline sto
6466
| STRING | `STRING` |
6567
| INT32 | `INT64 / INTEGER` |
6668
| INT64 | `INT64 / INTEGER` |
69+
| UNIX_TIMESTAMP | `INT64 / INTEGER` |
6770
| DOUBLE | `FLOAT64 / FLOAT` |
6871
| FLOAT | `FLOAT64 / FLOAT` |
6972
| BOOL | `BOOL`|
7073
| BYTES\_LIST | `ARRAY<BYTES>` |
7174
| STRING\_LIST | `ARRAY<STRING>`|
7275
| INT32\_LIST | `ARRAY<INT64>`|
7376
| INT64\_LIST | `ARRAY<INT64>`|
77+
| UNIX_TIMESTAMP\_LIST | `ARRAY<INT64>`|
7478
| DOUBLE\_LIST | `ARRAY<FLOAT64>`|
7579
| FLOAT\_LIST | `ARRAY<FLOAT64>`|
7680
| BOOL\_LIST | `ARRAY<BOOL>`|

‎docs/specs/online_store_format.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,13 +107,15 @@ message ValueType {
107107
DOUBLE = 5;
108108
FLOAT = 6;
109109
BOOL = 7;
110+
UNIX_TIMESTAMP = 8;
110111
BYTES_LIST = 11;
111112
STRING_LIST = 12;
112113
INT32_LIST = 13;
113114
INT64_LIST = 14;
114115
DOUBLE_LIST = 15;
115116
FLOAT_LIST = 16;
116117
BOOL_LIST = 17;
118+
UNIX_TIMESTAMP_LIST = 18;
117119
}
118120
}
119121
@@ -128,13 +130,15 @@ message Value {
128130
double double_val = 5;
129131
float float_val = 6;
130132
bool bool_val = 7;
133+
int64 unix_timestamp_val = 8;
131134
BytesList bytes_list_val = 11;
132135
StringList string_list_val = 12;
133136
Int32List int32_list_val = 13;
134137
Int64List int64_list_val = 14;
135138
DoubleList double_list_val = 15;
136139
FloatList float_list_val = 16;
137140
BoolList bool_list_val = 17;
141+
Int64List unix_timestamp_list_val = 18;
138142
}
139143
}
140144

‎protos/feast/types/Value.proto‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,15 @@ message ValueType {
3232
DOUBLE = 5;
3333
FLOAT = 6;
3434
BOOL = 7;
35+
UNIX_TIMESTAMP = 8;
3536
BYTES_LIST = 11;
3637
STRING_LIST = 12;
3738
INT32_LIST = 13;
3839
INT64_LIST = 14;
3940
DOUBLE_LIST = 15;
4041
FLOAT_LIST = 16;
4142
BOOL_LIST = 17;
43+
UNIX_TIMESTAMP_LIST = 18;
4244
}
4345
}
4446

@@ -53,13 +55,15 @@ message Value {
5355
double double_val = 5;
5456
float float_val = 6;
5557
bool bool_val = 7;
58+
int64 unix_timestamp_val = 8;
5659
BytesList bytes_list_val = 11;
5760
StringList string_list_val = 12;
5861
Int32List int32_list_val = 13;
5962
Int64List int64_list_val = 14;
6063
DoubleList double_list_val = 15;
6164
FloatList float_list_val = 16;
6265
BoolList bool_list_val = 17;
66+
Int64List unix_timestamp_list_val = 18;
6367
}
6468
}
6569

‎sdk/python/feast/type_map.py‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -107,9 +107,9 @@ def python_type_to_feast_value_type(
107107
"uint8": ValueType.INT32,
108108
"int8": ValueType.INT32,
109109
"bool": ValueType.BOOL,
110-
"timedelta": ValueType.INT64,
111-
"datetime64[ns]": ValueType.INT64,
112-
"datetime64[ns, tz]": ValueType.INT64,
110+
"timedelta": ValueType.UNIX_TIMESTAMP,
111+
"datetime64[ns]": ValueType.UNIX_TIMESTAMP,
112+
"datetime64[ns, tz]": ValueType.UNIX_TIMESTAMP,
113113
"category": ValueType.STRING,
114114
}
115115

@@ -252,6 +252,18 @@ def _python_value_to_proto_value(feast_value_type, value) -> ProtoValue:
252252
)
253253
)
254254

255+
if feast_value_type == ValueType.UNIX_TIMESTAMP_LIST:
256+
return ProtoValue(
257+
int64_list_val=Int64List(
258+
val=[
259+
item
260+
if type(item) in [np.int64, np.int32]
261+
else _type_err(item, np.int64)
262+
for item in value
263+
]
264+
)
265+
)
266+
255267
if feast_value_type == ValueType.STRING_LIST:
256268
return ProtoValue(
257269
string_list_val=StringList(
@@ -296,6 +308,8 @@ def _python_value_to_proto_value(feast_value_type, value) -> ProtoValue:
296308
return ProtoValue(int32_val=int(value))
297309
elif feast_value_type == ValueType.INT64:
298310
return ProtoValue(int64_val=int(value))
311+
elif feast_value_type == ValueType.UNIX_TIMESTAMP:
312+
return ProtoValue(int64_val=int(value))
299313
elif feast_value_type == ValueType.FLOAT:
300314
return ProtoValue(float_val=float(value))
301315
elif feast_value_type == ValueType.DOUBLE:

‎sdk/python/feast/value_type.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,15 @@ class ValueType(enum.Enum):
2929
DOUBLE = 5
3030
FLOAT = 6
3131
BOOL = 7
32+
UNIX_TIMESTAMP = 8
3233
BYTES_LIST = 11
3334
STRING_LIST = 12
3435
INT32_LIST = 13
3536
INT64_LIST = 14
3637
DOUBLE_LIST = 15
3738
FLOAT_LIST = 16
3839
BOOL_LIST = 17
40+
UNIX_TIMESTAMP_LIST = 18
3941

4042
def to_tfx_schema_feature_type(self):
4143
if self.value in [
@@ -49,9 +51,14 @@ def to_tfx_schema_feature_type(self):
4951
ValueType.DOUBLE_LIST.value,
5052
ValueType.FLOAT_LIST.value,
5153
ValueType.BOOL_LIST.value,
54+
ValueType.UNIX_TIMESTAMP_LIST.value,
5255
]:
5356
return schema_pb2.FeatureType.BYTES
54-
elif self.value in [ValueType.INT32.value, ValueType.INT64.value]:
57+
elif self.value in [
58+
ValueType.INT32.value,
59+
ValueType.INT64.value,
60+
ValueType.UNIX_TIMESTAMP.value,
61+
]:
5562
return schema_pb2.FeatureType.INT
5663
elif self.value in [ValueType.DOUBLE.value, ValueType.FLOAT.value]:
5764
return schema_pb2.FeatureType.FLOAT

‎sdk/python/tensorflow_metadata/proto/v0/path_pb2.py‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎sdk/python/tensorflow_metadata/proto/v0/path_pb2.pyi‎

Lines changed: 36 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,45 @@
22
@generated by mypy-protobuf. Do not edit manually!
33
isort:skip_file
44
"""
5-
import builtins
6-
import google.protobuf.descriptor
7-
import google.protobuf.internal.containers
8-
import google.protobuf.message
9-
import typing
10-
import typing_extensions
5+
from google.protobuf.descriptor import (
6+
Descriptor as google___protobuf___descriptor___Descriptor,
7+
FileDescriptor as google___protobuf___descriptor___FileDescriptor,
8+
)
119

12-
DESCRIPTOR: google.protobuf.descriptor.FileDescriptor = ...
10+
from google.protobuf.internal.containers import (
11+
RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer,
12+
)
1313

14-
class Path(google.protobuf.message.Message):
15-
DESCRIPTOR: google.protobuf.descriptor.Descriptor = ...
16-
STEP_FIELD_NUMBER: builtins.int
17-
step: google.protobuf.internal.containers.RepeatedScalarFieldContainer[typing.Text] = ...
14+
from google.protobuf.message import (
15+
Message as google___protobuf___message___Message,
16+
)
17+
18+
from typing import (
19+
Iterable as typing___Iterable,
20+
Optional as typing___Optional,
21+
Text as typing___Text,
22+
)
23+
24+
from typing_extensions import (
25+
Literal as typing_extensions___Literal,
26+
)
27+
28+
29+
builtin___bool = bool
30+
builtin___bytes = bytes
31+
builtin___float = float
32+
builtin___int = int
33+
34+
35+
DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
36+
37+
class Path(google___protobuf___message___Message):
38+
DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
39+
step: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] = ...
1840

1941
def __init__(self,
2042
*,
21-
step : typing.Optional[typing.Iterable[typing.Text]] = ...,
43+
step : typing___Optional[typing___Iterable[typing___Text]] = None,
2244
) -> None: ...
23-
def ClearField(self, field_name: typing_extensions.Literal[u"step",b"step"]) -> None: ...
24-
global___Path = Path
45+
def ClearField(self, field_name: typing_extensions___Literal[u"step",b"step"]) -> None: ...
46+
type___Path = Path

‎sdk/python/tensorflow_metadata/proto/v0/schema_pb2.py‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)