Skip to content

Commit 7249bc1

Browse files
oavdeevoavdeev
authored
Add feature views (#1386)
* add featureviews Signed-off-by: Oleg Avdeev <oleg.v.avdeev@gmail.com> * fix comment Signed-off-by: Oleg Avdeev <oleg.v.avdeev@gmail.com> * fix comment Signed-off-by: Oleg Avdeev <oleg.v.avdeev@gmail.com>
1 parent 8ada7ae commit 7249bc1

File tree

14 files changed

+370
-113
lines changed

14 files changed

+370
-113
lines changed

‎protos/feast/core/FeatureView.proto‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
//
2+
// Copyright 2020 The Feast Authors
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
//
8+
// https://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
//
16+
17+
18+
syntax = "proto3";
19+
package feast.core;
20+
21+
option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core";
22+
option java_outer_classname = "FeatureViewProto";
23+
option java_package = "feast.proto.core";
24+
25+
import "google/protobuf/duration.proto";
26+
import "google/protobuf/timestamp.proto";
27+
import "feast/core/DataSource.proto";
28+
import "feast/core/Feature.proto";
29+
30+
message FeatureView {
31+
// User-specified specifications of this feature table.
32+
FeatureViewSpec spec = 1;
33+
34+
// System-populated metadata for this feature table.
35+
FeatureViewMeta meta = 2;
36+
string project = 3;
37+
}
38+
39+
message FeatureViewSpec {
40+
// Name of the feature table. Must be unique. Not updated.
41+
string name = 1;
42+
43+
// List names of entities to associate with the Features defined in this
44+
// Feature View. Not updatable.
45+
repeated string entities = 3;
46+
47+
// List of features specifications for each feature defined with this feature table.
48+
repeated FeatureSpecV2 features = 4;
49+
50+
// User defined metadata
51+
map<string,string> tags = 5;
52+
53+
// Features in this feature table can only be retrieved from online serving
54+
// younger than ttl. Ttl is measured as the duration of time between
55+
// the feature's event timestamp and when the feature is retrieved
56+
// Feature values outside ttl will be returned as unset values and indicated to end user
57+
google.protobuf.Duration ttl = 6;
58+
59+
DataSource input = 7;
60+
61+
bool online = 8;
62+
}
63+
64+
message FeatureViewMeta {
65+
// Time where this Feature View is created
66+
google.protobuf.Timestamp created_timestamp = 1;
67+
68+
// Time where this Feature View is last updated
69+
google.protobuf.Timestamp last_updated_timestamp = 2;
70+
}

‎protos/feast/core/Registry.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,13 @@ option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core";
2323

2424
import "feast/core/Entity.proto";
2525
import "feast/core/FeatureTable.proto";
26+
import "feast/core/FeatureView.proto";
2627
import "google/protobuf/timestamp.proto";
2728

2829
message Registry {
2930
repeated Entity entities = 1;
3031
repeated FeatureTable feature_tables = 2;
32+
repeated FeatureView feature_views = 6;
3133

3234
string registry_schema_version = 3; // to support migrations; incremented when schema is changed
3335
string version_id = 4; // version id, random string generated on each update of the data; now used only for debugging purposes

‎sdk/python/feast/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from .entity import Entity
1212
from .feature import Feature
1313
from .feature_table import FeatureTable
14+
from .feature_view import FeatureView
1415
from .value_type import ValueType
1516

1617
try:
@@ -28,6 +29,7 @@
2829
"KinesisSource",
2930
"Feature",
3031
"FeatureTable",
32+
"FeatureView",
3133
"SourceType",
3234
"ValueType",
3335
]

‎sdk/python/feast/big_query_source.py‎

Lines changed: 0 additions & 57 deletions
This file was deleted.

‎sdk/python/feast/data_source.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -579,6 +579,10 @@ def __eq__(self, other):
579579

580580
return True
581581

582+
@property
583+
def table_ref(self):
584+
return self._bigquery_options.table_ref
585+
582586
@property
583587
def bigquery_options(self):
584588
"""

‎sdk/python/feast/feature_view.py‎

Lines changed: 110 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -11,43 +11,138 @@
1111
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
14-
from datetime import datetime
15-
from typing import Dict, List
14+
from datetime import timedelta
15+
from typing import Dict, List, Optional, Union
1616

17-
from feast.big_query_source import BigQuerySource
18-
from feast.entity import Entity
17+
from google.protobuf.duration_pb2 import Duration
18+
from google.protobuf.timestamp_pb2 import Timestamp
19+
20+
from feast.core.FeatureView_pb2 import FeatureView as FeatureViewProto
21+
from feast.core.FeatureView_pb2 import FeatureViewMeta as FeatureViewMetaProto
22+
from feast.core.FeatureView_pb2 import FeatureViewSpec as FeatureViewSpecProto
23+
from feast.data_source import BigQuerySource, DataSource
1924
from feast.feature import Feature
25+
from feast.value_type import ValueType
2026

2127

2228
class FeatureView:
2329
"""
2430
A FeatureView defines a logical grouping of servable features.
2531
"""
2632

33+
name: str
34+
entities: List[str]
35+
features: List[Feature]
36+
tags: Dict[str, str]
37+
ttl: Optional[Duration]
38+
online: bool
39+
input: BigQuerySource
40+
41+
created_timestamp: Optional[Timestamp] = None
42+
last_updated_timestamp: Optional[Timestamp] = None
43+
2744
def __init__(
2845
self,
2946
name: str,
30-
entities: List[Entity],
47+
entities: List[str],
3148
features: List[Feature],
3249
tags: Dict[str, str],
33-
ttl: str,
50+
ttl: Optional[Union[Duration, timedelta]],
3451
online: bool,
35-
inputs: BigQuerySource,
36-
feature_start_time: datetime,
52+
input: BigQuerySource,
3753
):
38-
cols = [entity.name for entity in entities] + [feat.name for feat in features]
54+
cols = [entity for entity in entities] + [feat.name for feat in features]
3955
for col in cols:
40-
if inputs.field_mapping is not None and col in inputs.field_mapping.keys():
56+
if input.field_mapping is not None and col in input.field_mapping.keys():
4157
raise ValueError(
42-
f"The field {col} is mapped to {inputs.field_mapping[col]} for this data source. Please either remove this field mapping or use {inputs.field_mapping[col]} as the Entity or Feature name."
58+
f"The field {col} is mapped to {input.field_mapping[col]} for this data source. Please either remove this field mapping or use {input.field_mapping[col]} as the Entity or Feature name."
4359
)
4460

4561
self.name = name
4662
self.entities = entities
4763
self.features = features
4864
self.tags = tags
49-
self.ttl = ttl
65+
if isinstance(ttl, timedelta):
66+
proto_ttl = Duration()
67+
proto_ttl.FromTimedelta(ttl)
68+
self.ttl = proto_ttl
69+
else:
70+
self.ttl = ttl
71+
5072
self.online = online
51-
self.inputs = inputs
52-
self.feature_start_time = feature_start_time
53-
return
73+
self.input = input
74+
75+
def is_valid(self):
76+
"""
77+
Validates the state of a feature view locally. Raises an exception
78+
if feature view is invalid.
79+
"""
80+
81+
if not self.name:
82+
raise ValueError("Feature view needs a name")
83+
84+
if not self.entities:
85+
raise ValueError("Feature view has no entities")
86+
87+
def to_proto(self) -> FeatureViewProto:
88+
"""
89+
Converts an feature view object to its protobuf representation.
90+
91+
Returns:
92+
FeatureViewProto protobuf
93+
"""
94+
95+
meta = FeatureViewMetaProto(
96+
created_timestamp=self.created_timestamp,
97+
last_updated_timestamp=self.last_updated_timestamp,
98+
)
99+
100+
spec = FeatureViewSpecProto(
101+
name=self.name,
102+
entities=self.entities,
103+
features=[feature.to_proto() for feature in self.features],
104+
tags=self.tags,
105+
ttl=self.ttl,
106+
online=self.online,
107+
input=self.input.to_proto(),
108+
)
109+
110+
return FeatureViewProto(spec=spec, meta=meta)
111+
112+
@classmethod
113+
def from_proto(cls, feature_view_proto: FeatureViewProto):
114+
"""
115+
Creates a feature view from a protobuf representation of a feature view
116+
117+
Args:
118+
feature_view_proto: A protobuf representation of a feature view
119+
120+
Returns:
121+
Returns a FeatureViewProto object based on the feature view protobuf
122+
"""
123+
124+
feature_view = cls(
125+
name=feature_view_proto.spec.name,
126+
entities=[entity for entity in feature_view_proto.spec.entities],
127+
features=[
128+
Feature(
129+
name=feature.name,
130+
dtype=ValueType(feature.value_type),
131+
labels=feature.labels,
132+
)
133+
for feature in feature_view_proto.spec.features
134+
],
135+
tags=dict(feature_view_proto.spec.tags),
136+
online=feature_view_proto.spec.online,
137+
ttl=(
138+
None
139+
if feature_view_proto.spec.ttl.seconds == 0
140+
and feature_view_proto.spec.ttl.nanos == 0
141+
else feature_view_proto.spec.ttl
142+
),
143+
input=DataSource.from_proto(feature_view_proto.spec.input),
144+
)
145+
146+
feature_view.created_timestamp = feature_view_proto.meta.created_timestamp
147+
148+
return feature_view

‎sdk/python/feast/infra/gcp.py‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
from datetime import datetime
2-
from typing import Dict, List, Optional, Tuple
2+
from typing import Dict, List, Optional, Tuple, Union
33

44
import mmh3
55
from pytz import utc
66

7-
from feast import FeatureTable
7+
from feast import FeatureTable, FeatureView
88
from feast.infra.provider import Provider
99
from feast.repo_config import DatastoreOnlineStoreConfig
1010
from feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
@@ -66,8 +66,8 @@ def _initialize_client(self):
6666
def update_infra(
6767
self,
6868
project: str,
69-
tables_to_delete: List[FeatureTable],
70-
tables_to_keep: List[FeatureTable],
69+
tables_to_delete: List[Union[FeatureTable, FeatureView]],
70+
tables_to_keep: List[Union[FeatureTable, FeatureView]],
7171
):
7272
from google.cloud import datastore
7373

@@ -88,7 +88,9 @@ def update_infra(
8888
key = client.key("Project", project, "Table", table.name)
8989
client.delete(key)
9090

91-
def teardown_infra(self, project: str, tables: List[FeatureTable]) -> None:
91+
def teardown_infra(
92+
self, project: str, tables: List[Union[FeatureTable, FeatureView]]
93+
) -> None:
9294
client = self._initialize_client()
9395

9496
for table in tables:
@@ -103,7 +105,7 @@ def teardown_infra(self, project: str, tables: List[FeatureTable]) -> None:
103105
def online_write_batch(
104106
self,
105107
project: str,
106-
table: FeatureTable,
108+
table: Union[FeatureTable, FeatureView],
107109
data: List[Tuple[EntityKeyProto, Dict[str, ValueProto], datetime]],
108110
created_ts: datetime,
109111
) -> None:
@@ -143,7 +145,10 @@ def online_write_batch(
143145
client.put(entity)
144146

145147
def online_read(
146-
self, project: str, table: FeatureTable, entity_key: EntityKeyProto
148+
self,
149+
project: str,
150+
table: Union[FeatureTable, FeatureView],
151+
entity_key: EntityKeyProto,
147152
) -> Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]:
148153
client = self._initialize_client()
149154

0 commit comments

Comments
 (0)