forked from feathr-ai/feathr
-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathmodels.py
More file actions
146 lines (115 loc) · 4.77 KB
/
Copy pathmodels.py
File metadata and controls
146 lines (115 loc) · 4.77 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
from pydantic import BaseModel
from typing import List
"""
This file defines abstract backend data models for feature registry.
Backend data models will be used by backend API server to talk to feature registry backend.
Purpose of this is to decouple backend data models from API specific data models.
For each feature registry provider/implementation, they will extend this abstract
data models and backend API.
Diagram of the data models: ./data-model-diagram.md
"""
class FeatureId(BaseModel):
"""
Id for Feature, it's unique ID represents Feature.
Id can be a simple string, int or complex key.
"""
id: str # id of a feature
class FeatureNameId(BaseModel):
"""
Id for FeatureName, it's unique ID represents FeatureName.
Id can be a simple string, int or complex key.
"""
id: str # id of a FeatureName
class AnchorId(BaseModel):
"""
Id for Anchor, it's unique ID represents Anchor.
Id can be a simple string, int or complex key.
"""
id: str # id of a anchor
class ProjectId(BaseModel):
"""
Id for Project, it's unique ID represents Project.
Id can be a simple string, int or complex key.
"""
id: str # id of a project
class Source(BaseModel):
"""
Source of the feature.
It defines where the feature is extracted or derived from.
"""
pass
class DataSource(Source):
"""
Data source of the feature.
It defines the raw data source the feature is extracted from.
"""
pass
class FeatureSource(BaseModel):
"""
Represents a feature source for a derived feature. That is, it is a source 'FeatureName' which is used for
creating other derived features.
"""
input_feature_name_id: FeatureNameId # Input feature name Key
class MultiFeatureSource(Source):
"""
Feature sources of the feature.
It defines one to many features where the feature is derived from.
"""
sources: List[FeatureSource] # All source features which the feature is derived from
pass
class Transformation(BaseModel):
"""
The transformation of a Feature.
A transformation function represents the transformation logic to produce feature value from the source of FeatureAnchor
"""
pass
class Feature(BaseModel):
"""
Actual implementation of FeatureName.
An implementation defines where a feature is extracted from (Source) and how it is computed (Transformation).
The Source of a feature can be raw data sources and/or other features.
"""
id: FeatureId # Unique ID for Feature
feature_name_id: FeatureNameId # Id of the feature name that the feature belongs to
source: Source # Source can be either data source or feature source
transformation: Transformation # transformation logic to produce feature value
class AnchorFeature(Feature):
"""
Feature implementation of FeatureName which anchored to a data source.
"""
source: DataSource # Raw data source where the feature is extracted from
class DerivedFeature(Feature):
"""
Feature implementation that is derived from other FeatureNames.
"""
source: MultiFeatureSource # Source features where the feature is derived from
class FeatureName(BaseModel):
"""
Named Feature Interface that can be backed by multiple Feature implementations across
different environments accessing different sources (data lake access for batch training,
KV store access for online serving). Each FeatureName is defined by feature producer.
Feature consumers reference a feature by that name to access that feature data,
agnostic of runtime environment. Each FeatureName also encloses attributes that does not
change across implementations.
"""
id: FeatureNameId # unique ID for FeatureName, used to extract data for current FeatureName
project_id: ProjectId # ID of the project the FeatureName belongs to
feature_ids: List[FeatureId] # List of ids of feature that the FeatureName has
class Project(BaseModel):
"""
Group of FeatureNames. It can be a project the team is working on,
or a namespace which related FeatureNames have.
"""
id: ProjectId # Unique ID of the project.
feature_name_ids: List[FeatureNameId] # List of feature name ids that the project has
anchor_ids: List[AnchorId] # List of Anchor ids that the project has
class Anchor(BaseModel):
"""
Group of AnchorFeatures which anchored on same DataSource.
This is mainly used by feature producer gather information about DataSource
and FeatureImplementations associated with the DataSource.
"""
id: AnchorId # Unique ID for Anchor
project_id: ProjectId # ID of Project that the anchor belongs to
source: DataSource # data source of the Anchor
anchor_feature_ids: List[FeatureId] # List of anchor features that the anchor has