From c6acae3119b252c90d653a8637e0bf0dc6eea935 Mon Sep 17 00:00:00 2001 From: Helen Yang Date: Mon, 12 Sep 2022 17:26:07 -0700 Subject: [PATCH 01/80] Add Data Models in Feathr This RB is to create data models based on proposal: https://microsoft-my.sharepoint.com/:w:/g/personal/djkim_linkedin_biz/EZspGt7jJlRAqHTICZg3UbcBgQQ_VncOgM48hKW--T8qkg?e=T4N3zw --- registry/data-models/__init__.py | 0 registry/data-models/models.py | 102 +++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 registry/data-models/__init__.py create mode 100644 registry/data-models/models.py diff --git a/registry/data-models/__init__.py b/registry/data-models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/registry/data-models/models.py b/registry/data-models/models.py new file mode 100644 index 000000000..e9599cbc3 --- /dev/null +++ b/registry/data-models/models.py @@ -0,0 +1,102 @@ +from pydantic import BaseModel +from typing import List, Optional + + +class FeatureValue(BaseModel): + """ + A value of a Feature. + NOTE: FeatureValue is to be customized in each registry + """ + pass + + +class FeatureValueType(BaseModel): + """ + A value of a Feature. + NOTE: FeatureValue is to be customized in each registry + """ + pass + + +class Source(BaseModel): + """ + Type of FeatureValue. + It defines where the feature is extracted from. It can be online, offline, Restli, or other FeatureInterfaces. + """ + pass + + +class DataSource(Source): + pass + + +class FeatureSource(Source): + pass + + +class Transformation(BaseModel): + """ + The transformation of a Feature. + A transformation function represents the transformation logic to produce feature value from the source of FeatureAnchor + NOTE: Transformation is to be customized in each registry + """ + pass + + +class Project(BaseModel): + """ + Group of FeatureInterfaces. It can be a project the team is working on, + or a namespace which related FeatureInterfaces have. + """ + id: str + name: str + + +class FeatureInterface(BaseModel): + """ + Named Feature Interface of FeatureImplementations. + Each FeatureInterface is defined by feature producer and interact + by feature consumers.Each FeatureInterface enclosed attributes that don't change across + implementations. (eg. name, default value, value type, format, status, ownership). + One FeatureInterface can have multiple Features for different environments. + """ + default_value: Optional[FeatureValue] # Optional default value + id: str + value_type: FeatureValueType # Feature value type + + +class Anchor(BaseModel): + """ + Group of AnchorFeatures which anchored on same DataSource. + This is mainly used by feature producer gather information about DataSource + and FeatureImplementations associated with the DataSource. + """ + id: str + source: DataSource + + +class Feature(BaseModel): + """ + Actual implementation of FeatureInterface. + An implementation defines where a feature is extracted from (Source) and how it is computed (Transformation). + The Source of a feature can be raw data sources (like Rest.li, HDFS) and/or other features. + """ + feature_interface_id: str # ID of the feature interface that the feature anchor belongs to + id: str + name: str + source: Source # Source can be either data source or feature source + transformation: Transformation # transformation logic to produce feature value + + +class AnchorFeature(Feature): + """ + Feature implementation of FeatureInterface which anchored to a data source. + """ + pass + + +class DerivedFeature(Feature): + """ + Feature implementation that is derived from other FeatureInterfaces. + """ + input_feature_interface_ids: List[str] # List of input feature interface IDs From 45a05acb2a9bf79bf82f9876573910e27cf09749 Mon Sep 17 00:00:00 2001 From: Helen Yang Date: Wed, 14 Sep 2022 17:21:14 -0700 Subject: [PATCH 02/80] Update models.py --- registry/data-models/models.py | 100 +++++++++++++++++++++++---------- 1 file changed, 70 insertions(+), 30 deletions(-) diff --git a/registry/data-models/models.py b/registry/data-models/models.py index e9599cbc3..0b103d434 100644 --- a/registry/data-models/models.py +++ b/registry/data-models/models.py @@ -1,6 +1,40 @@ from pydantic import BaseModel from typing import List, Optional +""" +This file defines abstract backend data models for feature registry. +Backend data models will be used by backend API server to talk to feature registry backend. +Purpose of this is to decouple backend data models from API specific data models. +For each feature registry provider/implementation, they will extend this abstract +data models and backend API. +Diagram of the data models: + +-----------------+ + | Project | + +-----------------+ + | + | + | + +------------------------------+ + | | + | 1:n | 1:n + +------------------+ +------------------+ + | FeatureInterface | | Anchor | + +------------------+ +------------------+ + ｜ ｜ | + ｜ 1:n ｜1:n | + +------------------------------+ | + ｜ | + ｜1:n | contains + +---------------+ +---------------+ + | Feature | --------- | Source | + +---------------+ contains +---------------+ + | + | + +----------------+ + | Transformation | + +----------------+ +""" + class FeatureValue(BaseModel): """ @@ -31,6 +65,7 @@ class DataSource(Source): class FeatureSource(Source): + input_feature_interface_ids: List[str] # List of input feature interface IDs pass @@ -43,13 +78,31 @@ class Transformation(BaseModel): pass -class Project(BaseModel): +class Feature(BaseModel): """ - Group of FeatureInterfaces. It can be a project the team is working on, - or a namespace which related FeatureInterfaces have. + Actual implementation of FeatureInterface. + An implementation defines where a feature is extracted from (Source) and how it is computed (Transformation). + The Source of a feature can be raw data sources (like Rest.li, HDFS) and/or other features. """ + feature_interface_id: str # ID of the feature interface that the feature belongs to id: str name: str + source: Source # Source can be either data source or feature source + transformation: Transformation # transformation logic to produce feature value + + +class AnchorFeature(Feature): + """ + Feature implementation of FeatureInterface which anchored to a data source. + """ + source: DataSource + + +class DerivedFeature(Feature): + """ + Feature implementation that is derived from other FeatureInterfaces. + """ + source: FeatureSource class FeatureInterface(BaseModel): @@ -63,40 +116,27 @@ class FeatureInterface(BaseModel): default_value: Optional[FeatureValue] # Optional default value id: str value_type: FeatureValueType # Feature value type + project_id: str + features: List[Feature] # List of features that the FeatureInterface has -class Anchor(BaseModel): - """ - Group of AnchorFeatures which anchored on same DataSource. - This is mainly used by feature producer gather information about DataSource - and FeatureImplementations associated with the DataSource. - """ - id: str - source: DataSource - - -class Feature(BaseModel): +class Project(BaseModel): """ - Actual implementation of FeatureInterface. - An implementation defines where a feature is extracted from (Source) and how it is computed (Transformation). - The Source of a feature can be raw data sources (like Rest.li, HDFS) and/or other features. + Group of FeatureInterfaces. It can be a project the team is working on, + or a namespace which related FeatureInterfaces have. """ - feature_interface_id: str # ID of the feature interface that the feature anchor belongs to id: str name: str - source: Source # Source can be either data source or feature source - transformation: Transformation # transformation logic to produce feature value + feature_interfaces: List[FeatureInterface] # List of feature interfaces that the project has -class AnchorFeature(Feature): - """ - Feature implementation of FeatureInterface which anchored to a data source. - """ - pass - - -class DerivedFeature(Feature): +class Anchor(BaseModel): """ - Feature implementation that is derived from other FeatureInterfaces. + Group of AnchorFeatures which anchored on same DataSource. + This is mainly used by feature producer gather information about DataSource + and FeatureImplementations associated with the DataSource. """ - input_feature_interface_ids: List[str] # List of input feature interface IDs + id: str + project_id: str # ID of Project that the anchor belongs to + source: DataSource # data source of the Anchor + anchor_features: List[AnchorFeature] # List of anchor features that the anchor has From 06966486ba8685f2a731802e0164b125b2235b8b Mon Sep 17 00:00:00 2001 From: Helen Yang Date: Mon, 19 Sep 2022 19:32:26 -0700 Subject: [PATCH 03/80] Update models.py --- registry/data-models/models.py | 142 +++++++++++++++++++-------------- 1 file changed, 84 insertions(+), 58 deletions(-) diff --git a/registry/data-models/models.py b/registry/data-models/models.py index 0b103d434..6cbeb55c8 100644 --- a/registry/data-models/models.py +++ b/registry/data-models/models.py @@ -1,5 +1,5 @@ from pydantic import BaseModel -from typing import List, Optional +from typing import List """ This file defines abstract backend data models for feature registry. @@ -8,46 +8,76 @@ For each feature registry provider/implementation, they will extend this abstract data models and backend API. Diagram of the data models: - +-----------------+ - | Project | - +-----------------+ - | - | - | - +------------------------------+ - | | - | 1:n | 1:n - +------------------+ +------------------+ - | FeatureInterface | | Anchor | - +------------------+ +------------------+ - ｜ ｜ | - ｜ 1:n ｜1:n | - +------------------------------+ | - ｜ | - ｜1:n | contains - +---------------+ +---------------+ - | Feature | --------- | Source | - +---------------+ contains +---------------+ - | - | - +----------------+ - | Transformation | - +----------------+ + +-----------------+ + | Project | + +-----------------+ + | + | + | + +------------------------------+ + | | + | 1:n | 1:n + +------------------+ +------------------+ + +---------- | FeatureName | | Anchor | + | 1:n +------------------+ +------------------+ + | ｜ ｜ | + | ｜ 1:n ｜1:n | + | +------------------------------+ | + | ｜ | + | ｜1:n \|/ has + | +--------------+ +---------------+ +----------------+ + | |Transformation|----| Feature | --------------- | DataSource | + | +--------------+ has+---------------+ +----------------+ + | /|\ | /|\ |extends + | | | | | + | | +----------------------|---------+ | + | | | | | + | +--------------------------------------+ | has| \|/ + | | | | +----------+ + | |extends |extends |has | Source | + | +----------------+ +-----------------+ +----------+ + | | DerivedFeature | | AnchorFeature | /|\ + | +----------------+ +-----------------+ | + | | | + | | |extends + | | +------------------+ + | +-------------------------------------------------------| FeatureSource | + | has +------------------+ + | | + +-----------------------------------------------------------------------------| + + + + + + """ -class FeatureValue(BaseModel): +class FeatureId(BaseModel): """ - A value of a Feature. - NOTE: FeatureValue is to be customized in each registry + Id for Feature, it's unique ID represents Feature. """ pass -class FeatureValueType(BaseModel): +class FeatureNameId(BaseModel): """ - A value of a Feature. - NOTE: FeatureValue is to be customized in each registry + Id for FeatureName, it's unique ID represents FeatureName. + """ + pass + + +class AnchorId(BaseModel): + """ + Id for Anchor, it's unique ID represents Anchor. + """ + pass + + +class ProjectId(BaseModel): + """ + Id for Project, it's unique ID represents Project. """ pass @@ -55,7 +85,7 @@ class FeatureValueType(BaseModel): class Source(BaseModel): """ Type of FeatureValue. - It defines where the feature is extracted from. It can be online, offline, Restli, or other FeatureInterfaces. + It defines where the feature is extracted from. """ pass @@ -65,7 +95,7 @@ class DataSource(Source): class FeatureSource(Source): - input_feature_interface_ids: List[str] # List of input feature interface IDs + input_feature_name_ids: List[FeatureNameId] # List of input feature name Keys pass @@ -73,51 +103,47 @@ class Transformation(BaseModel): """ The transformation of a Feature. A transformation function represents the transformation logic to produce feature value from the source of FeatureAnchor - NOTE: Transformation is to be customized in each registry """ pass class Feature(BaseModel): """ - Actual implementation of FeatureInterface. + Actual implementation of FeatureName. An implementation defines where a feature is extracted from (Source) and how it is computed (Transformation). - The Source of a feature can be raw data sources (like Rest.li, HDFS) and/or other features. + The Source of a feature can be raw data sources and/or other features. """ - feature_interface_id: str # ID of the feature interface that the feature belongs to - id: str - name: str + id: FeatureId # Unique ID for Feature + feature_name_id: FeatureNameId # Id of the feature name that the feature belongs to source: Source # Source can be either data source or feature source transformation: Transformation # transformation logic to produce feature value class AnchorFeature(Feature): """ - Feature implementation of FeatureInterface which anchored to a data source. + Feature implementation of FeatureName which anchored to a data source. """ source: DataSource class DerivedFeature(Feature): """ - Feature implementation that is derived from other FeatureInterfaces. + Feature implementation that is derived from other FeatureNames. """ source: FeatureSource -class FeatureInterface(BaseModel): +class FeatureName(BaseModel): """ Named Feature Interface of FeatureImplementations. - Each FeatureInterface is defined by feature producer and interact - by feature consumers.Each FeatureInterface enclosed attributes that don't change across - implementations. (eg. name, default value, value type, format, status, ownership). - One FeatureInterface can have multiple Features for different environments. + Each FeatureName is defined by feature producer and interact + by feature consumers.Each FeatureName enclosed attributes that don't change across + implementations. + One FeatureName can have multiple Features for different environments. """ - default_value: Optional[FeatureValue] # Optional default value - id: str - value_type: FeatureValueType # Feature value type - project_id: str - features: List[Feature] # List of features that the FeatureInterface has + id: FeatureNameId # unique ID for FeatureName, used to extract data for current FeatureName + project_id: ProjectId # ID of the project the FeatureName belongs to + features: List[FeatureId] # List of ids of feature that the FeatureName has class Project(BaseModel): @@ -125,9 +151,9 @@ class Project(BaseModel): Group of FeatureInterfaces. It can be a project the team is working on, or a namespace which related FeatureInterfaces have. """ - id: str - name: str - feature_interfaces: List[FeatureInterface] # List of feature interfaces that the project has + id: ProjectId # Unique ID of the project. + feature_names: List[FeatureNameId] # List of feature name ids that the project has + anchor_ids: List[AnchorId] # List of Anchor ids that the project has class Anchor(BaseModel): @@ -136,7 +162,7 @@ class Anchor(BaseModel): This is mainly used by feature producer gather information about DataSource and FeatureImplementations associated with the DataSource. """ - id: str - project_id: str # ID of Project that the anchor belongs to + id: AnchorId # Unique ID for Anchor + project_id: ProjectId # ID of Project that the anchor belongs to source: DataSource # data source of the Anchor - anchor_features: List[AnchorFeature] # List of anchor features that the anchor has + anchor_features: List[FeatureId] # List of anchor features that the anchor has From 5cc163d3480f6b7f40631103d6d589759663b34b Mon Sep 17 00:00:00 2001 From: Helen Yang Date: Tue, 20 Sep 2022 18:48:00 -0700 Subject: [PATCH 04/80] Update models.py --- registry/data-models/models.py | 63 ++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/registry/data-models/models.py b/registry/data-models/models.py index 6cbeb55c8..c59fbccc5 100644 --- a/registry/data-models/models.py +++ b/registry/data-models/models.py @@ -26,13 +26,13 @@ | ｜ | | ｜1:n \|/ has | +--------------+ +---------------+ +----------------+ - | |Transformation|----| Feature | --------------- | DataSource | + | |Transformation|----| Feature | | DataSource | | +--------------+ has+---------------+ +----------------+ - | /|\ | /|\ |extends - | | | | | - | | +----------------------|---------+ | - | | | | | - | +--------------------------------------+ | has| \|/ + | /|\ /|\ |extends + | | | | + | | | | + | | | | + | +--------------------------------------+ | \|/ | | | | +----------+ | |extends |extends |has | Source | | +----------------+ +-----------------+ +----------+ @@ -41,16 +41,10 @@ | | | | | |extends | | +------------------+ - | +-------------------------------------------------------| FeatureSource | + | +-------------------------------------------------------| FeaturesSource | | has +------------------+ | | - +-----------------------------------------------------------------------------| - - - - - - + +-----------------------------------------------------------------------------| """ @@ -58,43 +52,51 @@ class FeatureId(BaseModel): """ Id for Feature, it's unique ID represents Feature. """ - pass + id: str # id of a feature class FeatureNameId(BaseModel): """ Id for FeatureName, it's unique ID represents FeatureName. """ - pass + id: str # id of a FeatureName class AnchorId(BaseModel): """ Id for Anchor, it's unique ID represents Anchor. """ - pass + id: str # id of a anchor class ProjectId(BaseModel): """ Id for Project, it's unique ID represents Project. """ - pass + id: str # id of a project class Source(BaseModel): """ - Type of FeatureValue. - It defines where the feature is extracted from. + Source of the feature. + It defines where the feature is extracted or derived from. """ pass class DataSource(Source): + """ + Data source of the feature. + It defines the raw data source the feature is extracted from. + """ pass -class FeatureSource(Source): +class FeaturesSource(Source): + """ + Feature source of the feature. + It defines one of multiple features where the feature is derived from. + """ input_feature_name_ids: List[FeatureNameId] # List of input feature name Keys pass @@ -123,23 +125,24 @@ class AnchorFeature(Feature): """ Feature implementation of FeatureName which anchored to a data source. """ - source: DataSource + source: DataSource # Raw data source where the feature is extracted from class DerivedFeature(Feature): """ Feature implementation that is derived from other FeatureNames. """ - source: FeatureSource + source: FeaturesSource # Source features where the feature is derived from class FeatureName(BaseModel): """ - Named Feature Interface of FeatureImplementations. - Each FeatureName is defined by feature producer and interact - by feature consumers.Each FeatureName enclosed attributes that don't change across - implementations. - One FeatureName can have multiple Features for different environments. + Named Feature Interface that can be backed by multiple Feature implementations across + different environments accessing different sources (data lake access for batch training, + KV store access for online serving). Each FeatureName is defined by feature producer. + Feature consumers reference a feature by that name to access that feature data, + agnostic of runtime environment. Each FeatureName also encloses attributes that does not + change across implementations. """ id: FeatureNameId # unique ID for FeatureName, used to extract data for current FeatureName project_id: ProjectId # ID of the project the FeatureName belongs to @@ -148,8 +151,8 @@ class FeatureName(BaseModel): class Project(BaseModel): """ - Group of FeatureInterfaces. It can be a project the team is working on, - or a namespace which related FeatureInterfaces have. + Group of FeatureNames. It can be a project the team is working on, + or a namespace which related FeatureNames have. """ id: ProjectId # Unique ID of the project. feature_names: List[FeatureNameId] # List of feature name ids that the project has From 53dfb5e796034bd72b8290f39611a3335b67e716 Mon Sep 17 00:00:00 2001 From: Helen Yang Date: Wed, 21 Sep 2022 10:16:27 -0700 Subject: [PATCH 05/80] Update models.py --- registry/data-models/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/registry/data-models/models.py b/registry/data-models/models.py index c59fbccc5..92f7f9004 100644 --- a/registry/data-models/models.py +++ b/registry/data-models/models.py @@ -146,7 +146,7 @@ class FeatureName(BaseModel): """ id: FeatureNameId # unique ID for FeatureName, used to extract data for current FeatureName project_id: ProjectId # ID of the project the FeatureName belongs to - features: List[FeatureId] # List of ids of feature that the FeatureName has + feature_ids: List[FeatureId] # List of ids of feature that the FeatureName has class Project(BaseModel): @@ -155,7 +155,7 @@ class Project(BaseModel): or a namespace which related FeatureNames have. """ id: ProjectId # Unique ID of the project. - feature_names: List[FeatureNameId] # List of feature name ids that the project has + feature_name_ids: List[FeatureNameId] # List of feature name ids that the project has anchor_ids: List[AnchorId] # List of Anchor ids that the project has @@ -168,4 +168,4 @@ class Anchor(BaseModel): id: AnchorId # Unique ID for Anchor project_id: ProjectId # ID of Project that the anchor belongs to source: DataSource # data source of the Anchor - anchor_features: List[FeatureId] # List of anchor features that the anchor has + anchor_feature_ids: List[FeatureId] # List of anchor features that the anchor has From 9d9af5f9621174955a49a268011c6b1c31d39367 Mon Sep 17 00:00:00 2001 From: Helen Yang Date: Wed, 21 Sep 2022 10:16:27 -0700 Subject: [PATCH 06/80] Update models.py --- registry/data-models/data-model-diagram.md | 59 ++++++++++++++++++++++ registry/data-models/models.py | 52 +++---------------- 2 files changed, 66 insertions(+), 45 deletions(-) create mode 100644 registry/data-models/data-model-diagram.md diff --git a/registry/data-models/data-model-diagram.md b/registry/data-models/data-model-diagram.md new file mode 100644 index 000000000..31e3fbff7 --- /dev/null +++ b/registry/data-models/data-model-diagram.md @@ -0,0 +1,59 @@ + +# Feathr Abstract backend Data Model Diagram + +This file defines abstract backend data models diagram for feature registry. +[Python Code](./models.py) + +```mermaid +classDiagram + Project "1" --> "n" FeatureName : contains + Project "1" --> "n" Anchor : contains + FeatureName "1" --> "n" Feature : contains + Anchor "1" --> "n" Feature : contains + Feature <|-- AnchorFeature : extends + Feature <|-- DerivedFeature: extends + Feature --> Transformation + Feature --> Transformation : contains + Source <|-- DataSource: extends + Source <|-- FeatureSource: extends + AnchorFeature --> DataSource : contains + DerivedFeature "1" --> "1..n" FeatureSource: contains + + class Source{ + } + + class DataSource{ + } + + class FeatureSource{ + +feature_name_id FeatureNameId + } + class Feature{ + +FeatureId id + +FeatureNameId feature_namme_id + +Source source + +Transformation transformation + } + class AnchorFeature{ + +DataSource source + } + class DerivedFeature{ + +List[FeatureSource] source + } + class FeatureName{ + +FeatureNameId id + +ProjectId project_id + +List[FeatureId] feature_ids + } + class Project{ + +ProjectId id + +List[FeatureNameId] feature_name_ids + +List[AnchorId] anchor_ids + } + class Anchor{ + +AnchorId id + +ProjectId project_id + +DataSource source + +List[FeatureId] anchor_feature_ids + } +``` \ No newline at end of file diff --git a/registry/data-models/models.py b/registry/data-models/models.py index c59fbccc5..c50b6a1c0 100644 --- a/registry/data-models/models.py +++ b/registry/data-models/models.py @@ -7,44 +7,7 @@ Purpose of this is to decouple backend data models from API specific data models. For each feature registry provider/implementation, they will extend this abstract data models and backend API. -Diagram of the data models: - +-----------------+ - | Project | - +-----------------+ - | - | - | - +------------------------------+ - | | - | 1:n | 1:n - +------------------+ +------------------+ - +---------- | FeatureName | | Anchor | - | 1:n +------------------+ +------------------+ - | ｜ ｜ | - | ｜ 1:n ｜1:n | - | +------------------------------+ | - | ｜ | - | ｜1:n \|/ has - | +--------------+ +---------------+ +----------------+ - | |Transformation|----| Feature | | DataSource | - | +--------------+ has+---------------+ +----------------+ - | /|\ /|\ |extends - | | | | - | | | | - | | | | - | +--------------------------------------+ | \|/ - | | | | +----------+ - | |extends |extends |has | Source | - | +----------------+ +-----------------+ +----------+ - | | DerivedFeature | | AnchorFeature | /|\ - | +----------------+ +-----------------+ | - | | | - | | |extends - | | +------------------+ - | +-------------------------------------------------------| FeaturesSource | - | has +------------------+ - | | - +-----------------------------------------------------------------------------| +Diagram of the data models: """ @@ -89,15 +52,14 @@ class DataSource(Source): Data source of the feature. It defines the raw data source the feature is extracted from. """ - pass class FeaturesSource(Source): """ Feature source of the feature. - It defines one of multiple features where the feature is derived from. + It defines one of features where the feature is derived from. """ - input_feature_name_ids: List[FeatureNameId] # List of input feature name Keys + input_feature_name_id: FeatureNameId # Input feature name Keys pass @@ -132,7 +94,7 @@ class DerivedFeature(Feature): """ Feature implementation that is derived from other FeatureNames. """ - source: FeaturesSource # Source features where the feature is derived from + source: List[FeaturesSource] # Source features where the feature is derived from class FeatureName(BaseModel): @@ -146,7 +108,7 @@ class FeatureName(BaseModel): """ id: FeatureNameId # unique ID for FeatureName, used to extract data for current FeatureName project_id: ProjectId # ID of the project the FeatureName belongs to - features: List[FeatureId] # List of ids of feature that the FeatureName has + feature_ids: List[FeatureId] # List of ids of feature that the FeatureName has class Project(BaseModel): @@ -155,7 +117,7 @@ class Project(BaseModel): or a namespace which related FeatureNames have. """ id: ProjectId # Unique ID of the project. - feature_names: List[FeatureNameId] # List of feature name ids that the project has + feature_name_ids: List[FeatureNameId] # List of feature name ids that the project has anchor_ids: List[AnchorId] # List of Anchor ids that the project has @@ -168,4 +130,4 @@ class Anchor(BaseModel): id: AnchorId # Unique ID for Anchor project_id: ProjectId # ID of Project that the anchor belongs to source: DataSource # data source of the Anchor - anchor_features: List[FeatureId] # List of anchor features that the anchor has + anchor_feature_ids: List[FeatureId] # List of anchor features that the anchor has From 5358bd90dce2d3d8a142dd5e3ca8af6377ec213a Mon Sep 17 00:00:00 2001 From: Helen Yang Date: Thu, 20 Oct 2022 16:24:48 -0700 Subject: [PATCH 07/80] Add attributes to data models Add data attributes to data models --- registry/data-models/common/__init__.py | 0 registry/data-models/common/models.py | 139 ++++++++++++++ registry/data-models/data-model-diagram.md | 181 +++++++++++++++++- registry/data-models/models.py | 28 +-- .../data-models/transformation/__init__.py | 0 registry/data-models/transformation/models.py | 84 ++++++++ 6 files changed, 411 insertions(+), 21 deletions(-) create mode 100644 registry/data-models/common/__init__.py create mode 100644 registry/data-models/common/models.py create mode 100644 registry/data-models/transformation/__init__.py create mode 100644 registry/data-models/transformation/models.py diff --git a/registry/data-models/common/__init__.py b/registry/data-models/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/registry/data-models/common/models.py b/registry/data-models/common/models.py new file mode 100644 index 000000000..846d2cc6a --- /dev/null +++ b/registry/data-models/common/models.py @@ -0,0 +1,139 @@ +from pydantic import BaseModel +from typing import Dict, Optional, List, Union +import json +from enum import Enum + + +class ValueType(Enum): + """ + Type of the feature. + """ + INT = "int" + LONG = "long" + FLOAT = "float" + DOUBLE = "double" + STRING = "string" + BOOLEAN = "boolean" + BYTES = "bytes" + + +class DimensionType(Enum): + """ + Supported dimension types for tensors in Feathr. + """ + INT = "int" + LONG = "long" + STRING = "string" + BOOLEAN = "boolean" + BYTES = "bytes" + + +class TensorCategory(Enum): + """ + Supported Tensor categories in Feathr. + """ + DENSE = "dense" # Dense tensors store values in a contiguous sequential block of memory where all values are represented. + SPARSE = "sparse" # Sparse tensor represents a dataset in which most of the entries are zero. + RAGGED = "ragged" # Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions. + + +class FeatureValueType(Enum): + """ + The high level types associated with a feature. + This represents the high level semantic types supported by early versions of Frame. + """ + BOOLEAN = "boolean" # Boolean valued feature + NUMERIC = "numeric" # Numerically valued feature + CATEGORICAL = "categorical" # Represent a feature that consists of a single category + CATEGORICAL_SET = "categorical_set" # Represent a feature that consists of multiple categories + DENSE_VECTOR = "dense_vector" # Represent a feature in vector format where the majority of the elements are non-zero + TERM_VECTOR = "term_vector" # Represent features that has string terms and numeric value + TENSOR = "tensor" # Represent tensor based features. + UNSPECIFIED = "unspecified" # Placeholder for when no types are specified + + +class Dimension(BaseModel): + """ + Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. + """ + type: DimensionType # Type of the dimension in the tensor. Each dimension can have a different type. + shape: Optional[int] # Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime. + + +class TensorFeatureFormat(BaseModel): + """ + Defines the format of feature data. Feature data is produced by applying transformation on source, in a Feature. + Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially + higher dimensions. + """ + tensorCategory: TensorCategory # Type of the tensor. + valueType: ValueType # Type of the value column. + dimensions: List[Dimension] # A feature data can have zero or more dimensions (columns that represent keys). + + +class FeatureType(BaseModel): + """ + Information about a featureName. It defines the type, format and default value. + Tensor is the next generation representation of the features, so using + Tensor type w TensorFeatureFormat would be preferable FeatureType. + """ + type: FeatureValueType # Defines the high level semantic type of feature. + format: Optional[TensorFeatureFormat] # Defines the format of feature data. + defaultValue: Union[bool, int, float, str, bytes] + + +class Clazz(BaseModel): + """ + Reference to a class by fully-qualified name + """ + fullyQualifiedName: str # A fully-qualified class name including paths. + + +class Function(BaseModel): + """ + Base model for all functions + """ + expression: str # Expression in str format + + +class MvelExpression(Function): + """ + An expression in MVEL language. + """ + mvel: str # The MVEL expression + + +class UserDefinedFunction(Function): + """ + User defined function that can be used in feature extraction or derivation. + """ + clazz: Clazz # Reference to the class that implements the user defined function. + parameters: Dict[str, json] = {} # This field defines the custom parameters of the user defined function + + +class SparkSqlExpression(Function): + """ + An expression in Spark SQL. + """ + sql: str # Spark SQl expression + + +class SemanticVersion(BaseModel): + """ + A representation of a semantic version (see https://semver.org/) + """ + majorVersion: int # The major version of this version. This is the x in x.y.z. + minorVersion: int # The minor version of this version. This is the y in x.y.z + patchVersion: int # The patch version of this version. This is the z in x.y.z + metadata: Optional[str] # Optional build metadata attached to this version. + + +class FeathrModel(BaseModel): + """ + Base model for feathr entity which will be displayed in Feathr UI + """ + displayName: str # name of the entity showed on UI + typeName: str # type of entity in str format, will be displayed in UI + # Represents zero, one or multiple keyPlaceholderRefs which are used as the + # identifiers to reference KeyPlaceholders of the FeatureSource + keyPlaceholderRefs: List[str] = [] diff --git a/registry/data-models/data-model-diagram.md b/registry/data-models/data-model-diagram.md index e43ffa0af..e2cf59c54 100644 --- a/registry/data-models/data-model-diagram.md +++ b/registry/data-models/data-model-diagram.md @@ -6,30 +6,188 @@ This file defines abstract backend data models diagram for feature registry. ```mermaid classDiagram - Project "1" --> "n" FeatureName : contains - Project "1" --> "n" Anchor : contains - FeatureName "1" --> "n" Feature : contains - Anchor "1" --> "n" Feature : contains - Feature <|-- AnchorFeature : extends + Project "1" --> "n" FeatureName: contains + Project "1" --> "n" Anchor: contains + FeatureName "1" --> "n" Feature: contains + FeatureName --> "Optional" SemanticVersion: contains + FeatureName --> "Optional" FeatureType: contains + Anchor "1" --> "n" AnchorFeature: contains + Anchor --> DataSource: contains + Feature <|-- AnchorFeature: extends Feature <|-- DerivedFeature: extends - Feature --> Transformation - Feature --> Transformation : contains + Feature --> Transformation: contains + Feature --> Source: contains + Transformation --> Function: contains Source <|-- DataSource: extends + DataSource --> "Optional" Clazz: contains + DataSource --> "Optional" Function: contains Source <|-- MultiFeatureSource: extends MultiFeatureSource "1" --> "1..n" FeatureSource: contains - AnchorFeature --> DataSource : contains + AnchorFeature --> DataSource: contains DerivedFeature --> MultiFeatureSource: contains - + FeathrModel <|-- Project: extends + FeathrModel <|-- FeatureName: extends + FeathrModel <|-- Anchor: extends + FeathrModel <|-- Feature: extends + FeathrModel <|-- Source: extends + Dimension --> DimensionType: contains + TensorFeatureFormat --> TensorCategory: contains + TensorFeatureFormat --> ValueType: contains + TensorFeatureFormat "1" --> "1..n" Dimension: contains + FeatureType --> FeatureValueType: contains + FeatureType --> "Optional" TensorFeatureFormat: contains + Window --> WindowTimeUnit: contains + Function <|-- MvelExpression: extends + Function <|-- UserDefinedFunction: extends + Function <|-- SparkSqlExpression: extends + SlidingWindowAggregation --> SparkSqlExpression: contains + SlidingWindowAggregation --> SlidingWindowAggregationType: contains + SlidingWindowAggregation --> Window: contains + SlidingWindowEmbeddingAggregation --> SparkSqlExpression: contains + SlidingWindowEmbeddingAggregation --> SlidingWindowEmbeddingAggregationType: contains + SlidingWindowEmbeddingAggregation --> Window: contains + SlidingWindowLatestAvailable --> SparkSqlExpression: contains + SlidingWindowLatestAvailable --> Window: contains + Function <|-- SlidingWindowAggregation: extends + Function <|-- SlidingWindowEmbeddingAggregation: extends + Function <|-- SlidingWindowLatestAvailable: extends + + class ValueType{ + <> + INT + LONG + FLOAT + DOUBLE + STRING + BOOLEAN + BYTES + } + class DimensionType{ + <> + INT + LONG + STRING + BOOLEAN + BYTES + } + class TensorCategory{ + <> + DENSE + SPARSE + RAGGED + } + class FeatureValueType{ + <> + BOOLEAN + NUMERIC + CATEGORICAL + CATEGORICAL_SET + DENSE_VECTOR + TERM_VECTOR + TENSOR + UNSPECIFIED + } + class Dimension{ + +DimensionType type + +Optional[str] shape + } + class TensorFeatureFormat{ + +TensorCategory tensorCategory + +ValueType valueType + +List[Dimension] dimensions + } + class FeatureType{ + +FeatureValueType type + +Optional[TensorFeatureFormat] format + +Union[bool, int, float, str, types] defaultValue + } + class Clazz{ + +str fullyQualifiedName + } + class Function{ + +str expression + } + class MvelExpression{ + +str mvel + } + class UserDefinedFunction{ + +str sql + } + class SemanticVersion{ + +int majorVersion + +int minorVersion + +int patchVersion + +Optional[str] metadata + } + class FeathrModel{ + +str displayName + +str typeName + +List[str] keyPlaceholderRefs + } + class SlidingWindowAggregationType{ + <> + SUM + COUNT + MAX + MIN + AVG + } + class SlidingWindowEmbeddingAggregationType{ + <> + MAX_POOLING + MIN_POOLING + AVG_POOLING + } + class WindowTimeUnit{ + <> + DAY + HOUR + MINUTE + SECOND + } + class Window{ + +int size + +WindowTimeUnit unit + } + class SlidingWindowAggregation{ + +SlidingWindowAggregationType aggregationType + +Window window + +SparkSqlExpression targetColumn + +Optional[SparkSqlExpression] filter + +Optional[SparkSqlExpression] groupBy + +Optional[int] limit + } + class SlidingWindowEmbeddingAggregation{ + +SlidingWindowEmbeddingAggregationType aggregationType + +Window window + +SparkSqlExpression targetColumn + +Optional[SparkSqlExpression] filter + +Optional[SparkSqlExpression] groupBy + } + class SlidingWindowLatestAvailable{ + +Optional[Window] window + +SparkSqlExpression targetColumn + +Optional[SparkSqlExpression] filter + +Optional[SparkSqlExpression] groupBy + +Optional[int] limit + } class Source{ + +Optional[str] sourceRef } class DataSource{ + +Optional[Clazz] clazz + +Optional[Function] keyFunction } class FeatureSource{ - +FeatureNameId feature_name_id + +FeatureNameId input_feature_name_id + +Optional[str] alias } class MultiFeatureSource{ +List[FeatureSource] sources } + class Transformation{ + +Function transformationFunction + } class Feature{ +FeatureId id +FeatureNameId feature_namme_id @@ -37,6 +195,7 @@ classDiagram +Transformation transformation } class AnchorFeature{ + +AnchorId anchor_id +DataSource source } class DerivedFeature{ @@ -46,6 +205,8 @@ classDiagram +FeatureNameId id +ProjectId project_id +List[FeatureId] feature_ids + +Optional[SemanticVersion] semanticVersion + +Optional[FeatureType] featureType } class Project{ +ProjectId id diff --git a/registry/data-models/models.py b/registry/data-models/models.py index c4ae31f68..a6e7fa2b1 100644 --- a/registry/data-models/models.py +++ b/registry/data-models/models.py @@ -1,12 +1,13 @@ +from registry.data-models.transformation.models import * +from registry.data-models.common.models import SemanticVersion, FeathrModel, Function +from typing import Optional from pydantic import BaseModel -from typing import List + """ This file defines abstract backend data models for feature registry. Backend data models will be used by backend API server to talk to feature registry backend. Purpose of this is to decouple backend data models from API specific data models. -For each feature registry provider/implementation, they will extend this abstract -data models and backend API. Diagram of the data models: ./data-model-diagram.md """ @@ -43,12 +44,12 @@ class ProjectId(BaseModel): id: str # id of a project -class Source(BaseModel): +class Source(FeathrModel): """ Source of the feature. It defines where the feature is extracted or derived from. """ - pass + sourceRef: Optional[str] # Represents the identifier of a Source object. class DataSource(Source): @@ -56,7 +57,8 @@ class DataSource(Source): Data source of the feature. It defines the raw data source the feature is extracted from. """ - pass + clazz: Optional[Clazz] # Fully qualified Java class name for data model + keyFunction: Optional[Function] class FeatureSource(BaseModel): @@ -65,6 +67,7 @@ class FeatureSource(BaseModel): creating other derived features. """ input_feature_name_id: FeatureNameId # Input feature name Key + alias: Optional[str] # A feature's alias to be used in transformation function. class MultiFeatureSource(Source): @@ -81,10 +84,10 @@ class Transformation(BaseModel): The transformation of a Feature. A transformation function represents the transformation logic to produce feature value from the source of FeatureAnchor """ - pass + transformationFunction: Function -class Feature(BaseModel): +class Feature(FeathrModel): """ Actual implementation of FeatureName. An implementation defines where a feature is extracted from (Source) and how it is computed (Transformation). @@ -100,6 +103,7 @@ class AnchorFeature(Feature): """ Feature implementation of FeatureName which anchored to a data source. """ + anchor_id: AnchorId # ID of the anchor this feature belongs to source: DataSource # Raw data source where the feature is extracted from @@ -110,7 +114,7 @@ class DerivedFeature(Feature): source: MultiFeatureSource # Source features where the feature is derived from -class FeatureName(BaseModel): +class FeatureName(FeathrModel): """ Named Feature Interface that can be backed by multiple Feature implementations across different environments accessing different sources (data lake access for batch training, @@ -122,9 +126,11 @@ class FeatureName(BaseModel): id: FeatureNameId # unique ID for FeatureName, used to extract data for current FeatureName project_id: ProjectId # ID of the project the FeatureName belongs to feature_ids: List[FeatureId] # List of ids of feature that the FeatureName has + semanticVersion: Optional[SemanticVersion] # Semantic version associated with this FeatureName + featureType: Optional[FeatureType] # Information about featureName, like feature type, format and value. -class Project(BaseModel): +class Project(FeathrModel): """ Group of FeatureNames. It can be a project the team is working on, or a namespace which related FeatureNames have. @@ -134,7 +140,7 @@ class Project(BaseModel): anchor_ids: List[AnchorId] # List of Anchor ids that the project has -class Anchor(BaseModel): +class Anchor(FeathrModel): """ Group of AnchorFeatures which anchored on same DataSource. This is mainly used by feature producer gather information about DataSource diff --git a/registry/data-models/transformation/__init__.py b/registry/data-models/transformation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/registry/data-models/transformation/models.py b/registry/data-models/transformation/models.py new file mode 100644 index 000000000..a8e740447 --- /dev/null +++ b/registry/data-models/transformation/models.py @@ -0,0 +1,84 @@ +from registry.data-models.common.models import * +from typing import Optional + + +class SlidingWindowAggregationType(Enum): + """ + Represents supported types of aggregation. + """ + SUM = "sum" + COUNT = "count" + MAX = "maximum" + MIN = "minium" + AVG = "average" + + +class SlidingWindowEmbeddingAggregationType(Enum): + """ + Represents supported types for embedding aggregation. + Pooling is a sample-based discretization process. The objective is to down-sample an input + representation and reduce its dimensionality. + """ + MAX_POOLING = "max_pooling" # Max pooling is done by applying a max filter to (usually) non-overlapping subregions of the initial representation + MIN_POOLING = "min_pooling" # Min pooling is done by applying a min filter to (usually) non-overlapping subregions of the initial representation. + AVG_POOLING = "avg_pooling" # Average pooling is done by applying a average filter to (usually) non-overlapping subregions of the initial representation + + +class WindowTimeUnit(Enum): + """ + Represents a unit of time. + """ + DAY = "day" + HOUR = "hour" + MINUTE = "minute" + SECOND = "second" + + +class Window(BaseModel): + """ + Represents a time window used in sliding window algorithms. + """ + size: int # Represents the duration of the window. + unit: WindowTimeUnit + + +class SlidingWindowAggregation(Function): + """ + Sliding window aggregation produces feature data by aggregating a collection of data within a given time + interval into an aggregate value. It ensures point-in-time correctness, when joining with label data, + it looks back the configurable time window from each entry's timestamp and compute the aggregate value. + This class can be extended to support LateralView in aggregation. + """ + aggregationType: SlidingWindowAggregationType # Represents supported types of aggregation. + window: Window # Represents the time window to look back from label data's timestamp. + targetColumn: SparkSqlExpression # The target column to perform aggregation against. + filter: Optional[SparkSqlExpression] # Represents the filter statement before the aggregation. + groupBy: Optional[SparkSqlExpression] # Represents the target to be grouped by before aggregation. + limit: Optional[int] # Represents the max number of groups (with aggregation results) to return. + + +class SlidingWindowEmbeddingAggregation(Function): + """ + Sliding window embedding aggregation produces a single embedding by performing element-wise operations or + discretion on a collection of embeddings within a given time interval. It ensures point-in-time correctness, + when joining with label data, Frame looks back the configurable time window from each entry's timestamp and produce + the aggregated embedding. + """ + aggregationType: SlidingWindowEmbeddingAggregationType # Represents supported types for embedding aggregation. + window: Window # Represents the time window to look back from label data's timestamp. + targetColumn: SparkSqlExpression # The target column to perform aggregation against. + filter: Optional[SparkSqlExpression] # Represents the filter statement before the aggregation. + groupBy: Optional[SparkSqlExpression] # Represents the target to be grouped by before aggregation. + + +class SlidingWindowLatestAvailable(Function): + """ + This sliding window algorithm picks the latest available feature data from the source data. + Note the latest here means event time instead of processing time. + This class can be extended to support LateralView in aggregation. + """ + window: Optional[Window] # Represents the time window to look back from label data's timestamp. + targetColumn: SparkSqlExpression # The target column to perform aggregation against. + filter: Optional[SparkSqlExpression] # Represents the filter statement before the aggregation. + groupBy: Optional[SparkSqlExpression] # Represents the target to be grouped by before aggregation. + limit: Optional[int] # Represents the max number of groups (with aggregation results) to return. From 4b05e534eb53d1476eb4d99708c825609a1e7a68 Mon Sep 17 00:00:00 2001 From: Chang Yong Lik <51813538+ahlag@users.noreply.github.com> Date: Wed, 14 Sep 2022 00:40:00 +0900 Subject: [PATCH 08/80] Added _scproxy necessary for MacOS (#651) * Added _scproxy necessary for MacOS Signed-off-by: changyonglik * Changed to conditional import Signed-off-by: changyonglik * Added comments Signed-off-by: changyonglik Signed-off-by: changyonglik --- registry/sql-registry/registry/database.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/registry/sql-registry/registry/database.py b/registry/sql-registry/registry/database.py index 39bab8ec4..21b8a2aca 100644 --- a/registry/sql-registry/registry/database.py +++ b/registry/sql-registry/registry/database.py @@ -3,6 +3,13 @@ import logging import threading import os + +# Checks if the platform is Max (Darwin). +# If so, imports _scproxy that is necessary for pymssql to work on MacOS +import platform +if platform.system().lower().startswith('dar'): + import _scproxy + import pymssql @@ -53,7 +60,7 @@ def __init__(self, params): self.params = params self.make_connection() self.mutex = threading.Lock() - + def make_connection(self): self.conn = pymssql.connect(**self.params) @@ -85,10 +92,10 @@ def transaction(self): """ Start a transaction so we can run multiple SQL in one batch. User should use `with` with the returned value, look into db_registry.py for more real usage. - + NOTE: `self.query` and `self.execute` will use a different MSSQL connection so any change made in this transaction will *not* be visible in these calls. - + The minimal implementation could look like this if the underlying engine doesn't support transaction. ``` @contextmanager @@ -125,4 +132,4 @@ def connect(*args, **kargs): ret = p.connect(*args, **kargs) if ret is not None: return ret - raise RuntimeError("Cannot connect to database") \ No newline at end of file + raise RuntimeError("Cannot connect to database") From 6d930ea1f0e37820c441065017d2ad30adfc9d27 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Tue, 13 Sep 2022 21:16:04 -0700 Subject: [PATCH 09/80] Add docs for consuming features in online environment (#609) * Create consume-features.md * Update consume-features.md * rename docs * Update model-inference-with-feathr.md * Update README.md * update docs per feedback * Update streaming-source-ingestion.md * update docs * update docs * Update azure-deployment-arm.md * Update model-inference-with-feathr.md * add sign off message Signed-off-by: Xiaoyong Zhu xiaoyzhu@outlook.com * fix comments * Delete deploy-feathr-api-as-webapp.md * Update model-inference-with-feathr.md Signed-off-by: Xiaoyong Zhu xiaoyzhu@outlook.com --- docs/README.md | 30 ++--- ...d-and-push-feathr-registry-docker-image.md | 34 ++++- docs/dev_guide/cloud_resource_provision.md | 4 +- docs/dev_guide/deploy-feathr-api-as-webapp.md | 122 ------------------ docs/dev_guide/feathr-core-code-structure.md | 2 +- docs/how-to-guides/azure-deployment-arm.md | 5 +- .../azure_resource_provision.json | 4 +- .../model-inference-with-feathr.md | 56 ++++++++ .../streaming-source-ingestion.md | 18 +-- 9 files changed, 115 insertions(+), 160 deletions(-) delete mode 100644 docs/dev_guide/deploy-feathr-api-as-webapp.md create mode 100644 docs/how-to-guides/model-inference-with-feathr.md diff --git a/docs/README.md b/docs/README.md index 958da9b49..2d98def0a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -170,26 +170,26 @@ Follow the [quick start Jupyter Notebook](./samples/product_recommendation_demo. ![Architecture Diagram](./images/architecture.png) -| Feathr component | Cloud Integrations | -| ------------------------------- | --------------------------------------------------------------------------- | -| Offline store – Object Store | Azure Blob Storage, Azure ADLS Gen2, AWS S3 | -| Offline store – SQL | Azure SQL DB, Azure Synapse Dedicated SQL Pools, Azure SQL in VM, Snowflake | -| Streaming Source | Kafka, EventHub | -| Online store | Redis, Azure Cosmos DB (coming soon), Aerospike (coming soon) | -| Feature Registry and Governance | Azure Purview, ANSI SQL such as Azure SQL Server | -| Compute Engine | Azure Synapse Spark Pools, Databricks | -| Machine Learning Platform | Azure Machine Learning, Jupyter Notebook, Databricks Notebook | -| File Format | Parquet, ORC, Avro, JSON, Delta Lake, CSV | -| Credentials | Azure Key Vault | +| Feathr component | Cloud Integrations | +| ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| Offline store – Object Store | Azure Blob Storage, Azure ADLS Gen2, AWS S3 | +| Offline store – SQL | Azure SQL DB, Azure Synapse Dedicated SQL Pools, Azure SQL in VM, Snowflake | +| Streaming Source | Kafka, EventHub | +| Online store | Redis, [Azure Cosmos DB](https://feathr-ai.github.io/feathr/how-to-guides/jdbc-cosmos-notes.html#using-cosmosdb-as-the-online-store), Aerospike (coming soon) | +| Feature Registry and Governance | Azure Purview, ANSI SQL such as Azure SQL Server | +| Compute Engine | Azure Synapse Spark Pools, Databricks | +| Machine Learning Platform | Azure Machine Learning, Jupyter Notebook, Databricks Notebook | +| File Format | Parquet, ORC, Avro, JSON, Delta Lake, CSV | +| Credentials | Azure Key Vault | ## 🚀 Roadmap -For a complete roadmap with estimated dates, please [visit this page](https://github.com/linkedin/feathr/milestones?direction=asc&sort=title&state=open). - -- [x] Support streaming -- [x] Support common data sources +- [x] Support streaming features with transformation +- [x] Support common data sources and sinks. Read more in the [Cloud Integrations and Architecture](#️-cloud-integrations-and-architecture) part. - [x] Support feature store UI, including Lineage and Search functionalities +- [ ] Support a sandbox Feathr environment for better getting started experience - [ ] Support online transformation +- [ ] More Feathr online client libraries such as Java - [ ] Support feature versioning - [ ] Support feature monitoring - [ ] Support feature data deletion and retention diff --git a/docs/dev_guide/build-and-push-feathr-registry-docker-image.md b/docs/dev_guide/build-and-push-feathr-registry-docker-image.md index 034b502df..873c6a141 100644 --- a/docs/dev_guide/build-and-push-feathr-registry-docker-image.md +++ b/docs/dev_guide/build-and-push-feathr-registry-docker-image.md @@ -6,7 +6,7 @@ parent: Developer Guides # How to build and push feathr registry docker image -This doc shows how to build feathr registry docker image locally and publish to registry. +This doc shows how to build feathr registry docker image locally and publish to DockerHub. ## Prerequisites @@ -28,32 +28,52 @@ Run **docker images** command, you will see newly created image listed in output docker images ``` -Run **docker run** command to test docker image locally: +Run **docker run** command to test docker image locally. + +### Test SQL-based registry + +You need to setup the connection string `CONNECTION_STR` for the docker container, so that it knows which SQL-based registry is connected to. The connection string will be something like this: + +```bash +"Server=tcp:testregistry.database.windows.net,1433;Initial Catalog=testsql;Persist Security Info=False;User ID=feathr@feathrtestsql;Password=StrongPassword;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" +``` + +Then you can test the docker locally by running this command: -### Test SQL registry ```bash docker run --env CONNECTION_STR= --env API_BASE=api/v1 -it --rm -p 3000:80 feathrfeaturestore/sql-registry ``` ### Test Purview registry + +You need to setup a few environment variables, include: + +- `PURVIEW_NAME` indicates the Purview service name +- `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_CLIENT_SECRET` indicates the service principal account to talk with Purview service. + ```bash docker run --env PURVIEW_NAME= --env AZURE_CLIENT_ID= --env AZURE_TENANT_ID= --env AZURE_CLIENT_SECRET= --env API_BASE=api/v1 -it --rm -p 3000:80 feathrfeaturestore/feathr-registry ``` ### Test SQL registry + RBAC + ```bash docker run --env REACT_APP_ENABLE_RBAC=true --env REACT_APP_AZURE_CLIENT_ID= --env REACT_APP_AZURE_TENANT_ID= --env CONNECTION_STR= --env API_BASE=api/v1 -it --rm -p 3000:80 feathrfeaturestore/feathr-registry ``` -After docker image launched, open web browser and navigate to ，verify both UI and backend api can work correctly. +After docker image launched, open web browser and navigate to ，verify both the Feathr UI and the registry backend (SQL/Purview) can work correctly. + +## Upload to DockerHub (For Feathr Release Manager) -## Upload to DockerHub Registry +The Feathr repository already have automatic CD pipelines to publish the docker image to DockerHub on release branches. Please checkout [docker publish workflow](https://github.com/feathr-ai/feathr/blob/main/.github/workflows/docker-publish.yml) for details -Login with feathrfeaturestore account and then run **docker push** command to publish docker image to DockerHub. Contact Feathr Team (@jainr, @blrchen) for credentials. +In case if the Feathr release manager wants to do it manually, login with feathrfeaturestore account and then run **docker push** command to publish docker image to DockerHub. Contact Feathr Team (@jainr, @blrchen) for credentials. ```bash docker login -docker push feathrfeaturestore/sql-registry +docker push feathrfeaturestore/feathr-registry ``` +## Published Feathr Registry Image +The published feathr feature registry is located in [DockerHub here](https://hub.docker.com/r/feathrfeaturestore/feathr-registry). \ No newline at end of file diff --git a/docs/dev_guide/cloud_resource_provision.md b/docs/dev_guide/cloud_resource_provision.md index 8ac07ac43..033030694 100644 --- a/docs/dev_guide/cloud_resource_provision.md +++ b/docs/dev_guide/cloud_resource_provision.md @@ -29,12 +29,12 @@ Invoke Deployment Script from GitHub Repo with parameter for Azure Region. Available regions can be checked with this command ```powershell - Get-AzLocation | select displayname,location +Get-AzLocation | select displayname,location ``` ```powershell - iwr https://raw.githubusercontent.com/linkedin/feathr/main/docs/how-to-guides/deployFeathr.ps1 -outfile ./deployFeathr.ps1; ./deployFeathr.ps1 -AzureRegion '{Assign Your Region}' +iwr https://raw.githubusercontent.com/linkedin/feathr/main/docs/how-to-guides/deployFeathr.ps1 -outfile ./deployFeathr.ps1; ./deployFeathr.ps1 -AzureRegion '{Assign Your Region}' ``` diff --git a/docs/dev_guide/deploy-feathr-api-as-webapp.md b/docs/dev_guide/deploy-feathr-api-as-webapp.md deleted file mode 100644 index a25abf817..000000000 --- a/docs/dev_guide/deploy-feathr-api-as-webapp.md +++ /dev/null @@ -1,122 +0,0 @@ ---- -layout: default -title: Feathr REST API Deployment -parent: Developer Guides ---- - -# Feathr REST API - -The REST API currently supports following functionalities: - -1. Get Feature by Qualified Name -2. Get Feature by GUID -3. Get List of Features -4. Get Lineage for a Feature - -## Build and run locally - -### Install - -**NOTE:** You can run the following command in your local python environment or in your Azure Virtual machine. -You can install dependencies through the requirements file - -```bash -pip install -r requirements.txt -``` - -### Run - -This command will start the uvicorn server locally and will dynamically load your changes. - -```bash -uvicorn api:app --port 8080 --reload -``` - -## Build and deploy on Azure - -Here are the steps to build the API as a docker container, push it to Azure Container registry and then deploy it as webapp. The instructions below are for Mac/Linux but should work on Windows too. You might have to use sudo command or run docker as administrator on windows if you don't have right privileges. - -1. Install Azure CLI by following instructions [here](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest) - -1. Create Azure Container Registry. First create the resource group. - - ```bash - az group create --name --location - ``` - - Then create the container registry - - ```bash - az acr create --resource-group --name --sku Basic - ``` - -1. Login to your Azure container registry (ACR) account. - - ```bash - $ az acr login --name - ``` - -1. Clone the repository and navigate to api folder - - ```bash - $ git clone git@github.com:linkedin/feathr.git - - $ cd feathr_project/feathr/api - - ``` - -1. Build the docker container locally, you need to have docker installed locally and have it running. To set up docker on your machine follow the instructions [here](https://docs.docker.com/get-started/) - **Note: Note: /image_name is not a mandatory format for specifying the name of the image.It’s just a useful convention to avoid tagging your image again when you need to push it to a registry. It can be anything you want in the format below** - - ```bash - $ docker build -t feathr/api . - ``` - -1. Run docker images command and you will see your newly created image - - ```bash - $ docker images - - REPOSITORY TAG IMAGE ID CREATED SIZE - feathr/api latest a647ea749b9b 5 minutes ago 529MB - ``` - -1. Before you can push an image to your registry, you must tag it with the fully qualified name of your ACR login server. The login server name is in the format .azurecr.io (all lowercase), for example, mycontainerregistry007.azurecr.io. Tag the image - ```bash - $ docker tag feathr/api:latest feathracr.azurecr.io/feathr/api:latest - ``` -1. Push the image to the registry - ```bash - $ docker push feathracr.azurecr.io/feathr/api:latest - ``` -1. List the images from your registry to see your recently pushed image - ``` - az acr repository list --name feathracr --output table - ``` - Output: - ``` - Result - ---------- - feathr/api - ``` - -## Deploy image to Azure WebApp for Containers - -1. Go to [Azure portal](https://portal.azure.com) and search for your container registry -1. Select repositories from the left pane and click latest tag. Click on the three dots on right side of the tag and select **Deploy to WebApp** option. If you see the **Deploy to WebApp** option greyed out, you would have to enable Admin User on the registry by Updating it. - - ![Container Image 1](../images/feathr_api_image_latest.png) - - ![Container Image 2](../images/feathr_api_image_latest_options.png) - -1. Provide a name for the deployed webapp, along with the subscription to deploy app into, the resource group and the appservice plan - - ![Container Image](../images/feathr_api_image_latest_deployment.png) - -1. You will get the notification that your app has been successfully deployed, click on **Go to Resource** button. - -1. On the App overview page go to the URL (https://.azurewebsites.net/docs) for deployed app (it's under URL on the app overview page) and you should see the API documentation. - - ![API docs](../images/api-docs.png) - -Congratulations you have successfully deployed the Feathr API. diff --git a/docs/dev_guide/feathr-core-code-structure.md b/docs/dev_guide/feathr-core-code-structure.md index ab812f32e..acf0c8c93 100644 --- a/docs/dev_guide/feathr-core-code-structure.md +++ b/docs/dev_guide/feathr-core-code-structure.md @@ -1,6 +1,6 @@ --- layout: default -title: Documentation Guideline +title: Feathr Core Code Structure parent: Developer Guides --- diff --git a/docs/how-to-guides/azure-deployment-arm.md b/docs/how-to-guides/azure-deployment-arm.md index bfb748d67..9245db91d 100644 --- a/docs/how-to-guides/azure-deployment-arm.md +++ b/docs/how-to-guides/azure-deployment-arm.md @@ -17,7 +17,9 @@ The provided Azure Resource Manager (ARM) template deploys the following resourc 7. Azure Event Hub 8. Azure Redis -Please note, you need to have **owner access** in the resource group you are deploying this in. Owner access is required to assign role to managed identity within ARM template so it can access key vault and store secrets. +Please note, you need to have **owner access** in the resource group you are deploying this in. Owner access is required to assign role to managed identity within ARM template so it can access key vault and store secrets. If you don't have such permission, you might want to contact your IT admin to see if they can do that. + +Although we recommend end users deploy the resources using the ARM template, we understand that in many situations where users want to reuse existing resources instead of creating new resources; or users have many other permission issues. See [Manually connecting existing resources](#manually-connecting-existing-resources) for more details. ## Architecture @@ -111,7 +113,6 @@ https://{resource_prefix}webapp.azurewebsites.net ![feathr ui landing page](../images/feathr-ui-landingpage.png) - ### 5. Initialize RBAC access table (Optional) If you want to use RBAC access for your deployment, you also need to manually initialize the user access table. Replace `[your-email-account]` with the email account that you are currently using, and this email will be the global admin for Feathr feature registry. diff --git a/docs/how-to-guides/azure_resource_provision.json b/docs/how-to-guides/azure_resource_provision.json index 827757b8c..58300fae4 100644 --- a/docs/how-to-guides/azure_resource_provision.json +++ b/docs/how-to-guides/azure_resource_provision.json @@ -35,13 +35,13 @@ "sqlAdminUsername": { "type": "String", "metadata": { - "description": "Specifies the username for admin" + "description": "Specifies the username for SQL Database admin" } }, "sqlAdminPassword": { "type": "SecureString", "metadata": { - "description": "Specifies the password for admin" + "description": "Specifies the password for SQL Database admin" } }, "registryBackend": { diff --git a/docs/how-to-guides/model-inference-with-feathr.md b/docs/how-to-guides/model-inference-with-feathr.md new file mode 100644 index 000000000..c2b5a8e7c --- /dev/null +++ b/docs/how-to-guides/model-inference-with-feathr.md @@ -0,0 +1,56 @@ +--- +layout: default +title: Online Model Inference with Features from Feathr +parent: How-to Guides +--- + +# Online Model Inference with Features from Feathr + +After you have materialized features in online store such as Redis or Azure Cosmos DB, usually end users want to consume those features in production environment for model inference. + +With Feathr's [online client](https://feathr.readthedocs.io/en/latest/#feathr.FeathrClient.get_online_features), it is quite straightforward to do that. The sample code is as below, where users only need to configure the online store endpoint (if using Redis), and call `client.get_online_features()` to get the features for a particular key. + +```python + +## put the section below into the initialization handler +import os +from feathr import FeathrClient + +# Set Redis endpoint +os.environ['online_store__redis__host'] = ".redis.cache.windows.net" +os.environ['online_store__redis__port'] = "6380" +os.environ['online_store__redis__ssl_enabled'] = "True" +os.environ['REDIS_PASSWORD'] = "" + +client = FeathrClient() + + +# put this section in the model inference handler +feature = client.get_online_features(feature_table="nycTaxiCITable", + key='2020-04-15', + feature_names=['f_is_long_trip_distance', 'f_day_of_week']) +# `res` will be an array representing the features of that particular key. + + +# `model` will be a ML model that is loaded previously. +result = model.predict(feature) +``` + +## Best Practices + +Usually for ML platforms such as Azure Machine Learning, Sagemaker, or DataRobot, there are options where you can "bring your own container" or using "container inference". Basically it requires end users to write an "entry script" and provide a few functions. In those cases, there are usually two handlers: + +- an initialization handler to allow users to load configurations. For example, in Azure Machine Learning, it is a function called `init()`, and in Sagemaker, it is `model_fn()`. +- a model inference handler to do the model inference. For example, in Azure Machine Learning, it is called `init()`, and in Sagemaker, it is called `predict_fn()`. + +In the initialization handler, initialize the environment variables and initialize `FeathrClient` as shown in the above script; in the inference handler, call this line: + +```python +# put this section in the model inference handler +feature = client.get_online_features(feature_table="nycTaxiCITable", + key='2020-04-15', + feature_names=['f_is_long_trip_distance', 'f_day_of_week']) +# `res` will be an array representing the features of that particular key. +# `model` will be a ML model that is loaded previously. +result = model.predict(feature) +``` diff --git a/docs/how-to-guides/streaming-source-ingestion.md b/docs/how-to-guides/streaming-source-ingestion.md index 4a59abc48..499efef5c 100644 --- a/docs/how-to-guides/streaming-source-ingestion.md +++ b/docs/how-to-guides/streaming-source-ingestion.md @@ -1,12 +1,12 @@ --- layout: default -title: Streaming Source Ingestion +title: Streaming Source Ingestion and Feature Definition parent: How-to Guides --- -# Streaming feature ingestion +# Streaming Source Ingestion and Feature Definition -Feathr supports defining features from a stream source (for example Kafka) and sink the features into an online store (such as Redis). This is very useful if you need up-to-date features for online store, for example when user clicks on the website, that web log event is usually sent to Kafka, and data scientists might need some features immediately, such as the browser used in this particular event. The steps are as below: +Feathr supports defining features from a stream source (for example Kafka) with transformations, and sink the features into an online store (such as Redis). This is very useful if you need up-to-date features for online store, for example when user clicks on the website, that web log event is usually sent to Kafka, and data scientists might need some features immediately, such as the browser used in this particular event. The steps are as below: ## Define Kafka streaming input source @@ -35,13 +35,13 @@ stream_source = KafKaSource(name="kafkaStreamingSource", ) ``` -You may need to produce data and send them into Kafka as this data source in advance. Please check [Kafka data source producer](../../feathr_project/test/prep_azure_kafka_test_data.py) as a reference. Also you should keep this producer running which means there are data stream keep coming into Kafka while calling the 'materialize_features' below. +You may need to produce data and send them into Kafka as this data source in advance. Please check [Kafka data source producer](https://github.com/linkedin/feathr/blob/main/feathr_project/test/prep_azure_kafka_test_data.py) as a reference. Also you should keep this producer running which means there are data stream keep coming into Kafka while calling the 'materialize_features' below. ## Define feature definition with the Kafka source You can then define features. They are mostly the same with the [regular feature definition](../concepts/feature-definition.md). -Note that for the `transform` part, only row level transformation is allowed in streaming anchor at the moment, i.e. the transformations listed in [Spark SQL Built-in Functions](https://spark.apache.org/docs/latest/api/sql/) are supported. Other transformations support are in the roadmap. +Note that for the `transform` part, only row level transformation is allowed in streaming anchor at the moment, i.e. the transformations listed in [Spark SQL Built-in Functions](https://spark.apache.org/docs/latest/api/sql/) are supported. Users can also define customized [Spark SQL functions](./feathr-spark-udf-advanced.md). For example, you can specify to do a row-level transformation like `trips_today + randn() * cos(trips_today)` for your input data. @@ -90,14 +90,14 @@ res = client.multi_get_online_features('kafkaSampleDemoFeature', ['1', '2'], ['f ``` -You can also refer to the [test case](../../feathr_project/test/test_azure_kafka_e2e.py) for more details. +You can also refer to the [test case](https://github.com/linkedin/feathr/blob/main/feathr_project/test/test_azure_kafka_e2e.py) for more details. ## Kafka configuration -Please refer to the [Feathr Configuration Doc](./feathr-configuration-and-env.md#kafkasasljaasconfig) for more details on the credentials. +Please refer to the [Feathr Configuration Doc](./feathr-configuration-and-env.md#KAFKA_SASL_JAAS_CONFIG) for more details on the credentials. -## Event Hub monitor +## Event Hub monitoring -Please check monitor panel on your 'Event Hub' overview page while running materialize to make sure there are both incoming and outgoing messages, like below graph. Otherwise, you may not get anything from 'get_online_features' since the source is empty. +If you feel something is wrong, you can check the monitor panel on your 'Event Hub' overview page while running the Feathr materialization job, to make sure there are both incoming and outgoing messages, like the graph below. Otherwise, you may not get anything from `get_online_features()` since the source is empty. ![Kafka Monitor Page](../images/kafka-messages-monitor.png) \ No newline at end of file From fe478ab3f6303e0f997259faff2054fd22920b7c Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Wed, 14 Sep 2022 01:05:13 -0700 Subject: [PATCH 10/80] Clean up after moving to LFAI (#665) * Clean up after moving to LFAI Clean up after moving to LFAI * Update README.md --- .github/pull_request_template.md | 2 +- CONTRIBUTING.md | 4 +- docs/README.md | 38 ++++++++++--------- docs/concepts/feature-registry.md | 2 +- docs/concepts/get-offline-features.md | 2 +- docs/dev_guide/cloud_integration_testing.md | 2 +- docs/dev_guide/cloud_resource_provision.md | 2 +- .../dev_guide/feathr_overall_release_guide.md | 26 ++++++------- docs/dev_guide/new_contributor_guide.md | 10 ++--- docs/dev_guide/publish_to_maven.md | 2 +- docs/dev_guide/scala_dev_guide.md | 2 +- docs/how-to-guides/azure-deployment-arm.md | 6 +-- .../feathr-configuration-and-env.md | 6 +-- .../streaming-source-ingestion.md | 4 +- docs/quickstart_synapse.md | 6 +-- feathr_project/feathr/client.py | 4 +- feathr_project/feathrcli/cli.py | 2 +- feathr_project/setup.py | 6 +-- 18 files changed, 64 insertions(+), 62 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 735b76e5a..9d2d9e746 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,6 @@ ## Description