diff --git a/registry/data-models/common/__init__.py b/registry/data-models/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/registry/data-models/common/models.py b/registry/data-models/common/models.py new file mode 100644 index 000000000..7c07a6542 --- /dev/null +++ b/registry/data-models/common/models.py @@ -0,0 +1,137 @@ +from pydantic import BaseModel +from typing import Dict, Optional, List, Union +import json +from enum import Enum + + +class ValueType(Enum): + """ + Type of the feature. + """ + INT = "int" + LONG = "long" + FLOAT = "float" + DOUBLE = "double" + STRING = "string" + BOOLEAN = "boolean" + BYTES = "bytes" + + +class DimensionType(Enum): + """ + Supported dimension types for tensors in Feathr. + """ + INT = "int" + LONG = "long" + STRING = "string" + BOOLEAN = "boolean" + BYTES = "bytes" + + +class TensorCategory(Enum): + """ + Supported Tensor categories in Feathr. + """ + DENSE = "dense" # Dense tensors store values in a contiguous sequential block of memory where all values are represented. + SPARSE = "sparse" # Sparse tensor represents a dataset in which most of the entries are zero. + RAGGED = "ragged" # Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions. + + +class FeatureValueType(Enum): + """ + The high level types associated with a feature. + This represents the high level semantic types supported by early versions of Frame. + """ + BOOLEAN = "boolean" # Boolean valued feature + NUMERIC = "numeric" # Numerically valued feature + CATEGORICAL = "categorical" # Represent a feature that consists of a single category + CATEGORICAL_SET = "categorical_set" # Represent a feature that consists of multiple categories + DENSE_VECTOR = "dense_vector" # Represent a feature in vector format where the majority of the elements are non-zero + TERM_VECTOR = "term_vector" # Represent features that has string terms and numeric value + TENSOR = "tensor" # Represent tensor based features. + UNSPECIFIED = "unspecified" # Placeholder for when no types are specified + + +class Dimension(BaseModel): + """ + Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. + """ + type: DimensionType # Type of the dimension in the tensor. Each dimension can have a different type. + shape: Optional[int] # Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime. + + +class TensorFeatureFormat(BaseModel): + """ + Defines the format of feature data. Feature data is produced by applying transformation on source, in a Feature. + Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially + higher dimensions. + """ + tensorCategory: TensorCategory # Type of the tensor. + valueType: ValueType # Type of the value column. + dimensions: List[Dimension] # A feature data can have zero or more dimensions (columns that represent keys). + + +class FeatureType(BaseModel): + """ + Information about a featureName. It defines the type, format and default value. + Tensor is the next generation representation of the features, so using + Tensor type w TensorFeatureFormat would be preferable FeatureType. + """ + type: FeatureValueType # Defines the high level semantic type of feature. + format: Optional[TensorFeatureFormat] # Defines the format of feature data. + defaultValue: Union[bool, int, float, str, bytes] + + +class Clazz(BaseModel): + """ + Reference to a class by fully-qualified name + """ + fullyQualifiedName: str # A fully-qualified class name including paths. + + +class Function(BaseModel): + """ + Base model for all functions + """ + expression: str # Expression in str format + functionType: str # Type of function in str format, will be used in UI + + +class MvelExpression(Function): + """ + An expression in MVEL language. + """ + mvel: str # The MVEL expression + + +class UserDefinedFunction(Function): + """ + User defined function that can be used in feature extraction or derivation. + """ + clazz: Clazz # Reference to the class that implements the user defined function. + parameters: Dict[str, json] = {} # This field defines the custom parameters of the user defined function + + +class SparkSqlExpression(Function): + """ + An expression in Spark SQL. + """ + sql: str # Spark SQl expression + + +class SemanticVersion(BaseModel): + """ + A representation of a semantic version (see https://semver.org/) + """ + majorVersion: int # The major version of this version. This is the x in x.y.z. + minorVersion: int # The minor version of this version. This is the y in x.y.z + patchVersion: int # The patch version of this version. This is the z in x.y.z + metadata: Optional[str] # Optional build metadata attached to this version. + + +class FeathrModel(BaseModel): + """ + Base model for feathr entity which will be displayed in Feathr UI + """ + displayName: str # name of the entity showed on UI + typeName: str # type of entity in str format, will be displayed in UI diff --git a/registry/data-models/data-model-diagram.md b/registry/data-models/data-model-diagram.md index e43ffa0af..d612e005f 100644 --- a/registry/data-models/data-model-diagram.md +++ b/registry/data-models/data-model-diagram.md @@ -6,30 +6,186 @@ This file defines abstract backend data models diagram for feature registry. ```mermaid classDiagram - Project "1" --> "n" FeatureName : contains - Project "1" --> "n" Anchor : contains - FeatureName "1" --> "n" Feature : contains - Anchor "1" --> "n" Feature : contains - Feature <|-- AnchorFeature : extends + Project "1" --> "n" FeatureName: contains + Project "1" --> "n" Anchor: contains + FeatureName "1" --> "n" Feature: contains + FeatureName --> "Optional" SemanticVersion: contains + FeatureName --> "Optional" FeatureType: contains + Anchor "1" --> "n" AnchorFeature: contains + Anchor --> DataSource: contains + Feature <|-- AnchorFeature: extends Feature <|-- DerivedFeature: extends - Feature --> Transformation - Feature --> Transformation : contains + Feature --> Transformation: contains + Feature --> Source: contains + Transformation --> Function: contains Source <|-- DataSource: extends + DataSource --> "Optional" Clazz: contains + DataSource --> "Optional" Function: contains Source <|-- MultiFeatureSource: extends MultiFeatureSource "1" --> "1..n" FeatureSource: contains - AnchorFeature --> DataSource : contains + AnchorFeature --> DataSource: contains DerivedFeature --> MultiFeatureSource: contains - + FeathrModel <|-- Project: extends + FeathrModel <|-- FeatureName: extends + FeathrModel <|-- Anchor: extends + FeathrModel <|-- Feature: extends + FeathrModel <|-- Source: extends + Dimension --> DimensionType: contains + TensorFeatureFormat --> TensorCategory: contains + TensorFeatureFormat --> ValueType: contains + TensorFeatureFormat "1" --> "1..n" Dimension: contains + FeatureType --> FeatureValueType: contains + FeatureType --> "Optional" TensorFeatureFormat: contains + Window --> WindowTimeUnit: contains + Function <|-- MvelExpression: extends + Function <|-- UserDefinedFunction: extends + Function <|-- SparkSqlExpression: extends + SlidingWindowAggregation --> SparkSqlExpression: contains + SlidingWindowAggregation --> SlidingWindowAggregationType: contains + SlidingWindowAggregation --> Window: contains + SlidingWindowEmbeddingAggregation --> SparkSqlExpression: contains + SlidingWindowEmbeddingAggregation --> SlidingWindowEmbeddingAggregationType: contains + SlidingWindowEmbeddingAggregation --> Window: contains + SlidingWindowLatestAvailable --> SparkSqlExpression: contains + SlidingWindowLatestAvailable --> Window: contains + Function <|-- SlidingWindowAggregation: extends + Function <|-- SlidingWindowEmbeddingAggregation: extends + Function <|-- SlidingWindowLatestAvailable: extends + + class ValueType{ + <> + INT + LONG + FLOAT + DOUBLE + STRING + BOOLEAN + BYTES + } + class DimensionType{ + <> + INT + LONG + STRING + BOOLEAN + BYTES + } + class TensorCategory{ + <> + DENSE + SPARSE + RAGGED + } + class FeatureValueType{ + <> + BOOLEAN + NUMERIC + CATEGORICAL + CATEGORICAL_SET + DENSE_VECTOR + TERM_VECTOR + TENSOR + UNSPECIFIED + } + class Dimension{ + +DimensionType type + +Optional[str] shape + } + class TensorFeatureFormat{ + +TensorCategory tensorCategory + +ValueType valueType + +List[Dimension] dimensions + } + class FeatureType{ + +FeatureValueType type + +Optional[TensorFeatureFormat] format + +Union[bool, int, float, str, types] defaultValue + } + class Clazz{ + +str fullyQualifiedName + } + class Function{ + +str expression + } + class MvelExpression{ + +str mvel + } + class UserDefinedFunction{ + +str sql + } + class SemanticVersion{ + +int majorVersion + +int minorVersion + +int patchVersion + +Optional[str] metadata + } + class FeathrModel{ + +str displayName + +str typeName + } + class SlidingWindowAggregationType{ + <> + SUM + COUNT + MAX + MIN + AVG + } + class SlidingWindowEmbeddingAggregationType{ + <> + MAX_POOLING + MIN_POOLING + AVG_POOLING + } + class WindowTimeUnit{ + <> + DAY + HOUR + MINUTE + SECOND + } + class Window{ + +int size + +WindowTimeUnit unit + } + class SlidingWindowAggregation{ + +SlidingWindowAggregationType aggregationType + +Window window + +SparkSqlExpression targetColumn + +Optional[SparkSqlExpression] filter + +Optional[SparkSqlExpression] groupBy + +Optional[int] limit + } + class SlidingWindowEmbeddingAggregation{ + +SlidingWindowEmbeddingAggregationType aggregationType + +Window window + +SparkSqlExpression targetColumn + +Optional[SparkSqlExpression] filter + +Optional[SparkSqlExpression] groupBy + } + class SlidingWindowLatestAvailable{ + +Optional[Window] window + +SparkSqlExpression targetColumn + +Optional[SparkSqlExpression] filter + +Optional[SparkSqlExpression] groupBy + +Optional[int] limit + } class Source{ } class DataSource{ + +Optional[Clazz] clazz + +Optional[Function] keyFunction } class FeatureSource{ - +FeatureNameId feature_name_id + +FeatureNameId input_feature_name_id + +Optional[str] alias } class MultiFeatureSource{ +List[FeatureSource] sources } + class Transformation{ + +Function transformationFunction + } class Feature{ +FeatureId id +FeatureNameId feature_namme_id @@ -37,6 +193,7 @@ classDiagram +Transformation transformation } class AnchorFeature{ + +AnchorId anchor_id +DataSource source } class DerivedFeature{ @@ -46,6 +203,8 @@ classDiagram +FeatureNameId id +ProjectId project_id +List[FeatureId] feature_ids + +Optional[SemanticVersion] semanticVersion + +Optional[FeatureType] featureType } class Project{ +ProjectId id diff --git a/registry/data-models/models.py b/registry/data-models/models.py index c4ae31f68..c230240ab 100644 --- a/registry/data-models/models.py +++ b/registry/data-models/models.py @@ -1,12 +1,13 @@ +from registry.data-models.transformation.models import * +from registry.data-models.common.models import SemanticVersion, FeathrModel, Function +from typing import Optional from pydantic import BaseModel -from typing import List + """ This file defines abstract backend data models for feature registry. Backend data models will be used by backend API server to talk to feature registry backend. Purpose of this is to decouple backend data models from API specific data models. -For each feature registry provider/implementation, they will extend this abstract -data models and backend API. Diagram of the data models: ./data-model-diagram.md """ @@ -43,11 +44,7 @@ class ProjectId(BaseModel): id: str # id of a project -class Source(BaseModel): - """ - Source of the feature. - It defines where the feature is extracted or derived from. - """ +class Source(FeathrModel): pass @@ -56,7 +53,8 @@ class DataSource(Source): Data source of the feature. It defines the raw data source the feature is extracted from. """ - pass + clazz: Optional[Clazz] # Fully qualified Java class name for data model + keyFunction: Optional[Function] class FeatureSource(BaseModel): @@ -65,6 +63,7 @@ class FeatureSource(BaseModel): creating other derived features. """ input_feature_name_id: FeatureNameId # Input feature name Key + alias: Optional[str] # A feature's alias to be used in transformation function. class MultiFeatureSource(Source): @@ -73,7 +72,6 @@ class MultiFeatureSource(Source): It defines one to many features where the feature is derived from. """ sources: List[FeatureSource] # All source features which the feature is derived from - pass class Transformation(BaseModel): @@ -81,10 +79,10 @@ class Transformation(BaseModel): The transformation of a Feature. A transformation function represents the transformation logic to produce feature value from the source of FeatureAnchor """ - pass + transformationFunction: Function -class Feature(BaseModel): +class Feature(FeathrModel): """ Actual implementation of FeatureName. An implementation defines where a feature is extracted from (Source) and how it is computed (Transformation). @@ -100,6 +98,7 @@ class AnchorFeature(Feature): """ Feature implementation of FeatureName which anchored to a data source. """ + anchor_id: AnchorId # ID of the anchor this feature belongs to source: DataSource # Raw data source where the feature is extracted from @@ -110,7 +109,7 @@ class DerivedFeature(Feature): source: MultiFeatureSource # Source features where the feature is derived from -class FeatureName(BaseModel): +class FeatureName(FeathrModel): """ Named Feature Interface that can be backed by multiple Feature implementations across different environments accessing different sources (data lake access for batch training, @@ -122,9 +121,11 @@ class FeatureName(BaseModel): id: FeatureNameId # unique ID for FeatureName, used to extract data for current FeatureName project_id: ProjectId # ID of the project the FeatureName belongs to feature_ids: List[FeatureId] # List of ids of feature that the FeatureName has + semanticVersion: Optional[SemanticVersion] # Semantic version associated with this FeatureName + featureType: Optional[FeatureType] # Information about featureName, like feature type, format and value. -class Project(BaseModel): +class Project(FeathrModel): """ Group of FeatureNames. It can be a project the team is working on, or a namespace which related FeatureNames have. @@ -134,7 +135,7 @@ class Project(BaseModel): anchor_ids: List[AnchorId] # List of Anchor ids that the project has -class Anchor(BaseModel): +class Anchor(FeathrModel): """ Group of AnchorFeatures which anchored on same DataSource. This is mainly used by feature producer gather information about DataSource diff --git a/registry/data-models/transformation/__init__.py b/registry/data-models/transformation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/registry/data-models/transformation/models.py b/registry/data-models/transformation/models.py new file mode 100644 index 000000000..b721d174e --- /dev/null +++ b/registry/data-models/transformation/models.py @@ -0,0 +1,84 @@ +from registry.data-models.common.models import * +from typing import Optional + + +class SlidingWindowAggregationType(Enum): + """ + Represents supported types of aggregation. + """ + SUM = "sum" + COUNT = "count" + MAX = "maximum" + MIN = "minium" + AVG = "average" + + +class SlidingWindowEmbeddingAggregationType(Enum): + """ + Represents supported types for embedding aggregation. + Pooling is a sample-based discretization process. The objective is to down-sample an input + representation and reduce its dimensionality. + """ + MAX_POOLING = "max_pooling" # Max pooling is done by applying a max filter to (usually) non-overlapping subregions of the initial representation + MIN_POOLING = "min_pooling" # Min pooling is done by applying a min filter to (usually) non-overlapping subregions of the initial representation. + AVG_POOLING = "avg_pooling" # Average pooling is done by applying a average filter to (usually) non-overlapping subregions of the initial representation + + +class WindowTimeUnit(Enum): + """ + Represents a unit of time. + """ + DAY = "day" + HOUR = "hour" + MINUTE = "minute" + SECOND = "second" + + +class Window(BaseModel): + """ + Represents a time window used in sliding window algorithms. + """ + size: int # Represents the duration of the window. + unit: WindowTimeUnit + + +class SlidingWindowAggregation(Function): + """ + Sliding window aggregation produces feature data by aggregating a collection of data within a given time + interval into an aggregate value. It ensures point-in-time correctness, when joining with label data, + it looks back the configurable time window from each entry's timestamp and compute the aggregate value. + This class can be extended to support LateralView in aggregation. + """ + aggregationType: SlidingWindowAggregationType # Represents supported types of aggregation. + window: Window # Represents the time window to look back from label data's timestamp. + targetColumn: SparkSqlExpression # The target column to perform aggregation against. + filter: Optional[SparkSqlExpression] # Represents the filter statement before the aggregation. + groupBy: Optional[SparkSqlExpression] # Represents the target to be grouped by before aggregation. + limit: Optional[int] # Represents the max number of groups (with aggregation results) to return. + + +class SlidingWindowEmbeddingAggregation(Function): + """ + Sliding window embedding aggregation produces a single embedding by performing element-wise operations or + discretion on a collection of embeddings within a given time interval. It ensures point-in-time correctness, + when joining with label data, Frame looks back the configurable time window from each entry's timestamp and produce + the aggregated embedding. + """ + aggregationType: SlidingWindowEmbeddingAggregationType # Represents supported types for embedding aggregation. + window: Window # Represents the time window to look back from label data's timestamp. + targetColumn: SparkSqlExpression # The target column to perform aggregation against. + filter: Optional[SparkSqlExpression] # Represents the filter statement before the aggregation. + groupBy: Optional[SparkSqlExpression] # Represents the target to be grouped by before aggregation. + + +class SlidingWindowLatestAvailable(Function): + """ + This sliding window algorithm picks the latest available feature data from the source data. + Note the latest here means event time instead of processing time. + This class can be extended to support LateralView in aggregation. + """ + window: Optional[Window] # Represents the time window to look back from label data's timestamp. + targetColumn: SparkSqlExpression # The target column to perform aggregation against. + filter: Optional[SparkSqlExpression] # Represents the filter statement before the aggregation. + groupBy: Optional[SparkSqlExpression] # Represents the target to be grouped by before aggregation. + limit: Optional[int] # Represents the max number of groups (with aggregation results) to return.