feathr-ai · windoze · Nov 9, 2022 · Sep 13, 2022 · Sep 15, 2022 · Sep 20, 2022
diff --git a/registry/data-models/common/__init__.py b/registry/data-models/common/__init__.py
diff --git a/registry/data-models/common/models.py b/registry/data-models/common/models.py
@@ -0,0 +1,137 @@
+from pydantic import BaseModel
+from typing import Dict, Optional, List, Union
+import json
+from enum import Enum
+
+
+class ValueType(Enum):
+    """
+    Type of the feature.
+    """
+    INT = "int"
+    LONG = "long"
+    FLOAT = "float"
+    DOUBLE = "double"
+    STRING = "string"
+    BOOLEAN = "boolean"
+    BYTES = "bytes"
+
+
+class DimensionType(Enum):
+    """
+    Supported dimension types for tensors in Feathr.
+    """
+    INT = "int"
+    LONG = "long"
+    STRING = "string"
+    BOOLEAN = "boolean"
+    BYTES = "bytes"
+
+
+class TensorCategory(Enum):
+    """
+    Supported Tensor categories in Feathr.
+    """
+    DENSE = "dense"  # Dense tensors store values in a contiguous sequential block of memory where all values are represented.
+    SPARSE = "sparse"  # Sparse tensor represents a dataset in which most of the entries are zero.
+    RAGGED = "ragged"  # Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.
+
+
+class FeatureValueType(Enum):
+    """
+    The high level types associated with a feature.
+    This represents the high level semantic types supported by early versions of Frame.
+    """
+    BOOLEAN = "boolean"  # Boolean valued feature
+    NUMERIC = "numeric"  # Numerically valued feature
+    CATEGORICAL = "categorical"  # Represent a feature that consists of a single category
+    CATEGORICAL_SET = "categorical_set"  # Represent a feature that consists of multiple categories
+    DENSE_VECTOR = "dense_vector"  # Represent a feature in vector format where the majority of the elements are non-zero
+    TERM_VECTOR = "term_vector"  # Represent features that has string terms and numeric value
+    TENSOR = "tensor"  # Represent tensor based features.
+    UNSPECIFIED = "unspecified"  # Placeholder for when no types are specified
+
+
+class Dimension(BaseModel):
+    """
+    Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions.
+    """
+    type: DimensionType  # Type of the dimension in the tensor. Each dimension can have a different type.
+    shape: Optional[int]  # Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.
+
+
+class TensorFeatureFormat(BaseModel):
+    """
+    Defines the format of feature data. Feature data is produced by applying transformation on source, in a Feature.
+    Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially
+    higher dimensions.
+    """
+    tensorCategory: TensorCategory  # Type of the tensor.
+    valueType: ValueType  # Type of the value column.
+    dimensions: List[Dimension]  # A feature data can have zero or more dimensions (columns that represent keys).
+
+
+class FeatureType(BaseModel):
+    """
+   Information about a featureName. It defines the type, format and default value.
+   Tensor is the next generation representation of the features, so using
+   Tensor type w TensorFeatureFormat would be preferable FeatureType.
+    """
+    type: FeatureValueType  # Defines the high level semantic type of feature.
+    format: Optional[TensorFeatureFormat]  # Defines the format of feature data.
+    defaultValue: Union[bool, int, float, str, bytes]
+
+
+class Clazz(BaseModel):
+    """
+    Reference to a class by fully-qualified name
+    """
+    fullyQualifiedName: str  # A fully-qualified class name including paths.
+
+
+class Function(BaseModel):
+    """
+    Base model for all functions
+    """
+    expression: str  # Expression in str format
+    functionType: str # Type of function in str format, will be used in UI
+
+
+class MvelExpression(Function):
+    """
+    An expression in MVEL language.
+    """
+    mvel: str  # The MVEL expression
+
+
+class UserDefinedFunction(Function):
+    """
+    User defined function that can be used in feature extraction or derivation.
+    """
+    clazz: Clazz  # Reference to the class that implements the user defined function.
+    parameters: Dict[str, json] = {}  # This field defines the custom parameters of the user defined function
+
+
+class SparkSqlExpression(Function):
+    """
+    An expression in Spark SQL.
+    """
+    sql: str  # Spark SQl expression
+
+
+class SemanticVersion(BaseModel):
+    """
+    A representation of a semantic version (see https://semver.org/)
+    """
+    majorVersion: int  # The major version of this version. This is the x in x.y.z.
+    minorVersion: int  # The minor version of this version. This is the y in x.y.z
+    patchVersion: int  # The patch version of this version. This is the z in x.y.z
+    metadata: Optional[str]  # Optional build metadata attached to this version.
+
+
+class FeathrModel(BaseModel):
+    """
+    Base model for feathr entity which will be displayed in Feathr UI
+    """
+    displayName: str  # name of the entity showed on UI
+    typeName: str  # type of entity in str format, will be displayed in UI
diff --git a/registry/data-models/data-model-diagram.md b/registry/data-models/data-model-diagram.md
@@ -6,37 +6,194 @@ This file defines abstract backend data models diagram for feature registry.
 
 ```mermaid
 classDiagram
-    Project "1" --> "n" FeatureName : contains
-    Project "1" --> "n" Anchor : contains
-    FeatureName "1" --> "n" Feature : contains
-    Anchor "1" --> "n" Feature : contains
-    Feature <|-- AnchorFeature : extends
+   Project "1" --> "n" FeatureName: contains
+    Project "1" --> "n" Anchor: contains
+    FeatureName "1" --> "n" Feature: contains
+    FeatureName --> "Optional" SemanticVersion: contains
+    FeatureName --> "Optional" FeatureType: contains
+    Anchor "1" --> "n" AnchorFeature: contains
+    Anchor -->  DataSource: contains    
+    Feature <|-- AnchorFeature: extends
     Feature <|-- DerivedFeature: extends
-    Feature --> Transformation
-    Feature  -->  Transformation : contains
+    Feature --> Transformation: contains
+    Feature --> Source: contains    
+    Transformation --> Function: contains
     Source <|-- DataSource: extends
+    DataSource --> "Optional" Clazz: contains
+    DataSource --> "Optional" Function: contains
     Source <|-- MultiFeatureSource: extends
     MultiFeatureSource "1" --> "1..n" FeatureSource: contains
-    AnchorFeature  -->  DataSource : contains
+    AnchorFeature  -->  DataSource: contains
     DerivedFeature -->  MultiFeatureSource: contains
-
+    FeathrModel <|-- Project: extends
+    FeathrModel <|-- FeatureName: extends
+    FeathrModel <|-- Anchor: extends
+    FeathrModel <|-- Feature: extends
+    FeathrModel <|-- Source: extends
+    Dimension --> DimensionType: contains
+    TensorFeatureFormat --> TensorCategory: contains
+    TensorFeatureFormat --> ValueType: contains
+    TensorFeatureFormat "1" --> "1..n" Dimension: contains
+    FeatureType --> FeatureValueType: contains
+    FeatureType --> "Optional" TensorFeatureFormat: contains
+    Window --> WindowTimeUnit: contains
+    Function <|-- MvelExpression: extends
+    Function <|-- UserDefinedFunction: extends
+    Function <|-- SparkSqlExpression: extends
+    SlidingWindowAggregation --> SparkSqlExpression: contains
+    SlidingWindowAggregation --> SlidingWindowAggregationType: contains
+    SlidingWindowAggregation --> Window: contains
+    SlidingWindowEmbeddingAggregation --> SparkSqlExpression: contains
+    SlidingWindowEmbeddingAggregation --> SlidingWindowEmbeddingAggregationType: contains
+    SlidingWindowEmbeddingAggregation --> Window: contains
+    SlidingWindowLatestAvailable --> SparkSqlExpression: contains
+    SlidingWindowLatestAvailable --> Window: contains
+    Function <|-- SlidingWindowAggregation: extends
+    Function <|-- SlidingWindowEmbeddingAggregation: extends
+    Function <|-- SlidingWindowLatestAvailable: extends
+
+    class ValueType{
+        <<enumeration>>
+        INT
+        LONG
+        FLOAT
+        DOUBLE
+        STRING
+        BOOLEAN
+        BYTES
+    }
+    class DimensionType{
+        <<enumeration>>
+        INT
+        LONG
+        STRING
+        BOOLEAN
+        BYTES
+    }
+    class TensorCategory{
+        <<enumeration>>
+        DENSE
+        SPARSE
+        RAGGED
+    }
+    class FeatureValueType{
+        <<enumeration>>
+        BOOLEAN
+        NUMERIC
+        CATEGORICAL
+        CATEGORICAL_SET
+        DENSE_VECTOR
+        TERM_VECTOR
+        TENSOR
+        UNSPECIFIED
+    }
+    class Dimension{
+        +DimensionType type
+        +Optional[str] shape
+    }
+    class TensorFeatureFormat{
+        +TensorCategory tensorCategory
+        +ValueType valueType
+        +List[Dimension] dimensions
+    }
+    class FeatureType{
+        +FeatureValueType type
+        +Optional[TensorFeatureFormat] format
+        +Union[bool, int, float, str, types] defaultValue
+    }
+    class Clazz{
+        +str fullyQualifiedName
+    }
+    class Function{
+        +str expression
+    }
+    class MvelExpression{
+        +str mvel
+    }
+    class UserDefinedFunction{
+        +str sql
+    }
+    class SemanticVersion{
+        +int majorVersion
+        +int minorVersion
+        +int patchVersion
+        +Optional[str] metadata
+    }
+    class FeathrModel{
+        +str displayName
+        +str typeName
+    }
+    class SlidingWindowAggregationType{
+        <<enumeration>>
+        SUM
+        COUNT
+        MAX
+        MIN
+        AVG
+    }
+    class SlidingWindowEmbeddingAggregationType{
+        <<enumeration>>
+        MAX_POOLING
+        MIN_POOLING
+        AVG_POOLING
+    }
+    class WindowTimeUnit{
+        <<enumeration>>
+        DAY
+        HOUR
+        MINUTE
+        SECOND
+    }
+    class Window{
+        +int size
+        +WindowTimeUnit unit     
+    }
+    class SlidingWindowAggregation{
+        +SlidingWindowAggregationType aggregationType
+        +Window window
+        +SparkSqlExpression targetColumn
+        +Optional[SparkSqlExpression] filter
+        +Optional[SparkSqlExpression] groupBy
+        +Optional[int] limit        
+    }
+    class SlidingWindowEmbeddingAggregation{
+        +SlidingWindowEmbeddingAggregationType aggregationType
+        +Window window
+        +SparkSqlExpression targetColumn
+        +Optional[SparkSqlExpression] filter
+        +Optional[SparkSqlExpression] groupBy
+    }
+    class SlidingWindowLatestAvailable{
+        +Optional[Window] window
+        +SparkSqlExpression targetColumn
+        +Optional[SparkSqlExpression] filter
+        +Optional[SparkSqlExpression] groupBy
+        +Optional[int] limit        
+    }
     class Source{
     }
     class DataSource{
+        +Optional[Clazz] clazz
+        +Optional[Function] keyFunction
     }
     class FeatureSource{
-        +FeatureNameId feature_name_id
+        +FeatureNameId input_feature_name_id
+        +Optional[str] alias
     }
     class MultiFeatureSource{
         +List[FeatureSource] sources
     }
+    class Transformation{
+        +Function transformationFunction
+    }
     class Feature{
         +FeatureId id
         +FeatureNameId feature_namme_id
         +Source source
         +Transformation transformation
     }
     class AnchorFeature{
+        +AnchorId  anchor_id
         +DataSource source
     }
     class DerivedFeature{
@@ -46,6 +203,8 @@ classDiagram
         +FeatureNameId id
         +ProjectId project_id
         +List[FeatureId] feature_ids
+        +Optional[SemanticVersion] semanticVersion
+        +Optional[FeatureType] featureType
     }
     class Project{
         +ProjectId id