feat: Replace zod by protobufjs

Signed-off-by: Jérôme Viveret <jerome.viveret@onfido.com> Signed-off-by: Danny Chiao <danny@tecton.ai>
feast-dev · feast-ci-bot · Oct 5, 2022 · Sep 13, 2022 · Oct 5, 2022 · Oct 5, 2022
commit 51b8114b7adb20085e7cb4383e28c2e5a93b8af9
@@ -23,6 +23,7 @@ option java_outer_classname = "DataSourceProto";
 option java_package = "feast.proto.core";
 
 import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
 import "feast/core/DataFormat.proto";
 import "feast/types/Value.proto";
 import "feast/core/Feature.proto";
@@ -89,6 +90,12 @@ message DataSource {
   // Optional batch source for streaming sources for historical features and materialization.
   DataSource batch_source = 26;
 
+  SourceMeta meta = 50;
+
+  message SourceMeta {
+    google.protobuf.Timestamp earliestEventTimestamp = 1;
+    google.protobuf.Timestamp latestEventTimestamp = 2;
+  }
 
   // Defines options for DataSource that sources features from a file
   message FileOptions {
@@ -112,6 +119,8 @@ message DataSource {
     // SQL query that returns a table containing feature data. Must contain an event_timestamp column, and respective
     // entity columns
     string query = 2;
+
+    string dbtModelSerialized = 3;
   }
 
   // Defines options for DataSource that sources features from a Trino Query

@@ -0,0 +1,2 @@
+src/protos.d.ts
+src/protos.js
@@ -16,7 +16,9 @@
 def credit_profiler(ds: PandasDataset) -> ExpectationSuite:
     # simple checks on data consistency
     ds.expect_column_values_to_be_between(
-        "credit_card_due", min_value=0, mostly=0.99,  # allow some outliers
+        "credit_card_due",
+        min_value=0,
+        mostly=0.99,  # allow some outliers
     )
 
     ds.expect_column_values_to_be_between(
@@ -34,7 +36,10 @@ def generate_saved_dataset():
     entity_df = pd.read_parquet(path="data/loan_table.parquet")
 
     fs = store.get_feature_service("credit_score_v1")
-    job = store.get_historical_features(entity_df=entity_df, features=fs,)
+    job = store.get_historical_features(
+        entity_df=entity_df,
+        features=fs,
+    )
     store.create_saved_dataset(
         from_=job,
         name="my_training_ds",

@@ -25,8 +25,7 @@
     "react-router-dom": "6",
     "react-scripts": "^5.0.0",
     "typescript": "^4.4.2",
-    "use-query-params": "^1.2.3",
-    "zod": "^3.11.6"
+    "use-query-params": "^1.2.3"
   },
   "dependencies": {
     "@elastic/datemath": "^5.0.3",
@@ -41,20 +40,22 @@
     "inter-ui": "^3.19.3",
     "moment": "^2.29.1",
     "prop-types": "^15.8.1",
+    "protobufjs": "^7.1.1",
     "query-string": "^7.1.1",
     "react-query": "^3.34.12",
     "react-router-dom": "6",
     "react-scripts": "^5.0.0",
     "use-query-params": "^1.2.3",
-    "zod": "^3.11.6"
+    "zod": "^3.19.1"
   },
   "scripts": {
-    "start": "react-scripts start",
-    "build": "react-scripts build",
-    "build:lib": "rimraf ./dist && tsc && rollup -c",
-    "build:lib-dev": "rimraf ./dist && tsc && rollup -c && yalc publish -f",
-    "test": "react-scripts test",
-    "eject": "react-scripts eject"
+    "start": "npm run generate-protos && react-scripts start",
+    "build": "npm run generate-protos && react-scripts build",
+    "build:lib": "npm run generate-protos && rimraf ./dist && tsc && rollup -c",
+    "build:lib-dev": "npm run generate-protos && rimraf ./dist && tsc && rollup -c && yalc publish -f",
+    "test": "npm run generate-protos && react-scripts test",
+    "eject": "react-scripts eject",
+    "generate-protos": "pbjs --no-encode -o src/protos.js -w commonjs -t static-module `find ../protos/feast/ -iname *.proto` && pbts -n protos -o src/protos.d.ts src/protos.js"
   },
   "eslintConfig": {
     "extends": [
@@ -92,6 +93,7 @@
     "@testing-library/react": "^12.0.0",
     "@testing-library/user-event": "^13.2.1",
     "msw": "^0.36.8",
+    "protobufjs-cli": "^1.0.2",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
     "rimraf": "^3.0.2",

@@ -1,37 +1,16 @@
 import React from "react";
-import { z } from "zod";
 import { EuiBasicTable } from "@elastic/eui";
-import { FeastFeatureInServiceType } from "../parsers/feastFeatureServices";
 import EuiCustomLink from "./EuiCustomLink";
-import { FEAST_FEATURE_VALUE_TYPES } from "../parsers/types";
 import { useParams } from "react-router-dom";
+import { feast } from "../protos";
 
 interface FeatureViewsListInterace {
-  featureViews: FeastFeatureInServiceType[];
+  featureViews: feast.core.IFeatureViewProjection[];
 }
 
 const FeaturesInServiceList = ({ featureViews }: FeatureViewsListInterace) => {
   const { projectName } = useParams();
 
-  const FeatureInService = z.object({
-    featureViewName: z.string(),
-    featureColumnName: z.string(),
-    valueType: z.nativeEnum(FEAST_FEATURE_VALUE_TYPES),
-  });
-  type FeatureInServiceType = z.infer<typeof FeatureInService>;
-
-  var items: FeatureInServiceType[] = [];
-  featureViews.forEach((featureView) => {
-    featureView.featureColumns.forEach((featureColumn) => {
-      const row: FeatureInServiceType = {
-        featureViewName: featureView.featureViewName,
-        featureColumnName: featureColumn.name,
-        valueType: featureColumn.valueType,
-      };
-      items.push(row);
-    });
-  });
-
   const columns = [
     {
       name: "Feature View",
@@ -49,22 +28,22 @@ const FeaturesInServiceList = ({ featureViews }: FeatureViewsListInterace) => {
     },
     {
       name: "Feature Column",
-      field: "featureColumnName",
+      field: "featureColumn.name",
     },
     {
       name: "Value Type",
       field: "valueType",
     },
   ];
 
-  const getRowProps = (item: FeatureInServiceType) => {
+  const getRowProps = (item: feast.core.IFeatureViewProjection) => {
     return {
       "data-test-subj": `row-${item.featureViewName}`,
     };
   };
 
   return (
-    <EuiBasicTable columns={columns} items={items} rowProps={getRowProps} />
+    <EuiBasicTable columns={columns} items={featureViews} rowProps={getRowProps} />
   );
 };
 

@@ -1,15 +1,15 @@
 import React, { useContext } from "react";
 import { EuiBasicTable, EuiLoadingSpinner, EuiBadge } from "@elastic/eui";
-import { FeastFeatureColumnType } from "../parsers/feastFeatureViews";
 import useLoadFeatureViewSummaryStatistics from "../queries/useLoadFeatureViewSummaryStatistics";
 import SparklineHistogram from "./SparklineHistogram";
 import FeatureFlagsContext from "../contexts/FeatureFlagsContext";
 import EuiCustomLink from "./EuiCustomLink";
+import { feast } from "../protos";
 
 interface FeaturesListProps {
   projectName: string;
   featureViewName: string;
-  features: FeastFeatureColumnType[];
+  features: feast.core.IFeatureSpecV2[];
   link: boolean;
 }
 
@@ -19,16 +19,16 @@ const FeaturesList = ({ projectName, featureViewName, features, link }: Features
     useLoadFeatureViewSummaryStatistics(featureViewName);
 
   let columns: { name: string; render?: any; field: any }[] = [
-    { 
+    {
       name: "Name",
       field: "name",
-      render: (item: string) => ( 
-        <EuiCustomLink 
+      render: (item: string) => (
+        <EuiCustomLink
           href={`/p/${projectName}/feature-view/${featureViewName}/feature/${item}`}
           to={`/p/${projectName}/feature-view/${featureViewName}/feature/${item}`}>
           {item}
         </EuiCustomLink>
-      ) 
+      )
     },
     {
       name: "Value Type",
@@ -46,9 +46,9 @@ const FeaturesList = ({ projectName, featureViewName, features, link }: Features
         {
           name: "Sample",
           field: "",
-          render: (item: FeastFeatureColumnType) => {
+          render: (item: feast.core.IFeatureSpecV2) => {
             const statistics =
-              isSuccess && data && data.columnsSummaryStatistics[item.name];
+              isSuccess && data && data.columnsSummaryStatistics[item.name!];
 
             return (
               <React.Fragment>
@@ -64,9 +64,9 @@ const FeaturesList = ({ projectName, featureViewName, features, link }: Features
         {
           name: "Sparklines",
           field: "",
-          render: (item: FeastFeatureColumnType) => {
+          render: (item: feast.core.IFeatureSpecV2) => {
             const statistics =
-              isSuccess && data && data.columnsSummaryStatistics[item.name];
+              isSuccess && data && data.columnsSummaryStatistics[item?.name!];
 
             if (
               statistics &&
@@ -83,7 +83,7 @@ const FeaturesList = ({ projectName, featureViewName, features, link }: Features
     );
   }
 
-  const getRowProps = (item: FeastFeatureColumnType) => {
+  const getRowProps = (item: feast.core.IFeatureSpecV2) => {
     return {
       "data-test-subj": `row-${item.name}`,
     };

@@ -1,9 +1,9 @@
 import { encodeSearchQueryString } from "./encodeSearchQueryString";
 import { FEAST_FCO_TYPES } from "../parsers/types";
 
-import { FeastFeatureViewType } from "../parsers/feastFeatureViews";
 import { useParams } from "react-router-dom";
 import { useFeatureViewTagsAggregation } from "./useTagsAggregation";
+import { feast } from "../protos";
 
 interface ExplorationSuggestionItem {
   name: string;
@@ -66,14 +66,14 @@ const sortTagsByTotalUsage = <T>(
 };
 
 const generateExplorationSuggestions = (
-  tagAggregation: Record<string, Record<string, FeastFeatureViewType[]>>,
+  tagAggregation: Record<string, Record<string, feast.core.IFeatureView[]>>,
   projectName: string
 ) => {
   const suggestions: ExplorationSuggestion[] = [];
 
   if (tagAggregation) {
     const SortedCandidates =
-      sortTagByUniqueValues<FeastFeatureViewType>(tagAggregation);
+      sortTagByUniqueValues<feast.core.IFeatureView>(tagAggregation);
 
     SortedCandidates.slice(0, NUMBER_OF_SUGGESTION_GROUPS).forEach(
       ([selectedTag, selectedTagValuesMap]) => {

@@ -1,8 +1,7 @@
 import { useContext, useMemo } from "react";
 import RegistryPathContext from "../contexts/RegistryPathContext";
-import { FeastFeatureServiceType } from "../parsers/feastFeatureServices";
-import { FeastFeatureViewType } from "../parsers/feastFeatureViews";
 import useLoadRegistry from "../queries/useLoadRegistry";
+import { feast } from "../protos";
 
 // Usage of generic type parameter T
 // https://stackoverflow.com/questions/53203409/how-to-tell-typescript-that-im-returning-an-array-of-arrays-of-the-input-type
@@ -44,12 +43,12 @@ const useFeatureViewTagsAggregation = () => {
 
   const data = useMemo(() => {
     return query.data && query.data.objects && query.data.objects.featureViews
-      ? buildTagCollection<FeastFeatureViewType>(
-          query.data.objects.featureViews,
-          (fv) => {
-            return fv.spec.tags;
-          }
-        )
+      ? buildTagCollection<feast.core.IFeatureView>(
+        query.data.objects.featureViews!,
+        (fv) => {
+          return fv.spec?.tags!;
+        }
+      )
       : undefined;
   }, [query.data]);
 
@@ -67,12 +66,12 @@ const useFeatureServiceTagsAggregation = () => {
     return query.data &&
       query.data.objects &&
       query.data.objects.featureServices
-      ? buildTagCollection<FeastFeatureServiceType>(
-          query.data.objects.featureServices,
-          (fs) => {
-            return fs.spec.tags;
-          }
-        )
+      ? buildTagCollection<feast.core.IFeatureService>(
+        query.data.objects.featureServices,
+        (fs) => {
+          return fs.spec?.tags!;
+        }
+      )
       : undefined;
   }, [query.data]);
 

@@ -9,30 +9,11 @@ import {
   EuiSpacer,
   EuiTitle,
 } from "@elastic/eui";
+import { feast } from "../../protos";
+import { toDate } from "../../utils/timestamp";
 
 interface BatchSourcePropertiesViewProps {
-  batchSource: {
-    type?: string | undefined;
-    owner?: string | undefined;
-    description?: string | undefined;
-    dataSourceClassType?: string | undefined;
-    fileOptions?:
-      | {
-          uri?: string | undefined;
-        }
-      | undefined;
-    meta?:
-      | {
-          latestEventTimestamp?: Date | undefined;
-          earliestEventTimestamp?: Date | undefined;
-        }
-      | undefined;
-    bigqueryOptions?:
-      | {
-          dbtModelSerialized?: string | undefined;
-        }
-      | undefined;
-  };
+  batchSource: feast.core.IDataSource;
 }
 
 const BatchSourcePropertiesView = (props: BatchSourcePropertiesViewProps) => {
@@ -87,7 +68,7 @@ const BatchSourcePropertiesView = (props: BatchSourcePropertiesViewProps) => {
               <React.Fragment>
                 <EuiDescriptionListTitle>Latest Event</EuiDescriptionListTitle>
                 <EuiDescriptionListDescription>
-                  {batchSource.meta.latestEventTimestamp.toLocaleDateString(
+                  {toDate(batchSource.meta.latestEventTimestamp).toLocaleDateString(
                     "en-CA"
                   )}
                 </EuiDescriptionListDescription>
@@ -99,7 +80,7 @@ const BatchSourcePropertiesView = (props: BatchSourcePropertiesViewProps) => {
                   Earliest Event
                 </EuiDescriptionListTitle>
                 <EuiDescriptionListDescription>
-                  {batchSource.meta.earliestEventTimestamp.toLocaleDateString(
+                  {toDate(batchSource.meta?.earliestEventTimestamp).toLocaleDateString(
                     "en-CA"
                   )}
                 </EuiDescriptionListDescription>

@@ -77,12 +77,12 @@ const DataSourceOverviewTab = () => {
                       </EuiTitle>
                       <EuiHorizontalRule margin="xs"></EuiHorizontalRule>
                       <RequestDataSourceSchemaTable
-                        fields={data.requestDataOptions.schema.map((obj) => {
+                        fields={data?.requestDataOptions?.schema!.map((obj) => {
                           return {
-                            fieldName: obj.name,
-                            valueType: obj.valueType,
+                            fieldName: obj.name!,
+                            valueType: obj.valueType!,
                           };
-                        })}
+                        })!}
                       />
                     </EuiPanel>
                   ) : (