diff --git a/protos/feast/core/CoreService.proto b/protos/feast/core/CoreService.proto new file mode 100644 index 00000000..e9b7b4c4 --- /dev/null +++ b/protos/feast/core/CoreService.proto @@ -0,0 +1,306 @@ +// +// Copyright 2018 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; +package feast.core; + +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; +option java_outer_classname = "CoreServiceProto"; +option java_package = "feast.proto.core"; + +import "google/protobuf/timestamp.proto"; +import "tensorflow_metadata/proto/v0/statistics.proto"; +import "feast/core/Entity.proto"; +import "feast/core/Feature.proto"; +import "feast/core/FeatureTable.proto"; +import "feast/core/Store.proto"; + +service CoreService { + // Retrieve version information about this Feast deployment + rpc GetFeastCoreVersion (GetFeastCoreVersionRequest) returns (GetFeastCoreVersionResponse); + + // Returns a specific entity + rpc GetEntity (GetEntityRequest) returns (GetEntityResponse); + + // Returns all feature references and respective features matching that filter. If none are found + // an empty map will be returned + // If no filter is provided in the request, the response will contain all the features + // currently stored in the default project. + rpc ListFeatures (ListFeaturesRequest) returns (ListFeaturesResponse); + + // Retrieve store details given a filter. + // + // Returns all stores matching that filter. If none are found, an empty list will be returned. + // If no filter is provided in the request, the response will contain all the stores currently + // stored in the registry. + rpc ListStores (ListStoresRequest) returns (ListStoresResponse); + + // Create or update and existing entity. + // + // This function is idempotent - it will not create a new entity if schema does not change. + // Schema changes will update the entity if the changes are valid. + // Following changes are not valid: + // - Changes to name + // - Changes to type + rpc ApplyEntity (ApplyEntityRequest) returns (ApplyEntityResponse); + + // Returns all entity references and respective entities matching that filter. If none are found + // an empty map will be returned + // If no filter is provided in the request, the response will contain all the entities + // currently stored in the default project. + rpc ListEntities (ListEntitiesRequest) returns (ListEntitiesResponse); + + // Updates core with the configuration of the store. + // + // If the changes are valid, core will return the given store configuration in response, and + // start or update the necessary feature population jobs for the updated store. + rpc UpdateStore (UpdateStoreRequest) returns (UpdateStoreResponse); + + // Creates a project. Projects serve as namespaces within which resources like features will be + // created. Feature table names as must be unique within a project while field (Feature/Entity) names + // must be unique within a Feature Table. Project names themselves must be globally unique. + rpc CreateProject (CreateProjectRequest) returns (CreateProjectResponse); + + // Archives a project. Archived projects will continue to exist and function, but won't be visible + // through the Core API. Any existing ingestion or serving requests will continue to function, + // but will result in warning messages being logged. It is not possible to unarchive a project + // through the Core API + rpc ArchiveProject (ArchiveProjectRequest) returns (ArchiveProjectResponse); + + // Lists all projects active projects. + rpc ListProjects (ListProjectsRequest) returns (ListProjectsResponse); + + /* Feature Tables */ + // Create or update an existing feature table. + // This function is idempotent - it will not create a new feature table if the schema does not change. + // Schema changes will update the feature table if the changes are valid. + // All changes except the following are valid: + // - Changes to feature table name. + // - Changes to entities + // - Changes to feature name and type + rpc ApplyFeatureTable (ApplyFeatureTableRequest) returns (ApplyFeatureTableResponse); + + // List feature tables that match a given filter. + // Returns the references of the Feature Tables matching that filter. If none are found, + // an empty list will be returned. + // If no filter is provided in the request, the response will match all the feature + // tables currently stored in the registry. + rpc ListFeatureTables (ListFeatureTablesRequest) returns (ListFeatureTablesResponse); + + // Returns a specific feature table + rpc GetFeatureTable (GetFeatureTableRequest) returns (GetFeatureTableResponse); + + // Delete a specific feature table + rpc DeleteFeatureTable (DeleteFeatureTableRequest) returns (DeleteFeatureTableResponse); + +} + +// Request for a single entity +message GetEntityRequest { + // Name of entity (required). + string name = 1; + + // Name of project the entity belongs to. If omitted will default to 'default' project. + string project = 2; +} + +// Response containing a single entity +message GetEntityResponse { + feast.core.Entity entity = 1; +} + +// Retrieves details for all versions of a specific entity +message ListEntitiesRequest { + Filter filter = 1; + + message Filter { + // Optional. Specifies the name of the project to list Entities in. + // It is NOT possible to provide an asterisk with a string in order to do pattern matching. + // If unspecified, this field will default to the default project 'default'. + string project = 3; + + // Optional. User defined metadata for entity. + // Entities with all matching labels will be returned. + map labels = 4; + } +} + +message ListEntitiesResponse { + repeated feast.core.Entity entities = 1; +} + +message ListFeaturesRequest { + message Filter { + // User defined metadata for feature. + // Features with all matching labels will be returned. + map labels = 1; + + // List of entities contained within the featureSet that the feature belongs to. + // Only feature tables with these entities will be searched for features. + repeated string entities = 2; + + // Name of project that the feature tables belongs to. Filtering on projects is disabled. + // It is NOT possible to provide an asterisk with a string in order to do pattern matching. + // If unspecified this field will default to the default project 'default'. + string project = 3; + } + + Filter filter = 1; +} + +message ListFeaturesResponse { + reserved 1; + + map features = 2; +} + +message ListStoresRequest { + message Filter { + // Name of desired store. Regex is not supported in this query. + string name = 1; + } + + Filter filter = 1; +} + +message ListStoresResponse { + repeated feast.core.Store store = 1; +} + +message ApplyEntityRequest { + // If project is unspecified, will default to 'default' project. + // If project specified does not exist, the project would be automatically created. + feast.core.EntitySpecV2 spec = 1; + + // Name of project that this entity belongs to. + string project = 2; +} + +message ApplyEntityResponse { + feast.core.Entity entity = 1; +} + +message GetFeastCoreVersionRequest { +} + +message GetFeastCoreVersionResponse { + string version = 1; +} + +message UpdateStoreRequest { + feast.core.Store store = 1; +} + +message UpdateStoreResponse { + enum Status { + // Existing store config matching the given store id is identical to the given store config. + NO_CHANGE = 0; + + // New store created or existing config updated. + UPDATED = 1; + } + feast.core.Store store = 1; + Status status = 2; +} + +// Request to create a project +message CreateProjectRequest { + // Name of project (required) + string name = 1; +} + +// Response for creation of a project +message CreateProjectResponse { +} + +// Request for the archival of a project +message ArchiveProjectRequest { + // Name of project to be archived + string name = 1; +} + +// Response for archival of a project +message ArchiveProjectResponse { +} + +// Request for listing of projects +message ListProjectsRequest { +} + +// Response for listing of projects +message ListProjectsResponse { + // List of project names (archived projects are filtered out) + repeated string projects = 1; +} + +message UpdateFeatureSetStatusResponse {} + +message ApplyFeatureTableRequest { + // Optional. Name of the Project to apply the Feature Table to. + // If unspecified, will apply FeatureTable to the default project. + string project = 1; + // Feature Table specification to apply + FeatureTableSpec table_spec = 2; +} + +message ApplyFeatureTableResponse { + FeatureTable table = 1; +} + +message GetFeatureTableRequest { + // Optional. Name of the Project to retrieve the Feature Table from. + // If unspecified, will apply FeatureTable to the default project. + string project = 1; + + // Name of the FeatureTable to retrieve. + string name = 2; +} + +message GetFeatureTableResponse { + // The Feature Table retrieved. + FeatureTable table = 1; +} + +message ListFeatureTablesRequest { + message Filter { + // Optional. Specifies the name of the project to list Feature Tables in. + // If unspecified would list Feature Tables in the default project. + string project = 1; + + // Optional. Feature Tables with all matching labels will be returned. + // If unspecified would list Feature Tables without filtering by labels. + map labels = 3; + } + + // Filter used when listing Feature Tables + Filter filter = 1; +} + +message ListFeatureTablesResponse { + // List of matching Feature Tables + repeated FeatureTable tables = 1; +} + +message DeleteFeatureTableRequest { + // Optional. Name of the Project to delete the Feature Table from. + // If unspecified, will delete FeatureTable from the default project. + string project = 1; + + // Name of the FeatureTable to delete. + string name = 2; +} + +message DeleteFeatureTableResponse {} diff --git a/protos/feast/core/DataFormat.proto b/protos/feast/core/DataFormat.proto new file mode 100644 index 00000000..2926e08c --- /dev/null +++ b/protos/feast/core/DataFormat.proto @@ -0,0 +1,56 @@ +// +// Copyright 2020 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + + +syntax = "proto3"; +package feast.core; + +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; +option java_outer_classname = "DataFormatProto"; +option java_package = "feast.proto.core"; + +// Defines the file format encoding the features/entity data in files +message FileFormat { + // Defines options for the Parquet data format + message ParquetFormat {} + + oneof format { + ParquetFormat parquet_format = 1; + } +} + +// Defines the data format encoding features/entity data in data streams +message StreamFormat { + // Defines options for the protobuf data format + message ProtoFormat { + // Classpath to the generated Java Protobuf class that can be used to decode + // Feature data from the obtained stream message + string class_path = 1; + } + + // Defines options for the avro data format + message AvroFormat { + // Optional if used in a File DataSource as schema is embedded in avro file. + // Specifies the schema of the Avro message as JSON string. + string schema_json = 1; + } + + // Specifies the data format and format specific options + oneof format { + AvroFormat avro_format = 1; + ProtoFormat proto_format = 2; + } +} diff --git a/protos/feast/core/DataSource.proto b/protos/feast/core/DataSource.proto new file mode 100644 index 00000000..c06a74b1 --- /dev/null +++ b/protos/feast/core/DataSource.proto @@ -0,0 +1,106 @@ +// +// Copyright 2020 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + + +syntax = "proto3"; +package feast.core; + +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; +option java_outer_classname = "DataSourceProto"; +option java_package = "feast.proto.core"; + +import "feast/core/DataFormat.proto"; + +// Defines a Data Source that can be used source Feature data +message DataSource { + // Type of Data Source. + enum SourceType { + INVALID = 0; + BATCH_FILE = 1; + BATCH_BIGQUERY = 2; + STREAM_KAFKA = 3; + STREAM_KINESIS = 4; + } + SourceType type = 1; + + // Defines mapping between fields in the sourced data + // and fields in parent FeatureTable. + map field_mapping = 2; + + // Must specify event timestamp column name + string event_timestamp_column = 3; + + // (Optional) Specify partition column + // useful for file sources + string date_partition_column = 4; + + // Must specify creation timestamp column name + string created_timestamp_column = 5; + + // Defines options for DataSource that sources features from a file + message FileOptions { + FileFormat file_format = 1; + + // Target URL of file to retrieve and source features from. + // s3://path/to/file for AWS S3 storage + // gs://path/to/file for GCP GCS storage + // file:///path/to/file for local storage + string file_url = 2; + } + + // Defines options for DataSource that sources features from a BigQuery Query + message BigQueryOptions { + // Full table reference in the form of [project:dataset.table] + string table_ref = 1; + } + + // Defines options for DataSource that sources features from Kafka messages. + // Each message should be a Protobuf that can be decoded with the generated + // Java Protobuf class at the given class path + message KafkaOptions { + // Comma separated list of Kafka bootstrap servers. Used for feature tables without a defined source host[:port]] + string bootstrap_servers = 1; + + // Kafka topic to collect feature data from. + string topic = 2; + + // Defines the stream data format encoding feature/entity data in Kafka messages. + StreamFormat message_format = 3; + } + + // Defines options for DataSource that sources features from Kinesis records. + // Each record should be a Protobuf that can be decoded with the generated + // Java Protobuf class at the given class path + message KinesisOptions { + // AWS region of the Kinesis stream + string region = 1; + + // Name of the Kinesis stream to obtain feature data from. + string stream_name = 2; + + // Defines the data format encoding the feature/entity data in Kinesis records. + // Kinesis Data Sources support Avro and Proto as data formats. + StreamFormat record_format = 3; + } + + // DataSource options. + oneof options { + FileOptions file_options = 11; + BigQueryOptions bigquery_options = 12; + KafkaOptions kafka_options = 13; + KinesisOptions kinesis_options = 14; + } +} diff --git a/protos/feast/core/Entity.proto b/protos/feast/core/Entity.proto new file mode 100644 index 00000000..ec7b2a07 --- /dev/null +++ b/protos/feast/core/Entity.proto @@ -0,0 +1,51 @@ +// +// * Copyright 2020 The Feast Authors +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * https://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// + +syntax = "proto3"; + +package feast.core; +option java_package = "feast.proto.core"; +option java_outer_classname = "EntityProto"; +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; + +import "feast/types/Value.proto"; +import "google/protobuf/timestamp.proto"; + +message Entity { + // User-specified specifications of this entity. + EntitySpecV2 spec = 1; + // System-populated metadata for this entity. + EntityMeta meta = 2; +} + +message EntitySpecV2 { + // Name of the entity. + string name = 1; + + // Type of the entity. + feast.types.ValueType.Enum value_type = 2; + + // Description of the entity. + string description = 3; + + // User defined metadata + map labels = 8; +} + +message EntityMeta { + google.protobuf.Timestamp created_timestamp = 1; + google.protobuf.Timestamp last_updated_timestamp = 2; +} diff --git a/protos/feast/core/Feature.proto b/protos/feast/core/Feature.proto new file mode 100644 index 00000000..ea0d340a --- /dev/null +++ b/protos/feast/core/Feature.proto @@ -0,0 +1,36 @@ +// +// Copyright 2020 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; +package feast.core; + + +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; +option java_outer_classname = "FeatureProto"; +option java_package = "feast.proto.core"; + +import "feast/types/Value.proto"; + +message FeatureSpecV2 { + // Name of the feature. Not updatable. + string name = 1; + + // Value type of the feature. Not updatable. + feast.types.ValueType.Enum value_type = 2; + + // Labels for user defined metadata on a feature + map labels = 3; +} diff --git a/protos/feast/core/FeatureTable.proto b/protos/feast/core/FeatureTable.proto new file mode 100644 index 00000000..13780f0a --- /dev/null +++ b/protos/feast/core/FeatureTable.proto @@ -0,0 +1,83 @@ +// +// Copyright 2020 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + + +syntax = "proto3"; +package feast.core; + + +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; +option java_outer_classname = "FeatureTableProto"; +option java_package = "feast.proto.core"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; +import "feast/core/DataSource.proto"; +import "feast/core/Feature.proto"; + +message FeatureTable { + // User-specified specifications of this feature table. + FeatureTableSpec spec = 1; + + // System-populated metadata for this feature table. + FeatureTableMeta meta = 2; +} + +message FeatureTableSpec { + // Name of the feature table. Must be unique. Not updated. + string name = 1; + + // List names of entities to associate with the Features defined in this + // Feature Table. Not updatable. + repeated string entities = 3; + + // List of features specifications for each feature defined with this feature table. + repeated FeatureSpecV2 features = 4; + + // User defined metadata + map labels = 5; + + // Features in this feature table can only be retrieved from online serving + // younger than max age. Age is measured as the duration of time between + // the feature's event timestamp and when the feature is retrieved + // Feature values outside max age will be returned as unset values and indicated to end user + google.protobuf.Duration max_age = 6; + + // Batch/Offline DataSource to source batch/offline feature data. + // Only batch DataSource can be specified + // (ie source type should start with 'BATCH_') + DataSource batch_source = 7; + + // Stream/Online DataSource to source stream/online feature data. + // Only stream DataSource can be specified + // (ie source type should start with 'STREAM_') + DataSource stream_source = 8; +} + +message FeatureTableMeta { + // Time where this Feature Table is created + google.protobuf.Timestamp created_timestamp = 1; + + // Time where this Feature Table is last updated + google.protobuf.Timestamp last_updated_timestamp = 2; + + // Auto incrementing revision no. of this Feature Table + int64 revision = 3; + + // Hash entities, features, batch_source and stream_source to inform JobService if + // jobs should be restarted should hash change + string hash = 4; +} diff --git a/protos/feast/core/JobService.proto b/protos/feast/core/JobService.proto new file mode 100644 index 00000000..d79cb9e1 --- /dev/null +++ b/protos/feast/core/JobService.proto @@ -0,0 +1,210 @@ +// +// Copyright 2018 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; +package feast.core; + +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; +option java_outer_classname = "JobServiceProto"; +option java_package = "feast.proto.core"; + +import "google/protobuf/timestamp.proto"; +import "feast/core/DataSource.proto"; + + +service JobService { + // Start job to ingest data from offline store into online store + rpc StartOfflineToOnlineIngestionJob (StartOfflineToOnlineIngestionJobRequest) returns (StartOfflineToOnlineIngestionJobResponse); + + // Produce a training dataset, return a job id that will provide a file reference + rpc GetHistoricalFeatures (GetHistoricalFeaturesRequest) returns (GetHistoricalFeaturesResponse); + + // Start job to ingest data from stream into online store + rpc StartStreamToOnlineIngestionJob (StartStreamToOnlineIngestionJobRequest) returns (StartStreamToOnlineIngestionJobResponse); + + // List all types of jobs + rpc ListJobs (ListJobsRequest) returns (ListJobsResponse); + + // Cancel a single job + rpc CancelJob (CancelJobRequest) returns (CancelJobResponse); + + // Get details of a single job + rpc GetJob (GetJobRequest) returns (GetJobResponse); +} + + +enum JobType { + INVALID_JOB = 0; + BATCH_INGESTION_JOB = 1; + STREAM_INGESTION_JOB = 2; + RETRIEVAL_JOB = 4; +} + +enum JobStatus { + JOB_STATUS_INVALID = 0; + // The Job has be registered and waiting to get scheduled to run + JOB_STATUS_PENDING = 1; + // The Job is currently processing its task + JOB_STATUS_RUNNING = 2; + // The Job has successfully completed its task + JOB_STATUS_DONE = 3; + // The Job has encountered an error while processing its task + JOB_STATUS_ERROR = 4; +} + +message Job { + // Identifier of the Job + string id = 1; + // Type of the Job + JobType type = 2; + // Current job status + JobStatus status = 3; + // Deterministic hash of the Job + string hash = 4; + // Start time of the Job + google.protobuf.Timestamp start_time = 5; + + message RetrievalJobMeta { + string output_location = 1; + } + + message OfflineToOnlineMeta { + string table_name = 1; + } + + message StreamToOnlineMeta { + string table_name = 1; + } + + // JobType specific metadata on the job + oneof meta { + RetrievalJobMeta retrieval = 6; + OfflineToOnlineMeta batch_ingestion = 7; + StreamToOnlineMeta stream_ingestion = 8; + } + + // Path to Spark job logs, if available + string log_uri = 9; +} + +// Ingest data from offline store into online store +message StartOfflineToOnlineIngestionJobRequest { + // Feature table to ingest + string project = 1; + string table_name = 2; + + // Start of time range for source data from offline store + google.protobuf.Timestamp start_date = 3; + + // End of time range for source data from offline store + google.protobuf.Timestamp end_date = 4; +} + +message StartOfflineToOnlineIngestionJobResponse { + // Job ID assigned by Feast + string id = 1; + + // Job start time + google.protobuf.Timestamp job_start_time = 2; + + // Feature table associated with the job + string table_name = 3; + + // Path to Spark job logs, if available + string log_uri = 4; +} + +message GetHistoricalFeaturesRequest { + // List of feature references that are being retrieved + repeated string feature_refs = 1; + + // Batch DataSource that can be used to obtain entity values for historical retrieval. + // For each entity value, a feature value will be retrieved for that value/timestamp + // Only 'BATCH_*' source types are supported. + // Currently only BATCH_FILE source type is supported. + DataSource entity_source = 2; + + // Optional field to specify project name override. If specified, uses the + // given project for retrieval. Overrides the projects specified in + // Feature References if both are specified. + string project = 3; + + // Specifies the path in a bucket to write the exported feature data files + // Export to AWS S3 - s3://path/to/features + // Export to GCP GCS - gs://path/to/features + string output_location = 4; + + // Specify format name for output, eg. parquet + string output_format = 5; +} + +message GetHistoricalFeaturesResponse { + // Export Job with ID assigned by Feast + string id = 1; + + // Uri to the join result output file + string output_file_uri = 2; + + // Job start time + google.protobuf.Timestamp job_start_time = 3; + + // Path to Spark job logs, if available + string log_uri = 4; + +} + +message StartStreamToOnlineIngestionJobRequest { + // Feature table to ingest + string project = 1; + string table_name = 2; +} + +message StartStreamToOnlineIngestionJobResponse { + // Job ID assigned by Feast + string id = 1; + + // Job start time + google.protobuf.Timestamp job_start_time = 2; + + // Feature table associated with the job + string table_name = 3; + + // Path to Spark job logs, if available + string log_uri = 4; +} + +message ListJobsRequest { + bool include_terminated = 1; + string table_name = 2; +} + +message ListJobsResponse { + repeated Job jobs = 1; +} + +message GetJobRequest { + string job_id = 1; +} + +message GetJobResponse { + Job job = 1; +} + +message CancelJobRequest{ + string job_id = 1; +} + +message CancelJobResponse {} \ No newline at end of file diff --git a/protos/feast/core/Store.proto b/protos/feast/core/Store.proto new file mode 100644 index 00000000..58d02874 --- /dev/null +++ b/protos/feast/core/Store.proto @@ -0,0 +1,134 @@ +// +// * Copyright 2019 The Feast Authors +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * https://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// + +syntax = "proto3"; +package feast.core; + +option java_package = "feast.proto.core"; +option java_outer_classname = "StoreProto"; +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; + +import "google/protobuf/duration.proto"; + +// Store provides a location where Feast reads and writes feature values. +// Feature values will be written to the Store in the form of FeatureRow elements. +// The way FeatureRow is encoded and decoded when it is written to and read from +// the Store depends on the type of the Store. +// +message Store { + + enum StoreType { + // These positions should not be reused. + reserved 2, 3, 12, 13; + + INVALID = 0; + + // Redis stores a FeatureRow element as a key, value pair. + // + // The Redis data types used (https://redis.io/topics/data-types): + // - key: STRING + // - value: STRING + // + // Encodings: + // - key: byte array of RedisKey (refer to feast.storage.RedisKeyV2) + // - value: Redis hashmap + // + REDIS = 1; + + REDIS_CLUSTER = 4; + } + + message RedisConfig { + string host = 1; + int32 port = 2; + // Optional. The number of milliseconds to wait before retrying failed Redis connection. + // By default, Feast uses exponential backoff policy and "initial_backoff_ms" sets the initial wait duration. + int32 initial_backoff_ms = 3; + // Optional. Maximum total number of retries for connecting to Redis. Default to zero retries. + int32 max_retries = 4; + // Optional. How often flush data to redis + int32 flush_frequency_seconds = 5; + // Optional. Connect over SSL. + bool ssl = 6; + } + + message RedisClusterConfig { + // List of Redis Uri for all the nodes in Redis Cluster, comma separated. Eg. host1:6379, host2:6379 + string connection_string = 1; + int32 initial_backoff_ms = 2; + int32 max_retries = 3; + // Optional. How often flush data to redis + int32 flush_frequency_seconds = 4; + // Optional. Append a prefix to the Redis Key + string key_prefix = 5; + // Optional. Enable fallback to another key prefix if the original key is not present. + // Useful for migrating key prefix without re-ingestion. Disabled by default. + bool enable_fallback = 6; + // Optional. This would be the fallback prefix to use if enable_fallback is true. + string fallback_prefix = 7; + + // Optional. Priority of nodes when reading from cluster + enum ReadFrom { + MASTER = 0; + MASTER_PREFERRED = 1; + REPLICA = 2; + REPLICA_PREFERRED = 3; + } + ReadFrom read_from = 8; + // Optional. Timeout on waiting response from redis node + google.protobuf.Duration timeout = 9; + } + + message Subscription { + // Name of project that the feature sets belongs to. This can be one of + // - [project_name] + // - * + // If an asterisk is provided, filtering on projects will be disabled. All projects will + // be matched. It is NOT possible to provide an asterisk with a string in order to do + // pattern matching. + string project = 3; + + // Name of the desired feature set. Asterisks can be used as wildcards in the name. + // Matching on names is only permitted if a specific project is defined. It is disallowed + // If the project name is set to "*" + // e.g. + // - * can be used to match all feature sets + // - my-feature-set* can be used to match all features prefixed by "my-feature-set" + // - my-feature-set-6 can be used to select a single feature set + string name = 1; + + // All matches with exclude enabled will be filtered out instead of added + bool exclude = 4; + + // Feature set version was removed in v0.5.0. + reserved 2; + } + + // Name of the store. + string name = 1; + + // Type of store. + StoreType type = 2; + + // Feature sets to subscribe to. + repeated Subscription subscriptions = 4; + + // Configuration to connect to the store. Required. + oneof config { + RedisConfig redis_config = 11; + RedisClusterConfig redis_cluster_config = 14; + } +} diff --git a/protos/feast/serving/ServingService.proto b/protos/feast/serving/ServingService.proto new file mode 100644 index 00000000..e69de29b diff --git a/protos/feast/storage/Redis.proto b/protos/feast/storage/Redis.proto new file mode 100644 index 00000000..a662e352 --- /dev/null +++ b/protos/feast/storage/Redis.proto @@ -0,0 +1,34 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +import "feast/types/Field.proto"; +import "feast/types/Value.proto"; + +package feast.storage; + +option java_outer_classname = "RedisProto"; +option java_package = "feast.proto.storage"; +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/storage"; + +message RedisKeyV2 { + string project = 1; + + repeated string entity_names = 2; + + repeated feast.types.Value entity_values = 3; +} diff --git a/protos/feast/third_party/grpc/health/v1/HealthService.proto b/protos/feast/third_party/grpc/health/v1/HealthService.proto new file mode 100644 index 00000000..342db35d --- /dev/null +++ b/protos/feast/third_party/grpc/health/v1/HealthService.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package grpc.health.v1; + +option java_package = "io.grpc.health.v1"; +option java_outer_classname = "HealthProto"; + +message HealthCheckRequest { + string service = 1; +} + +enum ServingStatus { + UNKNOWN = 0; + SERVING = 1; + NOT_SERVING = 2; +} + +message HealthCheckResponse { + ServingStatus status = 1; +} + +service Health { + rpc Check(HealthCheckRequest) returns (HealthCheckResponse); +} \ No newline at end of file diff --git a/protos/feast/types/Field.proto b/protos/feast/types/Field.proto new file mode 100644 index 00000000..3b8416c2 --- /dev/null +++ b/protos/feast/types/Field.proto @@ -0,0 +1,30 @@ +/* + * Copyright 2018 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +import "feast/types/Value.proto"; + +package feast.types; + +option java_package = "feast.proto.types"; +option java_outer_classname = "FieldProto"; +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/types"; + +message Field { + string name = 1; + feast.types.Value value = 2; +} diff --git a/protos/feast/types/Value.proto b/protos/feast/types/Value.proto new file mode 100644 index 00000000..b0beabd7 --- /dev/null +++ b/protos/feast/types/Value.proto @@ -0,0 +1,92 @@ +/* + * Copyright 2018 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package feast.types; + +option java_package = "feast.proto.types"; +option java_outer_classname = "ValueProto"; +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/types"; + +message ValueType { + enum Enum { + INVALID = 0; + BYTES = 1; + STRING = 2; + INT32 = 3; + INT64 = 4; + DOUBLE = 5; + FLOAT = 6; + BOOL = 7; + BYTES_LIST = 11; + STRING_LIST = 12; + INT32_LIST = 13; + INT64_LIST = 14; + DOUBLE_LIST = 15; + FLOAT_LIST = 16; + BOOL_LIST = 17; + } +} + +message Value { + // ValueType is referenced by the metadata types, FeatureInfo and EntityInfo. + // The enum values do not have to match the oneof val field ids, but they should. + oneof val { + bytes bytes_val = 1; + string string_val = 2; + int32 int32_val = 3; + int64 int64_val = 4; + double double_val = 5; + float float_val = 6; + bool bool_val = 7; + BytesList bytes_list_val = 11; + StringList string_list_val = 12; + Int32List int32_list_val = 13; + Int64List int64_list_val = 14; + DoubleList double_list_val = 15; + FloatList float_list_val = 16; + BoolList bool_list_val = 17; + } +} + +message BytesList { + repeated bytes val = 1; +} + +message StringList { + repeated string val = 1; +} + +message Int32List { + repeated int32 val = 1; +} + +message Int64List { + repeated int64 val = 1; +} + +message DoubleList { + repeated double val = 1; +} + +message FloatList { + repeated float val = 1; +} + +message BoolList { + repeated bool val = 1; +} diff --git a/python/setup.py b/python/setup.py index 81b1367b..b9c77093 100644 --- a/python/setup.py +++ b/python/setup.py @@ -81,12 +81,9 @@ class BuildProtoCommand(Command): description = "Builds the proto files into python files." def initialize_options(self): - import feast - self.protoc = ["python", "-m", "grpc_tools.protoc"] # find_executable("protoc") self.proto_folder = os.path.join(repo_root, "protos") self.this_package = os.path.dirname(__file__) or os.getcwd() - self.feast_protos = os.path.join(os.path.dirname(feast.__file__), 'protos') self.sub_folders = ["api"] def finalize_options(self): @@ -97,7 +94,6 @@ def _generate_protos(self, path): subprocess.check_call(self.protoc + [ '-I', self.proto_folder, - '-I', self.feast_protos, '--python_out', self.this_package, '--grpc_python_out', self.this_package, '--mypy_out', self.this_package] + proto_files) @@ -153,7 +149,7 @@ def run(self): ], entry_points={"console_scripts": ["feast-spark=feast_spark.cli:cli"]}, use_scm_version={"root": "../", "relative_to": __file__, "tag_regex": TAG_REGEX}, - setup_requires=["setuptools_scm", "grpcio-tools==1.31.0", "google-auth==1.21.1", "feast>=0.9.8,<0.10.0", "mypy-protobuf==2.5"], + setup_requires=["setuptools_scm", "grpcio-tools==1.31.0", "google-auth==1.21.1", "mypy-protobuf==2.5"], cmdclass={ "build_proto": BuildProtoCommand, "build_py": BuildCommand,