From 95e5f3269cbd823cc7ef70f0578c634c35ec7f00 Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Thu, 11 Feb 2021 10:34:49 +0800 Subject: [PATCH 1/5] Remove protos Signed-off-by: Terence Lim --- protos/feast/core/JobService.proto | 211 ------------------ .../grpc/health/v1/HealthService.proto | 24 -- 2 files changed, 235 deletions(-) delete mode 100644 protos/feast/core/JobService.proto delete mode 100644 protos/feast/third_party/grpc/health/v1/HealthService.proto diff --git a/protos/feast/core/JobService.proto b/protos/feast/core/JobService.proto deleted file mode 100644 index d698c50e00b..00000000000 --- a/protos/feast/core/JobService.proto +++ /dev/null @@ -1,211 +0,0 @@ -// -// Copyright 2018 The Feast Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -syntax = "proto3"; -package feast.core; - -option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; -option java_outer_classname = "JobServiceProto"; -option java_package = "feast.proto.core"; - -import "google/protobuf/timestamp.proto"; -import "feast/core/DataSource.proto"; - - -service JobService { - // Start job to ingest data from offline store into online store - rpc StartOfflineToOnlineIngestionJob (StartOfflineToOnlineIngestionJobRequest) returns (StartOfflineToOnlineIngestionJobResponse); - - // Produce a training dataset, return a job id that will provide a file reference - rpc GetHistoricalFeatures (GetHistoricalFeaturesRequest) returns (GetHistoricalFeaturesResponse); - - // Start job to ingest data from stream into online store - rpc StartStreamToOnlineIngestionJob (StartStreamToOnlineIngestionJobRequest) returns (StartStreamToOnlineIngestionJobResponse); - - // List all types of jobs - rpc ListJobs (ListJobsRequest) returns (ListJobsResponse); - - // Cancel a single job - rpc CancelJob (CancelJobRequest) returns (CancelJobResponse); - - // Get details of a single job - rpc GetJob (GetJobRequest) returns (GetJobResponse); -} - - -enum JobType { - INVALID_JOB = 0; - BATCH_INGESTION_JOB = 1; - STREAM_INGESTION_JOB = 2; - RETRIEVAL_JOB = 4; -} - -enum JobStatus { - JOB_STATUS_INVALID = 0; - // The Job has be registered and waiting to get scheduled to run - JOB_STATUS_PENDING = 1; - // The Job is currently processing its task - JOB_STATUS_RUNNING = 2; - // The Job has successfully completed its task - JOB_STATUS_DONE = 3; - // The Job has encountered an error while processing its task - JOB_STATUS_ERROR = 4; -} - -message Job { - // Identifier of the Job - string id = 1; - // Type of the Job - JobType type = 2; - // Current job status - JobStatus status = 3; - // Deterministic hash of the Job - string hash = 4; - // Start time of the Job - google.protobuf.Timestamp start_time = 5; - - message RetrievalJobMeta { - string output_location = 1; - } - - message OfflineToOnlineMeta { - string table_name = 1; - } - - message StreamToOnlineMeta { - string table_name = 1; - } - - // JobType specific metadata on the job - oneof meta { - RetrievalJobMeta retrieval = 6; - OfflineToOnlineMeta batch_ingestion = 7; - StreamToOnlineMeta stream_ingestion = 8; - } - - // Path to Spark job logs, if available - string log_uri = 9; -} - -// Ingest data from offline store into online store -message StartOfflineToOnlineIngestionJobRequest { - // Feature table to ingest - string project = 1; - string table_name = 2; - - // Start of time range for source data from offline store - google.protobuf.Timestamp start_date = 3; - - // End of time range for source data from offline store - google.protobuf.Timestamp end_date = 4; -} - -message StartOfflineToOnlineIngestionJobResponse { - // Job ID assigned by Feast - string id = 1; - - // Job start time - google.protobuf.Timestamp job_start_time = 2; - - // Feature table associated with the job - string table_name = 3; - - // Path to Spark job logs, if available - string log_uri = 4; -} - -message GetHistoricalFeaturesRequest { - // List of feature references that are being retrieved - repeated string feature_refs = 1; - - // Batch DataSource that can be used to obtain entity values for historical retrieval. - // For each entity value, a feature value will be retrieved for that value/timestamp - // Only 'BATCH_*' source types are supported. - // Currently only BATCH_FILE source type is supported. - DataSource entity_source = 2; - - // Optional field to specify project name override. If specified, uses the - // given project for retrieval. Overrides the projects specified in - // Feature References if both are specified. - string project = 3; - - // Specifies the path in a bucket to write the exported feature data files - // Export to AWS S3 - s3://path/to/features - // Export to GCP GCS - gs://path/to/features - string output_location = 4; - - // Specify format name for output, eg. parquet - string output_format = 5; -} - -message GetHistoricalFeaturesResponse { - // Export Job with ID assigned by Feast - string id = 1; - - // Uri to the join result output file - string output_file_uri = 2; - - // Job start time - google.protobuf.Timestamp job_start_time = 3; - - // Path to Spark job logs, if available - string log_uri = 4; - -} - -message StartStreamToOnlineIngestionJobRequest { - // Feature table to ingest - string project = 1; - string table_name = 2; -} - -message StartStreamToOnlineIngestionJobResponse { - // Job ID assigned by Feast - string id = 1; - - // Job start time - google.protobuf.Timestamp job_start_time = 2; - - // Feature table associated with the job - string table_name = 3; - - // Path to Spark job logs, if available - string log_uri = 4; -} - -message ListJobsRequest { - bool include_terminated = 1; - string table_name = 2; - string project = 3; -} - -message ListJobsResponse { - repeated Job jobs = 1; -} - -message GetJobRequest { - string job_id = 1; -} - -message GetJobResponse { - Job job = 1; -} - -message CancelJobRequest{ - string job_id = 1; -} - -message CancelJobResponse {} \ No newline at end of file diff --git a/protos/feast/third_party/grpc/health/v1/HealthService.proto b/protos/feast/third_party/grpc/health/v1/HealthService.proto deleted file mode 100644 index 342db35d4c3..00000000000 --- a/protos/feast/third_party/grpc/health/v1/HealthService.proto +++ /dev/null @@ -1,24 +0,0 @@ -syntax = "proto3"; - -package grpc.health.v1; - -option java_package = "io.grpc.health.v1"; -option java_outer_classname = "HealthProto"; - -message HealthCheckRequest { - string service = 1; -} - -enum ServingStatus { - UNKNOWN = 0; - SERVING = 1; - NOT_SERVING = 2; -} - -message HealthCheckResponse { - ServingStatus status = 1; -} - -service Health { - rpc Check(HealthCheckRequest) returns (HealthCheckResponse); -} \ No newline at end of file From 01e228b49fbf5bf32a625dc1231e2e33d20a5770 Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Thu, 11 Feb 2021 10:35:35 +0800 Subject: [PATCH 2/5] Update workflow Signed-off-by: Terence Lim --- .prow.yaml | 2 +- Makefile | 3 +++ sdk/python/setup.py | 7 +++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.prow.yaml b/.prow.yaml index 876fb2a6453..367f9311b1b 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -108,7 +108,7 @@ postsubmits: - sh - -c - | - make compile-protos-python && infra/scripts/publish-python-sdk.sh \ + make package-protos && make compile-protos-python && infra/scripts/publish-python-sdk.sh \ --directory-path sdk/python --repository pypi volumeMounts: - name: pypirc diff --git a/Makefile b/Makefile index 89b9433bfa6..fbc458ee5f6 100644 --- a/Makefile +++ b/Makefile @@ -64,6 +64,9 @@ build-java-no-tests: install-python-ci-dependencies: pip install --no-cache-dir -r sdk/python/requirements-ci.txt +package-protos: + cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos + compile-protos-python: install-python-ci-dependencies @$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ feast/$(dir)/*.proto;) @$(foreach dir,$(PROTO_SERVICE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/ feast/$(dir)/*.proto;) diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 47f0030b428..c8ceb5d4791 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -94,4 +94,11 @@ entry_points={"console_scripts": ["feast=feast.cli:cli"]}, use_scm_version={"root": "../..", "relative_to": __file__, "tag_regex": TAG_REGEX}, setup_requires=["setuptools_scm"], + package_data={ + "": [ + "protos/feast/**/*.proto", + "protos/feast/third_party/grpc/health/v1/*.proto", + "protos/tensorflow_metadata/proto/v0/*.proto", + ], + }, ) From 9d50d622543331de669105abbd150ecc64ecfce6 Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Thu, 11 Feb 2021 10:59:03 +0800 Subject: [PATCH 3/5] Update Makefile Signed-off-by: Terence Lim --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index fbc458ee5f6..53116ef740c 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,6 @@ compile-protos-python: install-python-ci-dependencies @$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ feast/$(dir)/*.proto;) @$(foreach dir,$(PROTO_SERVICE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/ feast/$(dir)/*.proto;) cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ tensorflow_metadata/proto/v0/*.proto - cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --grpc_python_out=../sdk/python/ --mypy_out=../sdk/python/ feast/third_party/grpc/health/v1/*.proto install-python: compile-protos-python python -m pip install -e sdk/python From 31a83e4d0b764fa695f7c7537d32ff49c5f4b723 Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Thu, 11 Feb 2021 11:32:21 +0800 Subject: [PATCH 4/5] Remove unused import Signed-off-by: Terence Lim --- sdk/python/feast/client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 504e623be69..9fb5bee5c5c 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -51,7 +51,6 @@ ListProjectsResponse, ) from feast.core.CoreService_pb2_grpc import CoreServiceStub -from feast.core.JobService_pb2_grpc import JobServiceStub from feast.data_format import ParquetFormat from feast.data_source import BigQuerySource, FileSource from feast.entity import Entity @@ -114,7 +113,6 @@ def __init__(self, options: Optional[Dict[str, str]] = None, **kwargs): self._core_service_stub: Optional[CoreServiceStub] = None self._serving_service_stub: Optional[ServingServiceStub] = None - self._job_service_stub: Optional[JobServiceStub] = None self._auth_metadata: Optional[grpc.AuthMetadataPlugin] = None # Configure Auth Metadata Plugin if auth is enabled From b6fac5f8199fc2295b29c6edbeeffe006727514e Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Thu, 11 Feb 2021 11:44:45 +0800 Subject: [PATCH 5/5] Revert HealthService proto Signed-off-by: Terence Lim --- .../grpc/health/v1/HealthService.proto | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 protos/feast/third_party/grpc/health/v1/HealthService.proto diff --git a/protos/feast/third_party/grpc/health/v1/HealthService.proto b/protos/feast/third_party/grpc/health/v1/HealthService.proto new file mode 100644 index 00000000000..342db35d4c3 --- /dev/null +++ b/protos/feast/third_party/grpc/health/v1/HealthService.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package grpc.health.v1; + +option java_package = "io.grpc.health.v1"; +option java_outer_classname = "HealthProto"; + +message HealthCheckRequest { + string service = 1; +} + +enum ServingStatus { + UNKNOWN = 0; + SERVING = 1; + NOT_SERVING = 2; +} + +message HealthCheckResponse { + ServingStatus status = 1; +} + +service Health { + rpc Check(HealthCheckRequest) returns (HealthCheckResponse); +} \ No newline at end of file