From 1cf9e4cdf8f2a47ee9018b3ad02596b3e0f27485 Mon Sep 17 00:00:00 2001 From: Ches Martin Date: Tue, 24 Dec 2019 16:06:40 +0700 Subject: [PATCH] Introduce datatypes/java module for proto generation Rather than the Maven protobuf plugin running on the same symlinked definitions in several Java modules, localize this process into one module that the others depend on. This provides a single module that can be depended on by third-party extensions with the bare minimum of dependencies. Also removes proto files that are no longer used. --- core/pom.xml | 4 -- core/src/main/proto/feast | 1 - core/src/main/proto/third_party | 1 - datatypes/java/README.md | 43 +++++++++++ datatypes/java/pom.xml | 72 +++++++++++++++++++ {sdk => datatypes}/java/src/main/proto/feast | 0 datatypes/java/src/main/proto/third_party | 1 + ingestion/pom.xml | 10 +-- ingestion/src/main/proto/feast | 1 - .../feast_ingestion/types/CoalesceAccum.proto | 35 --------- .../feast_ingestion/types/CoalesceKey.proto | 25 ------- ingestion/src/main/proto/third_party | 1 - ingestion/src/test/proto/DriverArea.proto | 10 --- ingestion/src/test/proto/Ping.proto | 12 ---- pom.xml | 20 +----- sdk/java/pom.xml | 10 +-- serving/pom.xml | 10 +-- serving/src/main/proto/feast | 1 - serving/src/main/proto/third_party | 1 - 19 files changed, 135 insertions(+), 123 deletions(-) delete mode 120000 core/src/main/proto/feast delete mode 120000 core/src/main/proto/third_party create mode 100644 datatypes/java/README.md create mode 100644 datatypes/java/pom.xml rename {sdk => datatypes}/java/src/main/proto/feast (100%) create mode 120000 datatypes/java/src/main/proto/third_party delete mode 120000 ingestion/src/main/proto/feast delete mode 100644 ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto delete mode 100644 ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto delete mode 120000 ingestion/src/main/proto/third_party delete mode 100644 ingestion/src/test/proto/DriverArea.proto delete mode 100644 ingestion/src/test/proto/Ping.proto delete mode 120000 serving/src/main/proto/feast delete mode 120000 serving/src/main/proto/third_party diff --git a/core/pom.xml b/core/pom.xml index 954c7c00185..e1567ae8fe3 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -39,10 +39,6 @@ false - - org.xolstice.maven.plugins - protobuf-maven-plugin - diff --git a/core/src/main/proto/feast b/core/src/main/proto/feast deleted file mode 120000 index d520da9126b..00000000000 --- a/core/src/main/proto/feast +++ /dev/null @@ -1 +0,0 @@ -../../../../protos/feast \ No newline at end of file diff --git a/core/src/main/proto/third_party b/core/src/main/proto/third_party deleted file mode 120000 index 363d20598e6..00000000000 --- a/core/src/main/proto/third_party +++ /dev/null @@ -1 +0,0 @@ -../../../../protos/third_party \ No newline at end of file diff --git a/datatypes/java/README.md b/datatypes/java/README.md new file mode 100644 index 00000000000..f93bd99aa38 --- /dev/null +++ b/datatypes/java/README.md @@ -0,0 +1,43 @@ +Feast Data Types for Java +========================= + +This module produces Java class files for Feast's data type and gRPC service +definitions, from Protobuf IDL. These are used across Feast components for wire +interchange, contracts, etc. + +End users of Feast will be best served by our Java SDK which adds higher-level +conveniences, but the data types are published independently for custom needs, +without any additional dependencies the SDK may add. + +Dependency Coordinates +---------------------- + +```xml + + dev.feast + datatypes-java + 0.4.0-SNAPSHOT + +``` + +Using the `.proto` Definitions +------------------------------ + +The `.proto` definitions are packaged as resources within the Maven artifact, +which may be useful to `include` them in dependent Protobuf definitions in a +downstream project, or for other JVM languages to consume from their builds to +generate more idiomatic bindings. + +Google's Gradle plugin, for instance, [can use protos in dependencies][Gradle] +either for `include` or to compile with a different `protoc` plugin than Java. + +[sbt-protoc] offers similar functionality for sbt/Scala. + +[Gradle]: https://github.com/google/protobuf-gradle-plugin#protos-in-dependencies +[sbt-protoc]: https://github.com/thesamet/sbt-protoc + +Publishing +---------- + +TODO: this module should be published to Maven Central upon Feast releases—this +needs to be set up in POM configuration and release automation. diff --git a/datatypes/java/pom.xml b/datatypes/java/pom.xml new file mode 100644 index 00000000000..a6dfa8e345a --- /dev/null +++ b/datatypes/java/pom.xml @@ -0,0 +1,72 @@ + + + + 4.0.0 + + Feast Data Types for Java + + Data types and service contracts used throughout Feast components and + their interchanges. These are generated from Protocol Buffers and gRPC + definitions included in the package. + + datatypes-java + + + dev.feast + feast-parent + ${revision} + ../.. + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + + true + + com.google.protobuf:protoc:${protocVersion}:exe:${os.detected.classifier} + + grpc-java + + io.grpc:protoc-gen-grpc-java:${grpcVersion}:exe:${os.detected.classifier} + + + + + + compile + compile-custom + test-compile + + + + + + + + + + io.grpc + grpc-services + + + diff --git a/sdk/java/src/main/proto/feast b/datatypes/java/src/main/proto/feast similarity index 100% rename from sdk/java/src/main/proto/feast rename to datatypes/java/src/main/proto/feast diff --git a/datatypes/java/src/main/proto/third_party b/datatypes/java/src/main/proto/third_party new file mode 120000 index 00000000000..f015f8477d1 --- /dev/null +++ b/datatypes/java/src/main/proto/third_party @@ -0,0 +1 @@ +../../../../../protos/third_party \ No newline at end of file diff --git a/ingestion/pom.xml b/ingestion/pom.xml index 4908b546985..2e1dee65536 100644 --- a/ingestion/pom.xml +++ b/ingestion/pom.xml @@ -31,10 +31,6 @@ - - org.xolstice.maven.plugins - protobuf-maven-plugin - org.apache.maven.plugins maven-shade-plugin @@ -90,6 +86,12 @@ + + dev.feast + datatypes-java + ${project.version} + + org.glassfish javax.el diff --git a/ingestion/src/main/proto/feast b/ingestion/src/main/proto/feast deleted file mode 120000 index d520da9126b..00000000000 --- a/ingestion/src/main/proto/feast +++ /dev/null @@ -1 +0,0 @@ -../../../../protos/feast \ No newline at end of file diff --git a/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto b/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto deleted file mode 100644 index cb64dd715f6..00000000000 --- a/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto3"; - -import "google/protobuf/timestamp.proto"; -import "feast/types/Field.proto"; - -option java_package = "feast_ingestion.types"; -option java_outer_classname = "CoalesceAccumProto"; - -// Accumlator for merging feature rows. -message CoalesceAccum { - string entityKey = 1; - google.protobuf.Timestamp eventTimestamp = 3; - string entityName = 4; - - map features = 6; - // map of features to their counter values when they were last added to accumulator - map featureMarks = 7; - int64 counter = 8; -} \ No newline at end of file diff --git a/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto b/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto deleted file mode 100644 index 9730b49ec3b..00000000000 --- a/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto3"; - -option java_package = "feast_ingestion.types"; -option java_outer_classname = "CoalesceKeyProto"; - -message CoalesceKey { - string entityName = 1; - string entityKey = 2; -} \ No newline at end of file diff --git a/ingestion/src/main/proto/third_party b/ingestion/src/main/proto/third_party deleted file mode 120000 index 363d20598e6..00000000000 --- a/ingestion/src/main/proto/third_party +++ /dev/null @@ -1 +0,0 @@ -../../../../protos/third_party \ No newline at end of file diff --git a/ingestion/src/test/proto/DriverArea.proto b/ingestion/src/test/proto/DriverArea.proto deleted file mode 100644 index fee838b9e17..00000000000 --- a/ingestion/src/test/proto/DriverArea.proto +++ /dev/null @@ -1,10 +0,0 @@ -syntax = "proto3"; - -package feast; - -option java_outer_classname = "DriverAreaProto"; - -message DriverArea { - int32 driverId = 1; - int32 areaId = 2; -} \ No newline at end of file diff --git a/ingestion/src/test/proto/Ping.proto b/ingestion/src/test/proto/Ping.proto deleted file mode 100644 index b1069afa5bd..00000000000 --- a/ingestion/src/test/proto/Ping.proto +++ /dev/null @@ -1,12 +0,0 @@ -syntax = "proto3"; - -package feast; -import "google/protobuf/timestamp.proto"; - -option java_outer_classname = "PingProto"; - -message Ping { - double lat = 1; - double lng = 2; - google.protobuf.Timestamp timestamp = 3; -} diff --git a/pom.xml b/pom.xml index 05fb701ac44..939dc8507c7 100644 --- a/pom.xml +++ b/pom.xml @@ -28,6 +28,7 @@ pom + datatypes/java ingestion core serving @@ -542,25 +543,6 @@ org.xolstice.maven.plugins protobuf-maven-plugin 0.6.1 - - true - - com.google.protobuf:protoc:${protocVersion}:exe:${os.detected.classifier} - - grpc-java - - io.grpc:protoc-gen-grpc-java:${grpcVersion}:exe:${os.detected.classifier} - - - - - - compile - compile-custom - test-compile - - - diff --git a/sdk/java/pom.xml b/sdk/java/pom.xml index 2970dae3ee2..e8a82a485fc 100644 --- a/sdk/java/pom.xml +++ b/sdk/java/pom.xml @@ -21,6 +21,12 @@ + + dev.feast + datatypes-java + ${project.version} + + io.grpc @@ -79,10 +85,6 @@ - - org.xolstice.maven.plugins - protobuf-maven-plugin - org.apache.maven.plugins diff --git a/serving/pom.xml b/serving/pom.xml index dc3391df62f..c15881030e2 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -47,10 +47,6 @@ false - - org.xolstice.maven.plugins - protobuf-maven-plugin - org.apache.maven.plugins maven-failsafe-plugin @@ -74,6 +70,12 @@ + + dev.feast + datatypes-java + ${project.version} + + org.slf4j diff --git a/serving/src/main/proto/feast b/serving/src/main/proto/feast deleted file mode 120000 index d520da9126b..00000000000 --- a/serving/src/main/proto/feast +++ /dev/null @@ -1 +0,0 @@ -../../../../protos/feast \ No newline at end of file diff --git a/serving/src/main/proto/third_party b/serving/src/main/proto/third_party deleted file mode 120000 index 363d20598e6..00000000000 --- a/serving/src/main/proto/third_party +++ /dev/null @@ -1 +0,0 @@ -../../../../protos/third_party \ No newline at end of file