Support multiple application properties files (incl from classpath) (feast-dev#2187)

pyalex · web-flow · commit 94c780bf419c · 2022-01-04T08:11:56.000-08:00
* support spring-like properties paths

Signed-off-by: pyalex &lt;moskalenko.alexey@gmail.com&gt;

* typo

Signed-off-by: pyalex &lt;moskalenko.alexey@gmail.com&gt;
diff --git a/infra/charts/feast/charts/feature-server/templates/deployment.yaml b/infra/charts/feast/charts/feature-server/templates/deployment.yaml
@@ -89,8 +89,7 @@ spec:
         - java
         - -jar
         - /opt/feast/feast-serving.jar
-        - --spring.config.location=
-          {{- if index .Values "application.yaml" "enabled" -}}
+        - {{- if index .Values "application.yaml" "enabled" -}}
           classpath:/application.yml
           {{- end }}
           {{- if index .Values "application-generated.yaml" "enabled" -}}
diff --git a/infra/docker-compose/docker-compose.yml b/infra/docker-compose/docker-compose.yml
@@ -16,7 +16,7 @@ services:
       - java
       - -jar
       - /opt/feast/feast-core.jar
-      - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml
+      - classpath:/application.yml,file:/etc/feast/application.yml
 
   jobservice:
     image: gcr.io/kf-feast/feast-jobservice:${FEAST_VERSION}
@@ -104,7 +104,7 @@ services:
       - java
       - -jar
       - /opt/feast/feast-serving.jar
-      - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml
+      - classpath:/application.yml,file:/etc/feast/application.yml
 
   redis:
     image: redis:5-alpine
diff --git a/java/serving/src/main/java/feast/serving/ServingGuiceApplication.java b/java/serving/src/main/java/feast/serving/ServingGuiceApplication.java
@@ -27,7 +27,7 @@ public class ServingGuiceApplication {
   public static void main(String[] args) throws InterruptedException, IOException {
     if (args.length == 0) {
       throw new RuntimeException(
-          "Path to application configuration file needs to be specifed via CLI");
+          "Path to application configuration file needs to be specified via CLI");
     }
 
     final Injector i =
diff --git a/java/serving/src/main/java/feast/serving/config/ApplicationProperties.java b/java/serving/src/main/java/feast/serving/config/ApplicationProperties.java
@@ -21,6 +21,8 @@
 // https://www.baeldung.com/configuration-properties-in-spring-boot
 // https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-external-config.html#boot-features-external-config-typesafe-configuration-properties
 
+import com.fasterxml.jackson.annotation.JsonMerge;
+import com.fasterxml.jackson.annotation.OptBoolean;
 import feast.common.logging.config.LoggingProperties;
 import feast.storage.connectors.redis.retriever.RedisClusterStoreConfig;
 import feast.storage.connectors.redis.retriever.RedisStoreConfig;
@@ -83,6 +85,7 @@ public void setActiveStore(String activeStore) {
     /**
      * Collection of store configurations. The active store is selected by the "activeStore" field.
      */
+    @JsonMerge(OptBoolean.FALSE)
     private List<Store> stores = new ArrayList<>();
 
     /* Metric tracing properties. */
@@ -177,6 +180,9 @@ public static class Store {
 
     private Map<String, String> config = new HashMap<>();
 
+    // default construct for deserialization
+    public Store() {}
+
     public Store(String name, String type, Map<String, String> config) {
       this.name = name;
       this.type = type;
@@ -210,17 +216,17 @@ public StoreType getType() {
       return StoreType.valueOf(this.type);
     }
 
+    public void setType(String type) {
+      this.type = type;
+    }
+
     /**
      * Gets the configuration to this specific store. This is a map of strings. These options are
      * unique to the store. Please see protos/feast/core/Store.proto for the store specific
      * configuration options
      *
      * @return Returns the store specific configuration
      */
-    public Map<String, String> getConfig() {
-      return config;
-    }
-
     public RedisClusterStoreConfig getRedisClusterConfig() {
       return new RedisClusterStoreConfig(
           this.config.get("connection_string"),
@@ -235,6 +241,10 @@ public RedisStoreConfig getRedisConfig() {
           Boolean.valueOf(this.config.getOrDefault("ssl", "false")),
           this.config.getOrDefault("password", ""));
     }
+
+    public void setConfig(Map<String, String> config) {
+      this.config = config;
+    }
   }
 
   public static class Server {
diff --git a/java/serving/src/main/java/feast/serving/config/ApplicationPropertiesModule.java b/java/serving/src/main/java/feast/serving/config/ApplicationPropertiesModule.java
@@ -17,12 +17,15 @@
 package feast.serving.config;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.google.common.io.Resources;
 import com.google.inject.AbstractModule;
 import com.google.inject.Provides;
 import com.google.inject.Singleton;
-import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 
 public class ApplicationPropertiesModule extends AbstractModule {
   private final String[] args;
@@ -36,9 +39,37 @@ public ApplicationPropertiesModule(String[] args) {
   public ApplicationProperties provideApplicationProperties() throws IOException {
     ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
     mapper.findAndRegisterModules();
-    ApplicationProperties properties =
-        mapper.readValue(new File(this.args[0]), ApplicationProperties.class);
+    mapper.setDefaultMergeable(Boolean.TRUE);
+
+    ApplicationProperties properties = new ApplicationProperties();
+    ObjectReader objectReader = mapper.readerForUpdating(properties);
+
+    String[] filePaths = this.args[0].split(",");
+    for (String filePath : filePaths) {
+      objectReader.readValue(readPropertiesFile(filePath));
+    }
 
     return properties;
   }
+
+  /**
+   * Read file path in spring compatible format, eg classpath:/application.yml or
+   * file:/path/application.yml
+   */
+  private byte[] readPropertiesFile(String filePath) throws IOException {
+    if (filePath.startsWith("classpath:")) {
+      filePath = filePath.substring("classpath:".length());
+      if (filePath.startsWith("/")) {
+        filePath = filePath.substring(1);
+      }
+
+      return Resources.toByteArray(Resources.getResource(filePath));
+    }
+
+    if (filePath.startsWith("file")) {
+      filePath = filePath.substring("file:".length());
+    }
+
+    return Files.readAllBytes(Path.of(filePath));
+  }
 }
diff --git a/java/serving/src/main/java/feast/serving/config/ServerModule.java b/java/serving/src/main/java/feast/serving/config/ServerModule.java
@@ -17,6 +17,7 @@
 package feast.serving.config;
 
 import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
 import feast.serving.grpc.OnlineServingGrpcServiceV2;
 import io.grpc.Server;
 import io.grpc.ServerBuilder;
@@ -30,7 +31,7 @@ protected void configure() {
     bind(OnlineServingGrpcServiceV2.class);
   }
 
-  //  @Provides
+  @Provides
   public Server provideGrpcServer(
       ApplicationProperties applicationProperties,
       OnlineServingGrpcServiceV2 onlineServingGrpcServiceV2,

Original file line number	Diff line number	Diff line change
`@@ -27,7 +27,7 @@ public class ServingGuiceApplication {`
`27`	`27`	`public static void main(String[] args) throws InterruptedException, IOException {`
`28`	`28`	`if (args.length == 0) {`
`29`	`29`	`throw new RuntimeException(`
`30`		`- "Path to application configuration file needs to be specifed via CLI");`
	`30`	`+ "Path to application configuration file needs to be specified via CLI");`
`31`	`31`	`}`
`32`	`32`
`33`	`33`	`final Injector i =`