From f77a2020a9666538a908647b44c595b3c01bb64f Mon Sep 17 00:00:00 2001 From: Shift Date: Tue, 18 Sep 2018 14:18:39 +0000 Subject: [PATCH 01/10] docs(README): Bring README up to date * Add Secrets Manager adapter documentation * Make failure scenarios and fallback behaviour clear and explicit * Add Contribution guidelines * Add Maven Central badge Co-authored-by: Ina Tsetsova Co-authored-by: Alex Wilson --- README.md | 166 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 133 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 75ea6d8..53a0823 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,83 @@ # java-config-fallback [![Build Status](https://travis-ci.org/unruly/java-config-fallback.svg?branch=master)](https://travis-ci.org/unruly/java-config-fallback) +[![Release Version](https://img.shields.io/maven-central/v/co.unruly/java-config-fallback.svg)](https://search.maven.org/#search%7Cgav%7C1%7Cg%3A%22co.unruly%22%20AND%20a%3A%22java-config-fallback%22) -A lightweight library to facilitate extracting configuration from code into configuration files. +A lightweight library to query a hierarchy of configuration sources -## Install from Maven Central +```java +Configuration config = Configuration.of( + secretsManager("my-secret-name", "eu-west-1"), // { "username": "user" } + properties("/etc/my-app-config.properties"), // { "username": "anotherUser", "password": "secret" } +); + +Optional username = config.get("username"); // Optional[user] +Optional password = config.get("password"); // Optional[secret] +``` + +## Contents + +- [Install](#install) +- [Usage](#usage) + - [Fallback Behaviour](#fallback-behaviour) +- [Supported Configuration Sources](#supported-configuration-sources) + - [Map](#map) + - [Properties](#properties) + - [Environment](#environment) + - [AWS Secrets Manager](#aws-secrets-manager) +- [Extending with custom Configuration Sources](#extending-with-custom-configuration-sources) +- [Contributing](#contributing) + - [Code of Conduct](#code-of-conduct) + - [Getting Started](#getting-started) + - [Releasing a Change](#releasing-a-change) +- [License](#license) +- [Design Decisions](#design-decisions) + +## Install + +This library is available on Maven Central ```xml     co.unruly     java-config-fallback -    0.1.5 +    0.1.7 ``` ## Usage -```java -Map credentials = new HashMap<>(); -credentials.put("username","foo"); -credentials.put("password","this-is-a-secret!"); +### Fallback Behaviour -Configuration config = Configuration.of( - properties("/etc/my-app-config.properties"), - map(credentials) -); +When queried for a key, a `Configuration` instance will query each `ConfigurationSource` passed to its builder **in order** until it finds a key. -// look in properties file first, else look in the map, -// and if still not found, return Optional.empty() -Optional username = config.get("username"); -``` - -You can also provide a use-site default value: +Example: ```java -String username = config.get("username", "Titus Andromedon"); +Configuration config = Configuration.of( + properties("high-priority-config.properties"), + properties("default.properties"), + environment() +); ``` -Or state that a value is _required_, and throw a `ConfigurationMissing` exception if if is not defined: +* This configuration will first look for a key in `high-priority-config.properties`. +* If it is not there, it will then look for that key in `default.properties` +* If it is not in either of those, then it will check the application's environment variables. -```java -String username = config.require("username"); -``` +Behaviour if a key cannot be retrieved from any of the provided sources (whether from missing key or missing file or network timeout) depends on the method used to query. -## Configuration Sources +| Method | Parameters | Success Behaviour | Failure Behaviour | +|:-------------|:-----------------------------|:---------------------------------------|:-----------------------------------------------| +| `.get()` | `String key` | Returns `Optional.of()` | Returns `Optional.empty()` | +| `.get()` | `String key, String default` | Returns `Optional.of()` | Returns `Optional.of()` | +| `.require()` | `String key` | Returns `` as a String | Throws `ConfigurationMissing` exception | -### In-code map +## Supported Configuration Sources -`map(Map)` - an in-code map of keys to values +### Map + +The `.map()` configuration source takes a `Map` as a datasource. ```java Map credentials = new HashMap<>(); @@ -60,9 +90,9 @@ String username = config.get("username"); // Optional[foo] String password = config.get("password"); // Optional[this-is-a-secret!] ``` -### Properties file +### Properties -`properties(String pathToFile)` - a .properties file in the filesystem +The `.properties()` configuration source takes a `String filePath` pointing to a `.properties` file as a datasource. ```java Configuration config = Configuration.of( @@ -72,22 +102,87 @@ Configuration config = Configuration.of( String password = config.get("password"); // Optional[new-password] ``` -### Environment variables +### Environment -`environment()` - environment variables set for this application. +The `.environment()` configuration source uses environment variables set for this application. -This assumes environment variables are UPPER_CASE and converts queried keys to UPPER_CASE before lookup. +**NOTE**: This assumes environment variables are UPPER_CASE and converts queried keys to UPPER_CASE before lookup. ```java // With VARIABLE=foo set -Configuration config = Configuration.of( - environment() -); +Configuration config = Configuration.of(environment()); String password = config.get("VARIABLE"); // Optional[foo] String password = config.get("variable"); // Same as above ``` + +### AWS Secrets Manager + +The `.secretsManager()` configuration source uses AWS Secrets Manager as its datasource. + +This can be configured with: + + * `.secretsManager(String secretName, String region)` - uses default AWS client to query Secrets Manager. + * `.secretsManager(String secretName, String region, AWSSecretsManager client)` - uses provided client to query Secrets Manager. Use this if you want to provide custom client behaviour e.g. using a specific set of credentials or instance-role. + +**NOTE**: This configuration source only supports key-value pairs as a return type - plaintext secrets will be ignored. + +```java +Configuration config = Configuration.of( + secretsManager("my-secret", "eu-west-1") // { "username": "user", "password": "secret" } +); + +String password = config.get("username"); // Optional[user] +String password = config.get("password"); // Optional[secret] +``` + +## Extending with custom Configuration Sources + +You can define your own sources of configuration by implementing the `ConfigurationSource` functional interface. + +The contract between this and the `Configuration` class is: + * A `ConfigurationSource` will not throw exceptions when queried. + * A `ConfigurationSource` will return `null` if the requested key cannot be found or other error occurs. + * A `ConfigurationSource` will return a `String` representing the value of a successful lookup. + +For example, a custom `ConfigurationSource` that returns a different random string every time it's queried: + +```java +import java.util.UUID; + +public class RandomUUIDSource implements ConfigurationSource { + @Override + public String get(String key) { + return UUID.randomUUID().toString(); + } +} +``` + +**NOTE**: Don't actually do this. + +## Contributing + +### Code of Conduct + +Everyone interacting with this project is required to follow the [Code of Conduct](./CODE_OF_CONDUCT.md). + +### Getting Started + +You'll need Maven and Java 8+ installed + +```bash +$ git clone git@github.com:unruly/java-config-fallback.git +$ mvn clean install +``` + +You can run `mvn clean test` in the project root to run JUnit tests. + +### Releasing a Change + +- To release a new version: + - Run `mvn release:prepare release:perform` - NOTE: this requires signing the release with a GPG key. + ## Design Decisions There are already libraries (such as [properlty](https://github.com/ufoscout/properlty)) that do something very similar — why build our own? @@ -108,3 +203,8 @@ Configuration config = Configuration.of( * **Do one thing well**: The library doesn’t coerce types (opting for String key, String value) or assume behaviour if a source cannot be queried (missing file or slow network call). This is up to the user to decide, and simplifies a lot of our other design choices. * **Eat our own dog-food**: This library is in use at Unruly, helping us transition our app configuration and advance our architecture. We don’t want to release anything we wouldn’t use ourselves. + +## License + +This project is available as open source under the terms of the [MIT License](./LICENSE). + From f3a133bacfcb7ae4ee9e8a0d3d18518b1d1ceb2a Mon Sep 17 00:00:00 2001 From: Shift Date: Tue, 18 Sep 2018 14:24:53 +0000 Subject: [PATCH 02/10] docs(README): Reflect correct return type in code examples * Make custom configuration sources section header slightly neater Co-authored-by: Ina Tsetsova Co-authored-by: Alex Wilson --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 53a0823..5ddcc8c 100644 --- a/README.md +++ b/README.md @@ -86,8 +86,8 @@ credentials.put("password","this-is-a-secret!"); Configuration config = Configuration.of(map(credentials)); -String username = config.get("username"); // Optional[foo] -String password = config.get("password"); // Optional[this-is-a-secret!] +Optional username = config.get("username"); // Optional[foo] +Optional password = config.get("password"); // Optional[this-is-a-secret!] ``` ### Properties @@ -99,7 +99,7 @@ Configuration config = Configuration.of( properties("/etc/my-app/config.properties"), // password=new-password ); -String password = config.get("password"); // Optional[new-password] +Optional password = config.get("password"); // Optional[new-password] ``` ### Environment @@ -113,8 +113,8 @@ The `.environment()` configuration source uses environment variables set for thi Configuration config = Configuration.of(environment()); -String password = config.get("VARIABLE"); // Optional[foo] -String password = config.get("variable"); // Same as above +Optional password = config.get("VARIABLE"); // Optional[foo] +Optional password = config.get("variable"); // Same as above ``` ### AWS Secrets Manager @@ -133,11 +133,11 @@ Configuration config = Configuration.of( secretsManager("my-secret", "eu-west-1") // { "username": "user", "password": "secret" } ); -String password = config.get("username"); // Optional[user] -String password = config.get("password"); // Optional[secret] +Optional password = config.get("username"); // Optional[user] +Optional password = config.get("password"); // Optional[secret] ``` -## Extending with custom Configuration Sources +## Adding custom Configuration Sources You can define your own sources of configuration by implementing the `ConfigurationSource` functional interface. From 82b9adcde82649b4a36beffd97f26dfcebdb9f8f Mon Sep 17 00:00:00 2001 From: Shift Date: Thu, 20 Sep 2018 10:23:53 +0000 Subject: [PATCH 03/10] feat(system-properties-adapter): Create ConfigurationSource for system properties * This is similar to the existing environment() ConfigurationSource but exposes properties available through System.getProperty() Co-authored-by: Sarah Young Co-authored-by: Alex Wilson --- .../java/co/unruly/config/Configuration.java | 4 ++++ .../co/unruly/config/ConfigurationTest.java | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/main/java/co/unruly/config/Configuration.java b/src/main/java/co/unruly/config/Configuration.java index b0e7541..53c12bb 100644 --- a/src/main/java/co/unruly/config/Configuration.java +++ b/src/main/java/co/unruly/config/Configuration.java @@ -65,6 +65,10 @@ public static ConfigurationSource properties(Properties properties) { return properties::getProperty; } + public static ConfigurationSource systemProperties() { + return System::getProperty; + } + public static ConfigurationSource environment() { return (key) -> System.getenv(key.toUpperCase()); } diff --git a/src/test/java/co/unruly/config/ConfigurationTest.java b/src/test/java/co/unruly/config/ConfigurationTest.java index 005fbba..39f06c6 100644 --- a/src/test/java/co/unruly/config/ConfigurationTest.java +++ b/src/test/java/co/unruly/config/ConfigurationTest.java @@ -165,4 +165,22 @@ public void shouldReturnEmptyOptionalIfNotFoundInSecretsManager() { assertThat(config.get("pass"), is(empty())); } + @Test + public void shouldUseSystemProperties() { + System.setProperty("my.system.property", "my-value"); + + Configuration config = Configuration.of(systemProperties()); + + assertThat(config.get("my.system.property"), contains("my-value")); + } + + @Test + public void shouldUseSystemProperties_BehavesCorrectlyWhenMissing() { + System.clearProperty("my.system.property"); + + Configuration config = Configuration.of(systemProperties()); + + assertThat(config.get("my.system.property"), is(empty())); + } + } From cf741ca34d97bd45a3f802d5acb2425d55f10568 Mon Sep 17 00:00:00 2001 From: Shift Date: Thu, 20 Sep 2018 10:35:05 +0000 Subject: [PATCH 04/10] docs(system-properties-adapter): Add documentation for systemProperties() * Add examples for systemProperties to demonstrate both user-specified and system default properties * Rename test and add comment to explicitly capture why the test exists. Co-authored-by: Sarah Young Co-authored-by: Alex Wilson --- README.md | 17 +++++++++++++++++ .../co/unruly/config/ConfigurationTest.java | 4 +++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5ddcc8c..647c9b6 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Optional password = config.get("password"); // Optional[secret] - [Supported Configuration Sources](#supported-configuration-sources) - [Map](#map) - [Properties](#properties) + - [System Properties](#system-properties) - [Environment](#environment) - [AWS Secrets Manager](#aws-secrets-manager) - [Extending with custom Configuration Sources](#extending-with-custom-configuration-sources) @@ -102,6 +103,22 @@ Configuration config = Configuration.of( Optional password = config.get("password"); // Optional[new-password] ``` +### System Properties + +The `.systemProperties()` configuration source uses `System.getProperty()` as a datasource. + +```java +Configuration config = Configuration.of( + systemProperties() +); + +// Running code with `mvn clean test -Dmy.property=hello-world +Optional myProperty = config.get("my.property"); // Optional[hello-world] + +// Default system properties in every JVM +Optional javaVersion = config.get("java.version"); // Optional[1.8.0_12345] +``` + ### Environment The `.environment()` configuration source uses environment variables set for this application. diff --git a/src/test/java/co/unruly/config/ConfigurationTest.java b/src/test/java/co/unruly/config/ConfigurationTest.java index 39f06c6..dbd60ba 100644 --- a/src/test/java/co/unruly/config/ConfigurationTest.java +++ b/src/test/java/co/unruly/config/ConfigurationTest.java @@ -175,7 +175,9 @@ public void shouldUseSystemProperties() { } @Test - public void shouldUseSystemProperties_BehavesCorrectlyWhenMissing() { + public void shouldUseSystemProperties_isEmptyWhenMissing() { + // This is a separate test because we want to explicitly clear my.system.property + // Otherwise running e.g. `mvn clean test -Dmy.system.property=foo` would cause this test to fail System.clearProperty("my.system.property"); Configuration config = Configuration.of(systemProperties()); From 08ae4bcdcc6b3cf09b57f8b063da697e46b9f358 Mon Sep 17 00:00:00 2001 From: ShiftDevelopers Date: Thu, 20 Sep 2018 10:36:30 +0000 Subject: [PATCH 05/10] [maven-release-plugin] prepare release java-config-fallback-1.0.0 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 964b3d0..c1cb8d2 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ co.unruly java-config-fallback - 0.1.8-SNAPSHOT + 1.0.0 jar java-config-fallback @@ -45,7 +45,7 @@ scm:git:git@github.com:unruly/${project.artifactId}.git scm:git:git@github.com:unruly/${project.artifactId}.git git@github.com:unruly/${project.artifactId}.git - HEAD + java-config-fallback-1.0.0 From a08809f129cbc06f91e327281119aa14f7b81da1 Mon Sep 17 00:00:00 2001 From: ShiftDevelopers Date: Thu, 20 Sep 2018 10:36:37 +0000 Subject: [PATCH 06/10] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index c1cb8d2..9843ec3 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ co.unruly java-config-fallback - 1.0.0 + 1.0.1-SNAPSHOT jar java-config-fallback @@ -45,7 +45,7 @@ scm:git:git@github.com:unruly/${project.artifactId}.git scm:git:git@github.com:unruly/${project.artifactId}.git git@github.com:unruly/${project.artifactId}.git - java-config-fallback-1.0.0 + HEAD From 7bdaf75759f138f8650d666cde016dcd745e12b1 Mon Sep 17 00:00:00 2001 From: Shift Date: Thu, 20 Sep 2018 10:40:40 +0000 Subject: [PATCH 07/10] docs(README): Reflect release version in README Co-authored-by: Sarah Young Co-authored-by: Alex Wilson --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 647c9b6..c6763c8 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ This library is available on Maven Central     co.unruly     java-config-fallback -    0.1.7 +    1.0.0 ``` From b88cc54d3f9b7010f6158cb69f01f24ed303539e Mon Sep 17 00:00:00 2001 From: Jahed Ahmed Date: Fri, 21 Sep 2018 15:54:48 +0100 Subject: [PATCH 08/10] docs: move password to secretsManager in example Secrets Manager is made to store secrets like passwords, so it makes more sense to show it there in the example snippet. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c6763c8..976617e 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ A lightweight library to query a hierarchy of configuration sources ```java Configuration config = Configuration.of( - secretsManager("my-secret-name", "eu-west-1"), // { "username": "user" } - properties("/etc/my-app-config.properties"), // { "username": "anotherUser", "password": "secret" } + secretsManager("my-secret-name", "eu-west-1"), // { "username": "user", "password": "secret" } + properties("/etc/my-app-config.properties"), // { "username": "anotherUser" } ); Optional username = config.get("username"); // Optional[user] From c2ba47664ec6178294259e17262be379344db667 Mon Sep 17 00:00:00 2001 From: Jahed Ahmed Date: Fri, 21 Sep 2018 16:01:43 +0100 Subject: [PATCH 09/10] docs: clarify how secretsManager makes requests to AWS --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 976617e..197801a 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,9 @@ This can be configured with: * `.secretsManager(String secretName, String region)` - uses default AWS client to query Secrets Manager. * `.secretsManager(String secretName, String region, AWSSecretsManager client)` - uses provided client to query Secrets Manager. Use this if you want to provide custom client behaviour e.g. using a specific set of credentials or instance-role. + +Secrets are only requested from AWS the first time they're accessed. Subsequent +requests using the same configuration will use a cached response. **NOTE**: This configuration source only supports key-value pairs as a return type - plaintext secrets will be ignored. From 1e7e7327c35a0afbbcdc4163b69e13baddb56095 Mon Sep 17 00:00:00 2001 From: Jahed Ahmed Date: Fri, 21 Sep 2018 17:43:52 +0100 Subject: [PATCH 10/10] fix: create AWS client on demand to avoid side effects in constructor AWSSecretsManagerClient prints debug logs when it's initialised which may be triggered before logging preferences are processed in a static context. Fixes #8 --- .../java/co/unruly/config/SecretsManager.java | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/main/java/co/unruly/config/SecretsManager.java b/src/main/java/co/unruly/config/SecretsManager.java index 1d00739..a16fdec 100644 --- a/src/main/java/co/unruly/config/SecretsManager.java +++ b/src/main/java/co/unruly/config/SecretsManager.java @@ -17,16 +17,18 @@ public class SecretsManager implements ConfigurationSource { private final String secretName; private final String region; - private final AWSSecretsManager client; + private final Optional client; private Map credentials; public SecretsManager(String secretName, String region) { - this.secretName = secretName; - this.region = region; - this.client = setupClient(); + this(secretName, region, Optional.empty()); } public SecretsManager(String secretName, String region, AWSSecretsManager client) { + this(secretName, region, Optional.of(client)); + } + + private SecretsManager(String secretName, String region, Optional client) { this.secretName = secretName; this.region = region; this.client = client; @@ -50,13 +52,6 @@ private Map fetchCredentials() { .orElse(emptyResults); } - - private AWSSecretsManager setupClient() { - return AWSSecretsManagerClientBuilder.standard() - .withRegion(this.region) - .build(); - } - private Optional getSecretValueFromRequest() { return getSecretValue(createRequest()); } @@ -67,12 +62,21 @@ private GetSecretValueRequest createRequest() { private Optional getSecretValue(GetSecretValueRequest getSecretValueRequest) { try { - return Optional.ofNullable(client.getSecretValue(getSecretValueRequest)); + return Optional.of(getOrCreateClient()) + .map(client -> client.getSecretValue(getSecretValueRequest)); } catch (ResourceNotFoundException e) { return Optional.empty(); } } + private AWSSecretsManager getOrCreateClient() { + return client.orElseGet(() -> ( + AWSSecretsManagerClientBuilder.standard() + .withRegion(this.region) + .build() + )); + } + private Optional> parseJSON(String jsonInput) { final ObjectMapper mapper = new ObjectMapper(); final TypeReference> parseAsMap = new TypeReference>() {};