diff --git a/README.md b/README.md new file mode 100644 index 0000000..52ab583 --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ +Prometheus Monitoring for Java Developers +========================================= + +Notes and demo code. The Java demos are in separate branches: + +* [01-hello-world](https://github.com/fstab/prometheus-for-java-developers/tree/01-hello-world) +* [02a-direct-instrumentation](https://github.com/fstab/prometheus-for-java-developers/tree/02a-direct-instrumentation) +* [02b-direct-instrumentation-with-spring-boot-endpoint](https://github.com/fstab/prometheus-for-java-developers/tree/02b-direct-instrumentation-with-spring-boot-endpoint) +* [03a-spring-boot-actuator-enabled](https://github.com/fstab/prometheus-for-java-developers/tree/03a-spring-boot-actuator-enabled) +* [03b-spring-boot-actuator-custom-metric](https://github.com/fstab/prometheus-for-java-developers/tree/03b-spring-boot-actuator-custom-metric) +* [03c-spring-boot-actuator-prometheus-bridge](https://github.com/fstab/prometheus-for-java-developers/tree/03c-spring-boot-actuator-prometheus-bridge) +* [04a-jmx-enabled](https://github.com/fstab/prometheus-for-java-developers/tree/04a-jmx-enabled) +* [04b-jmx-custom-metric](https://github.com/fstab/prometheus-for-java-developers/tree/04b-jmx-custom-metric) +* [04c-jmx-remote-prometheus-bridge](https://github.com/fstab/prometheus-for-java-developers/tree/04c-jmx-remote-prometheus-bridge) +* [04d-jmx-agent-prometheus-bridge](https://github.com/fstab/prometheus-for-java-developers/tree/04d-jmx-agent-prometheus-bridge) +* [05a-dropwizard-enabled](https://github.com/fstab/prometheus-for-java-developers/tree/05a-dropwizard-enabled) +* [05b-dropwizard-prometheus-bridge](https://github.com/fstab/prometheus-for-java-developers/tree/05b-dropwizard-prometheus-bridge) + +With each branch, an example is added, so that the demo in the last branch contains all examples in one single Java class. In practice, you would choose only one of the alternatives and use only one alternative in your Java class. + +Notes are here: + +* [doc/00-AGENDA.md](https://github.com/fstab/prometheus-for-java-developers/blob/doc/doc/00-AGENDA.md) +* [doc/01-OVERVIEW.md](https://github.com/fstab/prometheus-for-java-developers/blob/doc/doc/01-OVERVIEW.md) +* [doc/02-PROMETHEUS-DEMO.md](https://github.com/fstab/prometheus-for-java-developers/blob/doc/doc/02-PROMETHEUS-DEMO.md) +* [doc/03-ALERTING-EXAMPLE.md](https://github.com/fstab/prometheus-for-java-developers/blob/doc/doc/03-ALERTING-EXAMPLE.md) +* [doc/04-PUSH-vs-PULL.md](https://github.com/fstab/prometheus-for-java-developers/blob/doc/doc/04-PUSH-vs-PULL.md) +* [doc/05-JAVA-DEMO.md](https://github.com/fstab/prometheus-for-java-developers/blob/doc/doc/05-JAVA-DEMO.md) +* [doc/06-JMX-AGENT.md](https://github.com/fstab/prometheus-for-java-developers/blob/doc/doc/06-JMX-AGENT.md) +* [doc/xx_PROMETHEUS-DEMO-NOTES.md](https://github.com/fstab/prometheus-for-java-developers/blob/doc/doc/xx_PROMETHEUS-DEMO-NOTES.md) +* [doc/xy_JAVA-DEMO-NOTES.md](https://github.com/fstab/prometheus-for-java-developers/blob/doc/doc/xy_JAVA-DEMO-NOTES.md) diff --git a/doc/00-AGENDA.md b/doc/00-AGENDA.md new file mode 100644 index 0000000..18dd644 --- /dev/null +++ b/doc/00-AGENDA.md @@ -0,0 +1,8 @@ +Prometheus Monitoring for Java Developers +========================================= + +1. Prometheus Overview +2. Instrumenting a Java application + +About me: Fabian Stäber, ConSol Software GmbH, Munich, Germany +github.com/fstab/prometheus-for-java-developers diff --git a/doc/01-OVERVIEW.md b/doc/01-OVERVIEW.md new file mode 100644 index 0000000..1734f8f --- /dev/null +++ b/doc/01-OVERVIEW.md @@ -0,0 +1,8 @@ +Prometheus Overview +=================== + +* Open Source monitoring tool, started at Souncloud by ex-Googlers +* Core component is time-series DB and query language (Prometheus Server) +* Everything else is modular: Exporters, Dashboard, Alertmanager, ... +* Very active community and ecosystem +* White box monitoring approach diff --git a/doc/02-PROMETHEUS-DEMO.md b/doc/02-PROMETHEUS-DEMO.md new file mode 100644 index 0000000..79e89a9 --- /dev/null +++ b/doc/02-PROMETHEUS-DEMO.md @@ -0,0 +1,13 @@ +Prometheus Demo +=============== + +* `node_exporter`: Provides metrics +* `prometheus`: Metric database and query language +* `grafana`: Dashboard +* `alertmanager`: Handles alerts + +``` +node_exporter <--- Prometheus Server <--- Grafana Dashboard + | + -----> Alertmanager +``` diff --git a/doc/03-ALERTING-EXAMPLE.md b/doc/03-ALERTING-EXAMPLE.md new file mode 100644 index 0000000..c08e0ad --- /dev/null +++ b/doc/03-ALERTING-EXAMPLE.md @@ -0,0 +1,13 @@ +Alerting Example +================ + +``` +# Alert for any instance that have a median request latency >1s. +ALERT APIHighRequestLatency + IF api_http_request_latencies_second{quantile="0.5"} > 1 + FOR 1m + ANNOTATIONS { + summary = "High request latency on {{ $labels.instance }}", + description = "{{ $labels.instance }} has a median request latency above 1s (current value: {{ $value }}s)", + } +``` diff --git a/doc/04-PUSH-vs-PULL.md b/doc/04-PUSH-vs-PULL.md new file mode 100644 index 0000000..1b0e4d6 --- /dev/null +++ b/doc/04-PUSH-vs-PULL.md @@ -0,0 +1,12 @@ +Push vs Pull +============ + +``` +node_exporter <----HTTP---- prometheus server +port: 9000 +``` + +* Prometheus server pulls metrics from exporters + - Pro: Exporters don't need to know anything about the Prometheus Server(s) + - Con: Service discovery needed when adding services/exporters +* exporter per service, not per machine diff --git a/doc/05-JAVA-DEMO.md b/doc/05-JAVA-DEMO.md new file mode 100644 index 0000000..481fe52 --- /dev/null +++ b/doc/05-JAVA-DEMO.md @@ -0,0 +1,9 @@ +Instrumenting a Java application +================================ + +Examples: + + * Direct instrumentation with Prometheus client library + * Spring boot actuator to Prometheus bridge + * Java management extensions (JMX) to Prometheus bridge + * Dropwizard to Prometheus bridge diff --git a/doc/06-JMX-AGENT.md b/doc/06-JMX-AGENT.md new file mode 100644 index 0000000..03be4d5 --- /dev/null +++ b/doc/06-JMX-AGENT.md @@ -0,0 +1,19 @@ +JMX Prometheus Bridge +===================== + +JMX Remote +---------- + +``` +Application <----- JMX Exporter <--------- Prometheus +RMI Port 5555 (*) HTTP Port 9200 +``` + +JMX Agent +--------- + +``` +Application <---------------------------------- Prometheus ++ Prometheus Agent +HTTP Port 9200 +``` diff --git a/doc/xx_PROMETHEUS-DEMO-NOTES.md b/doc/xx_PROMETHEUS-DEMO-NOTES.md new file mode 100644 index 0000000..462b290 --- /dev/null +++ b/doc/xx_PROMETHEUS-DEMO-NOTES.md @@ -0,0 +1,124 @@ +Prometheus for Java Developers +============================== + +Docker image with all components downloaded: + +```bash +docker run --rm -t \ + -p 9100:9100 -p 9090:9090 -p 3000:3000 -p 9093:9093 -p 8080:8080 -p 9200:9200 \ + -i fstab/prometheus-demo +``` + +Demo runs in a screen session, so we can run each tool in its own terminal. + +```bash +screen +``` + +node_exporter +------------- + +```console +> tar xfz node_exporter-0.12.0.linux-amd64.tar.gz +> cd node_exporter-0.12.0.linux-amd64 +> ./node_exporter +``` + +View on [http://localhost:9100/metrics](http://localhost:9100/metrics) + +Notes: + +* As most Go programs, `node_exporter` is a single executable +* What is a TSDB? Basic datatype is float64 with timestamp. +* Browser shows text format, but Prometheus uses protobuf. ASCII format from OpenTSDB. +* `node_exporter` gets most info from `/proc` fs, which is mainly intended for native Linux hosts. For Docker containers look at `cAdviser`. + +Prometheus Server +----------------- + +```console +> tar xfz prometheus-1.2.1.linux-amd64.tar.gz +> cd prometheus-1.2.1.linux-amd64 +``` + +Edit `prometheus.yml` and add config for node: + +```yaml +- job_name: 'node' + static_configs: + - targets: ['localhost:9100'] +``` + +View on [http://localhost:9090](http://localhost:9090) + +* Show _Status_ -> _Configuration_ +* Show _Status_ -> _Targets_ + +Notes: + +* As most Go programs, `prometheus` is a single executable +* `scrape_interval` configured in `prometheus.yml` + +PromQL Query Language +--------------------- + +``` +node_network_transmit_bytes +node_network_transmit_bytes{device='eth0'} +node_network_transmit_bytes{device=~"eth.*"} +sum(node_network_transmit_bytes) +sum(node_network_transmit_bytes) by (device) +sum(node_network_transmit_bytes) by (instance) +sum(node_network_transmit_bytes + node_network_receive_bytes) by (device) +sum(node_network_transmit_bytes + node_network_receive_bytes) by (device) / 1024 / 1024 + +# All values that were recorded in the last 5m +node_network_transmit_bytes{device='eth0'}[5m] + +# Per-second average rate of last 5m +rate(node_network_transmit_bytes{device='eth0'}[5m]) +``` + +PromQL is well documented on [http://prometheus.io](http://prometheus.io) + +Grafana +------- + +```console +> tar xfz grafana-3.1.1-1470047149.linux-x64.tar.gz +> cd grafana-3.1.1-1470047149 +> ./bin/grafana-server +``` + +View on [http://localhost:3000](http://localhost:3000), user _admin_, password _admin_ + +* Create Prometheus datasource +* Show Prometheus default dashboard +* Create dashboard with query form above: `rate(node_network_transmit_bytes{device='eth0'}[5m])` +* Show export as JSON and grafana.net + +Alertmanager +============ + +```console +> tar xfz alertmanager-0.4.2.linux-amd64.tar.gz +> cd alertmanager-0.4.2.linux-amd64 +> ./alertmanager -config.file simple.yml +``` + +View on [http://localhost:9093/](http://localhost:9093/) + +Example alert: + +``` +# Alert for any instance that have a median request latency >1s. +ALERT APIHighRequestLatency + IF api_http_request_latencies_second{quantile="0.5"} > 1 + FOR 1m + ANNOTATIONS { + summary = "High request latency on {{ $labels.instance }}", + description = "{{ $labels.instance }} has a median request latency above 1s (current value: {{ $value }}s)", + } +``` + +We skip the alerting demo, because we need to define alerts in Prometheus server, configure alertmanager URL, have a metric that triggers the alert, route the alert somewhere, which takes a while to configure. diff --git a/doc/xy_JAVA-DEMO-NOTES.md b/doc/xy_JAVA-DEMO-NOTES.md new file mode 100644 index 0000000..737fd44 --- /dev/null +++ b/doc/xy_JAVA-DEMO-NOTES.md @@ -0,0 +1,662 @@ +Prometheus for Java Developers +============================== + +Java examples: + +* Direct instrumentation with the Prometheus client library +* Prometheus bridges for the two big frameworks: + * Spring: Spring Boot Actuator to Prometheus bridge example + * Java EE: JMX to Prometheus bridge example +* Prometheus bridges for independent 3rd party frameworks: + * Dropwizard to Prometheus bridge example + +The goal is to show different alternatives. In the demo, all alternatives will be added to the same application. In practice, you would choose one of the alternatives and use only one of these alternatives in your application. + +Hello, World! +------------- + +[https://github.com/fstab/prometheus-demo-new/tree/01-hello-world](https://github.com/fstab/prometheus-demo-new/tree/01-hello-world) + +pom.xml + +```xml + + + 4.0.0 + + de.consol.labs.promdemo + hello-world + 0.1-SNAPSHOT + + + org.springframework.boot + spring-boot-starter-parent + 1.4.5.RELEASE + + + + 1.8 + + + + + + + org.springframework.boot + spring-boot-starter-web + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + +``` + +src/main/java/de/consol/labs/promdemo/HelloWorldController.java + +```java +package de.consol.labs.promdemo; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; + +@Controller +@SpringBootApplication +public class HelloWorldController { + + @RequestMapping(path = "/hello-world") + public @ResponseBody String sayHello() { + return "hello, world"; + } + + public static void main(String[] args) { + SpringApplication.run(HelloWorldController.class, args); + } +} +``` + +test + +```console +> curl http://localhost:8080/hello-world +``` + +Direct Instrumentation w/ Prometheus client library +--------------------------------------------------- + +New dependency + +```xml + + io.prometheus + simpleclient_common + 0.0.21 + +``` + +src/main/java/de/consol/labs/promdemo/HelloWorldController.java + +```java +package de.consol.labs.promdemo; + +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.Counter; +import io.prometheus.client.exporter.common.TextFormat; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; + +import java.io.IOException; +import java.io.Writer; + +@Controller +@SpringBootApplication +public class HelloWorldController { + + private final Counter promRequestsTotal = Counter.build() + .name("requests_total") + .help("Total number of requests.") + .register(); + + @RequestMapping(path = "/hello-world") + public @ResponseBody String sayHello() { + promRequestsTotal.inc(); + return "hello, world"; + } + + @RequestMapping(path = "/prometheus") + public void metrics(Writer responseWriter) throws IOException { + TextFormat.write004(responseWriter, CollectorRegistry.defaultRegistry.metricFamilySamples()); + responseWriter.close(); + } + + public static void main(String[] args) { + SpringApplication.run(HelloWorldController.class, args); + } +} +``` + +test + +```console +> curl http://localhost:8080/hello-world +> curl http://localhost:8080/prometheus +``` + +Direct Instrumentation w/ Prometheus client library and Spring Boot Endpoint +---------------------------------------------------------------------------- + +New dependency + +```xml + + io.prometheus + simpleclient_spring_boot + 0.0.21 + +``` + +src/main/java/de/consol/labs/promdemo/HelloWorldController.java + +```java +package de.consol.labs.promdemo; + +import io.prometheus.client.Counter; +import io.prometheus.client.spring.boot.EnablePrometheusEndpoint; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; + +@Controller +@SpringBootApplication +@EnablePrometheusEndpoint +public class HelloWorldController { + + private final Counter promRequestsTotal = Counter.build() + .name("requests_total") + .help("Total number of requests.") + .register(); + + @RequestMapping(path = "/hello-world") + public @ResponseBody String sayHello() { + promRequestsTotal.inc(); + return "hello, world"; + } + + public static void main(String[] args) { + SpringApplication.run(HelloWorldController.class, args); + } +} +``` + +We removed the request handler for `/prometheus` endpoint (`metrics()` method) and annotate class with `@EnablePrometheusEndpoint`. +As a result, the `/prometheus` endpoint will be implicitly created, the application behaves like in the previous example. + +Spring Boot Actuator +-------------------- + +### Enabling the Spring Boot Actuator + +New dependency + +```xml + + org.springframework.boot + spring-boot-starter-actuator + +``` + +test + +```console +> curl http://localhost:8080/metrics +``` + +### Adding a Custom Metric to Spring Boot Actuator + +src/main/java/de/consol/labs/promdemo/HelloWorldController.java + +```java +package de.consol.labs.promdemo; + +import io.prometheus.client.Counter; +import io.prometheus.client.spring.boot.EnablePrometheusEndpoint; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.actuate.metrics.CounterService; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; + +@Controller +@SpringBootApplication +@EnablePrometheusEndpoint +public class HelloWorldController { + + private final Counter promRequestsTotal = Counter.build() + .name("requests_total") + .help("Total number of requests.") + .register(); + private final CounterService springRequestsTotal; + + public HelloWorldController(@Autowired CounterService sprintRequestsTotal) { + this.springRequestsTotal = sprintRequestsTotal; + } + + @RequestMapping(path = "/hello-world") + public @ResponseBody String sayHello() { + promRequestsTotal.inc(); + springRequestsTotal.increment("counter.calls.promdemo.hello_world"); + return "hello, world"; + } + + public static void main(String[] args) { + SpringApplication.run(HelloWorldController.class, args); + } +} +``` + +### Spring Boot Actuator to Prometheus Bridge + +src/main/java/de/consol/labs/promdemo/HelloWorldController.java + +```java +package de.consol.labs.promdemo; + +import io.prometheus.client.Counter; +import io.prometheus.client.spring.boot.EnablePrometheusEndpoint; +import io.prometheus.client.spring.boot.SpringBootMetricsCollector; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.actuate.endpoint.PublicMetrics; +import org.springframework.boot.actuate.metrics.CounterService; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; + +import java.util.Collection; + +@Controller +@SpringBootApplication +@EnablePrometheusEndpoint +public class HelloWorldController { + + private final Counter promRequestsTotal = Counter.build() + .name("requests_total") + .help("Total number of requests.") + .register(); + private final CounterService springRequestsTotal; + + public HelloWorldController(@Autowired Collection publicMetrics, @Autowired CounterService sprintRequestsTotal) { + this.springRequestsTotal = sprintRequestsTotal; + new SpringBootMetricsCollector(publicMetrics).register(); + } + + @RequestMapping(path = "/hello-world") + public @ResponseBody String sayHello() { + promRequestsTotal.inc(); + springRequestsTotal.increment("counter.calls.promdemo.hello_world"); + return "hello, world"; + } + + public static void main(String[] args) { + SpringApplication.run(HelloWorldController.class, args); + } +} +``` + +test + +```console +> curl http://localhost:8080/prometheus-metrics +``` + +Java Management Extensions (JMX) +-------------------------------- + +### Enable JMX + +Run the demo application with JMX enabled: + +```console +> mvn clean package +> java \ + -Djava.net.preferIPv4Stack=true \ + -Djava.rmi.server.hostname=localhost \ + -Dcom.sun.management.jmxremote \ + -Dcom.sun.management.jmxremote.port=5555 \ + -Dcom.sun.management.jmxremote.authenticate=false \ + -Dcom.sun.management.jmxremote.ssl=false \ + -jar target/hello-world-0.1-SNAPSHOT.jar +``` + +Run `jvisualvm`. When running for the first time +* Under 'tools -> plugins': Install VisualVM-MBeans +* Under 'file -> add jmx connecVon': localhost:5555, Do not require SSL + +For the demo: +* Go to `java.lang Memory` (on the MBeans tab) +* Click on the `javax.management.openmbean.CompositeDataSupport` bean for `HeapMemoryUsage`. Documentation for `committed`, `init`, `max`, `used` can be found on [https://docs.oracle.com/javase/8/docs/api/java/lang/management/MemoryUsage.html](https://docs.oracle.com/javase/8/docs/api/java/lang/management/MemoryUsage.html) + +By default Spring Boot will expose management endpoints as JMX MBeans under the `org.springframework.boot` domain. + +### Custom JMX Metric (MBean) + +src/main/java/de/consol/labs/promdemo/HelloWorldController.java + +```java +package de.consol.labs.promdemo; + +import io.prometheus.client.Counter; +import io.prometheus.client.spring.boot.EnablePrometheusEndpoint; +import io.prometheus.client.spring.boot.SpringBootMetricsCollector; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.actuate.endpoint.PublicMetrics; +import org.springframework.boot.actuate.metrics.CounterService; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.jmx.export.annotation.ManagedAttribute; +import org.springframework.jmx.export.annotation.ManagedResource; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; + +import java.util.Collection; +import java.util.concurrent.atomic.AtomicInteger; + +@Controller +@SpringBootApplication +@ManagedResource(objectName="de.consol.labs:name=PromDemoBean") +@EnablePrometheusEndpoint +public class HelloWorldController { + + private final Counter promRequestsTotal = Counter.build() + .name("requests_total") + .help("Total number of requests.") + .register(); + private final CounterService springRequestsTotal; + private final AtomicInteger jmxRequestsTotal = new AtomicInteger(); + + public HelloWorldController(@Autowired Collection publicMetrics, @Autowired CounterService sprintRequestsTotal) { + this.springRequestsTotal = sprintRequestsTotal; + new SpringBootMetricsCollector(publicMetrics).register(); + } + + @RequestMapping(path = "/hello-world") + public @ResponseBody String sayHello() { + promRequestsTotal.inc(); + springRequestsTotal.increment("counter.calls.promdemo.hello_world"); + jmxRequestsTotal.incrementAndGet(); + return "hello, world"; + } + + @ManagedAttribute(description="jmx_requests_total") + public int getJmxRequestsTotal() { + return jmxRequestsTotal.get(); + } + + public static void main(String[] args) { + SpringApplication.run(HelloWorldController.class, args); + } +} +``` + +### JMX Prometheus Bridge + +config-remote.yml + +```yml +--- +hostPort: 127.0.0.1:5555 +rules: +- pattern: ".*" +``` + +jmx-exporter.sh + +```bash +#!/bin/bash + +# mkdir tmp +# cd tmp +# git clone https://github.com/prometheus/jmx_exporter +# cd jmx_exporter +# mvn clean install +# cd ../.. +# rm -rf tmp + +export JMX_EXPORTER=$HOME/.m2/repository/io/prometheus/jmx/jmx_prometheus_httpserver/0.7-SNAPSHOT/jmx_prometheus_httpserver-0.7-SNAPSHOT-jar-with-dependencies.jar + +java -jar $JMX_EXPORTER 9200 config-remote.yml +``` + +test + +```console +> curl http://localhost:9200/metrics +``` + +Notes + +* With each scrape, Prometheus loads all JMX beans, then applies matching rules. If this takes long (because MBeans calculate data on the fly), use the JMX exporter's blacklist / whitelist facility, which is applied before MBeans are loaded. + +### JMX Prometheus Agent + +config-agent.yml + +```yml +--- +rules: +- pattern: ".*" +``` + +jmx-agent.sh + +```bash +#!/bin/bash + +# mkdir tmp +# cd tmp +# git clone https://github.com/prometheus/jmx_exporter +# cd jmx_exporter +# mvn clean install +# cd ../.. +# rm -rf tmp + +mvn clean package + +export AGENT_JAR=$HOME/.m2/repository/io/prometheus/jmx/jmx_prometheus_javaagent/0.7-SNAPSHOT/jmx_prometheus_javaagent-0.7-SNAPSHOT.jar + +java -javaagent:$AGENT_JAR=9200:config-agent.yml -jar target/hello-world-0.1-SNAPSHOT.jar +``` + +test + +```console +> curl http://localhost:9200/metrics +``` + +Dropwizard +---------- + +### Create a Custom Metric with Dropwizard + +New dependency: + +```xml + + io.dropwizard.metrics + metrics-core + 3.1.2 + +``` + +src/main/java/de/consol/labs/promdemo/HelloWorldController.java + +```java +package de.consol.labs.promdemo; + +import com.codahale.metrics.ConsoleReporter; +import com.codahale.metrics.MetricRegistry; +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.Counter; +import io.prometheus.client.exporter.common.TextFormat; +import io.prometheus.client.spring.boot.EnablePrometheusEndpoint; +import io.prometheus.client.spring.boot.SpringBootMetricsCollector; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.actuate.endpoint.PublicMetrics; +import org.springframework.boot.actuate.metrics.CounterService; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.jmx.export.annotation.ManagedAttribute; +import org.springframework.jmx.export.annotation.ManagedResource; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; + +import java.io.IOException; +import java.io.Writer; +import java.util.Collection; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@Controller +@SpringBootApplication +@ManagedResource(objectName="de.consol.labs:name=PromDemoBean") +@EnablePrometheusEndpoint +public class HelloWorldController { + + private final Counter promRequestsTotal = Counter.build() + .name("requests_total") + .help("Total number of requests.") + .register(); + private final CounterService springRequestsTotal; + private final AtomicInteger jmxRequestsTotal = new AtomicInteger(); + private final com.codahale.metrics.Counter dwRequestsTotal; + + public HelloWorldController(@Autowired Collection publicMetrics, @Autowired CounterService sprintRequestsTotal) { + this.springRequestsTotal = sprintRequestsTotal; + new SpringBootMetricsCollector(publicMetrics).register(); + MetricRegistry dropwizardRegistry = new MetricRegistry(); + dwRequestsTotal = dropwizardRegistry.counter("dropwizard_requests_total"); + ConsoleReporter.forRegistry(dropwizardRegistry).build().start(5, TimeUnit.SECONDS); + } + + @RequestMapping(path = "/hello-world") + public @ResponseBody String sayHello() { + promRequestsTotal.inc(); + springRequestsTotal.increment("counter.calls.promdemo.hello_world"); + jmxRequestsTotal.incrementAndGet(); + dwRequestsTotal.inc(); + return "hello, world"; + } + + @ManagedAttribute(description="jmx_requests_total") + public int getJmxRequestsTotal() { + return jmxRequestsTotal.get(); + } + + public static void main(String[] args) { + SpringApplication.run(HelloWorldController.class, args); + } +} +``` + +### Dropwizard Prometheus Bridge + +New dependency + +```xml + + io.prometheus + simpleclient_dropwizard + 0.0.21 + +``` + +src/main/java/de/consol/labs/promdemo/HelloWorldController.java + +```java +package de.consol.labs.promdemo; + +import com.codahale.metrics.ConsoleReporter; +import com.codahale.metrics.MetricRegistry; +import io.prometheus.client.Counter; +import io.prometheus.client.dropwizard.DropwizardExports; +import io.prometheus.client.spring.boot.EnablePrometheusEndpoint; +import io.prometheus.client.spring.boot.SpringBootMetricsCollector; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.actuate.endpoint.PublicMetrics; +import org.springframework.boot.actuate.metrics.CounterService; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.jmx.export.annotation.ManagedAttribute; +import org.springframework.jmx.export.annotation.ManagedResource; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; + +import java.util.Collection; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@Controller +@SpringBootApplication +@ManagedResource(objectName="de.consol.labs:name=PromDemoBean") +@EnablePrometheusEndpoint +public class HelloWorldController { + + private final Counter promRequestsTotal = Counter.build() + .name("requests_total") + .help("Total number of requests.") + .register(); + private final CounterService springRequestsTotal; + private final AtomicInteger jmxRequestsTotal = new AtomicInteger(); + private final com.codahale.metrics.Counter dwRequestsTotal; + + public HelloWorldController(@Autowired Collection publicMetrics, @Autowired CounterService sprintRequestsTotal) { + this.springRequestsTotal = sprintRequestsTotal; + new SpringBootMetricsCollector(publicMetrics).register(); + MetricRegistry dropwizardRegistry = new MetricRegistry(); + dwRequestsTotal = dropwizardRegistry.counter("dropwizard_requests_total"); + ConsoleReporter.forRegistry(dropwizardRegistry).build().start(5, TimeUnit.SECONDS); + new DropwizardExports(dropwizardRegistry).register(); + } + + @RequestMapping(path = "/hello-world") + public @ResponseBody String sayHello() { + promRequestsTotal.inc(); + springRequestsTotal.increment("counter.calls.promdemo.hello_world"); + jmxRequestsTotal.incrementAndGet(); + dwRequestsTotal.inc(); + return "hello, world"; + } + + @ManagedAttribute(description="jmx_requests_total") + public int getJmxRequestsTotal() { + return jmxRequestsTotal.get(); + } + + public static void main(String[] args) { + SpringApplication.run(HelloWorldController.class, args); + } +} +``` diff --git a/jmx-agent.sh b/jmx-agent.sh new file mode 100755 index 0000000..0f48952 --- /dev/null +++ b/jmx-agent.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# mkdir tmp +# cd tmp +# git clone https://github.com/prometheus/jmx_exporter +# cd jmx_exporter +# mvn clean install +# cd ../.. +# rm -rf tmp + +mvn clean package + +export AGENT_JAR=$HOME/.m2/repository/io/prometheus/jmx/jmx_prometheus_javaagent/0.7-SNAPSHOT/jmx_prometheus_javaagent-0.7-SNAPSHOT.jar + +java -javaagent:$AGENT_JAR=9200:config-agent.yml -jar target/hello-world-0.1-SNAPSHOT.jar diff --git a/jmx-exporter.sh b/jmx-exporter.sh new file mode 100755 index 0000000..a757c9e --- /dev/null +++ b/jmx-exporter.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# mkdir tmp +# cd tmp +# git clone https://github.com/prometheus/jmx_exporter +# cd jmx_exporter +# mvn clean install +# cd ../.. +# rm -rf tmp + +export JMX_EXPORTER=$HOME/.m2/repository/io/prometheus/jmx/jmx_prometheus_httpserver/0.7-SNAPSHOT/jmx_prometheus_httpserver-0.7-SNAPSHOT-jar-with-dependencies.jar + +java -jar $JMX_EXPORTER 9200 config-remote.yml diff --git a/jmx-remote.sh b/jmx-remote.sh new file mode 100755 index 0000000..97ab06c --- /dev/null +++ b/jmx-remote.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +mvn clean package + +java \ + -Djava.net.preferIPv4Stack=true \ + -Djava.rmi.server.hostname=localhost \ + -Dcom.sun.management.jmxremote \ + -Dcom.sun.management.jmxremote.port=5555 \ + -Dcom.sun.management.jmxremote.authenticate=false \ + -Dcom.sun.management.jmxremote.ssl=false \ + -jar target/hello-world-0.1-SNAPSHOT.jar diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..0003c8c --- /dev/null +++ b/pom.xml @@ -0,0 +1,66 @@ + + + 4.0.0 + + de.consol.labs.promdemo + hello-world + 0.1-SNAPSHOT + + + org.springframework.boot + spring-boot-starter-parent + 1.4.5.RELEASE + + + + 1.8 + + + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-actuator + + + + + io.dropwizard.metrics + metrics-core + 3.1.2 + + + + + io.prometheus + simpleclient_common + 0.0.21 + + + io.prometheus + simpleclient_spring_boot + 0.0.21 + + + io.prometheus + simpleclient_dropwizard + 0.0.21 + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + diff --git a/src/main/java/de/consol/labs/promdemo/HelloWorldController.java b/src/main/java/de/consol/labs/promdemo/HelloWorldController.java new file mode 100644 index 0000000..01f3128 --- /dev/null +++ b/src/main/java/de/consol/labs/promdemo/HelloWorldController.java @@ -0,0 +1,64 @@ +package de.consol.labs.promdemo; + +import com.codahale.metrics.ConsoleReporter; +import com.codahale.metrics.MetricRegistry; +import io.prometheus.client.Counter; +import io.prometheus.client.dropwizard.DropwizardExports; +import io.prometheus.client.spring.boot.EnablePrometheusEndpoint; +import io.prometheus.client.spring.boot.SpringBootMetricsCollector; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.actuate.endpoint.PublicMetrics; +import org.springframework.boot.actuate.metrics.CounterService; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.jmx.export.annotation.ManagedAttribute; +import org.springframework.jmx.export.annotation.ManagedResource; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseBody; + +import java.util.Collection; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@Controller +@SpringBootApplication +@ManagedResource(objectName="de.consol.labs:name=PromDemoBean") +@EnablePrometheusEndpoint +public class HelloWorldController { + + private final Counter promRequestsTotal = Counter.build() + .name("requests_total") + .help("Total number of requests.") + .register(); + private final CounterService springRequestsTotal; + private final AtomicInteger jmxRequestsTotal = new AtomicInteger(); + private final com.codahale.metrics.Counter dwRequestsTotal; + + public HelloWorldController(@Autowired Collection publicMetrics, @Autowired CounterService sprintRequestsTotal) { + this.springRequestsTotal = sprintRequestsTotal; + new SpringBootMetricsCollector(publicMetrics).register(); + MetricRegistry dropwizardRegistry = new MetricRegistry(); + dwRequestsTotal = dropwizardRegistry.counter("dropwizard_requests_total"); + ConsoleReporter.forRegistry(dropwizardRegistry).build().start(5, TimeUnit.SECONDS); + new DropwizardExports(dropwizardRegistry).register(); + } + + @RequestMapping(path = "/hello-world") + public @ResponseBody String sayHello() { + promRequestsTotal.inc(); + springRequestsTotal.increment("counter.calls.promdemo.hello_world"); + jmxRequestsTotal.incrementAndGet(); + dwRequestsTotal.inc(); + return "hello, world"; + } + + @ManagedAttribute(description="jmx_requests_total") + public int getJmxRequestsTotal() { + return jmxRequestsTotal.get(); + } + + public static void main(String[] args) { + SpringApplication.run(HelloWorldController.class, args); + } +}