diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index abb94d786793..cad758fcffa3 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -23,6 +23,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import javax.inject.Inject; @@ -102,6 +103,7 @@ public String toString() { } private static List metricsItems = new ArrayList<>(); + private volatile long lastMetricsUpdateTime = 0L; @Inject private DataCenterDao dcDao; @@ -502,7 +504,15 @@ private void addVMsBySizeMetrics(final List metricsList, final long dcId, } @Override - public void updateMetrics() { + public synchronized void updateMetrics() { + final long minIntervalMs = TimeUnit.SECONDS.toMillis(PrometheusExporterServer.PrometheusExporterMinRefreshInterval.value()); + final long now = System.currentTimeMillis(); + if (now - lastMetricsUpdateTime < minIntervalMs) { + logger.debug("Skipping metrics recomputation, last update was " + (now - lastMetricsUpdateTime) + "ms ago (min interval: " + minIntervalMs + "ms)"); + return; + } + + final long startNanos = System.nanoTime(); final List latestMetricsItems = new ArrayList(); try { for (final DataCenterVO dc : dcDao.listAll()) { @@ -523,8 +533,12 @@ public void updateMetrics() { addDomainResourceCount(latestMetricsItems); } catch (Exception e) { logger.warn("Getting metrics failed ", e); + } finally { + final long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); + logger.info("Prometheus metrics update completed in " + elapsedMs + " ms"); } metricsItems = latestMetricsItems; + lastMetricsUpdateTime = System.currentTimeMillis(); } @Override diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServer.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServer.java index f0f5e3c6987b..f171b9ca4d7f 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServer.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServer.java @@ -33,4 +33,7 @@ public interface PrometheusExporterServer extends Manager { ConfigKey PrometheusExporterOfferingCountLimit = new ConfigKey<>("Advanced", Integer.class, "prometheus.exporter.offering.output.limit", "-1", "Limit the number of output for cloudstack_vms_total_by_size to the provided value. -1 for unlimited output.", true); + + ConfigKey PrometheusExporterMinRefreshInterval = new ConfigKey<>("Advanced", Integer.class, "prometheus.exporter.metrics.min.refresh.interval", "5", + "Minimum interval in seconds between metrics recomputations. Scrapes arriving faster than this interval reuse the previously computed metrics.", true, EnablePrometheusExporter.key()); } diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java index d9f25d2f5772..f67a68019061 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java @@ -29,10 +29,13 @@ import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; public class PrometheusExporterServerImpl extends ManagerBase implements PrometheusExporterServer, Configurable { private static HttpServer httpServer; + private ExecutorService httpExecutor; @Inject private PrometheusExporter prometheusExporter; @@ -79,6 +82,8 @@ public boolean start() { if (EnablePrometheusExporter.value()) { try { httpServer = HttpServer.create(new InetSocketAddress(PrometheusExporterServerPort.value()), 0); + httpExecutor = Executors.newFixedThreadPool(2); + httpServer.setExecutor(httpExecutor); httpServer.createContext("/metrics", new ExporterHandler(prometheusExporter)); httpServer.createContext("/", new HttpHandler() { @Override @@ -108,6 +113,10 @@ public boolean stop() { httpServer.stop(0); logger.debug("Stopped Prometheus exporter http server"); } + if (httpExecutor != null) { + httpExecutor.shutdownNow(); + logger.debug("Shut down Prometheus exporter http executor"); + } return true; } @@ -122,7 +131,8 @@ public ConfigKey[] getConfigKeys() { EnablePrometheusExporter, PrometheusExporterServerPort, PrometheusExporterAllowedAddresses, - PrometheusExporterOfferingCountLimit + PrometheusExporterOfferingCountLimit, + PrometheusExporterMinRefreshInterval }; } } diff --git a/plugins/integrations/prometheus/src/test/java/org/apache/cloudstack/metrics/PrometheusExporterImplTest.java b/plugins/integrations/prometheus/src/test/java/org/apache/cloudstack/metrics/PrometheusExporterImplTest.java index 40490c46f56e..c6e5afc6b95f 100644 --- a/plugins/integrations/prometheus/src/test/java/org/apache/cloudstack/metrics/PrometheusExporterImplTest.java +++ b/plugins/integrations/prometheus/src/test/java/org/apache/cloudstack/metrics/PrometheusExporterImplTest.java @@ -18,6 +18,15 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.util.Collections; + +import com.cloud.dc.dao.DataCenterDao; import org.junit.Test; @@ -105,4 +114,66 @@ public void testItemHostCertExpiryContainsTimestampValue() { assertTrue("Metric should contain correct timestamp value", metricsString.endsWith(" " + CERT_EXPIRY_EPOCH)); } + + /** + * Two rapid calls to updateMetrics() within the min refresh interval + * should result in only one actual recomputation (one call to dcDao.listAll()). + */ + @Test + public void testUpdateMetricsTTLGuardSkipsSecondCall() throws Exception { + PrometheusExporterImpl exporter = new PrometheusExporterImpl(); + + DataCenterDao mockDcDao = mock(DataCenterDao.class); + when(mockDcDao.listAll()).thenReturn(Collections.emptyList()); + setField(exporter, "dcDao", mockDcDao); + + // First call should trigger recomputation + exporter.updateMetrics(); + // Second immediate call should be skipped by the TTL guard + exporter.updateMetrics(); + + verify(mockDcDao, times(1)).listAll(); + } + + /** + * After the min refresh interval has elapsed, updateMetrics() should + * trigger a fresh recomputation. + */ + @Test + public void testUpdateMetricsTTLGuardAllowsAfterInterval() throws Exception { + PrometheusExporterImpl exporter = new PrometheusExporterImpl(); + + DataCenterDao mockDcDao = mock(DataCenterDao.class); + when(mockDcDao.listAll()).thenReturn(Collections.emptyList()); + setField(exporter, "dcDao", mockDcDao); + + // First call + exporter.updateMetrics(); + + // Simulate that the min interval has already elapsed by resetting lastMetricsUpdateTime + setField(exporter, "lastMetricsUpdateTime", 0L); + + // Second call should now trigger recomputation + exporter.updateMetrics(); + + verify(mockDcDao, times(2)).listAll(); + } + + private static void setField(Object target, String fieldName, Object value) throws Exception { + Field field = null; + Class clazz = target.getClass(); + while (clazz != null) { + try { + field = clazz.getDeclaredField(fieldName); + break; + } catch (NoSuchFieldException e) { + clazz = clazz.getSuperclass(); + } + } + if (field == null) { + throw new NoSuchFieldException(fieldName); + } + field.setAccessible(true); + field.set(target, value); + } }