diff --git a/CHANGELOG.md b/CHANGELOG.md index e5140b06..3a343de0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [2.3.2](https://github.com/google/slo-generator/compare/v2.3.1...v2.3.2) (2022-10-29) + + +### Bug Fixes + +* remove calls to list and create metric descriptors in Cloud Monitoring exporter to prevent Quota Exceeded errors ([#286](https://github.com/google/slo-generator/issues/286)) ([0a6a0fb](https://github.com/google/slo-generator/commit/0a6a0fb75d6c83deddbf81288c4d020e22bbd6d5)) + + +### Documentation + +* update docs and samples with all backends ([#283](https://github.com/google/slo-generator/issues/283)) ([61f2f32](https://github.com/google/slo-generator/commit/61f2f3291671fbfc1afc607255c1366f90c55b98)) + ## [2.3.1](https://github.com/google/slo-generator/compare/v2.3.0...v2.3.1) (2022-10-27) diff --git a/samples/README.md b/samples/README.md index 201181fe..b2732c69 100644 --- a/samples/README.md +++ b/samples/README.md @@ -1,98 +1,152 @@ # SLO Library -This folder is an SLO library to facilitate writing new SLOs by starting from -already written SLO configurations. - -All samples are classified into a folder named after their respective backend -or exporter class. - -Each sample contains environmental variables that should be set prior to -running it. - -## Environmental variables - -The following is listing all environmental variables found in the SLO configs, -per backend: - -`cloud_monitoring/`: - - `WORKSPACE_PROJECT_ID`: Cloud Monitoring host project id. - - `LOG_METRIC_NAME`: Cloud Logging log-based metric name. - - `GAE_PROJECT_ID`: Google App Engine application project id. - - `GAE_MODULE_ID`: Google App Engine application module id. - - `PUBSUB_PROJECT_ID`: Pub/Sub project id. - - `PUBSUB_TOPIC_NAME`: Pub/Sub topic name. - -`cloud_service_monitoring/`: - - `WORKSPACE_PROJECT_ID`: Cloud Monitoring host project id. - - `LOG_METRIC_NAME`: Cloud Logging log-based metric name. - - `GAE_PROJECT_ID`: Google App Engine application project id. - - `GAE_MODULE_ID`: Google App Engine application module id. - - `PUBSUB_PROJECT_ID`: Pub/Sub project id. - - `PUBSUB_TOPIC_NAME`: Pub/Sub topic name. - - `GKE_PROJECT_ID`: GKE project id. - - `GKE_LOCATION`: GKE location. - - `GKE_CLUSTER_NAME`: GKE cluster name. - - `GKE_SERVICE_NAMESPACE`: GKE service namespace. - - `GKE_SERVICE_NAME`: GKE service name. - -`elasticsearch/`: - - `ELASTICSEARCH_URL`: ElasticSearch instance URL. - -`prometheus/`: - - `PROMETHEUS_URL`: Prometheus instance URL. - - `PROMETHEUS_PUSHGATEWAY_URL`: Prometheus Pushgateway instance URL. - -You can either set those variables for the backends you want to try, or set all -of those in an `.env` file and then `source` it. Note that the actual GCP resources -you're pointing to need to exist. +This folder is an SLO library to facilitate writing new SLOs by starting from already written SLO configurations. + +All samples are classified within folders named after their respective backend or exporter class. + +Each sample references environment variables that must be set prior to running it. + +## Environment variables + +The following is listing all environment variables found in the SLO configs, per backend: + +You can either set those variables for the backends you want to try, or set all of those in an `.env` file and then `source` it. Note that the actual GCP resources you're pointing to need to exist. + +### `cloud_monitoring` + +| Environment variable | Description | +| --- | --- | +| `WORKSPACE_PROJECT_ID` | Cloud Monitoring host project ID | +| `LOG_METRIC_NAME` | Cloud Logging log-based metric name | +| `GAE_PROJECT_ID` | Google App Engine application project ID | +| `GAE_MODULE_ID` | Google App Engine application module ID | +| `PUBSUB_PROJECT_ID` | Pub/Sub project ID | +| `PUBSUB_TOPIC_NAME` | Pub/Sub topic name | + +### `cloud_monitoring_mql` + +| Environment variable | Description | +| --- | --- | +| `WORKSPACE_PROJECT_ID` | Cloud Monitoring host project ID | +| `LOG_METRIC_NAME` | Cloud Logging log-based metric name | +| `GAE_PROJECT_ID` | Google App Engine application project ID | +| `GAE_MODULE_ID` | Google App Engine application module ID | +| `PUBSUB_PROJECT_ID` | Pub/Sub project ID | +| `PUBSUB_TOPIC_NAME` | Pub/Sub topic name | + +### `cloud_service_monitoring` + +| Environment variable | Description | +| --- | --- | +| `WORKSPACE_PROJECT_ID` | Cloud Monitoring host project ID | +| `LOG_METRIC_NAME` | Cloud Logging log-based metric name | +| `GAE_PROJECT_ID` | Google App Engine application project ID | +| `GAE_MODULE_ID` | Google App Engine application module ID | +| `PUBSUB_PROJECT_ID` | Pub/Sub project ID | +| `PUBSUB_TOPIC_NAME` | Pub/Sub topic name | +| `GKE_PROJECT_ID` | GKE project ID | +| `GKE_LOCATION` | GKE location | +| `GKE_CLUSTER_NAME` | GKE cluster name | +| `GKE_SERVICE_NAMESPACE` | GKE service namespace | +| `GKE_SERVICE_NAME` | GKE service name | + +### `datadog` + +| Environment variable | Description | +| --- | --- | +| `DATADOG_API_KEY` | Datadog API key | +| `DATADOG_APP_KEY` | Datadog APP key | + +### `dynatrace` + +| Environment variable | Description | +| --- | --- | +| `DYNATRACE_API_URL` | Dynatrace API URL | +| `DYNATRACE_API_TOKEN` | Dynatrace API token | + +### `elasticsearch` + +| Environment variable | Description | +| --- | --- | +| `ELASTICSEARCH_URL` | ElasticSearch instance URL | + +### `prometheus` + +| Environment variable | Description | +| --- | --- | +| `PROMETHEUS_URL` | Prometheus instance URL | +| `PROMETHEUS_PUSHGATEWAY_URL` | Prometheus Pushgateway instance URL | ## Running the samples To run one sample: -``` + +```sh slo-generator -f samples/cloud_monitoring/.yaml ``` To run all the samples for a backend: -``` +```sh slo-generator -f samples/ -b samples/ ``` + *where:* -* `` is the backend name (lowercase) -* `` is the path to the error budget policy YAML file. -***Note:*** *if you want to enable the exporters as well, you can add the -`--export` flag.* +- `` is the backend name (lowercase) +- `` is the path to the error budget policy YAML file. +***Note:*** *if you want to enable the exporters as well, you can add the `--export` flag.* ### Examples -##### Cloud Monitoring -``` +#### Cloud Monitoring (MQF) + +```sh slo-generator -f samples/cloud_monitoring -b error_budget_policy.yaml ``` -##### Cloud Service Monitoring +#### Cloud Monitoring (MQL) + +```sh +slo-generator -f samples/cloud_monitoring_mql -b error_budget_policy.yaml ``` + +#### Cloud Service Monitoring + +```sh slo-generator -f samples/cloud_service_monitoring -b error_budget_policy_ssm.yaml ``` -***Note:*** *the Error Budget Policy is different for this backend, because it only -supports steps where the `window` is a multiple of 24 hours.* +***Note:*** *the Error Budget Policy is different for this backend, because it only supports steps where `window` is a multiple of 24 hours.* -##### Prometheus -``` -slo-generator -f samples/prometheus -b error_budget_policy.yaml +#### Datadog + +```sh +slo-generator -f samples/datadog -b error_budget_policy.yaml ``` -##### Elasticsearch +#### Dynatrace + +```sh +slo-generator -f samples/dynatrace -b error_budget_policy.yaml ``` + +#### Elasticsearch + +```sh slo-generator -f samples/elasticsearch -b error_budget_policy.yaml ``` -##### Custom Class +#### Prometheus + +```sh +slo-generator -f samples/prometheus -b error_budget_policy.yaml ``` + +#### Custom Class + +```sh cd samples/ slo-generator -f custom -b error_budget_policy.yaml -e ``` diff --git a/samples/config.yaml b/samples/config.yaml index 9ef03fc8..5978cdad 100644 --- a/samples/config.yaml +++ b/samples/config.yaml @@ -4,6 +4,8 @@ default_exporters: [cloudevent] backends: cloud_monitoring: project_id: ${STACKDRIVER_HOST_PROJECT_ID} + cloud_monitoring_mql: + project_id: ${STACKDRIVER_HOST_PROJECT_ID} cloud_service_monitoring: project_id: ${STACKDRIVER_HOST_PROJECT_ID} samples.custom.custom_backend.CustomBackend: {} diff --git a/setup.cfg b/setup.cfg index da3ee1ff..d1df59a6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,7 @@ [metadata] name = slo-generator -version = 2.3.1 +version = 2.3.2 author = Google Inc. author_email = olivier.cervello@gmail.com maintainer = Laurent VAYLET diff --git a/slo_generator/exporters/cloud_monitoring.py b/slo_generator/exporters/cloud_monitoring.py index f45a803e..71e49e98 100644 --- a/slo_generator/exporters/cloud_monitoring.py +++ b/slo_generator/exporters/cloud_monitoring.py @@ -17,8 +17,6 @@ """ import logging -import google.api_core.exceptions -from google.api import metric_pb2 as ga_metric from google.cloud import monitoring_v3 from .base import MetricsExporter @@ -36,8 +34,7 @@ def __init__(self): self.client = monitoring_v3.MetricServiceClient() def export_metric(self, data: dict): - """Export metric to Cloud Monitoring. Create metric descriptor if - it doesn't exist. + """Export metric to Cloud Monitoring. Args: data (dict): Data to send to Cloud Monitoring. @@ -45,8 +42,6 @@ def export_metric(self, data: dict): Returns: object: Cloud Monitoring API result. """ - if not self.get_metric_descriptor(data): - self.create_metric_descriptor(data) self.create_timeseries(data) def create_timeseries(self, data: dict): @@ -101,44 +96,3 @@ def create_timeseries(self, data: dict): f"{labels['slo_name']}-{labels['error_budget_policy_step_name']}" ) # pylint: enable=E1101 - - def get_metric_descriptor(self, data: dict): - """Get Cloud Monitoring metric descriptor. - - Args: - data (dict): Metric data. - - Returns: - object: Metric descriptor (or None if not found). - """ - project_id = data["project_id"] - metric_id = data["name"] - request = monitoring_v3.GetMetricDescriptorRequest( - name=f"projects/{project_id}/metricDescriptors/{metric_id}" - ) - try: - return self.client.get_metric_descriptor(request) - except google.api_core.exceptions.NotFound: - return None - - def create_metric_descriptor(self, data: dict): - """Create Cloud Monitoring metric descriptor. - - Args: - data (dict): Metric data. - - Returns: - object: Metric descriptor. - """ - project = self.client.common_project_path(data["project_id"]) - descriptor = ga_metric.MetricDescriptor() - descriptor.type = data["name"] - # pylint: disable=E1101 - descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE - descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE - # pylint: enable=E1101 - descriptor.description = data["description"] - descriptor = self.client.create_metric_descriptor( - name=project, metric_descriptor=descriptor - ) - return descriptor