send nightly builds statistics to monitoring #1335

UnitTestBot · SBOne-Kenobi · Nov 24, 2022 · Nov 11, 2022 · Nov 14, 2022 · Nov 14, 2022
commit 1692824ed95de78d06c531b05e97e428dba369f8
diff --git a/.github/workflows/collect-statistics.yml b/.github/workflows/collect-statistics.yml
@@ -62,6 +62,9 @@ env:
   data_path: monitoring/data
   monitoring_properties: monitoring/monitoring.properties
   push_script: monitoring/push_with_rebase.sh
+  PUSHGATEWAY_HOSTNAME: monitoring.utbot.org
+  PUSHGATEWAY_ADDITIONAL_PATH: /pushgateway-custom
+  PROM_ADDITIONAL_LABELS: /service/github
 
 jobs:
   setup_matrix:
@@ -184,7 +187,8 @@ jobs:
         if: ${{ inputs.send_to_grafana }}
         run: |
           python monitoring/prepare_metrics.py --stats_file $stats_file --output_file grafana_metrics.json
-          echo "TODO send metrics to grafana"
+          chmod +x scripts/project/json_to_prometheus.py
+          python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/nightly-statistics${PROM_ADDITIONAL_LABELS}
         env:
           stats_file: ${{ steps.insert.outputs.output }}
 

diff --git a/scripts/project/json_to_prometheus.py b/scripts/project/json_to_prometheus.py
@@ -0,0 +1,36 @@
+import sys
+import json
+
+with open(sys.argv[1]) as metrics_raw:
+    metrics_json = json.load(metrics_raw)
+
+# metrics is a json list e.g.:
+#     [
+#         {
+#             "metric": "total_classes",
+#             "labels": {
+#                 "project": "guava",
+#                 "fuzzing_ratio": 0.1
+#             },
+#             "value": 20
+#         },
+#         {
+#             "metric": "testcases_generated",
+#             "labels": {
+#                 "project": "guava",
+#                 "fuzzing_ratio": 0.1
+#             },
+#             "value": 1042
+#         }
+#     ]
+#
+# the loop below iterates over each list item and constructs metrics set
+metrics_set_str = ""
+for metric in metrics_json:
+    labels_set_str = ""
+    comma = ""
+    for label, value in metric['labels'].items():
+        labels_set_str = f'{labels_set_str}{comma}{label}=\"{value}\"'
+        comma = ","
+    metrics_set_str += f'{metric["metric"]}{{{labels_set_str}}} {metric["value"]}\n'
+print(metrics_set_str)