feat(api): add prometheus metric coderd_workspace_builds_total (coder#6314)

johnstcn · web-flow · commit 43e8ba08113c · 2023-02-23T01:28:10.000Z
This PR adds the prometheus metric coderd_workspace_builds_total.
It measures the total number of workspace builds, along with a number of labels intended to be useful for an operator debugging a failed workspace build trying to discover the scope of the issue.
diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go
@@ -210,6 +210,8 @@ func (server *Server) AcquireJob(ctx context.Context, _ *proto.Empty) (*proto.Ac
 					WorkspaceOwnerEmail: owner.Email,
 					WorkspaceId:         workspace.ID.String(),
 					WorkspaceOwnerId:    owner.ID.String(),
+					TemplateName:        template.Name,
+					TemplateVersion:     templateVersion.Name,
 				},
 			},
 		}
diff --git a/coderd/provisionerdserver/provisionerdserver_test.go b/coderd/provisionerdserver/provisionerdserver_test.go
@@ -206,6 +206,8 @@ func TestAcquireJob(t *testing.T) {
 					WorkspaceOwnerEmail: user.Email,
 					WorkspaceId:         workspace.ID.String(),
 					WorkspaceOwnerId:    user.ID.String(),
+					TemplateName:        template.Name,
+					TemplateVersion:     version.Name,
 				},
 			},
 		})
diff --git a/docs/admin/prometheus.md b/docs/admin/prometheus.md
diff --git a/provisionerd/provisionerd.go b/provisionerd/provisionerd.go
@@ -157,6 +157,12 @@ func NewMetrics(reg prometheus.Registerer) Metrics {
 					60 * 60, // 1hr
 				},
 			}, []string{"provisioner", "status"}),
+			WorkspaceBuilds: auto.NewCounterVec(prometheus.CounterOpts{
+				Namespace: "coderd",
+				Subsystem: "", // Explicitly empty to make this a top-level metric.
+				Name:      "workspace_builds_total",
+				Help:      "The number of workspaces started, updated, or deleted.",
+			}, []string{"owner_email", "workspace_name", "template_name", "template_version", "action", "status"}),
 		},
 	}
 }
diff --git a/provisionerd/runner/runner.go b/provisionerd/runner/runner.go
@@ -77,6 +77,8 @@ type Metrics struct {
 	ConcurrentJobs *prometheus.GaugeVec
 	// JobTimings also counts the total amount of jobs.
 	JobTimings *prometheus.HistogramVec
+	// WorkspaceBuilds counts workspace build successes and failures.
+	WorkspaceBuilds *prometheus.CounterVec
 }
 
 type JobUpdater interface {
@@ -161,6 +163,16 @@ func (r *Runner) Run() {
 		}
 
 		concurrentGauge.Dec()
+		if build := r.job.GetWorkspaceBuild(); build != nil {
+			r.metrics.WorkspaceBuilds.WithLabelValues(
+				build.Metadata.WorkspaceOwnerEmail,
+				build.Metadata.WorkspaceName,
+				build.Metadata.TemplateName,
+				build.Metadata.TemplateVersion,
+				build.Metadata.WorkspaceTransition.String(),
+				status,
+			).Inc()
+		}
 		r.metrics.JobTimings.WithLabelValues(r.job.Provisioner, status).Observe(time.Since(start).Seconds())
 	}()
 
diff --git a/provisionersdk/proto/provisioner.pb.go b/provisionersdk/proto/provisioner.pb.go
diff --git a/provisionersdk/proto/provisioner.proto b/provisionersdk/proto/provisioner.proto
@@ -220,6 +220,8 @@ message Provision {
         string workspace_id = 5;
         string workspace_owner_id = 6;
         string workspace_owner_email = 7;
+        string template_name = 8;
+        string template_version = 9;
     }
 
     // Config represents execution configuration shared by both Plan and
diff --git a/scripts/metricsdocgen/metrics b/scripts/metricsdocgen/metrics
@@ -584,6 +584,11 @@ coderd_provisionerd_job_timings_seconds_count{provisioner="terraform",status="su
 # HELP coderd_provisionerd_jobs_current The number of currently running provisioner jobs.
 # TYPE coderd_provisionerd_jobs_current gauge
 coderd_provisionerd_jobs_current{provisioner="terraform"} 0
+# HELP coderd_workspace_builds_total The number of workspaces started, updated, or deleted.
+# TYPE coderd_workspace_builds_total counter
+coderd_workspace_builds_total{action="START",owner_email="admin@coder.com",status="failed",template_name="docker",template_version="gallant_wright0",workspace_name="test1"} 1
+coderd_workspace_builds_total{action="START",owner_email="admin@coder.com",status="success",template_name="docker",template_version="gallant_wright0",workspace_name="test1"} 1
+coderd_workspace_builds_total{action="STOP",owner_email="admin@coder.com",status="success",template_name="docker",template_version="gallant_wright0",workspace_name="test1"} 1
 # HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.
 # TYPE go_gc_duration_seconds summary
 go_gc_duration_seconds{quantile="0"} 2.4056e-05

Original file line number	Diff line number	Diff line change
`@@ -210,6 +210,8 @@ func (server Server) AcquireJob(ctx context.Context, _ proto.Empty) (*proto.Ac`
`210`	`210`	`WorkspaceOwnerEmail: owner.Email,`
`211`	`211`	`WorkspaceId: workspace.ID.String(),`
`212`	`212`	`WorkspaceOwnerId: owner.ID.String(),`
	`213`	`+ TemplateName: template.Name,`
	`214`	`+ TemplateVersion: templateVersion.Name,`
`213`	`215`	`},`
`214`	`216`	`},`
`215`	`217`	`}`
Original file line number	Diff line number	Diff line change
`@@ -157,6 +157,12 @@ func NewMetrics(reg prometheus.Registerer) Metrics {`
`157`	`157`	`60 * 60, // 1hr`
`158`	`158`	`},`
`159`	`159`	`}, []string{"provisioner", "status"}),`
	`160`	`+ WorkspaceBuilds: auto.NewCounterVec(prometheus.CounterOpts{`
	`161`	`+ Namespace: "coderd",`
	`162`	`+ Subsystem: "", // Explicitly empty to make this a top-level metric.`
	`163`	`+ Name: "workspace_builds_total",`
	`164`	`+ Help: "The number of workspaces started, updated, or deleted.",`
	`165`	`+ }, []string{"owner_email", "workspace_name", "template_name", "template_version", "action", "status"}),`
`160`	`166`	`},`
`161`	`167`	`}`
`162`	`168`	`}`
Original file line number	Diff line number	Diff line change
`@@ -220,6 +220,8 @@ message Provision {`
`220`	`220`	`string workspace_id = 5;`
`221`	`221`	`string workspace_owner_id = 6;`
`222`	`222`	`string workspace_owner_email = 7;`
	`223`	`+ string template_name = 8;`
	`224`	`+ string template_version = 9;`
`223`	`225`	`}`
`224`	`226`
`225`	`227`	`// Config represents execution configuration shared by both Plan and`