Skip to content

Commit d99949d

Browse files
johnstcnjohnstcn
authored
feat(scripts/develop): add --prometheus-server flag to run Prometheus UI (coder#24408)
Builds on coder#24389. Adds `--prometheus-server` flag that starts a `prom/prometheus:v3.11.2` container alongside the dev environment. > 🤖
1 parent 81bd78d commit d99949d

2 files changed

Lines changed: 391 additions & 49 deletions

File tree

‎scripts/develop/main.go‎

Lines changed: 242 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,18 @@ const (
4242
defaultAPIPort = "3000"
4343
defaultWebPort = "8080"
4444
defaultProxyPort = "3010"
45+
// prometheusServerPort is an int64 (not a string like the
46+
// user-facing defaults) because it has no corresponding CLI
47+
// flag; the Prometheus UI port is fixed at 9090.
48+
prometheusServerPort int64 = 9090
49+
// prometheusContainerName is the Docker container name for
50+
// the embedded Prometheus server, used for reuse detection
51+
// and explicit cleanup on shutdown.
52+
prometheusContainerName = "coder-prometheus"
4553
// defaultPrometheusPort avoids 2112 (agent prometheus) and
4654
// 2113 (agent debug) already bound inside Coder workspaces.
4755
defaultPrometheusPort = "2114"
56+
prometheusImage = "prom/prometheus:v3.11.2"
4857
defaultAccessURL = "http://127.0.0.1:%d"
4958
defaultPassword = "SomeSecurePassword!"
5059
defaultStarterTemplate = "docker"
@@ -85,7 +94,13 @@ func main() {
8594
Env: "CODER_DEV_PROMETHEUS_PORT",
8695
Default: defaultPrometheusPort,
8796
Description: "Prometheus metrics port. Set to 0 to disable.",
88-
Value: serpent.Int64Of(&cfg.prometheusPort),
97+
Value: serpent.Int64Of(&cfg.coderMetricsPort),
98+
},
99+
{
100+
Flag: "prometheus-server",
101+
Env: "CODER_DEV_PROMETHEUS_SERVER",
102+
Description: "Run a Prometheus server to scrape and visualize metrics. Requires Docker. Linux only.",
103+
Value: serpent.BoolOf(&cfg.prometheusServer),
89104
},
90105
{
91106
Flag: "agpl",
@@ -170,24 +185,25 @@ func main() {
170185
}
171186

172187
type devConfig struct {
173-
apiPort int64
174-
webPort int64
175-
proxyPort int64
176-
prometheusPort int64
177-
agpl bool
178-
accessURL string
179-
password string
180-
useProxy bool
181-
multiOrg bool
182-
debug bool
183-
starterTemplate string
184-
dbRollback bool
185-
dbReset bool
186-
dbContinue bool
187-
projectRoot string
188-
binaryPath string
189-
configDir string
190-
childEnv []string
188+
apiPort int64
189+
webPort int64
190+
proxyPort int64
191+
coderMetricsPort int64
192+
prometheusServer bool
193+
agpl bool
194+
accessURL string
195+
password string
196+
useProxy bool
197+
multiOrg bool
198+
debug bool
199+
starterTemplate string
200+
dbRollback bool
201+
dbReset bool
202+
dbContinue bool
203+
projectRoot string
204+
binaryPath string
205+
configDir string
206+
childEnv []string
191207
// Extra args after flags forwarded to "coder server".
192208
serverExtraArgs []string
193209
}
@@ -217,7 +233,7 @@ func (c *devConfig) validate() error {
217233
return xerrors.Errorf("%s must be between 1 and 65535", p.name)
218234
}
219235
}
220-
if c.prometheusPort < 0 || c.prometheusPort > 65535 {
236+
if c.coderMetricsPort < 0 || c.coderMetricsPort > 65535 {
221237
return xerrors.Errorf("--prometheus-port must be 0 (disabled) or between 1 and 65535")
222238
}
223239
if c.apiPort == c.webPort {
@@ -229,15 +245,39 @@ func (c *devConfig) validate() error {
229245
if c.useProxy && c.webPort == c.proxyPort {
230246
return xerrors.Errorf("--web-port %d conflicts with --proxy-port", c.webPort)
231247
}
232-
if c.prometheusPort != 0 {
233-
if c.prometheusPort == c.apiPort {
234-
return xerrors.Errorf("--prometheus-port %d conflicts with API server", c.prometheusPort)
248+
if c.coderMetricsPort != 0 {
249+
if c.coderMetricsPort == c.apiPort {
250+
return xerrors.Errorf("--prometheus-port %d conflicts with API server", c.coderMetricsPort)
251+
}
252+
if c.coderMetricsPort == c.webPort {
253+
return xerrors.Errorf("--prometheus-port %d conflicts with frontend dev server", c.coderMetricsPort)
254+
}
255+
if c.useProxy && c.coderMetricsPort == c.proxyPort {
256+
return xerrors.Errorf("--prometheus-port %d conflicts with workspace proxy", c.coderMetricsPort)
257+
}
258+
}
259+
if c.prometheusServer && c.coderMetricsPort == 0 {
260+
return xerrors.New("--prometheus-server requires prometheus to be enabled (--prometheus-port != 0)")
261+
}
262+
if c.prometheusServer {
263+
conflicts := []struct {
264+
flag string
265+
val int64
266+
}{
267+
{"--port", c.apiPort},
268+
{"--web-port", c.webPort},
269+
{"--prometheus-port", c.coderMetricsPort},
235270
}
236-
if c.prometheusPort == c.webPort {
237-
return xerrors.Errorf("--prometheus-port %d conflicts with frontend dev server", c.prometheusPort)
271+
if c.useProxy {
272+
conflicts = append(conflicts, struct {
273+
flag string
274+
val int64
275+
}{"--proxy-port", c.proxyPort})
238276
}
239-
if c.useProxy && c.prometheusPort == c.proxyPort {
240-
return xerrors.Errorf("--prometheus-port %d conflicts with workspace proxy", c.prometheusPort)
277+
for _, conflict := range conflicts {
278+
if prometheusServerPort == conflict.val {
279+
return xerrors.Errorf("%s %d conflicts with prometheus server", conflict.flag, conflict.val)
280+
}
241281
}
242282
}
243283
return nil
@@ -462,7 +502,17 @@ func develop(ctx context.Context, logger slog.Logger, cfg *devConfig) error {
462502
}
463503
}
464504

465-
printBanner(ctx, logger, cfg)
505+
var prometheusServerStarted bool
506+
if cfg.prometheusServer {
507+
started, err := startPrometheusServer(ctx, logger, cfg)
508+
if err != nil {
509+
logger.Warn(ctx, "prometheus server setup failed, continuing",
510+
slog.Error(err))
511+
}
512+
prometheusServerStarted = started
513+
}
514+
515+
printBanner(ctx, logger, cfg, prometheusServerStarted)
466516

467517
// Block until a signal fires or a child process exits.
468518
<-ctx.Done()
@@ -506,8 +556,8 @@ func preflight(ctx context.Context, logger slog.Logger, cfg *devConfig) error {
506556
if cfg.useProxy && isPortBusy(ctx, cfg.proxyPort) {
507557
return xerrors.Errorf("port %d is already in use (proxy)", cfg.proxyPort)
508558
}
509-
if cfg.prometheusPort != 0 && isPortBusy(ctx, cfg.prometheusPort) {
510-
return xerrors.Errorf("port %d is already in use (prometheus)", cfg.prometheusPort)
559+
if cfg.coderMetricsPort != 0 && isPortBusy(ctx, cfg.coderMetricsPort) {
560+
return xerrors.Errorf("port %d is already in use (prometheus)", cfg.coderMetricsPort)
511561
}
512562
return nil
513563
}
@@ -541,10 +591,10 @@ func startServer(cfg *devConfig, group *procGroup) error {
541591
"--dangerous-allow-cors-requests=true",
542592
"--enable-terraform-debug-mode",
543593
}
544-
if cfg.prometheusPort != 0 {
594+
if cfg.coderMetricsPort != 0 {
545595
serverArgs = append(serverArgs,
546596
"--prometheus-enable",
547-
"--prometheus-address", fmt.Sprintf("0.0.0.0:%d", cfg.prometheusPort),
597+
"--prometheus-address", fmt.Sprintf("0.0.0.0:%d", cfg.coderMetricsPort),
548598
"--prometheus-collect-agent-stats",
549599
"--prometheus-collect-db-metrics",
550600
)
@@ -896,6 +946,147 @@ func createTemplateInOrg(ctx context.Context, logger slog.Logger, client *coders
896946
return nil
897947
}
898948

949+
// startPrometheusServer runs the official Prometheus Docker image
950+
// with a generated config that scrapes the local Coder metrics
951+
// endpoint. It uses --net=host so the container can reach the
952+
// host-bound metrics port directly. Only supported on Linux;
953+
// returns false without error on other platforms.
954+
// Returns true if the server was started or is already running.
955+
func startPrometheusServer(ctx context.Context, logger slog.Logger, cfg *devConfig) (bool, error) {
956+
if runtime.GOOS != "linux" {
957+
logger.Warn(ctx, "prometheus server is only supported on Linux, skipping",
958+
slog.F("os", runtime.GOOS))
959+
return false, nil
960+
}
961+
962+
// Verify Docker is available before attempting anything.
963+
if err := exec.CommandContext(ctx, "docker", "info").Run(); err != nil {
964+
logger.Warn(ctx, "docker not available, skipping prometheus server",
965+
slog.Error(err))
966+
return false, nil
967+
}
968+
969+
// If the port is already in use, check whether it's our
970+
// container from a previous run. If so, reuse it.
971+
if isPortBusy(ctx, prometheusServerPort) {
972+
out, err := exec.CommandContext(ctx, "docker", "inspect",
973+
"-f", "{{.State.Running}}",
974+
prometheusContainerName).Output()
975+
if err == nil && strings.TrimSpace(string(out)) == "true" {
976+
logger.Info(ctx, "reusing existing prometheus server",
977+
slog.F("ui", fmt.Sprintf("http://localhost:%d", prometheusServerPort)),
978+
slog.F("note", fmt.Sprintf("scrape target may differ from current --prometheus-port %d; restart to apply", cfg.coderMetricsPort)))
979+
return true, nil
980+
}
981+
logger.Info(ctx, "prometheus server port already in use, skipping",
982+
slog.F("port", prometheusServerPort))
983+
return false, nil
984+
}
985+
986+
// Remove any stopped leftover container from a previous run.
987+
// Failure is fine; it just means the container doesn't exist.
988+
rmCmd := exec.CommandContext(ctx, "docker", "rm", "-f", prometheusContainerName) //nolint:gosec
989+
rmCmd.Stdout = nil
990+
rmCmd.Stderr = nil
991+
_ = rmCmd.Run()
992+
993+
// Persist TSDB data across dev environment restarts. The
994+
// container runs as nobody (UID 65534), so the directory must
995+
// be world-writable. os.MkdirAll applies the umask, so we
996+
// chmod explicitly after creation.
997+
prometheusDataDir := filepath.Join(cfg.configDir, "prometheus")
998+
if err := os.MkdirAll(prometheusDataDir, 0o777); err != nil {
999+
return false, xerrors.Errorf("creating prometheus data directory: %w", err)
1000+
}
1001+
if err := os.Chmod(prometheusDataDir, 0o777); err != nil {
1002+
return false, xerrors.Errorf("chmod prometheus data directory: %w", err)
1003+
}
1004+
1005+
// Write a minimal scrape config to a temp file.
1006+
promCfg := fmt.Sprintf(`global:
1007+
scrape_interval: 15s
1008+
1009+
scrape_configs:
1010+
- job_name: coder
1011+
scheme: http
1012+
static_configs:
1013+
- targets: ["127.0.0.1:%d"]
1014+
`, cfg.coderMetricsPort)
1015+
1016+
tmpFile, err := os.CreateTemp("", "coder-prometheus-*.yml")
1017+
if err != nil {
1018+
return false, xerrors.Errorf("creating prometheus config: %w", err)
1019+
}
1020+
// Stop the container and remove the temp file when the context is
1021+
// done. The stop must happen before the file removal so Prometheus
1022+
// is not holding the bind mount open when we delete the source.
1023+
// Registering this cleanup immediately after CreateTemp means every
1024+
// later failure path can simply return without its own cleanup call.
1025+
context.AfterFunc(ctx, func() {
1026+
stopCmd := exec.Command("docker", "stop", "-t", "5", prometheusContainerName) //nolint:gosec
1027+
stopCmd.Stdout = nil
1028+
stopCmd.Stderr = nil
1029+
_ = stopCmd.Run()
1030+
_ = os.Remove(tmpFile.Name())
1031+
})
1032+
1033+
if _, err := tmpFile.WriteString(promCfg); err != nil {
1034+
_ = tmpFile.Close()
1035+
return false, xerrors.Errorf("writing prometheus config: %w", err)
1036+
}
1037+
_ = tmpFile.Close()
1038+
1039+
// The Prometheus container runs as nobody, so the file must be
1040+
// world-readable. os.CreateTemp creates files with mode 0600.
1041+
if err := os.Chmod(tmpFile.Name(), 0o644); err != nil {
1042+
return false, xerrors.Errorf("chmod prometheus config: %w", err)
1043+
}
1044+
1045+
cmd := exec.CommandContext(ctx, "docker", "run", //nolint:gosec // args are all controlled constants or our own temp file path
1046+
"--rm",
1047+
"--name", prometheusContainerName,
1048+
"--net=host",
1049+
"-v", tmpFile.Name()+":/etc/prometheus/prometheus.yml:ro",
1050+
"-v", prometheusDataDir+":/prometheus",
1051+
prometheusImage,
1052+
"--config.file=/etc/prometheus/prometheus.yml",
1053+
fmt.Sprintf("--web.listen-address=0.0.0.0:%d", prometheusServerPort),
1054+
)
1055+
1056+
named := logger.Named("prometheus")
1057+
w := &logWriter{logger: named}
1058+
cmd.Stdout = w
1059+
cmd.Stderr = w
1060+
1061+
named.Info(ctx, "starting prometheus server",
1062+
slog.F("image", prometheusImage),
1063+
slog.F("scrape_target", fmt.Sprintf("127.0.0.1:%d", cfg.coderMetricsPort)),
1064+
slog.F("ui", fmt.Sprintf("http://localhost:%d", prometheusServerPort)),
1065+
)
1066+
1067+
if err := cmd.Start(); err != nil {
1068+
return false, xerrors.Errorf("starting prometheus container: %w", err)
1069+
}
1070+
1071+
// Wait for the container in a separate goroutine. Prometheus is
1072+
// optional, so if it dies we just log a warning rather than
1073+
// tearing down the entire dev environment.
1074+
go func() {
1075+
if err := cmd.Wait(); err != nil {
1076+
if ctx.Err() != nil {
1077+
// Normal shutdown: context was canceled.
1078+
named.Info(ctx, "prometheus server stopped")
1079+
return
1080+
}
1081+
named.Warn(ctx, "prometheus server exited", slog.Error(err))
1082+
} else {
1083+
named.Warn(ctx, "prometheus server exited unexpectedly")
1084+
}
1085+
}()
1086+
1087+
return true, nil
1088+
}
1089+
8991090
func pnpmCmd(ctx context.Context, cfg *devConfig) *exec.Cmd {
9001091
cmd := cfg.cmd(ctx, "pnpm", "--dir", "./site", "dev", "--host")
9011092
cmd.Env = append(cmd.Env,
@@ -905,7 +1096,22 @@ func pnpmCmd(ctx context.Context, cfg *devConfig) *exec.Cmd {
9051096
return cmd
9061097
}
9071098

908-
func printBanner(ctx context.Context, logger slog.Logger, cfg *devConfig) {
1099+
// prometheusBannerEntry decides which (if any) prometheus-related URL
1100+
// the dev banner should advertise. When the embedded Prometheus server
1101+
// is running we prefer its UI; otherwise fall back to the raw metrics
1102+
// endpoint. Returns an empty label when metrics are disabled entirely.
1103+
func prometheusBannerEntry(cfg *devConfig, prometheusServerStarted bool) (label string, port int64) {
1104+
switch {
1105+
case prometheusServerStarted:
1106+
return "Prometheus UI:", prometheusServerPort
1107+
case cfg.coderMetricsPort != 0:
1108+
return "Metrics:", cfg.coderMetricsPort
1109+
default:
1110+
return "", 0
1111+
}
1112+
}
1113+
1114+
func printBanner(ctx context.Context, logger slog.Logger, cfg *devConfig, prometheusServerStarted bool) {
9091115
ifaces := []string{"localhost"}
9101116
if addrs, err := net.InterfaceAddrs(); err == nil {
9111117
for _, addr := range addrs {
@@ -960,13 +1166,13 @@ func printBanner(ctx context.Context, logger slog.Logger, cfg *devConfig) {
9601166
line(indent(fmt.Sprintf("http://%s:%d", h, cfg.proxyPort)))
9611167
}
9621168
}
963-
if cfg.prometheusPort != 0 {
1169+
if label, port := prometheusBannerEntry(cfg, prometheusServerStarted); label != "" {
9641170
line(
9651171
"",
966-
"Metrics:",
1172+
label,
9671173
)
9681174
for _, h := range ifaces {
969-
line(indent(fmt.Sprintf("http://%s:%d", h, cfg.prometheusPort)))
1175+
line(indent(fmt.Sprintf("http://%s:%d", h, port)))
9701176
}
9711177
}
9721178
line(

0 commit comments

Comments
 (0)